From 075e95cc8f237f75696435e3e6cf51563999aed8 Mon Sep 17 00:00:00 2001 From: Ruinong Tian Date: Tue, 24 Dec 2024 21:02:12 +0000 Subject: [PATCH] reduce RUN commands in Dockerfile --- build_artifacts/v1/v1.10/v1.10.2/Dockerfile | 85 +++++++++----------- template/v1/Dockerfile | 86 +++++++++------------ 2 files changed, 73 insertions(+), 98 deletions(-) diff --git a/build_artifacts/v1/v1.10/v1.10.2/Dockerfile b/build_artifacts/v1/v1.10/v1.10.2/Dockerfile index ea41ae69..d0134b25 100644 --- a/build_artifacts/v1/v1.10/v1.10.2/Dockerfile +++ b/build_artifacts/v1/v1.10/v1.10.2/Dockerfile @@ -48,21 +48,17 @@ RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/pr # CodeEditor - create server, user data dirs RUN mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ - && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data - -# create dir to store user data files -RUN mkdir -p /opt/amazon/sagemaker/user-data \ - && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/user-data - - -# Merge in OS directory tree contents. -RUN mkdir -p ${DIRECTORY_TREE_STAGE_DIR} + && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data && \ + # create dir to store user data files + mkdir -p /opt/amazon/sagemaker/user-data \ + && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/user-data && \ + # Merge in OS directory tree contents. + mkdir -p ${DIRECTORY_TREE_STAGE_DIR} COPY dirs/ ${DIRECTORY_TREE_STAGE_DIR}/ RUN rsync -a ${DIRECTORY_TREE_STAGE_DIR}/ / && \ - rm -rf ${DIRECTORY_TREE_STAGE_DIR} - -# CodeEditor - download the extensions -RUN mkdir -p /etc/code-editor/extensions && \ + rm -rf ${DIRECTORY_TREE_STAGE_DIR} && \ + # CodeEditor - download the extensions + mkdir -p /etc/code-editor/extensions && \ while IFS= read -r url || [ -n "$url" ]; do \ echo "Downloading extension from ${url}..." && \ wget --no-check-certificate -P /etc/code-editor/extensions "${url}"; \ @@ -85,13 +81,10 @@ RUN micromamba install -y --name base --file /tmp/$ENV_IN_FILENAME && \ ARG MAMBA_DOCKERFILE_ACTIVATE=1 -RUN sudo ln -s $(which python3) /usr/bin/python - -# Update npm version -RUN npm update -g npm - -# Configure CodeEditor - Install extensions and set preferences -RUN \ +RUN sudo ln -s $(which python3) /usr/bin/python && \ + # Update npm version + npm update -g npm && \ + # Configure CodeEditor - Install extensions and set preferences extensionloc=/opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions && mkdir -p "${extensionloc}" \ # Loop through all vsix files in /etc/code-editor/extensions and install them && for ext in /etc/code-editor/extensions/*.vsix; do \ @@ -99,11 +92,10 @@ RUN \ sagemaker-code-editor --install-extension "${ext}" --extensions-dir "${extensionloc}" --server-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data; \ done \ # Copy the settings - && cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json - -# Install glue kernels, and move to shared directory -# Also patching base kernel so Studio background code doesn't start session silently -RUN install-glue-kernels && \ + && cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json && \ + # Install glue kernels, and move to shared directory + # Also patching base kernel so Studio background code doesn't start session silently + install-glue-kernels && \ SITE_PACKAGES=$(pip show aws-glue-sessions | grep Location | awk '{print $2}') && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_pyspark --user && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_spark --user && \ @@ -129,24 +121,20 @@ RUN HOME_DIR="/home/${NB_USER}/licenses" \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} python \ - && rm -rf ${HOME_DIR}/oss_compliance* - -# Create logging directories for supervisor -RUN mkdir -p $SAGEMAKER_LOGGING_DIR && \ + && rm -rf ${HOME_DIR}/oss_compliance* && \ + # Create logging directories for supervisor + mkdir -p $SAGEMAKER_LOGGING_DIR && \ chmod a+rw $SAGEMAKER_LOGGING_DIR && \ mkdir -p ${STUDIO_LOGGING_DIR} && \ - chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} - -# Clean up CodeEditor artifacts -RUN rm -rf /etc/code-editor - -# Create supervisord runtime directory -RUN mkdir -p /var/run/supervisord && \ - chmod a+rw /var/run/supervisord - -# Create root directory for DB -# Create logging directories for supervisor -RUN mkdir -p $DB_ROOT_DIR && \ + chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} && \ + # Clean up CodeEditor artifacts + rm -rf /etc/code-editor && \ + # Create supervisord runtime directory + mkdir -p /var/run/supervisord && \ + chmod a+rw /var/run/supervisord && \ + # Create root directory for DB + # Create logging directories for supervisor + mkdir -p $DB_ROOT_DIR && \ chmod a+rw $DB_ROOT_DIR USER $MAMBA_USER @@ -171,10 +159,10 @@ RUN INSTALLED_SSL=$(micromamba list | grep openssl | tr -s ' ' | cut -d ' ' -f 3 cp ../openssl-$FIPS_VALIDATED_SSL/providers/fipsmodule.cnf providers/. && \ make tests && cd ../openssl-$FIPS_VALIDATED_SSL && \ # After tests pass, install FIPS provider and remove source code - make install_fips && cd .. && rm -rf ./openssl-* -# Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS -# e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf -RUN cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ + make install_fips && cd .. && rm -rf ./openssl-* && \ + # Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS + # e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf + cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ sed -i "s:# .include fipsmodule.cnf:.include /opt/conda/ssl/fipsmodule.cnf:" /opt/conda/ssl/openssl-fips.cnf && \ sed -i 's:# fips = fips_sect:fips = fips_sect:' /opt/conda/ssl/openssl-fips.cnf ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ @@ -182,10 +170,9 @@ ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ # Install Kerberos. # Make sure no dependency is added/updated RUN pip install "krb5>=0.5.1,<0.6" && \ - pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' - -# https://stackoverflow.com/questions/122327 -RUN SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ + pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' && \ + # https://stackoverflow.com/questions/122327 + SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ # Remove SparkRKernel as it's not supported \ jupyter-kernelspec remove -f -y sparkrkernel && \ # Patch Sparkmagic lib to support Custom Certificates \ diff --git a/template/v1/Dockerfile b/template/v1/Dockerfile index 3519e830..1bab96b9 100644 --- a/template/v1/Dockerfile +++ b/template/v1/Dockerfile @@ -46,14 +46,12 @@ RUN apt-get update && apt-get upgrade -y && \ sudo ./aws/install && \ rm -rf aws awscliv2.zip && \ : -RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile - -# CodeEditor - create server, user data dirs -RUN mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ - && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data - -# create dir to store user data files -RUN mkdir -p /opt/amazon/sagemaker/user-data \ +RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile && \ + # CodeEditor - create server, user data dirs + mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \ + && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data && \ + # create dir to store user data files + mkdir -p /opt/amazon/sagemaker/user-data \ && chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/user-data @@ -61,10 +59,9 @@ RUN mkdir -p /opt/amazon/sagemaker/user-data \ RUN mkdir -p ${DIRECTORY_TREE_STAGE_DIR} COPY dirs/ ${DIRECTORY_TREE_STAGE_DIR}/ RUN rsync -a ${DIRECTORY_TREE_STAGE_DIR}/ / && \ - rm -rf ${DIRECTORY_TREE_STAGE_DIR} - -# CodeEditor - download the extensions -RUN mkdir -p /etc/code-editor/extensions && \ + rm -rf ${DIRECTORY_TREE_STAGE_DIR} && \ + # CodeEditor - download the extensions + mkdir -p /etc/code-editor/extensions && \ while IFS= read -r url || [ -n "$url" ]; do \ echo "Downloading extension from ${url}..." && \ wget --no-check-certificate -P /etc/code-editor/extensions "${url}"; \ @@ -87,14 +84,11 @@ RUN micromamba install -y --name base --file /tmp/$ENV_IN_FILENAME && \ ARG MAMBA_DOCKERFILE_ACTIVATE=1 -RUN sudo ln -s $(which python3) /usr/bin/python - -# Update npm version -RUN RUN npm update -g npm - -# Configure CodeEditor - Install extensions and set preferences -RUN \ - extensionloc=/opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions && mkdir -p "${extensionloc}" \ +RUN sudo ln -s $(which python3) /usr/bin/python && \ + # Update npm version + npm update -g npm && \ + # Configure CodeEditor - Install extensions and set preferences + xtensionloc=/opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions && mkdir -p "${extensionloc}" \ # Loop through all vsix files in /etc/code-editor/extensions and install them && for ext in /etc/code-editor/extensions/*.vsix; do \ echo "Installing extension ${ext}..."; \ @@ -102,11 +96,10 @@ RUN \ done \ # Copy the settings && cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json \ - && cp /etc/code-editor/code_editor_user_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/User/settings.json - -# Install glue kernels, and move to shared directory -# Also patching base kernel so Studio background code doesn't start session silently -RUN install-glue-kernels && \ + && cp /etc/code-editor/code_editor_user_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/User/settings.json && \ + # Install glue kernels, and move to shared directory + # Also patching base kernel so Studio background code doesn't start session silently + install-glue-kernels && \ SITE_PACKAGES=$(pip show aws-glue-sessions | grep Location | awk '{print $2}') && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_pyspark --user && \ jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_spark --user && \ @@ -132,24 +125,20 @@ RUN HOME_DIR="/home/${NB_USER}/licenses" \ && chmod +x /usr/local/bin/testOSSCompliance \ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} python \ - && rm -rf ${HOME_DIR}/oss_compliance* - -# Create logging directories for supervisor -RUN mkdir -p $SAGEMAKER_LOGGING_DIR && \ + && rm -rf ${HOME_DIR}/oss_compliance* && \ + # Create logging directories for supervisor + mkdir -p $SAGEMAKER_LOGGING_DIR && \ chmod a+rw $SAGEMAKER_LOGGING_DIR && \ mkdir -p ${STUDIO_LOGGING_DIR} && \ - chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} - -# Clean up CodeEditor artifacts -RUN rm -rf /etc/code-editor - -# Create supervisord runtime directory -RUN mkdir -p /var/run/supervisord && \ - chmod a+rw /var/run/supervisord - -# Create root directory for DB -# Create logging directories for supervisor -RUN mkdir -p $DB_ROOT_DIR && \ + chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR} && \ + # Clean up CodeEditor artifacts + rm -rf /etc/code-editor && \ + # Create supervisord runtime directory + mkdir -p /var/run/supervisord && \ + chmod a+rw /var/run/supervisord && \ + # Create root directory for DB + # Create logging directories for supervisor + mkdir -p $DB_ROOT_DIR && \ chmod a+rw $DB_ROOT_DIR USER $MAMBA_USER @@ -174,10 +163,10 @@ RUN INSTALLED_SSL=$(micromamba list | grep openssl | tr -s ' ' | cut -d ' ' -f 3 cp ../openssl-$FIPS_VALIDATED_SSL/providers/fipsmodule.cnf providers/. && \ make tests && cd ../openssl-$FIPS_VALIDATED_SSL && \ # After tests pass, install FIPS provider and remove source code - make install_fips && cd .. && rm -rf ./openssl-* -# Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS -# e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf -RUN cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ + make install_fips && cd .. && rm -rf ./openssl-* && \ + # Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS + # e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf + cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \ sed -i "s:# .include fipsmodule.cnf:.include /opt/conda/ssl/fipsmodule.cnf:" /opt/conda/ssl/openssl-fips.cnf && \ sed -i 's:# fips = fips_sect:fips = fips_sect:' /opt/conda/ssl/openssl-fips.cnf ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ @@ -185,10 +174,9 @@ ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/ # Install Kerberos. # Make sure no dependency is added/updated RUN pip install "krb5>=0.5.1,<0.6" && \ - pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' - -# https://stackoverflow.com/questions/122327 -RUN SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ + pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] ' && \ + # https://stackoverflow.com/questions/122327 + SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \ # Remove SparkRKernel as it's not supported \ jupyter-kernelspec remove -f -y sparkrkernel && \ # Patch Sparkmagic lib to support Custom Certificates \