Release 2.0.0-beta (#219)

.github/workflows/nightly-build-image.yml

@@ -0,0 +1,101 @@
+name: Build New Image Version
+on:
+  # Manually call
+  workflow_dispatch:
+    inputs:
+      release-type:
+        required: true
+        type: choice
+        description: Type of release
+        options:
+          - patch
+          - minor
+          - major
+      base-version:
+        required: true
+        description: Base version=
+  # Call from other workflow
+  workflow_call:
+    inputs:
+      release-type:
+        type: string
+        required: true
+      base-version:
+        type: string
+        required: true
+defaults:
+  run:
+    shell: bash -l {0}
+jobs:
+  open-pr:
+    name: Open Pull Request
+    runs-on: ubuntu-latest
+    if: github.repository == 'aws/sagemaker-distribution'
+    permissions:
+      pull-requests: write
+      contents: write
+    outputs:
+      pr_id: ${{ steps.get_pr_id.outputs.pr_id }}
+      target_version: ${{ steps.calc_target.outputs.target_version }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ./environment.yml
+          environment-name: sagemaker-distribution
+          init-shell: bash
+      - name: Free up disk space
+        run: rm -rf /opt/hostedtoolcache
+      - name: Activate sagemaker-distribution
+        run: micromamba activate sagemaker-distribution
+      - name: Calculate target version
+        id: calc_target
+        run: |
+          TARGET_VERSION=$(python -c 'import semver; print(semver.bump_${{ inputs.release-type }}("${{ inputs.base-version }}"))')
+          echo "target_version=$TARGET_VERSION" >> $GITHUB_OUTPUT
+      - name: Create new branch
+        # Note - CodeBuild depends on this branch name. Don't change without corresponding backend change.
+        run: git checkout -b 2.0.0-beta
+      - name: Generate artifacts
+        run: python ./src/main.py create-${{ inputs.release-type }}-version-artifacts --base-patch-version ${{ inputs.base-version }}
+      - name: Commit .in artifacts to branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add ./build_artifacts
+          git commit -m 'chore: Generate build artifacts for ${{ steps.calc_target.outputs.target_version }} release'
+          git push --set-upstream origin 2.0.0-beta
+      - name: Open pull request
+        id: get_pr_id
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # Note - CodeBuild depends on this PR title. Don't change without corresponding backend change.
+        run: |
+          URL=$(gh pr create -H 2.0.0-beta \
+                    --title 'release: v${{ steps.calc_target.outputs.target_version }}' -F ./.github/workflows/PR_TEMPLATE.md)
+          PR=$(echo $URL | sed 's:.*/::')
+          echo "pr_id=$PR" >> $GITHUB_OUTPUT
+  call-codebuild-project:
+    runs-on: ubuntu-latest
+    needs: open-pr
+    permissions:
+      pull-requests: write
+      contents: write
+      id-token: write
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::700843992353:role/codebuild-start-build-role
+          aws-region: us-west-2
+          # CodeBuild timeout of 8 hours
+          role-duration-seconds: 3840
+      - name: Run CodeBuild
+        uses: dark-mechanicum/aws-codebuild@v1
+        env:
+          CODEBUILD__sourceVersion: 'pr/${{ needs.open-pr.outputs.pr_id }}'
+        with:
+          projectName: 'buildtestpublicimage1C7307A-9AzES2hf19lW'
+          buildspec: '{"imageOverride": "aws/codebuild/standard:7.0"}'

.github/workflows/nightly-patch-release.yml

@@ -0,0 +1,46 @@
+name: Generate nightly patch release(s)
+#on:
+#  # Run every night at 5:15PM PST
+#  # Run before monthly, so we don't immediately patch a new minor version
+#  schedule:
+#    - cron: '0 15 17 ? * *'
+jobs:
+  generate-version-matrix:
+    name: Generate-matrix
+    runs-on: ubuntu-latest
+    if: github.repository == 'aws/sagemaker-distribution'
+    outputs:
+      matrix: ${{ steps.gen-mat.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Generate patch version matrix
+        id: gen-mat
+        # Output looks like :matrix={"version":["0.0.2","0.1.2",..."1.4.1"]}
+        # For each minor, get highest patch version. Use each patch
+        # as base version. Use this patch as base version.
+        run: |
+          versions=("{\"version\":[")
+          for minor in build_artifacts/v2/*; do
+            minor_version="${minor##*/}"
+            highest_patch=$(ls $minor | sort -t. -k3,3n | tail -n1)
+            versions+="\"${highest_patch#v}\""
+            versions+=","
+          done
+          versions=${versions::-1}
+          versions+="]}"
+          echo "matrix=$versions" >> $GITHUB_OUTPUT
+
+  start-nightly-patch:
+    name: Start nightly patch release
+    needs: generate-version-matrix
+    permissions:
+      pull-requests: write
+      contents: write
+      id-token: write
+    strategy:
+      matrix: ${{ fromJson(needs.generate-version-matrix.outputs.matrix) }}
+      fail-fast: false
+    uses: aws/sagemaker-distribution/.github/workflows/nightly-build-image.yml@main
+    with:
+      release-type: "patch"
+      base-version: ${{ matrix.version }}

build_artifacts/v2/v2.0/v2.0.0/v2.0.0-beta/CHANGELOG-cpu.md

@@ -0,0 +1,20 @@
+# Change log: 2.0.0-beta(cpu)
+
+## Upgrades: 
+
+Package | Previous Version | Current Version
+---|---|---
+boto3|1.28.64|1.34.51
+ipython|8.21.0|8.22.2
+scipy|1.11.4|1.12.0
+fastapi|0.103.2|0.110.0
+pip|23.3.2|24.0
+autogluon|0.8.2|1.0.0
+conda|23.11.0|24.1.2
+langchain|0.1.9|0.1.10
+jupyter-ai|2.9.1|2.10.0
+jupyterlab-lsp|5.0.3|5.1.0
+keras|2.12.0|2.15.0
+nodejs|18.18.2|18.18.2
+sagemaker-python-sdk|2.198.1|2.210.0
+tensorflow|2.12.1|2.15.0

build_artifacts/v2/v2.0/v2.0.0/v2.0.0-beta/CHANGELOG-gpu.md

@@ -0,0 +1,20 @@
+# Change log: 2.0.0-beta(gpu)
+
+## Upgrades: 
+
+Package | Previous Version | Current Version
+---|---|---
+boto3|1.28.64|1.34.51
+ipython|8.21.0|8.22.2
+scipy|1.11.4|1.12.0
+fastapi|0.103.2|0.110.0
+pip|23.3.2|24.0
+autogluon|0.8.2|1.0.0
+conda|23.11.0|24.1.2
+langchain|0.1.9|0.1.10
+jupyter-ai|2.9.1|2.10.0
+jupyterlab-lsp|5.0.3|5.1.0
+keras|2.12.0|2.15.0
+nodejs|18.18.2|18.18.2
+sagemaker-python-sdk|2.198.1|2.210.0
+tensorflow|2.12.1|2.15.0

build_artifacts/v2/v2.0/v2.0.0/v2.0.0-beta/Dockerfile

@@ -0,0 +1,127 @@
+ARG TAG_FOR_BASE_MICROMAMBA_IMAGE
+FROM mambaorg/micromamba:$TAG_FOR_BASE_MICROMAMBA_IMAGE
+
+ARG CUDA_MAJOR_MINOR_VERSION=''
+ARG ENV_IN_FILENAME
+ARG ARG_BASED_ENV_IN_FILENAME
+
+ARG AMZN_BASE="/opt/amazon/sagemaker"
+ARG DIRECTORY_TREE_STAGE_DIR="${AMZN_BASE}/dir-staging"
+
+ARG NB_USER="sagemaker-user"
+ARG NB_UID=1000
+ARG NB_GID=100
+
+ENV SAGEMAKER_LOGGING_DIR="/var/log/sagemaker/"
+ENV STUDIO_LOGGING_DIR="/var/log/studio/"
+
+USER root
+RUN usermod "--login=${NB_USER}" "--home=/home/${NB_USER}" --move-home "-u ${NB_UID}" "${MAMBA_USER}" && \
+    groupmod "--new-name=${NB_USER}" --non-unique "-g ${NB_GID}" "${MAMBA_USER}" && \
+    # Update the expected value of MAMBA_USER for the
+    # _entrypoint.sh consistency check.
+    echo "${NB_USER}" > "/etc/arg_mamba_user" && \
+    :
+ENV MAMBA_USER=$NB_USER
+ENV USER=$NB_USER
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo gettext-base wget curl unzip git rsync build-essential openssh-client nano && \
+    # We just install tzdata below but leave default time zone as UTC. This helps packages like Pandas to function correctly.
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata krb5-user libkrb5-dev libsasl2-dev libsasl2-modules && \
+    chmod g+w /etc/passwd && \
+    echo "ALL    ALL=(ALL)    NOPASSWD:    ALL" >> /etc/sudoers && \
+    touch /etc/krb5.conf.lock && chown ${NB_USER}:${MAMBA_USER} /etc/krb5.conf* && \
+    # Note that we do NOT run `rm -rf /var/lib/apt/lists/*` here. If we did, anyone building on top of our images will
+    # not be able to run any `apt-get install` commands and that would hamper customizability of the images.
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    sudo ./aws/install && \
+    rm -rf aws awscliv2.zip && \
+    :
+RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile
+
+USER $MAMBA_USER
+COPY --chown=$MAMBA_USER:$MAMBA_USER $ENV_IN_FILENAME *.in /tmp/
+
+# Make sure that $ENV_IN_FILENAME has a newline at the end before the `tee` command runs. Otherwise, nasty things
+# will happen.
+RUN if [[ -z $ARG_BASED_ENV_IN_FILENAME ]] ; \
+    then echo 'No ARG_BASED_ENV_IN_FILENAME passed' ; \
+    else envsubst < /tmp/$ARG_BASED_ENV_IN_FILENAME | tee --append /tmp/$ENV_IN_FILENAME ; \
+    fi
+
+ARG CONDA_OVERRIDE_CUDA=$CUDA_MAJOR_MINOR_VERSION
+RUN micromamba install -y --name base --file /tmp/$ENV_IN_FILENAME && \
+    micromamba clean --all --yes --force-pkgs-dirs && \
+    rm -rf /tmp/*.in
+
+
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+RUN sudo ln -s $(which python3) /usr/bin/python
+
+# Install glue kernels, and move to shared directory
+# Also patching base kernel so Studio background code doesn't start session silently
+RUN install-glue-kernels && \
+    SITE_PACKAGES=$(pip show aws-glue-sessions | grep Location | awk '{print $2}') && \
+    jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_pyspark --user && \
+    jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_spark --user && \
+    mv /home/sagemaker-user/.local/share/jupyter/kernels/glue_pyspark /opt/conda/share/jupyter/kernels && \
+    mv /home/sagemaker-user/.local/share/jupyter/kernels/glue_spark /opt/conda/share/jupyter/kernels && \
+    sed -i '/if not store_history and (/i\        if "sm_analytics_runtime_check" in code:\n            return await self._complete_cell()\n' \
+    "$SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_kernel_base/BaseKernel.py"
+
+
+# Patch glue kernels to use kernel wrapper
+COPY patch_glue_pyspark.json /opt/conda/share/jupyter/kernels/glue_pyspark/kernel.json
+COPY patch_glue_spark.json /opt/conda/share/jupyter/kernels/glue_spark/kernel.json
+
+USER root
+RUN HOME_DIR="/home/${NB_USER}/licenses" \
+    && mkdir -p ${HOME_DIR} \
+    && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \
+    && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \
+    && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \
+    && chmod +x /usr/local/bin/testOSSCompliance \
+    && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \
+    && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} python \
+    && rm -rf ${HOME_DIR}/oss_compliance*
+
+# Merge in OS directory tree contents.
+RUN mkdir -p ${DIRECTORY_TREE_STAGE_DIR}
+COPY dirs/ ${DIRECTORY_TREE_STAGE_DIR}/
+RUN rsync -a ${DIRECTORY_TREE_STAGE_DIR}/ / && \
+    rm -rf ${DIRECTORY_TREE_STAGE_DIR}
+
+# Create logging directories for supervisor
+RUN mkdir -p $SAGEMAKER_LOGGING_DIR && \
+    chmod a+rw $SAGEMAKER_LOGGING_DIR && \
+    mkdir -p ${STUDIO_LOGGING_DIR} && \
+    chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR}
+
+# Create supervisord runtime directory
+RUN mkdir -p /var/run/supervisord && \
+    chmod a+rw /var/run/supervisord
+
+USER $MAMBA_USER
+ENV PATH="/opt/conda/bin:/opt/conda/condabin:$PATH"
+WORKDIR "/home/${NB_USER}"
+
+# Install Kerberos.
+# Make sure no dependency is added/updated
+RUN pip install "krb5>=0.5.1,<0.6" && \
+    pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] '
+
+# https://stackoverflow.com/questions/122327
+RUN SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \
+    # Remove SparkRKernel as it's not supported \
+    jupyter-kernelspec remove -f -y sparkrkernel && \
+    # Patch Sparkmagic lib to support Custom Certificates \
+    # https://github.com/jupyter-incubator/sparkmagic/pull/435/files \
+    cp -a ${SYSTEM_PYTHON_PATH}/sagemaker_studio_analytics_extension/patches/configuration.py ${SYSTEM_PYTHON_PATH}/sparkmagic/utils/ && \
+    cp -a ${SYSTEM_PYTHON_PATH}/sagemaker_studio_analytics_extension/patches/reliablehttpclient.py ${SYSTEM_PYTHON_PATH}/sparkmagic/livyclientlib/reliablehttpclient.py && \
+    sed -i 's=  "python"=  "/opt/conda/bin/python"=g'  /opt/conda/share/jupyter/kernels/pysparkkernel/kernel.json /opt/conda/share/jupyter/kernels/sparkkernel/kernel.json && \
+    sed -i 's="Spark"="SparkMagic Spark"=g'  /opt/conda/share/jupyter/kernels/sparkkernel/kernel.json && \
+    sed -i 's="PySpark"="SparkMagic PySpark"=g'  /opt/conda/share/jupyter/kernels/pysparkkernel/kernel.json
+
+ENV SHELL=/bin/bash

build_artifacts/v2/v2.0/v2.0.0/v2.0.0-beta/RELEASE.md

@@ -0,0 +1,53 @@
+# Release notes: 2.0.0-beta
+
+Package | gpu| cpu
+---|---|---
+python|3.10.13|3.10.13
+numpy|1.26.4|1.26.4
+jinja2|3.1.3|3.1.3
+pandas|2.1.4|2.1.4
+altair|5.2.0|5.2.0
+boto3|1.34.51|1.34.51
+ipython|8.22.2|8.22.2
+jupyter-lsp|2.2.3|2.2.3
+jupyterlab|4.1.2|4.1.2
+amazon-codewhisperer-jupyterlab-ext|2.0.1|2.0.1
+jupyter-scheduler|2.5.1|2.5.1
+amazon-sagemaker-jupyter-scheduler|3.0.7|3.0.7
+scipy|1.12.0|1.12.0
+scikit-learn|1.4.1.post1|1.4.1.post1
+fastapi|0.110.0|0.110.0
+uvicorn|0.27.1|0.27.1
+pip|24.0|24.0
+torchvision|0.15.2|0.15.2
+autogluon|1.0.0|1.0.0
+ipywidgets|8.1.2|8.1.2
+notebook|7.1.1|7.1.1
+aws-glue-sessions|1.0.4|1.0.4
+conda|24.1.2|24.1.2
+langchain|0.1.10|0.1.10
+jupyter-ai|2.10.0|2.10.0
+jupyter-dash|0.4.2|0.4.2
+jupyter-server-proxy|4.1.0|4.1.0
+jupyterlab-git|0.50.0|0.50.0
+jupyterlab-lsp|5.1.0|5.1.0
+keras|2.15.0|2.15.0
+matplotlib|3.8.3|3.8.3
+nodejs|18.18.2|18.18.2
+py-xgboost-gpu|1.7.6| 
+thrift_sasl|0.4.3|0.4.3
+pyhive|0.7.0|0.7.0
+python-gssapi|1.8.3|1.8.3
+python-lsp-server|1.10.0|1.10.0
+pytorch-gpu|2.0.0| 
+sagemaker-headless-execution-driver|0.0.12|0.0.12
+sagemaker-jupyterlab-emr-extension|0.1.9|0.1.9
+sagemaker-jupyterlab-extension|0.2.0|0.2.0
+sagemaker-kernel-wrapper|0.0.2|0.0.2
+sagemaker-python-sdk|2.210.0|2.210.0
+sagemaker-studio-analytics-extension|0.0.21|0.0.21
+sasl|0.3.1|0.3.1
+supervisor|4.2.5|4.2.5
+tensorflow|2.15.0|2.15.0
+pytorch| |2.0.0
+py-xgboost-cpu| |1.7.6