diff --git a/.github/workflows/ci_code.yml b/.github/workflows/ci_code.yml index aa98b4d603..a531df7083 100644 --- a/.github/workflows/ci_code.yml +++ b/.github/workflows/ci_code.yml @@ -74,6 +74,10 @@ jobs: run: | make unit-testing PYTEST_ARGUMENTS="--cov=superduperdb --cov-report=xml" + - name: Cleanup Docker + run: | + docker system prune -af + - name: Integration Testing if: ${{ startsWith(matrix.python-version, '3.11') }} # Dask requires local python and Dockerfile to be in sync run: | diff --git a/.github/workflows/ci_docs.yaml b/.github/workflows/ci_docs.yaml index 039f282388..3d3a25eb7c 100644 --- a/.github/workflows/ci_docs.yaml +++ b/.github/workflows/ci_docs.yaml @@ -49,7 +49,6 @@ jobs: run: | # Build HR and API docs make build-docs - - name: Deploy uses: peaceiris/actions-gh-pages@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d65cd8cc7..c8c789d82f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Replaced JSON data with String format before storage in SQLAlchemy. - Implemented storage of byte data in base64 format. - Migrated MongoDB Atlas vector search as a standalone searcher like lance. +- Deprecated Demo Image. Now Notebooks run in Colab. +- Replace dask with ray compute backend #### New Features & Functionality @@ -27,7 +29,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `insert_to=` to `.predict` to allow single predictions to be saved. - Support vLLM (running locally or remotely on a ray cluster) - Support LLM service in OpenAI format -- Add ray compute backend #### Bug Fixes diff --git a/Makefile b/Makefile index 006c474458..f49d54c408 100644 --- a/Makefile +++ b/Makefile @@ -160,6 +160,9 @@ testenv_init: ## Initialize a local Testing environment SUPERDUPERDB_DATA_DIR=$(SUPERDUPERDB_DATA_DIR) docker compose -f deploy/testenv/docker-compose.yaml up --remove-orphans & + # Block waiting for the testenv to become ready. + @cd deploy/testenv/; ./wait_ready.sh + testenv_shutdown: ## Terminate the local Testing environment @echo "===> Shutting down the local Testing environment" docker compose -f deploy/testenv/docker-compose.yaml down @@ -179,10 +182,6 @@ unit-testing: ## Execute unit testing pytest $(PYTEST_ARGUMENTS) ./test/unittest/ integration-testing: ## Execute integration testing - # Block waiting for the testenv to become ready. - @cd deploy/testenv/; ./wait_ready.sh - - # Run the test pytest $(PYTEST_ARGUMENTS) ./test/integration test_notebooks: ## Test notebooks (argument: NOTEBOOKS=) diff --git a/deploy/images/superduperdb/Dockerfile b/deploy/images/superduperdb/Dockerfile index 03d8b86b65..16cee976cc 100644 --- a/deploy/images/superduperdb/Dockerfile +++ b/deploy/images/superduperdb/Dockerfile @@ -58,14 +58,13 @@ ENV PATH="${HOME}/.local/bin:$PATH" # Install common dependencies # --------------- -RUN pip install --upgrade setuptools pip -RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --upgrade --user \ +RUN pip install setuptools pip +RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip python -m pip install --user \ # JupyterLab extensions \ theme-darcula \ ipywidgets \ jupyterlab-lsp \ - 'python-lsp-server[all]' \ - "dask-labextension>=5" + 'python-lsp-server[all]' # Install Jupyterlab extensions @@ -97,10 +96,9 @@ ONBUILD ARG SUPERDUPERDB_EXTRAS='' ONBUILD COPY --chown=superduper ./ ${HOME}/superduperdb ONBUILD WORKDIR ${HOME}/superduperdb # Install project dependencies. -ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --user --editable \ - .[${SUPERDUPERDB_EXTRAS}] +ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip python -m pip install --user --editable .[${SUPERDUPERDB_EXTRAS}] # Install user-defined dependencies -ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --user -r requirements.txt +ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip python -m pip install --user -r requirements.txt # --------------- # Build Release @@ -111,8 +109,7 @@ ONBUILD ARG SUPERDUPERDB_EXTRAS='' ONBUILD COPY --chown=superduper ${PWD}/examples ./examples ONBUILD COPY --chown=superduper ${PWD}/contrib ./contrib # Drop cache to reduce image size. -ONBUILD RUN pip install --upgrade --user \ - superduperdb[${SUPERDUPERDB_EXTRAS}] \ +ONBUILD RUN python -m pip install --user superduperdb[${SUPERDUPERDB_EXTRAS}] \ # Purge pip cache && pip cache purge ONBUILD WORKDIR ${HOME}/examples