run testing within the sandbox image (#1768)

Signed-off-by: Fotis Nikolaidis <[email protected]>

.github/workflows/ci_code.yml

@@ -74,6 +74,10 @@ jobs:
         run: |
           make unit-testing PYTEST_ARGUMENTS="--cov=superduperdb --cov-report=xml"
 
+      - name: Cleanup Docker
+        run: |    
+          docker system prune -af
+
       - name: Integration Testing
         if: ${{ startsWith(matrix.python-version, '3.11') }} # Dask requires local python and Dockerfile to be in sync
         run: |

.github/workflows/ci_docs.yaml

@@ -49,7 +49,6 @@ jobs:
         run: |
           # Build HR and API docs
           make build-docs
-          
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3

CHANGELOG.md

@@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Replaced JSON data with String format before storage in SQLAlchemy.
 - Implemented storage of byte data in base64 format.
 - Migrated MongoDB Atlas vector search as a standalone searcher like lance.
+- Deprecated Demo Image. Now Notebooks run in Colab.
+- Replace dask with ray compute backend
 
 #### New Features & Functionality
 
@@ -27,7 +29,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add `insert_to=<table-or-collection>` to `.predict` to allow single predictions to be saved.
 - Support vLLM (running locally or remotely on a ray cluster)
 - Support LLM service in OpenAI format
-- Add ray compute backend
 
 #### Bug Fixes
 

Makefile

@@ -160,6 +160,9 @@ testenv_init: ## Initialize a local Testing environment
 
 	SUPERDUPERDB_DATA_DIR=$(SUPERDUPERDB_DATA_DIR) docker compose -f deploy/testenv/docker-compose.yaml up --remove-orphans &
 
+	# Block waiting for the testenv to become ready.
+	@cd deploy/testenv/; ./wait_ready.sh
+
 testenv_shutdown: ## Terminate the local Testing environment
 	@echo "===> Shutting down the local Testing environment"
 	docker compose -f deploy/testenv/docker-compose.yaml down
@@ -179,10 +182,6 @@ unit-testing: ## Execute unit testing
 	pytest $(PYTEST_ARGUMENTS) ./test/unittest/
 
 integration-testing: ## Execute integration testing
-	# Block waiting for the testenv to become ready.
-	@cd deploy/testenv/; ./wait_ready.sh
-
-	# Run the test
 	pytest $(PYTEST_ARGUMENTS) ./test/integration
 
 test_notebooks: ## Test notebooks (argument: NOTEBOOKS=<test|dir>)

deploy/images/superduperdb/Dockerfile

@@ -58,14 +58,13 @@ ENV PATH="${HOME}/.local/bin:$PATH"
 
 # Install common dependencies
 # ---------------
-RUN pip install --upgrade setuptools pip
-RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --upgrade --user \
+RUN pip install setuptools pip
+RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip python -m pip install --user \
     # JupyterLab extensions \
     theme-darcula \
     ipywidgets \
     jupyterlab-lsp \
-    'python-lsp-server[all]' \
-    "dask-labextension>=5"
+    'python-lsp-server[all]'
 
 
 # Install Jupyterlab extensions
@@ -97,10 +96,9 @@ ONBUILD ARG SUPERDUPERDB_EXTRAS=''
 ONBUILD COPY --chown=superduper ./ ${HOME}/superduperdb
 ONBUILD WORKDIR ${HOME}/superduperdb
 # Install project dependencies.
-ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --user --editable \
-    .[${SUPERDUPERDB_EXTRAS}]
+ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip python -m pip install --user --editable .[${SUPERDUPERDB_EXTRAS}]
 # Install user-defined dependencies
-ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --user  -r requirements.txt
+ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip python -m pip install --user  -r requirements.txt
 
 # ---------------
 # Build Release
@@ -111,8 +109,7 @@ ONBUILD ARG SUPERDUPERDB_EXTRAS=''
 ONBUILD COPY --chown=superduper ${PWD}/examples ./examples
 ONBUILD COPY --chown=superduper ${PWD}/contrib ./contrib
 # Drop cache to reduce image size.
-ONBUILD RUN pip install --upgrade --user \
-    superduperdb[${SUPERDUPERDB_EXTRAS}] \
+ONBUILD RUN python -m pip install --user superduperdb[${SUPERDUPERDB_EXTRAS}] \
     # Purge pip cache
     && pip cache purge
 ONBUILD WORKDIR ${HOME}/examples