Removed DevKit and added a deprecated warning for the plugins.

.github/workflows/ci_code.yml

@@ -66,10 +66,6 @@ jobs:
         # Once the local file database is complete, we may need to update this section.
         python -m pip install plugins/mongodb
 
-    - name: Install DevKit (docs, testing, etc)
-      run: |
-        make install_devkit
-
     - name: Lint and type-check
       run: |
         make lint-and-type-check

.github/workflows/ci_plugins.yaml

@@ -6,7 +6,7 @@ on:
       - main
 
 jobs:
-  prepare_matrix:
+  plugin_update_check:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -38,11 +38,11 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
 
   test_plugin:
-    needs: prepare_matrix
+    needs: plugin_update_check
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix: ${{fromJson(needs.prepare_matrix.outputs.matrix)}}
+      matrix: ${{fromJson(needs.plugin_update_check.outputs.matrix)}}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -66,27 +66,22 @@ jobs:
           # Install core and testsuite dependencies on the cached python environment.
           python -m pip install '.[test]'
 
-      - name: Install DevKit (docs, testing, etc)
-        run: |
-          make install_devkit
-
       - name: Lint and type-check
         run: |
           make lint-and-type-check DIRECTORIES="plugins/${{ matrix.plugin }}"
 
       - name: Install Plugin
         run: |
-          python -m pip install 'plugins/${{ matrix.plugin }}[test]'
+          echo "Installing local plugin dependencies..."
+          grep -o '#\s*:CI:\s*plugins/.*' plugins/${{ matrix.plugin }}/pyproject.toml | while read line; do
+            dep_path=${line##*# :CI: }
+            dep_path=${dep_path%%[[:space:]]*}
+            echo "Installing $dep_path for testing..."
+            python -m pip install "$dep_path[test]"
+          done
+          echo "Installing plugin..."
+          python -m pip install "plugins/${{ matrix.plugin }}[test]"
 
-      - name: Optionally run custom CI script
-        run: |
-          if [ -f "plugins/${{ matrix.plugin }}/.ci_extend.sh" ]; then
-            echo "Running custom CI script..."
-            bash ./plugins/${{ matrix.plugin }}/.ci_extend.sh
-          else
-            echo "No custom CI script found, skipping..."
-          fi
-   
       - name: Plugin Testing
         run: |
           export PYTHONPATH=./

Makefile

@@ -1,6 +1,7 @@
 DIRECTORIES ?= superduper test
 SUPERDUPER_CONFIG ?= test/configs/default.yaml
 PYTEST_ARGUMENTS ?=
+PLUGIN_NAME ?=
 
 # Export directories for data and artifacts
 export SUPERDUPER_DATA_DIR ?= ~/.cache/superduper/test_data
@@ -47,49 +48,6 @@ new_release: ## Release a new version of superduper.io
 	# Push the specific tag
 	git push origin $(RELEASE_VERSION)
 
-
-install_devkit: ## Add essential development tools
-	# Add pre-commit hooks to ensure that no strange stuff are being committed.
-	# https://stackoverflow.com/questions/3462955/putting-git-hooks-into-a-repository
-	python -m pip install pre-commit
-
-	@echo "Download Code Quality dependencies"
-	python -m pip install black==23.3 ruff==0.4.4 mypy types-PyYAML types-requests interrogate
-
-	@echo "Download Code Testing dependencies"
-	python -m pip install pytest pytest-cov "nbval>=0.10.0"
-
-install_plugin:
-	@if [ "$(filter-out $@,$(MAKECMDGOALS))" = "all" ]; then \
-		$(MAKE) install_all_plugins; \
-	else \
-		echo "Error: Cannot find plugin '$(filter-out $@,$(MAKECMDGOALS))'"; \
-	fi
-
-install_named_plugin:
-	@if [ -f "plugins/$(PLUGIN)/pyproject.toml" ]; then \
-		python -m pip install -e "plugins/$(PLUGIN)"; \
-	else \
-		echo "Error: Plugin '$(PLUGIN)' not found."; \
-	fi
-
-install_all_plugins:
-	@plugins=""; \
-	for plugin in $$(ls plugins); do \
-		if [ "$$plugin" != "template" -a -d "plugins/$$plugin" -a -f "plugins/$$plugin/pyproject.toml" ]; then \
-			plugins="$$plugins $$plugin"; \
-		fi \
-	done; \
-	echo "Found plugins:$$plugins"; \
-	for plugin in $$plugins; do \
-		echo "Installing $$plugin..."; \
-		python -m pip install -e "plugins/$$plugin"; \
-	done
-%:
-	@:
-
-
-
 ##@ Code Quality
 
 gen_docs: ## Generate Docs and API

plugins/ibis/pyproject.toml

@@ -25,6 +25,8 @@ dependencies = [
 [project.optional-dependencies]
 test = [
     "ibis-framework[sqlite]>=5.1.0",
+    # Annotation plugin dependencies will be installed in CI
+    # :CI: plugins/sqlalchemy
 ]
 
 [project.urls]

plugins/pillow/pyproject.toml

@@ -22,6 +22,13 @@ dependencies = [
     "pillow>=10.2.0",
 ]
 
+[project.optional-dependencies]
+test = [
+    "ibis-framework[sqlite]>=5.1.0",
+    # Annotation plugin dependencies will be installed in CI
+    # :CI: plugins/sqlalchemy
+]
+
 [project.urls]
 homepage = "https://www.superduper.com/"
 documentation = "https://docs.superduper.com"

plugins/sklearn/pyproject.toml

@@ -21,6 +21,12 @@ dynamic = ["version"]
 dependencies = [
 ]
 
+[project.optional-dependencies]
+test = [
+    # Annotation plugin dependencies will be installed in CI
+    # :CI: plugins/mongodb
+]
+
 [project.urls]
 homepage = "https://www.superduper.com/"
 documentation = "https://docs.superduper.com"

plugins/torch/pyproject.toml

@@ -23,6 +23,12 @@ dependencies = [
     "torchvision>=0.17.1",
 ]
 
+[project.optional-dependencies]
+test = [
+    # Annotation plugin dependencies will be installed in CI
+    # :CI: plugins/mongodb
+]
+
 [project.urls]
 homepage = "https://www.superduper.com/"
 documentation = "https://docs.superduper.com"

plugins/transformers/pyproject.toml

@@ -27,6 +27,8 @@ dependencies = [
 test = [
     "peft>=0.10.0",
     "trl>=0.8.0",
+    # Annotation plugin dependencies will be installed in CI
+    # :CI: plugins/mongodb
 ]
 
 [project.urls]

plugins/vllm/pyproject.toml

@@ -24,6 +24,8 @@ dependencies = [
 [project.optional-dependencies]
 test = [
     "vcrpy>=5.1.0",
+    # Annotation plugin dependencies will be installed in CI
+    # :CI: plugins/mongodb
 ]
 
 [project.urls]

pyproject.toml

@@ -61,6 +61,16 @@ dependencies = [
 test = [
     "scikit-learn>=1.1.3",
     "pandas",
+    "pre-commit",
+    "black==23.3",
+    "ruff==0.4.4",
+    "mypy",
+    "types-PyYAML",
+    "types-requests",
+    "interrogate",
+    "pytest",
+    "pytest-cov",
+    "nbval>=0.10.0",
 ]
 
 [project.urls]

superduper/backends/ibis/__init__.py

@@ -1 +1,5 @@
 from superduper_ibis import *  # noqa
+
+from superduper.misc.annotations import warn_plugin_deprecated
+
+warn_plugin_deprecated('ibis')

superduper/backends/mongodb/__init__.py

@@ -1 +1,5 @@
 from superduper_mongodb import *  # noqa
+
+from superduper.misc.annotations import warn_plugin_deprecated
+
+warn_plugin_deprecated('mongodb')

superduper/backends/sqlalchemy/__init__.py

@@ -0,0 +1,5 @@
+from superduper_sqlalchemy import *  # noqa
+
+from superduper.misc.annotations import warn_plugin_deprecated
+
+warn_plugin_deprecated('sqlalchemy')

superduper/backends/sqlalchemy/db_helper.py

@@ -1,119 +1 @@
-import json
-from typing import Tuple
-
-from sqlalchemy import (
-    Boolean,
-    DateTime,
-    Integer,
-    String,
-    Text,
-    TypeDecorator,
-)
-
-DEFAULT_LENGTH = 255
-
-
-class JsonMixin:
-    """Mixin for JSON type columns.
-
-    Converts dict to JSON strings before saving to database
-    and converts JSON strings to dict when loading from database.
-
-    # noqa
-    """
-
-    def process_bind_param(self, value, dialect):
-        """Convert dict to JSON string.
-
-        :param value: The dict to convert.
-        :param dialect: The dialect of the database.
-        """
-        if value is not None:
-            value = json.dumps(value)
-        return value
-
-    def process_result_value(self, value, dialect):
-        """Convert JSON string to dict.
-
-        :param value: The JSON string to convert.
-        :param dialect: The dialect of the database.
-        """
-        if value is not None:
-            value = json.loads(value)
-        return value
-
-
-class JsonAsString(JsonMixin, TypeDecorator):
-    """JSON type column for short JSON strings # noqa."""
-
-    impl = String(DEFAULT_LENGTH)
-
-
-class JsonAsText(JsonMixin, TypeDecorator):
-    """JSON type column for long JSON strings # noqa."""
-
-    impl = Text
-
-
-class DefaultConfig:
-    """Default configuration for database types # noqa."""
-
-    type_string = String(DEFAULT_LENGTH)
-    type_json_as_string = JsonAsString
-    type_json_as_text = JsonAsText
-    type_integer = Integer
-    type_datetime = DateTime
-    type_boolean = Boolean
-
-    query_id_table_args: Tuple = tuple()
-    job_table_args: Tuple = tuple()
-    parent_child_association_table_args: Tuple = tuple()
-    component_table_args: Tuple = tuple()
-    meta_table_args: Tuple = tuple()
-
-
-def create_clickhouse_config():
-    """Create configuration for ClickHouse database."""
-    # lazy import
-    try:
-        from clickhouse_sqlalchemy import engines, types
-    except ImportError:
-        raise ImportError(
-            'The clickhouse_sqlalchemy package is required to use the '
-            'clickhouse dialect. Please install it with pip install '
-            'clickhouse-sqlalchemy'
-        )
-
-    class ClickHouseConfig:
-        class JsonAsString(JsonMixin, TypeDecorator):
-            impl = types.String
-
-        class JsonAsText(JsonMixin, TypeDecorator):
-            impl = types.String
-
-        type_string = types.String
-        type_json_as_string = JsonAsString
-        type_json_as_text = JsonAsText
-        type_integer = types.Int32
-        type_datetime = types.DateTime
-        type_boolean = types.Boolean
-
-        # clickhouse need engine args to create table
-        query_id_table_args = (engines.MergeTree(order_by='query_id'),)
-        job_table_args = (engines.MergeTree(order_by='identifier'),)
-        parent_child_association_table_args = (engines.MergeTree(order_by='parent_id'),)
-        component_table_args = (engines.MergeTree(order_by='id'),)
-        meta_table_args = (engines.MergeTree(order_by='key'),)
-
-    return ClickHouseConfig
-
-
-def get_db_config(dialect):
-    """Get the configuration class for the specified dialect.
-
-    :param dialect: The dialect of the database.
-    """
-    if dialect == 'clickhouse':
-        return create_clickhouse_config()
-    else:
-        return DefaultConfig
+from superduper_sqlalchemy.db_helper import *  # noqa