From 23c21b329ad1e275773663717d6b569038df6b2b Mon Sep 17 00:00:00 2001 From: Fotis Nikolaidis Date: Fri, 24 Nov 2023 19:20:43 +0200 Subject: [PATCH] Notebook ci (#1398) Signed-off-by: Fotis Nikolaidis --- .github/workflows/ci_notebooks.yml | 4 +- .gitmodules | 4 +- Makefile | 2 + README.md | 24 +- contrib | 1 + examples | 1 - examples/.gitignore | 6 + examples/mnist_torch.ipynb | 992 ++++++++++++++++++++ examples/multimodal_image_search_clip.ipynb | 352 +++++++ examples/question_the_docs.ipynb | 450 +++++++++ examples/sql-example.ipynb | 339 +++++++ examples/transfer_learning.ipynb | 267 ++++++ examples/vector_search.ipynb | 324 +++++++ examples/video_search.ipynb | 399 ++++++++ examples/voice_memos.ipynb | 371 ++++++++ 15 files changed, 3519 insertions(+), 17 deletions(-) create mode 160000 contrib delete mode 160000 examples create mode 100644 examples/.gitignore create mode 100644 examples/mnist_torch.ipynb create mode 100644 examples/multimodal_image_search_clip.ipynb create mode 100644 examples/question_the_docs.ipynb create mode 100644 examples/sql-example.ipynb create mode 100644 examples/transfer_learning.ipynb create mode 100644 examples/vector_search.ipynb create mode 100644 examples/video_search.ipynb create mode 100644 examples/voice_memos.ipynb diff --git a/.github/workflows/ci_notebooks.yml b/.github/workflows/ci_notebooks.yml index 9be8db7707..fe2ef25008 100644 --- a/.github/workflows/ci_notebooks.yml +++ b/.github/workflows/ci_notebooks.yml @@ -6,7 +6,7 @@ on: branches: - main paths: - - 'examples/official-notebooks/**' + - 'examples/**' # Nightly is needed to ensure that code and notebooks are interopeable schedule: - cron: "30 1 * * *" @@ -56,4 +56,4 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | - make test_notebooks NOTEBOOKS=./examples/official-notebooks + make test_notebooks NOTEBOOKS=./examples diff --git a/.gitmodules b/.gitmodules index 18b5bd7278..1001b7ce99 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "examples"] - path = examples +[submodule "contrib"] + path = contrib url = https://github.com/SuperDuperDB/superduper-community-apps diff --git a/Makefile b/Makefile index 222ef81fd1..fac8f3b35a 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,8 @@ devkit: ## Add essential development tools # Download tools for code quality testing pip install .[quality] + # Set git to continuously update submodules + git config --global submodule.recurse true ##@ CI Doc Functions diff --git a/README.md b/README.md index cd4c6584c7..aef4157522 100644 --- a/README.md +++ b/README.md @@ -254,58 +254,58 @@ Also find use-cases and apps built by the community in the [superduper-community
- + - + - +
- Text-To-Image Search + Text-To-Image Search - Text-To-Video Search + Text-To-Video Search - Question the Docs + Question the Docs
- + - + - +
- Semantic Search Engine + Semantic Search Engine - Classical Machine Learning + Classical Machine Learning - Cross-Framework Transfer Learning + Cross-Framework Transfer Learning
diff --git a/contrib b/contrib new file mode 160000 index 0000000000..84f6c0bde8 --- /dev/null +++ b/contrib @@ -0,0 +1 @@ +Subproject commit 84f6c0bde8f82a3b011a72a1bee3ac83f47019ae diff --git a/examples b/examples deleted file mode 160000 index a908a70b42..0000000000 --- a/examples +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a908a70b422ad15ced59127e8a9d2e973cef995f diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000000..cdabd51a69 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,6 @@ +# Ignore everything +* +*/ + +# Except for notebook files +!*.ipynb \ No newline at end of file diff --git a/examples/mnist_torch.ipynb b/examples/mnist_torch.ipynb new file mode 100644 index 0000000000..b954a77f53 --- /dev/null +++ b/examples/mnist_torch.ipynb @@ -0,0 +1,992 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4b24af19", + "metadata": {}, + "source": [ + "# Training and Maintaining MNIST Predictions with SuperDuperDB" + ] + }, + { + "cell_type": "markdown", + "id": "8905783f", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "This notebook outlines the process of implementing a classic machine learning classification task - MNIST handwritten digit recognition, using a convolutional neural network. However, we introduce a unique twist by performing the task in a database using SuperDuperDB." + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Prerequisites\n", + "\n", + "Before diving into the implementation, ensure that you have the necessary libraries installed by running the following commands:" + ], + "metadata": { + "collapsed": false + }, + "id": "95f897a45b2a02cc" + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "a9897997-dee8-4947-9327-b96fe06a5a2c", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:48.403454042Z", + "start_time": "2023-11-24T16:52:43.394090934Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: superduperdb in /home/fnikol/Workspace/projects/superduper/superduperdb (0.0.16)\r\n", + "Requirement already satisfied: boto3>=1.16 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (1.28.82)\r\n", + "Requirement already satisfied: dask[distributed]>=2022.6.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (2023.10.1)\r\n", + "Requirement already satisfied: dill>=0.3.6 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (0.3.7)\r\n", + "Requirement already satisfied: loguru>=0.7.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (0.7.2)\r\n", + "Requirement already satisfied: loki-logger-handler>=0.1.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (0.1.1)\r\n", + "Requirement already satisfied: networkx>=2.8.8 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (3.2.1)\r\n", + "Requirement already satisfied: requests>=2.22 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (2.31.0)\r\n", + "Requirement already satisfied: tqdm>=4.64.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (4.66.1)\r\n", + "Requirement already satisfied: typer>=0.7.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (0.9.0)\r\n", + "Requirement already satisfied: pylance<=0.8.14,>=0.6.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (0.8.14)\r\n", + "Requirement already satisfied: readerwriterlock>=1.0.9 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (1.0.9)\r\n", + "Requirement already satisfied: pydantic>=1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (2.4.2)\r\n", + "Requirement already satisfied: pymongo>=4.3.3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (4.6.0)\r\n", + "Requirement already satisfied: numpy>=1.24.3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (1.26.1)\r\n", + "Requirement already satisfied: overrides>=7 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (7.4.0)\r\n", + "Requirement already satisfied: tenacity>=8.1.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (8.2.3)\r\n", + "Requirement already satisfied: scikit-learn>=1.1.3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (1.3.2)\r\n", + "Requirement already satisfied: pillow>=9.4.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (10.1.0)\r\n", + "Requirement already satisfied: mongomock>=4.1.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (4.1.2)\r\n", + "Requirement already satisfied: ibis-framework[sqlite]>=5.1.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (7.0.0)\r\n", + "Requirement already satisfied: PyYAML>=6.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from superduperdb) (6.0.1)\r\n", + "Requirement already satisfied: botocore<1.32.0,>=1.31.82 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from boto3>=1.16->superduperdb) (1.31.82)\r\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from boto3>=1.16->superduperdb) (1.0.1)\r\n", + "Requirement already satisfied: s3transfer<0.8.0,>=0.7.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from boto3>=1.16->superduperdb) (0.7.0)\r\n", + "Requirement already satisfied: click>=8.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (8.1.7)\r\n", + "Requirement already satisfied: cloudpickle>=1.5.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (3.0.0)\r\n", + "Requirement already satisfied: fsspec>=2021.09.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (2023.10.0)\r\n", + "Requirement already satisfied: packaging>=20.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (23.2)\r\n", + "Requirement already satisfied: partd>=1.2.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (1.4.1)\r\n", + "Requirement already satisfied: toolz>=0.10.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (0.12.0)\r\n", + "Requirement already satisfied: importlib-metadata>=4.13.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (6.8.0)\r\n", + "Requirement already satisfied: distributed==2023.10.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from dask[distributed]>=2022.6.0->superduperdb) (2023.10.1)\r\n", + "Requirement already satisfied: jinja2>=2.10.3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (3.1.2)\r\n", + "Requirement already satisfied: locket>=1.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (1.0.0)\r\n", + "Requirement already satisfied: msgpack>=1.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (1.0.7)\r\n", + "Requirement already satisfied: psutil>=5.7.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (5.9.6)\r\n", + "Requirement already satisfied: sortedcontainers>=2.0.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (2.4.0)\r\n", + "Requirement already satisfied: tblib>=1.6.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (3.0.0)\r\n", + "Requirement already satisfied: tornado>=6.0.4 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (6.3.3)\r\n", + "Requirement already satisfied: urllib3>=1.24.3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (1.26.18)\r\n", + "Requirement already satisfied: zict>=3.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (3.0.0)\r\n", + "Requirement already satisfied: atpublic<5,>=2.3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (4.0)\r\n", + "Requirement already satisfied: bidict<1,>=0.22.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (0.22.1)\r\n", + "Requirement already satisfied: duckdb<1,>=0.8.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (0.9.1)\r\n", + "Requirement already satisfied: duckdb-engine<1,>=0.1.8 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (0.9.2)\r\n", + "Requirement already satisfied: filelock<4,>=3.7.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (3.13.1)\r\n", + "Requirement already satisfied: multipledispatch<2,>=0.6 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (1.0.0)\r\n", + "Requirement already satisfied: pandas<3,>=1.2.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (2.1.2)\r\n", + "Requirement already satisfied: parsy<3,>=2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (2.1)\r\n", + "Requirement already satisfied: pins[gcs]<1,>=0.8.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (0.8.2)\r\n", + "Requirement already satisfied: pyarrow<14,>=2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (13.0.0)\r\n", + "Requirement already satisfied: python-dateutil<3,>=2.8.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (2.8.2)\r\n", + "Requirement already satisfied: pytz>=2022.7 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (2023.3.post1)\r\n", + "Requirement already satisfied: rich<14,>=12.4.4 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (13.6.0)\r\n", + "Requirement already satisfied: sqlalchemy<3,>=1.4 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (2.0.23)\r\n", + "Requirement already satisfied: sqlalchemy-views<1,>=0.3.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (0.3.2)\r\n", + "Requirement already satisfied: sqlglot<19,>=18.7.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (18.17.0)\r\n", + "Requirement already satisfied: typing-extensions<5,>=4.3.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (4.8.0)\r\n", + "Requirement already satisfied: regex>=2021.7.6 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from ibis-framework[sqlite]>=5.1.0->superduperdb) (2023.10.3)\r\n", + "Requirement already satisfied: sentinels in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from mongomock>=4.1.2->superduperdb) (1.0.0)\r\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pydantic>=1->superduperdb) (0.6.0)\r\n", + "Requirement already satisfied: pydantic-core==2.10.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pydantic>=1->superduperdb) (2.10.1)\r\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pymongo>=4.3.3->superduperdb) (2.4.2)\r\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests>=2.22->superduperdb) (3.3.2)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests>=2.22->superduperdb) (3.4)\r\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests>=2.22->superduperdb) (2023.7.22)\r\n", + "Requirement already satisfied: scipy>=1.5.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from scikit-learn>=1.1.3->superduperdb) (1.11.3)\r\n", + "Requirement already satisfied: joblib>=1.1.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from scikit-learn>=1.1.3->superduperdb) (1.3.2)\r\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from scikit-learn>=1.1.3->superduperdb) (3.2.0)\r\n", + "Requirement already satisfied: zipp>=0.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from importlib-metadata>=4.13.0->dask[distributed]>=2022.6.0->superduperdb) (3.17.0)\r\n", + "Requirement already satisfied: tzdata>=2022.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pandas<3,>=1.2.5->ibis-framework[sqlite]>=5.1.0->superduperdb) (2023.3)\r\n", + "Requirement already satisfied: xxhash>=1.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (3.4.1)\r\n", + "Requirement already satisfied: importlib-resources>=1.3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (6.1.1)\r\n", + "Requirement already satisfied: appdirs<2.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.4.4)\r\n", + "Requirement already satisfied: humanize>=1.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (4.8.0)\r\n", + "Requirement already satisfied: gcsfs in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (2023.10.0)\r\n", + "Requirement already satisfied: six>=1.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from python-dateutil<3,>=2.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.16.0)\r\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from rich<14,>=12.4.4->ibis-framework[sqlite]>=5.1.0->superduperdb) (3.0.0)\r\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from rich<14,>=12.4.4->ibis-framework[sqlite]>=5.1.0->superduperdb) (2.16.1)\r\n", + "Requirement already satisfied: greenlet!=0.4.17 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from sqlalchemy<3,>=1.4->ibis-framework[sqlite]>=5.1.0->superduperdb) (3.0.1)\r\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from jinja2>=2.10.3->distributed==2023.10.1->dask[distributed]>=2022.6.0->superduperdb) (2.1.3)\r\n", + "Requirement already satisfied: mdurl~=0.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14,>=12.4.4->ibis-framework[sqlite]>=5.1.0->superduperdb) (0.1.2)\r\n", + "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (3.8.6)\r\n", + "Requirement already satisfied: decorator>4.1.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (5.1.1)\r\n", + "Requirement already satisfied: google-auth>=1.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (2.23.4)\r\n", + "Requirement already satisfied: google-auth-oauthlib in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.1.0)\r\n", + "Requirement already satisfied: google-cloud-storage in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (2.13.0)\r\n", + "Requirement already satisfied: attrs>=17.3.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (23.1.0)\r\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (6.0.4)\r\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (4.0.3)\r\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.9.2)\r\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.4.0)\r\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.3.1)\r\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-auth>=1.2->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (5.3.2)\r\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-auth>=1.2->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (0.3.0)\r\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-auth>=1.2->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (4.9)\r\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-auth-oauthlib->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.3.1)\r\n", + "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-cloud-storage->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (2.13.0)\r\n", + "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-cloud-storage->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (2.3.3)\r\n", + "Requirement already satisfied: google-resumable-media>=2.6.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-cloud-storage->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (2.6.0)\r\n", + "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-cloud-storage->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.5.0)\r\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (1.61.0)\r\n", + "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (4.25.0)\r\n", + "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.2->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (0.5.0)\r\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib->gcsfs->pins[gcs]<1,>=0.8.2->ibis-framework[sqlite]>=5.1.0->superduperdb) (3.2.2)\r\n", + "\r\n", + "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.3.1\u001B[0m\r\n", + "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\r\n", + "Requirement already satisfied: torch in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (2.1.0)\r\n", + "Requirement already satisfied: torchvision in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (0.16.0)\r\n", + "Requirement already satisfied: matplotlib in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (3.8.1)\r\n", + "Requirement already satisfied: filelock in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (3.13.1)\r\n", + "Requirement already satisfied: typing-extensions in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (4.8.0)\r\n", + "Requirement already satisfied: sympy in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (1.12)\r\n", + "Requirement already satisfied: networkx in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (3.2.1)\r\n", + "Requirement already satisfied: jinja2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (3.1.2)\r\n", + "Requirement already satisfied: fsspec in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (2023.10.0)\r\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (12.1.105)\r\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (12.1.105)\r\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (12.1.105)\r\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (8.9.2.26)\r\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (12.1.3.1)\r\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (11.0.2.54)\r\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (10.3.2.106)\r\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (11.4.5.107)\r\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (12.1.0.106)\r\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (2.18.1)\r\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (12.1.105)\r\n", + "Requirement already satisfied: triton==2.1.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torch) (2.1.0)\r\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.3.52)\r\n", + "Requirement already satisfied: numpy in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torchvision) (1.26.1)\r\n", + "Requirement already satisfied: requests in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torchvision) (2.31.0)\r\n", + "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from torchvision) (10.1.0)\r\n", + "Requirement already satisfied: contourpy>=1.0.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from matplotlib) (1.2.0)\r\n", + "Requirement already satisfied: cycler>=0.10 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from matplotlib) (0.12.1)\r\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from matplotlib) (4.44.0)\r\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from matplotlib) (1.4.5)\r\n", + "Requirement already satisfied: packaging>=20.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from matplotlib) (23.2)\r\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from matplotlib) (3.1.1)\r\n", + "Requirement already satisfied: python-dateutil>=2.7 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from matplotlib) (2.8.2)\r\n", + "Requirement already satisfied: six>=1.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\r\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from jinja2->torch) (2.1.3)\r\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests->torchvision) (3.3.2)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests->torchvision) (3.4)\r\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests->torchvision) (1.26.18)\r\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from requests->torchvision) (2023.7.22)\r\n", + "Requirement already satisfied: mpmath>=0.19 in /home/fnikol/.pyenv/versions/3.11.5/envs/notebook-env/lib/python3.11/site-packages (from sympy->torch) (1.3.0)\r\n", + "\r\n", + "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.3.1\u001B[0m\r\n", + "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\r\n" + ] + } + ], + "source": [ + "!pip install superduperdb\n", + "!pip install torch torchvision matplotlib" + ] + }, + { + "cell_type": "markdown", + "id": "e3812091", + "metadata": {}, + "source": [ + "## Connect to datastore \n", + "\n", + "First, we need to establish a connection to a MongoDB datastore via SuperDuperDB. You can configure the `MongoDB_URI` based on your specific setup. \n", + "Here are some examples of MongoDB URIs:\n", + "\n", + "* For testing (default connection): `mongomock://test`\n", + "* Local MongoDB instance: `mongodb://localhost:27017`\n", + "* MongoDB with authentication: `mongodb://superduper:superduper@mongodb:27017/documents`\n", + "* MongoDB Atlas: `mongodb+srv://:@/`" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a28adbce", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:48.418849782Z", + "start_time": "2023-11-24T16:52:48.410870777Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:48.40\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.build\u001B[0m:\u001B[36m37 \u001B[0m | \u001B[34m\u001B[1mParsing data connection URI:mongomock://test\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:48.40\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.build\u001B[0m:\u001B[36m77 \u001B[0m | \u001B[1mData Client is ready. mongomock.MongoClient('localhost', 27017)\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:48.40\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m80 \u001B[0m | \u001B[1mBuilding Data Layer\u001B[0m\n" + ] + } + ], + "source": [ + "from superduperdb import superduper\n", + "from superduperdb.backends.mongodb import Collection\n", + "import os\n", + "\n", + "mongodb_uri = os.getenv(\"MONGODB_URI\",\"mongomock://test\")\n", + "db = superduper(mongodb_uri)\n", + "\n", + "# Create a collection for MNIST\n", + "mnist_collection = Collection('mnist')" + ] + }, + { + "cell_type": "markdown", + "id": "6233e891", + "metadata": {}, + "source": [ + "\n", + "## Load Dataset\n", + "\n", + "After connecting to MongoDB, we add the MNIST dataset. SuperDuperDB excels at handling \"difficult\" data types, and we achieve this using an `Encoder`, which works in tandem with the `Document` wrappers. Together, they enable Python dictionaries containing non-JSONable or bytes objects to be inserted into the underlying data infrastructure. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "bf0934cc", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:49.390789041Z", + "start_time": "2023-11-24T16:52:48.419404347Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:49.25\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m737 \u001B[0m | \u001B[34m\u001B[1mBuilding task workflow graph. Query:\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.25\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.backends.local.compute\u001B[0m:\u001B[36m32 \u001B[0m | \u001B[1mSubmitting job. function:\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.25\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.misc.download\u001B[0m:\u001B[36m338 \u001B[0m | \u001B[34m\u001B[1m{'cls': 'MongoCompoundSelect', 'dict': {'table_or_collection': {'cls': 'Collection', 'dict': {'identifier': 'mnist'}, 'module': 'superduperdb.backends.mongodb.query'}, 'pre_like': None, 'post_like': None, 'query_linker': {'cls': 'MongoQueryLinker', 'dict': {'table_or_collection': {'cls': 'Collection', 'dict': {'identifier': 'mnist'}, 'module': 'superduperdb.backends.mongodb.query'}, 'members': [{'cls': 'Find', 'dict': {'name': 'find', 'type': , 'args': [{}, {}], 'kwargs': {}, 'output_fields': None}, 'module': 'superduperdb.backends.mongodb.query'}]}, 'module': 'superduperdb.backends.mongodb.query'}}, 'module': 'superduperdb.backends.mongodb.query'}\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.25\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.misc.download\u001B[0m:\u001B[36m339 \u001B[0m | \u001B[34m\u001B[1m[ObjectId('6560d4e114097bdad056321a'), ObjectId('6560d4e114097bdad056321b'), ObjectId('6560d4e114097bdad056321c'), ObjectId('6560d4e114097bdad056321d'), ObjectId('6560d4e114097bdad056321e'), ObjectId('6560d4e114097bdad056321f'), ObjectId('6560d4e114097bdad0563220'), ObjectId('6560d4e114097bdad0563221'), ObjectId('6560d4e114097bdad0563222'), ObjectId('6560d4e114097bdad0563223'), ObjectId('6560d4e114097bdad0563224'), ObjectId('6560d4e114097bdad0563225'), ObjectId('6560d4e114097bdad0563226'), ObjectId('6560d4e114097bdad0563227'), ObjectId('6560d4e114097bdad0563228'), ObjectId('6560d4e114097bdad0563229'), ObjectId('6560d4e114097bdad056322a'), ObjectId('6560d4e114097bdad056322b'), ObjectId('6560d4e114097bdad056322c'), ObjectId('6560d4e114097bdad056322d'), ObjectId('6560d4e114097bdad056322e'), ObjectId('6560d4e114097bdad056322f'), ObjectId('6560d4e114097bdad0563230'), ObjectId('6560d4e114097bdad0563231'), ObjectId('6560d4e114097bdad0563232'), ObjectId('6560d4e114097bdad0563233'), ObjectId('6560d4e114097bdad0563234'), ObjectId('6560d4e114097bdad0563235'), ObjectId('6560d4e114097bdad0563236'), ObjectId('6560d4e114097bdad0563237'), ObjectId('6560d4e114097bdad0563238'), ObjectId('6560d4e114097bdad0563239'), ObjectId('6560d4e114097bdad056323a'), ObjectId('6560d4e114097bdad056323b'), ObjectId('6560d4e114097bdad056323c'), ObjectId('6560d4e114097bdad056323d'), ObjectId('6560d4e114097bdad056323e'), ObjectId('6560d4e114097bdad056323f'), ObjectId('6560d4e114097bdad0563240'), ObjectId('6560d4e114097bdad0563241'), ObjectId('6560d4e114097bdad0563242'), ObjectId('6560d4e114097bdad0563243'), ObjectId('6560d4e114097bdad0563244'), ObjectId('6560d4e114097bdad0563245'), ObjectId('6560d4e114097bdad0563246'), ObjectId('6560d4e114097bdad0563247'), ObjectId('6560d4e114097bdad0563248'), ObjectId('6560d4e114097bdad0563249'), ObjectId('6560d4e114097bdad056324a'), ObjectId('6560d4e114097bdad056324b'), ObjectId('6560d4e114097bdad056324c'), ObjectId('6560d4e114097bdad056324d'), ObjectId('6560d4e114097bdad056324e'), ObjectId('6560d4e114097bdad056324f'), ObjectId('6560d4e114097bdad0563250'), ObjectId('6560d4e114097bdad0563251'), ObjectId('6560d4e114097bdad0563252'), ObjectId('6560d4e114097bdad0563253'), ObjectId('6560d4e114097bdad0563254'), ObjectId('6560d4e114097bdad0563255'), ObjectId('6560d4e114097bdad0563256'), ObjectId('6560d4e114097bdad0563257'), ObjectId('6560d4e114097bdad0563258'), ObjectId('6560d4e114097bdad0563259'), ObjectId('6560d4e114097bdad056325a'), ObjectId('6560d4e114097bdad056325b'), ObjectId('6560d4e114097bdad056325c'), ObjectId('6560d4e114097bdad056325d'), ObjectId('6560d4e114097bdad056325e'), ObjectId('6560d4e114097bdad056325f'), ObjectId('6560d4e114097bdad0563260'), ObjectId('6560d4e114097bdad0563261'), ObjectId('6560d4e114097bdad0563262'), ObjectId('6560d4e114097bdad0563263'), ObjectId('6560d4e114097bdad0563264'), ObjectId('6560d4e114097bdad0563265'), ObjectId('6560d4e114097bdad0563266'), ObjectId('6560d4e114097bdad0563267'), ObjectId('6560d4e114097bdad0563268'), ObjectId('6560d4e114097bdad0563269'), ObjectId('6560d4e114097bdad056326a'), ObjectId('6560d4e114097bdad056326b'), ObjectId('6560d4e114097bdad056326c'), ObjectId('6560d4e114097bdad056326d'), ObjectId('6560d4e114097bdad056326e'), ObjectId('6560d4e114097bdad056326f'), ObjectId('6560d4e114097bdad0563270'), ObjectId('6560d4e114097bdad0563271'), ObjectId('6560d4e114097bdad0563272'), ObjectId('6560d4e114097bdad0563273'), ObjectId('6560d4e114097bdad0563274'), ObjectId('6560d4e114097bdad0563275'), ObjectId('6560d4e114097bdad0563276'), ObjectId('6560d4e114097bdad0563277'), ObjectId('6560d4e114097bdad0563278'), ObjectId('6560d4e114097bdad0563279'), ObjectId('6560d4e114097bdad056327a'), ObjectId('6560d4e114097bdad056327b'), ObjectId('6560d4e114097bdad056327c'), ObjectId('6560d4e114097bdad056327d'), ObjectId('6560d4e114097bdad056327e'), ObjectId('6560d4e114097bdad056327f'), ObjectId('6560d4e114097bdad0563280'), ObjectId('6560d4e114097bdad0563281'), ObjectId('6560d4e114097bdad0563282'), ObjectId('6560d4e114097bdad0563283'), ObjectId('6560d4e114097bdad0563284'), ObjectId('6560d4e114097bdad0563285'), ObjectId('6560d4e114097bdad0563286'), ObjectId('6560d4e114097bdad0563287'), ObjectId('6560d4e114097bdad0563288'), ObjectId('6560d4e114097bdad0563289'), ObjectId('6560d4e114097bdad056328a'), ObjectId('6560d4e114097bdad056328b'), ObjectId('6560d4e114097bdad056328c'), ObjectId('6560d4e114097bdad056328d'), ObjectId('6560d4e114097bdad056328e'), ObjectId('6560d4e114097bdad056328f'), ObjectId('6560d4e114097bdad0563290'), ObjectId('6560d4e114097bdad0563291'), ObjectId('6560d4e114097bdad0563292'), ObjectId('6560d4e114097bdad0563293'), ObjectId('6560d4e114097bdad0563294'), ObjectId('6560d4e114097bdad0563295'), ObjectId('6560d4e114097bdad0563296'), ObjectId('6560d4e114097bdad0563297'), ObjectId('6560d4e114097bdad0563298'), ObjectId('6560d4e114097bdad0563299'), ObjectId('6560d4e114097bdad056329a'), ObjectId('6560d4e114097bdad056329b'), ObjectId('6560d4e114097bdad056329c'), ObjectId('6560d4e114097bdad056329d'), ObjectId('6560d4e114097bdad056329e'), ObjectId('6560d4e114097bdad056329f'), ObjectId('6560d4e114097bdad05632a0'), ObjectId('6560d4e114097bdad05632a1'), ObjectId('6560d4e114097bdad05632a2'), ObjectId('6560d4e114097bdad05632a3'), ObjectId('6560d4e114097bdad05632a4'), ObjectId('6560d4e114097bdad05632a5'), ObjectId('6560d4e114097bdad05632a6'), ObjectId('6560d4e114097bdad05632a7'), ObjectId('6560d4e114097bdad05632a8'), ObjectId('6560d4e114097bdad05632a9'), ObjectId('6560d4e114097bdad05632aa'), ObjectId('6560d4e114097bdad05632ab'), ObjectId('6560d4e114097bdad05632ac'), ObjectId('6560d4e114097bdad05632ad'), ObjectId('6560d4e114097bdad05632ae'), ObjectId('6560d4e114097bdad05632af'), ObjectId('6560d4e114097bdad05632b0'), ObjectId('6560d4e114097bdad05632b1'), ObjectId('6560d4e114097bdad05632b2'), ObjectId('6560d4e114097bdad05632b3'), ObjectId('6560d4e114097bdad05632b4'), ObjectId('6560d4e114097bdad05632b5'), ObjectId('6560d4e114097bdad05632b6'), ObjectId('6560d4e114097bdad05632b7'), ObjectId('6560d4e114097bdad05632b8'), ObjectId('6560d4e114097bdad05632b9'), ObjectId('6560d4e114097bdad05632ba'), ObjectId('6560d4e114097bdad05632bb'), ObjectId('6560d4e114097bdad05632bc'), ObjectId('6560d4e114097bdad05632bd'), ObjectId('6560d4e114097bdad05632be'), ObjectId('6560d4e114097bdad05632bf'), ObjectId('6560d4e114097bdad05632c0'), ObjectId('6560d4e114097bdad05632c1'), ObjectId('6560d4e114097bdad05632c2'), ObjectId('6560d4e114097bdad05632c3'), ObjectId('6560d4e114097bdad05632c4'), ObjectId('6560d4e114097bdad05632c5'), ObjectId('6560d4e114097bdad05632c6'), ObjectId('6560d4e114097bdad05632c7'), ObjectId('6560d4e114097bdad05632c8'), ObjectId('6560d4e114097bdad05632c9'), ObjectId('6560d4e114097bdad05632ca'), ObjectId('6560d4e114097bdad05632cb'), ObjectId('6560d4e114097bdad05632cc'), ObjectId('6560d4e114097bdad05632cd'), ObjectId('6560d4e114097bdad05632ce'), ObjectId('6560d4e114097bdad05632cf'), ObjectId('6560d4e114097bdad05632d0'), ObjectId('6560d4e114097bdad05632d1'), ObjectId('6560d4e114097bdad05632d2'), ObjectId('6560d4e114097bdad05632d3'), ObjectId('6560d4e114097bdad05632d4'), ObjectId('6560d4e114097bdad05632d5'), ObjectId('6560d4e114097bdad05632d6'), ObjectId('6560d4e114097bdad05632d7'), ObjectId('6560d4e114097bdad05632d8'), ObjectId('6560d4e114097bdad05632d9'), ObjectId('6560d4e114097bdad05632da'), ObjectId('6560d4e114097bdad05632db'), ObjectId('6560d4e114097bdad05632dc'), ObjectId('6560d4e114097bdad05632dd'), ObjectId('6560d4e114097bdad05632de'), ObjectId('6560d4e114097bdad05632df'), ObjectId('6560d4e114097bdad05632e0'), ObjectId('6560d4e114097bdad05632e1'), ObjectId('6560d4e114097bdad05632e2'), ObjectId('6560d4e114097bdad05632e3'), ObjectId('6560d4e114097bdad05632e4'), ObjectId('6560d4e114097bdad05632e5'), ObjectId('6560d4e114097bdad05632e6'), ObjectId('6560d4e114097bdad05632e7'), ObjectId('6560d4e114097bdad05632e8'), ObjectId('6560d4e114097bdad05632e9'), ObjectId('6560d4e114097bdad05632ea'), ObjectId('6560d4e114097bdad05632eb'), ObjectId('6560d4e114097bdad05632ec'), ObjectId('6560d4e114097bdad05632ed'), ObjectId('6560d4e114097bdad05632ee'), ObjectId('6560d4e114097bdad05632ef'), ObjectId('6560d4e114097bdad05632f0'), ObjectId('6560d4e114097bdad05632f1'), ObjectId('6560d4e114097bdad05632f2'), ObjectId('6560d4e114097bdad05632f3'), ObjectId('6560d4e114097bdad05632f4'), ObjectId('6560d4e114097bdad05632f5'), ObjectId('6560d4e114097bdad05632f6'), ObjectId('6560d4e114097bdad05632f7'), ObjectId('6560d4e114097bdad05632f8'), ObjectId('6560d4e114097bdad05632f9'), ObjectId('6560d4e114097bdad05632fa'), ObjectId('6560d4e114097bdad05632fb'), ObjectId('6560d4e114097bdad05632fc'), ObjectId('6560d4e114097bdad05632fd'), ObjectId('6560d4e114097bdad05632fe'), ObjectId('6560d4e114097bdad05632ff'), ObjectId('6560d4e114097bdad0563300'), ObjectId('6560d4e114097bdad0563301'), ObjectId('6560d4e114097bdad0563302'), ObjectId('6560d4e114097bdad0563303'), ObjectId('6560d4e114097bdad0563304'), ObjectId('6560d4e114097bdad0563305'), ObjectId('6560d4e114097bdad0563306'), ObjectId('6560d4e114097bdad0563307'), ObjectId('6560d4e114097bdad0563308'), ObjectId('6560d4e114097bdad0563309'), ObjectId('6560d4e114097bdad056330a'), ObjectId('6560d4e114097bdad056330b'), ObjectId('6560d4e114097bdad056330c'), ObjectId('6560d4e114097bdad056330d'), ObjectId('6560d4e114097bdad056330e'), ObjectId('6560d4e114097bdad056330f'), ObjectId('6560d4e114097bdad0563310'), ObjectId('6560d4e114097bdad0563311'), ObjectId('6560d4e114097bdad0563312'), ObjectId('6560d4e114097bdad0563313'), ObjectId('6560d4e114097bdad0563314'), ObjectId('6560d4e114097bdad0563315'), ObjectId('6560d4e114097bdad0563316'), ObjectId('6560d4e114097bdad0563317'), ObjectId('6560d4e114097bdad0563318'), ObjectId('6560d4e114097bdad0563319'), ObjectId('6560d4e114097bdad056331a'), ObjectId('6560d4e114097bdad056331b'), ObjectId('6560d4e114097bdad056331c'), ObjectId('6560d4e114097bdad056331d'), ObjectId('6560d4e114097bdad056331e'), ObjectId('6560d4e114097bdad056331f'), ObjectId('6560d4e114097bdad0563320'), ObjectId('6560d4e114097bdad0563321'), ObjectId('6560d4e114097bdad0563322'), ObjectId('6560d4e114097bdad0563323'), ObjectId('6560d4e114097bdad0563324'), ObjectId('6560d4e114097bdad0563325'), ObjectId('6560d4e114097bdad0563326'), ObjectId('6560d4e114097bdad0563327'), ObjectId('6560d4e114097bdad0563328'), ObjectId('6560d4e114097bdad0563329'), ObjectId('6560d4e114097bdad056332a'), ObjectId('6560d4e114097bdad056332b'), ObjectId('6560d4e114097bdad056332c'), ObjectId('6560d4e114097bdad056332d'), ObjectId('6560d4e114097bdad056332e'), ObjectId('6560d4e114097bdad056332f'), ObjectId('6560d4e114097bdad0563330'), ObjectId('6560d4e114097bdad0563331'), ObjectId('6560d4e114097bdad0563332'), ObjectId('6560d4e114097bdad0563333'), ObjectId('6560d4e114097bdad0563334'), ObjectId('6560d4e114097bdad0563335'), ObjectId('6560d4e114097bdad0563336'), ObjectId('6560d4e114097bdad0563337'), ObjectId('6560d4e114097bdad0563338'), ObjectId('6560d4e114097bdad0563339'), ObjectId('6560d4e114097bdad056333a'), ObjectId('6560d4e114097bdad056333b'), ObjectId('6560d4e114097bdad056333c'), ObjectId('6560d4e114097bdad056333d'), ObjectId('6560d4e114097bdad056333e'), ObjectId('6560d4e114097bdad056333f'), ObjectId('6560d4e114097bdad0563340'), ObjectId('6560d4e114097bdad0563341'), ObjectId('6560d4e114097bdad0563342'), ObjectId('6560d4e114097bdad0563343'), ObjectId('6560d4e114097bdad0563344'), ObjectId('6560d4e114097bdad0563345'), ObjectId('6560d4e114097bdad0563346'), ObjectId('6560d4e114097bdad0563347'), ObjectId('6560d4e114097bdad0563348'), ObjectId('6560d4e114097bdad0563349'), ObjectId('6560d4e114097bdad056334a'), ObjectId('6560d4e114097bdad056334b'), ObjectId('6560d4e114097bdad056334c'), ObjectId('6560d4e114097bdad056334d'), ObjectId('6560d4e114097bdad056334e'), ObjectId('6560d4e114097bdad056334f'), ObjectId('6560d4e114097bdad0563350'), ObjectId('6560d4e114097bdad0563351'), ObjectId('6560d4e114097bdad0563352'), ObjectId('6560d4e114097bdad0563353'), ObjectId('6560d4e114097bdad0563354'), ObjectId('6560d4e114097bdad0563355'), ObjectId('6560d4e114097bdad0563356'), ObjectId('6560d4e114097bdad0563357'), ObjectId('6560d4e114097bdad0563358'), ObjectId('6560d4e114097bdad0563359'), ObjectId('6560d4e114097bdad056335a'), ObjectId('6560d4e114097bdad056335b'), ObjectId('6560d4e114097bdad056335c'), ObjectId('6560d4e114097bdad056335d'), ObjectId('6560d4e114097bdad056335e'), ObjectId('6560d4e114097bdad056335f'), ObjectId('6560d4e114097bdad0563360'), ObjectId('6560d4e114097bdad0563361'), ObjectId('6560d4e114097bdad0563362'), ObjectId('6560d4e114097bdad0563363'), ObjectId('6560d4e114097bdad0563364'), ObjectId('6560d4e114097bdad0563365'), ObjectId('6560d4e114097bdad0563366'), ObjectId('6560d4e114097bdad0563367'), ObjectId('6560d4e114097bdad0563368'), ObjectId('6560d4e114097bdad0563369'), ObjectId('6560d4e114097bdad056336a'), ObjectId('6560d4e114097bdad056336b'), ObjectId('6560d4e114097bdad056336c'), ObjectId('6560d4e114097bdad056336d'), ObjectId('6560d4e114097bdad056336e'), ObjectId('6560d4e114097bdad056336f'), ObjectId('6560d4e114097bdad0563370'), ObjectId('6560d4e114097bdad0563371'), ObjectId('6560d4e114097bdad0563372'), ObjectId('6560d4e114097bdad0563373'), ObjectId('6560d4e114097bdad0563374'), ObjectId('6560d4e114097bdad0563375'), ObjectId('6560d4e114097bdad0563376'), ObjectId('6560d4e114097bdad0563377'), ObjectId('6560d4e114097bdad0563378'), ObjectId('6560d4e114097bdad0563379'), ObjectId('6560d4e114097bdad056337a'), ObjectId('6560d4e114097bdad056337b'), ObjectId('6560d4e114097bdad056337c'), ObjectId('6560d4e114097bdad056337d'), ObjectId('6560d4e114097bdad056337e'), ObjectId('6560d4e114097bdad056337f'), ObjectId('6560d4e114097bdad0563380'), ObjectId('6560d4e114097bdad0563381'), ObjectId('6560d4e114097bdad0563382'), ObjectId('6560d4e114097bdad0563383'), ObjectId('6560d4e114097bdad0563384'), ObjectId('6560d4e114097bdad0563385'), ObjectId('6560d4e114097bdad0563386'), ObjectId('6560d4e114097bdad0563387'), ObjectId('6560d4e114097bdad0563388'), ObjectId('6560d4e114097bdad0563389'), ObjectId('6560d4e114097bdad056338a'), ObjectId('6560d4e114097bdad056338b'), ObjectId('6560d4e114097bdad056338c'), ObjectId('6560d4e114097bdad056338d'), ObjectId('6560d4e114097bdad056338e'), ObjectId('6560d4e114097bdad056338f'), ObjectId('6560d4e114097bdad0563390'), ObjectId('6560d4e114097bdad0563391'), ObjectId('6560d4e114097bdad0563392'), ObjectId('6560d4e114097bdad0563393'), ObjectId('6560d4e114097bdad0563394'), ObjectId('6560d4e114097bdad0563395'), ObjectId('6560d4e114097bdad0563396'), ObjectId('6560d4e114097bdad0563397'), ObjectId('6560d4e114097bdad0563398'), ObjectId('6560d4e114097bdad0563399'), ObjectId('6560d4e114097bdad056339a'), ObjectId('6560d4e114097bdad056339b'), ObjectId('6560d4e114097bdad056339c'), ObjectId('6560d4e114097bdad056339d'), ObjectId('6560d4e114097bdad056339e'), ObjectId('6560d4e114097bdad056339f'), ObjectId('6560d4e114097bdad05633a0'), ObjectId('6560d4e114097bdad05633a1'), ObjectId('6560d4e114097bdad05633a2'), ObjectId('6560d4e114097bdad05633a3'), ObjectId('6560d4e114097bdad05633a4'), ObjectId('6560d4e114097bdad05633a5'), ObjectId('6560d4e114097bdad05633a6'), ObjectId('6560d4e114097bdad05633a7'), ObjectId('6560d4e114097bdad05633a8'), ObjectId('6560d4e114097bdad05633a9'), ObjectId('6560d4e114097bdad05633aa'), ObjectId('6560d4e114097bdad05633ab'), ObjectId('6560d4e114097bdad05633ac'), ObjectId('6560d4e114097bdad05633ad'), ObjectId('6560d4e114097bdad05633ae'), ObjectId('6560d4e114097bdad05633af'), ObjectId('6560d4e114097bdad05633b0'), ObjectId('6560d4e114097bdad05633b1'), ObjectId('6560d4e114097bdad05633b2'), ObjectId('6560d4e114097bdad05633b3'), ObjectId('6560d4e114097bdad05633b4'), ObjectId('6560d4e114097bdad05633b5'), ObjectId('6560d4e114097bdad05633b6'), ObjectId('6560d4e114097bdad05633b7'), ObjectId('6560d4e114097bdad05633b8'), ObjectId('6560d4e114097bdad05633b9'), ObjectId('6560d4e114097bdad05633ba'), ObjectId('6560d4e114097bdad05633bb'), ObjectId('6560d4e114097bdad05633bc'), ObjectId('6560d4e114097bdad05633bd'), ObjectId('6560d4e114097bdad05633be'), ObjectId('6560d4e114097bdad05633bf'), ObjectId('6560d4e114097bdad05633c0'), ObjectId('6560d4e114097bdad05633c1'), ObjectId('6560d4e114097bdad05633c2'), ObjectId('6560d4e114097bdad05633c3'), ObjectId('6560d4e114097bdad05633c4'), ObjectId('6560d4e114097bdad05633c5'), ObjectId('6560d4e114097bdad05633c6'), ObjectId('6560d4e114097bdad05633c7'), ObjectId('6560d4e114097bdad05633c8'), ObjectId('6560d4e114097bdad05633c9'), ObjectId('6560d4e114097bdad05633ca'), ObjectId('6560d4e114097bdad05633cb'), ObjectId('6560d4e114097bdad05633cc'), ObjectId('6560d4e114097bdad05633cd'), ObjectId('6560d4e114097bdad05633ce'), ObjectId('6560d4e114097bdad05633cf'), ObjectId('6560d4e114097bdad05633d0'), ObjectId('6560d4e114097bdad05633d1'), ObjectId('6560d4e114097bdad05633d2'), ObjectId('6560d4e114097bdad05633d3'), ObjectId('6560d4e114097bdad05633d4'), ObjectId('6560d4e114097bdad05633d5'), ObjectId('6560d4e114097bdad05633d6'), ObjectId('6560d4e114097bdad05633d7'), ObjectId('6560d4e114097bdad05633d8'), ObjectId('6560d4e114097bdad05633d9'), ObjectId('6560d4e114097bdad05633da'), ObjectId('6560d4e114097bdad05633db'), ObjectId('6560d4e114097bdad05633dc'), ObjectId('6560d4e114097bdad05633dd'), ObjectId('6560d4e114097bdad05633de'), ObjectId('6560d4e114097bdad05633df'), ObjectId('6560d4e114097bdad05633e0'), ObjectId('6560d4e114097bdad05633e1'), ObjectId('6560d4e114097bdad05633e2'), ObjectId('6560d4e114097bdad05633e3'), ObjectId('6560d4e114097bdad05633e4'), ObjectId('6560d4e114097bdad05633e5'), ObjectId('6560d4e114097bdad05633e6'), ObjectId('6560d4e114097bdad05633e7'), ObjectId('6560d4e114097bdad05633e8'), ObjectId('6560d4e114097bdad05633e9'), ObjectId('6560d4e114097bdad05633ea'), ObjectId('6560d4e114097bdad05633eb'), ObjectId('6560d4e114097bdad05633ec'), ObjectId('6560d4e114097bdad05633ed'), ObjectId('6560d4e114097bdad05633ee'), ObjectId('6560d4e114097bdad05633ef'), ObjectId('6560d4e114097bdad05633f0'), ObjectId('6560d4e114097bdad05633f1'), ObjectId('6560d4e114097bdad05633f2'), ObjectId('6560d4e114097bdad05633f3'), ObjectId('6560d4e114097bdad05633f4'), ObjectId('6560d4e114097bdad05633f5'), ObjectId('6560d4e114097bdad05633f6'), ObjectId('6560d4e114097bdad05633f7'), ObjectId('6560d4e114097bdad05633f8'), ObjectId('6560d4e114097bdad05633f9'), ObjectId('6560d4e114097bdad05633fa'), ObjectId('6560d4e114097bdad05633fb'), ObjectId('6560d4e114097bdad05633fc'), ObjectId('6560d4e114097bdad05633fd'), ObjectId('6560d4e114097bdad05633fe'), ObjectId('6560d4e114097bdad05633ff'), ObjectId('6560d4e114097bdad0563400'), ObjectId('6560d4e114097bdad0563401'), ObjectId('6560d4e114097bdad0563402'), ObjectId('6560d4e114097bdad0563403'), ObjectId('6560d4e114097bdad0563404'), ObjectId('6560d4e114097bdad0563405'), ObjectId('6560d4e114097bdad0563406'), ObjectId('6560d4e114097bdad0563407'), ObjectId('6560d4e114097bdad0563408'), ObjectId('6560d4e114097bdad0563409'), ObjectId('6560d4e114097bdad056340a'), ObjectId('6560d4e114097bdad056340b'), ObjectId('6560d4e114097bdad056340c'), ObjectId('6560d4e114097bdad056340d'), ObjectId('6560d4e114097bdad056340e'), ObjectId('6560d4e114097bdad056340f'), ObjectId('6560d4e114097bdad0563410'), ObjectId('6560d4e114097bdad0563411'), ObjectId('6560d4e114097bdad0563412'), ObjectId('6560d4e114097bdad0563413'), ObjectId('6560d4e114097bdad0563414'), ObjectId('6560d4e114097bdad0563415'), ObjectId('6560d4e114097bdad0563416'), ObjectId('6560d4e114097bdad0563417'), ObjectId('6560d4e114097bdad0563418'), ObjectId('6560d4e114097bdad0563419'), ObjectId('6560d4e114097bdad056341a'), ObjectId('6560d4e114097bdad056341b'), ObjectId('6560d4e114097bdad056341c'), ObjectId('6560d4e114097bdad056341d'), ObjectId('6560d4e114097bdad056341e'), ObjectId('6560d4e114097bdad056341f'), ObjectId('6560d4e114097bdad0563420'), ObjectId('6560d4e114097bdad0563421'), ObjectId('6560d4e114097bdad0563422'), ObjectId('6560d4e114097bdad0563423'), ObjectId('6560d4e114097bdad0563424'), ObjectId('6560d4e114097bdad0563425'), ObjectId('6560d4e114097bdad0563426'), ObjectId('6560d4e114097bdad0563427'), ObjectId('6560d4e114097bdad0563428'), ObjectId('6560d4e114097bdad0563429'), ObjectId('6560d4e114097bdad056342a'), ObjectId('6560d4e114097bdad056342b'), ObjectId('6560d4e114097bdad056342c'), ObjectId('6560d4e114097bdad056342d'), ObjectId('6560d4e114097bdad056342e'), ObjectId('6560d4e114097bdad056342f'), ObjectId('6560d4e114097bdad0563430'), ObjectId('6560d4e114097bdad0563431'), ObjectId('6560d4e114097bdad0563432'), ObjectId('6560d4e114097bdad0563433'), ObjectId('6560d4e114097bdad0563434'), ObjectId('6560d4e114097bdad0563435'), ObjectId('6560d4e114097bdad0563436'), ObjectId('6560d4e114097bdad0563437'), ObjectId('6560d4e114097bdad0563438'), ObjectId('6560d4e114097bdad0563439'), ObjectId('6560d4e114097bdad056343a'), ObjectId('6560d4e114097bdad056343b'), ObjectId('6560d4e114097bdad056343c'), ObjectId('6560d4e114097bdad056343d'), ObjectId('6560d4e114097bdad056343e'), ObjectId('6560d4e114097bdad056343f'), ObjectId('6560d4e114097bdad0563440'), ObjectId('6560d4e114097bdad0563441'), ObjectId('6560d4e114097bdad0563442'), ObjectId('6560d4e114097bdad0563443'), ObjectId('6560d4e114097bdad0563444'), ObjectId('6560d4e114097bdad0563445'), ObjectId('6560d4e114097bdad0563446'), ObjectId('6560d4e114097bdad0563447'), ObjectId('6560d4e114097bdad0563448'), ObjectId('6560d4e114097bdad0563449'), ObjectId('6560d4e114097bdad056344a'), ObjectId('6560d4e114097bdad056344b'), ObjectId('6560d4e114097bdad056344c'), ObjectId('6560d4e114097bdad056344d'), ObjectId('6560d4e114097bdad056344e'), ObjectId('6560d4e114097bdad056344f'), ObjectId('6560d4e114097bdad0563450'), ObjectId('6560d4e114097bdad0563451'), ObjectId('6560d4e114097bdad0563452'), ObjectId('6560d4e114097bdad0563453'), ObjectId('6560d4e114097bdad0563454'), ObjectId('6560d4e114097bdad0563455'), ObjectId('6560d4e114097bdad0563456'), ObjectId('6560d4e114097bdad0563457'), ObjectId('6560d4e114097bdad0563458'), ObjectId('6560d4e114097bdad0563459'), ObjectId('6560d4e114097bdad056345a'), ObjectId('6560d4e114097bdad056345b'), ObjectId('6560d4e114097bdad056345c'), ObjectId('6560d4e114097bdad056345d'), ObjectId('6560d4e114097bdad056345e'), ObjectId('6560d4e114097bdad056345f'), ObjectId('6560d4e114097bdad0563460'), ObjectId('6560d4e114097bdad0563461'), ObjectId('6560d4e114097bdad0563462'), ObjectId('6560d4e114097bdad0563463'), ObjectId('6560d4e114097bdad0563464'), ObjectId('6560d4e114097bdad0563465'), ObjectId('6560d4e114097bdad0563466'), ObjectId('6560d4e114097bdad0563467'), ObjectId('6560d4e114097bdad0563468'), ObjectId('6560d4e114097bdad0563469'), ObjectId('6560d4e114097bdad056346a'), ObjectId('6560d4e114097bdad056346b'), ObjectId('6560d4e114097bdad056346c'), ObjectId('6560d4e114097bdad056346d'), ObjectId('6560d4e114097bdad056346e'), ObjectId('6560d4e114097bdad056346f'), ObjectId('6560d4e114097bdad0563470'), ObjectId('6560d4e114097bdad0563471'), ObjectId('6560d4e114097bdad0563472'), ObjectId('6560d4e114097bdad0563473'), ObjectId('6560d4e114097bdad0563474'), ObjectId('6560d4e114097bdad0563475'), ObjectId('6560d4e114097bdad0563476'), ObjectId('6560d4e114097bdad0563477'), ObjectId('6560d4e114097bdad0563478'), ObjectId('6560d4e114097bdad0563479'), ObjectId('6560d4e114097bdad056347a'), ObjectId('6560d4e114097bdad056347b'), ObjectId('6560d4e114097bdad056347c'), ObjectId('6560d4e114097bdad056347d'), ObjectId('6560d4e114097bdad056347e'), ObjectId('6560d4e114097bdad056347f'), ObjectId('6560d4e114097bdad0563480'), ObjectId('6560d4e114097bdad0563481'), ObjectId('6560d4e114097bdad0563482'), ObjectId('6560d4e114097bdad0563483'), ObjectId('6560d4e114097bdad0563484'), ObjectId('6560d4e114097bdad0563485'), ObjectId('6560d4e114097bdad0563486'), ObjectId('6560d4e114097bdad0563487'), ObjectId('6560d4e114097bdad0563488'), ObjectId('6560d4e114097bdad0563489'), ObjectId('6560d4e114097bdad056348a'), ObjectId('6560d4e114097bdad056348b'), ObjectId('6560d4e114097bdad056348c'), ObjectId('6560d4e114097bdad056348d'), ObjectId('6560d4e114097bdad056348e'), ObjectId('6560d4e114097bdad056348f'), ObjectId('6560d4e114097bdad0563490'), ObjectId('6560d4e114097bdad0563491'), ObjectId('6560d4e114097bdad0563492'), ObjectId('6560d4e114097bdad0563493'), ObjectId('6560d4e114097bdad0563494'), ObjectId('6560d4e114097bdad0563495'), ObjectId('6560d4e114097bdad0563496'), ObjectId('6560d4e114097bdad0563497'), ObjectId('6560d4e114097bdad0563498'), ObjectId('6560d4e114097bdad0563499'), ObjectId('6560d4e114097bdad056349a'), ObjectId('6560d4e114097bdad056349b'), ObjectId('6560d4e114097bdad056349c'), ObjectId('6560d4e114097bdad056349d'), ObjectId('6560d4e114097bdad056349e'), ObjectId('6560d4e114097bdad056349f'), ObjectId('6560d4e114097bdad05634a0'), ObjectId('6560d4e114097bdad05634a1'), ObjectId('6560d4e114097bdad05634a2'), ObjectId('6560d4e114097bdad05634a3'), ObjectId('6560d4e114097bdad05634a4'), ObjectId('6560d4e114097bdad05634a5'), ObjectId('6560d4e114097bdad05634a6'), ObjectId('6560d4e114097bdad05634a7'), ObjectId('6560d4e114097bdad05634a8'), ObjectId('6560d4e114097bdad05634a9'), ObjectId('6560d4e114097bdad05634aa'), ObjectId('6560d4e114097bdad05634ab'), ObjectId('6560d4e114097bdad05634ac'), ObjectId('6560d4e114097bdad05634ad'), ObjectId('6560d4e114097bdad05634ae'), ObjectId('6560d4e114097bdad05634af'), ObjectId('6560d4e114097bdad05634b0'), ObjectId('6560d4e114097bdad05634b1'), ObjectId('6560d4e114097bdad05634b2'), ObjectId('6560d4e114097bdad05634b3'), ObjectId('6560d4e114097bdad05634b4'), ObjectId('6560d4e114097bdad05634b5'), ObjectId('6560d4e114097bdad05634b6'), ObjectId('6560d4e114097bdad05634b7'), ObjectId('6560d4e114097bdad05634b8'), ObjectId('6560d4e114097bdad05634b9'), ObjectId('6560d4e114097bdad05634ba'), ObjectId('6560d4e114097bdad05634bb'), ObjectId('6560d4e114097bdad05634bc'), ObjectId('6560d4e114097bdad05634bd'), ObjectId('6560d4e114097bdad05634be'), ObjectId('6560d4e114097bdad05634bf'), ObjectId('6560d4e114097bdad05634c0'), ObjectId('6560d4e114097bdad05634c1'), ObjectId('6560d4e114097bdad05634c2'), ObjectId('6560d4e114097bdad05634c3'), ObjectId('6560d4e114097bdad05634c4'), ObjectId('6560d4e114097bdad05634c5'), ObjectId('6560d4e114097bdad05634c6'), ObjectId('6560d4e114097bdad05634c7'), ObjectId('6560d4e114097bdad05634c8'), ObjectId('6560d4e114097bdad05634c9'), ObjectId('6560d4e114097bdad05634ca'), ObjectId('6560d4e114097bdad05634cb'), ObjectId('6560d4e114097bdad05634cc'), ObjectId('6560d4e114097bdad05634cd'), ObjectId('6560d4e114097bdad05634ce'), ObjectId('6560d4e114097bdad05634cf'), ObjectId('6560d4e114097bdad05634d0'), ObjectId('6560d4e114097bdad05634d1'), ObjectId('6560d4e114097bdad05634d2'), ObjectId('6560d4e114097bdad05634d3'), ObjectId('6560d4e114097bdad05634d4'), ObjectId('6560d4e114097bdad05634d5'), ObjectId('6560d4e114097bdad05634d6'), ObjectId('6560d4e114097bdad05634d7'), ObjectId('6560d4e114097bdad05634d8'), ObjectId('6560d4e114097bdad05634d9'), ObjectId('6560d4e114097bdad05634da'), ObjectId('6560d4e114097bdad05634db'), ObjectId('6560d4e114097bdad05634dc'), ObjectId('6560d4e114097bdad05634dd'), ObjectId('6560d4e114097bdad05634de'), ObjectId('6560d4e114097bdad05634df'), ObjectId('6560d4e114097bdad05634e0'), ObjectId('6560d4e114097bdad05634e1'), ObjectId('6560d4e114097bdad05634e2'), ObjectId('6560d4e114097bdad05634e3'), ObjectId('6560d4e114097bdad05634e4'), ObjectId('6560d4e114097bdad05634e5'), ObjectId('6560d4e114097bdad05634e6'), ObjectId('6560d4e114097bdad05634e7'), ObjectId('6560d4e114097bdad05634e8'), ObjectId('6560d4e114097bdad05634e9'), ObjectId('6560d4e114097bdad05634ea'), ObjectId('6560d4e114097bdad05634eb'), ObjectId('6560d4e114097bdad05634ec'), ObjectId('6560d4e114097bdad05634ed'), ObjectId('6560d4e114097bdad05634ee'), ObjectId('6560d4e114097bdad05634ef'), ObjectId('6560d4e114097bdad05634f0'), ObjectId('6560d4e114097bdad05634f1'), ObjectId('6560d4e114097bdad05634f2'), ObjectId('6560d4e114097bdad05634f3'), ObjectId('6560d4e114097bdad05634f4'), ObjectId('6560d4e114097bdad05634f5'), ObjectId('6560d4e114097bdad05634f6'), ObjectId('6560d4e114097bdad05634f7'), ObjectId('6560d4e114097bdad05634f8'), ObjectId('6560d4e114097bdad05634f9'), ObjectId('6560d4e114097bdad05634fa'), ObjectId('6560d4e114097bdad05634fb'), ObjectId('6560d4e114097bdad05634fc'), ObjectId('6560d4e114097bdad05634fd'), ObjectId('6560d4e114097bdad05634fe'), ObjectId('6560d4e114097bdad05634ff'), ObjectId('6560d4e114097bdad0563500'), ObjectId('6560d4e114097bdad0563501'), ObjectId('6560d4e114097bdad0563502'), ObjectId('6560d4e114097bdad0563503'), ObjectId('6560d4e114097bdad0563504'), ObjectId('6560d4e114097bdad0563505'), ObjectId('6560d4e114097bdad0563506'), ObjectId('6560d4e114097bdad0563507'), ObjectId('6560d4e114097bdad0563508'), ObjectId('6560d4e114097bdad0563509'), ObjectId('6560d4e114097bdad056350a'), ObjectId('6560d4e114097bdad056350b'), ObjectId('6560d4e114097bdad056350c'), ObjectId('6560d4e114097bdad056350d'), ObjectId('6560d4e114097bdad056350e'), ObjectId('6560d4e114097bdad056350f'), ObjectId('6560d4e114097bdad0563510'), ObjectId('6560d4e114097bdad0563511'), ObjectId('6560d4e114097bdad0563512'), ObjectId('6560d4e114097bdad0563513'), ObjectId('6560d4e114097bdad0563514'), ObjectId('6560d4e114097bdad0563515'), ObjectId('6560d4e114097bdad0563516'), ObjectId('6560d4e114097bdad0563517'), ObjectId('6560d4e114097bdad0563518'), ObjectId('6560d4e114097bdad0563519'), ObjectId('6560d4e114097bdad056351a'), ObjectId('6560d4e114097bdad056351b'), ObjectId('6560d4e114097bdad056351c'), ObjectId('6560d4e114097bdad056351d'), ObjectId('6560d4e114097bdad056351e'), ObjectId('6560d4e114097bdad056351f'), ObjectId('6560d4e114097bdad0563520'), ObjectId('6560d4e114097bdad0563521'), ObjectId('6560d4e114097bdad0563522'), ObjectId('6560d4e114097bdad0563523'), ObjectId('6560d4e114097bdad0563524'), ObjectId('6560d4e114097bdad0563525'), ObjectId('6560d4e114097bdad0563526'), ObjectId('6560d4e114097bdad0563527'), ObjectId('6560d4e114097bdad0563528'), ObjectId('6560d4e114097bdad0563529'), ObjectId('6560d4e114097bdad056352a'), ObjectId('6560d4e114097bdad056352b'), ObjectId('6560d4e114097bdad056352c'), ObjectId('6560d4e114097bdad056352d'), ObjectId('6560d4e114097bdad056352e'), ObjectId('6560d4e114097bdad056352f'), ObjectId('6560d4e114097bdad0563530'), ObjectId('6560d4e114097bdad0563531'), ObjectId('6560d4e114097bdad0563532'), ObjectId('6560d4e114097bdad0563533'), ObjectId('6560d4e114097bdad0563534'), ObjectId('6560d4e114097bdad0563535'), ObjectId('6560d4e114097bdad0563536'), ObjectId('6560d4e114097bdad0563537'), ObjectId('6560d4e114097bdad0563538'), ObjectId('6560d4e114097bdad0563539'), ObjectId('6560d4e114097bdad056353a'), ObjectId('6560d4e114097bdad056353b'), ObjectId('6560d4e114097bdad056353c'), ObjectId('6560d4e114097bdad056353d'), ObjectId('6560d4e114097bdad056353e'), ObjectId('6560d4e114097bdad056353f'), ObjectId('6560d4e114097bdad0563540'), ObjectId('6560d4e114097bdad0563541'), ObjectId('6560d4e114097bdad0563542'), ObjectId('6560d4e114097bdad0563543'), ObjectId('6560d4e114097bdad0563544'), ObjectId('6560d4e114097bdad0563545'), ObjectId('6560d4e114097bdad0563546'), ObjectId('6560d4e114097bdad0563547'), ObjectId('6560d4e114097bdad0563548'), ObjectId('6560d4e114097bdad0563549'), ObjectId('6560d4e114097bdad056354a'), ObjectId('6560d4e114097bdad056354b'), ObjectId('6560d4e114097bdad056354c'), ObjectId('6560d4e114097bdad056354d'), ObjectId('6560d4e114097bdad056354e'), ObjectId('6560d4e114097bdad056354f'), ObjectId('6560d4e114097bdad0563550'), ObjectId('6560d4e114097bdad0563551'), ObjectId('6560d4e114097bdad0563552'), ObjectId('6560d4e114097bdad0563553'), ObjectId('6560d4e114097bdad0563554'), ObjectId('6560d4e114097bdad0563555'), ObjectId('6560d4e114097bdad0563556'), ObjectId('6560d4e114097bdad0563557'), ObjectId('6560d4e114097bdad0563558'), ObjectId('6560d4e114097bdad0563559'), ObjectId('6560d4e114097bdad056355a'), ObjectId('6560d4e114097bdad056355b'), ObjectId('6560d4e114097bdad056355c'), ObjectId('6560d4e114097bdad056355d'), ObjectId('6560d4e114097bdad056355e'), ObjectId('6560d4e114097bdad056355f'), ObjectId('6560d4e114097bdad0563560'), ObjectId('6560d4e114097bdad0563561'), ObjectId('6560d4e114097bdad0563562'), ObjectId('6560d4e114097bdad0563563'), ObjectId('6560d4e114097bdad0563564'), ObjectId('6560d4e114097bdad0563565'), ObjectId('6560d4e114097bdad0563566'), ObjectId('6560d4e114097bdad0563567'), ObjectId('6560d4e114097bdad0563568'), ObjectId('6560d4e114097bdad0563569'), ObjectId('6560d4e114097bdad056356a'), ObjectId('6560d4e114097bdad056356b'), ObjectId('6560d4e114097bdad056356c'), ObjectId('6560d4e114097bdad056356d'), ObjectId('6560d4e114097bdad056356e'), ObjectId('6560d4e114097bdad056356f'), ObjectId('6560d4e114097bdad0563570'), ObjectId('6560d4e114097bdad0563571'), ObjectId('6560d4e114097bdad0563572'), ObjectId('6560d4e114097bdad0563573'), ObjectId('6560d4e114097bdad0563574'), ObjectId('6560d4e114097bdad0563575'), ObjectId('6560d4e114097bdad0563576'), ObjectId('6560d4e114097bdad0563577'), ObjectId('6560d4e114097bdad0563578'), ObjectId('6560d4e114097bdad0563579'), ObjectId('6560d4e114097bdad056357a'), ObjectId('6560d4e114097bdad056357b'), ObjectId('6560d4e114097bdad056357c'), ObjectId('6560d4e114097bdad056357d'), ObjectId('6560d4e114097bdad056357e'), ObjectId('6560d4e114097bdad056357f'), ObjectId('6560d4e114097bdad0563580'), ObjectId('6560d4e114097bdad0563581'), ObjectId('6560d4e114097bdad0563582'), ObjectId('6560d4e114097bdad0563583'), ObjectId('6560d4e114097bdad0563584'), ObjectId('6560d4e114097bdad0563585'), ObjectId('6560d4e114097bdad0563586'), ObjectId('6560d4e114097bdad0563587'), ObjectId('6560d4e114097bdad0563588'), ObjectId('6560d4e114097bdad0563589'), ObjectId('6560d4e114097bdad056358a'), ObjectId('6560d4e114097bdad056358b'), ObjectId('6560d4e114097bdad056358c'), ObjectId('6560d4e114097bdad056358d'), ObjectId('6560d4e114097bdad056358e'), ObjectId('6560d4e114097bdad056358f'), ObjectId('6560d4e114097bdad0563590'), ObjectId('6560d4e114097bdad0563591'), ObjectId('6560d4e114097bdad0563592'), ObjectId('6560d4e114097bdad0563593'), ObjectId('6560d4e114097bdad0563594'), ObjectId('6560d4e114097bdad0563595'), ObjectId('6560d4e114097bdad0563596'), ObjectId('6560d4e114097bdad0563597'), ObjectId('6560d4e114097bdad0563598'), ObjectId('6560d4e114097bdad0563599'), ObjectId('6560d4e114097bdad056359a'), ObjectId('6560d4e114097bdad056359b'), ObjectId('6560d4e114097bdad056359c'), ObjectId('6560d4e114097bdad056359d')]\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.38\u001B[0m| \u001B[32m\u001B[1mSUCCESS \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.backends.local.compute\u001B[0m:\u001B[36m38 \u001B[0m | \u001B[32m\u001B[1mJob submitted. function: future:e3ca0b12-28bf-45d7-bf1f-b21e86022509\u001B[0m\n" + ] + }, + { + "data": { + "text/plain": "([ObjectId('6560d4e114097bdad056321a'),\n ObjectId('6560d4e114097bdad056321b'),\n ObjectId('6560d4e114097bdad056321c'),\n ObjectId('6560d4e114097bdad056321d'),\n ObjectId('6560d4e114097bdad056321e'),\n ObjectId('6560d4e114097bdad056321f'),\n ObjectId('6560d4e114097bdad0563220'),\n ObjectId('6560d4e114097bdad0563221'),\n ObjectId('6560d4e114097bdad0563222'),\n ObjectId('6560d4e114097bdad0563223'),\n ObjectId('6560d4e114097bdad0563224'),\n ObjectId('6560d4e114097bdad0563225'),\n ObjectId('6560d4e114097bdad0563226'),\n ObjectId('6560d4e114097bdad0563227'),\n ObjectId('6560d4e114097bdad0563228'),\n ObjectId('6560d4e114097bdad0563229'),\n ObjectId('6560d4e114097bdad056322a'),\n ObjectId('6560d4e114097bdad056322b'),\n ObjectId('6560d4e114097bdad056322c'),\n ObjectId('6560d4e114097bdad056322d'),\n ObjectId('6560d4e114097bdad056322e'),\n ObjectId('6560d4e114097bdad056322f'),\n ObjectId('6560d4e114097bdad0563230'),\n ObjectId('6560d4e114097bdad0563231'),\n ObjectId('6560d4e114097bdad0563232'),\n ObjectId('6560d4e114097bdad0563233'),\n ObjectId('6560d4e114097bdad0563234'),\n ObjectId('6560d4e114097bdad0563235'),\n ObjectId('6560d4e114097bdad0563236'),\n ObjectId('6560d4e114097bdad0563237'),\n ObjectId('6560d4e114097bdad0563238'),\n ObjectId('6560d4e114097bdad0563239'),\n ObjectId('6560d4e114097bdad056323a'),\n ObjectId('6560d4e114097bdad056323b'),\n ObjectId('6560d4e114097bdad056323c'),\n ObjectId('6560d4e114097bdad056323d'),\n ObjectId('6560d4e114097bdad056323e'),\n ObjectId('6560d4e114097bdad056323f'),\n ObjectId('6560d4e114097bdad0563240'),\n ObjectId('6560d4e114097bdad0563241'),\n ObjectId('6560d4e114097bdad0563242'),\n ObjectId('6560d4e114097bdad0563243'),\n ObjectId('6560d4e114097bdad0563244'),\n ObjectId('6560d4e114097bdad0563245'),\n ObjectId('6560d4e114097bdad0563246'),\n ObjectId('6560d4e114097bdad0563247'),\n ObjectId('6560d4e114097bdad0563248'),\n ObjectId('6560d4e114097bdad0563249'),\n ObjectId('6560d4e114097bdad056324a'),\n ObjectId('6560d4e114097bdad056324b'),\n ObjectId('6560d4e114097bdad056324c'),\n ObjectId('6560d4e114097bdad056324d'),\n ObjectId('6560d4e114097bdad056324e'),\n ObjectId('6560d4e114097bdad056324f'),\n ObjectId('6560d4e114097bdad0563250'),\n ObjectId('6560d4e114097bdad0563251'),\n ObjectId('6560d4e114097bdad0563252'),\n ObjectId('6560d4e114097bdad0563253'),\n ObjectId('6560d4e114097bdad0563254'),\n ObjectId('6560d4e114097bdad0563255'),\n ObjectId('6560d4e114097bdad0563256'),\n ObjectId('6560d4e114097bdad0563257'),\n ObjectId('6560d4e114097bdad0563258'),\n ObjectId('6560d4e114097bdad0563259'),\n ObjectId('6560d4e114097bdad056325a'),\n ObjectId('6560d4e114097bdad056325b'),\n ObjectId('6560d4e114097bdad056325c'),\n ObjectId('6560d4e114097bdad056325d'),\n ObjectId('6560d4e114097bdad056325e'),\n ObjectId('6560d4e114097bdad056325f'),\n ObjectId('6560d4e114097bdad0563260'),\n ObjectId('6560d4e114097bdad0563261'),\n ObjectId('6560d4e114097bdad0563262'),\n ObjectId('6560d4e114097bdad0563263'),\n ObjectId('6560d4e114097bdad0563264'),\n ObjectId('6560d4e114097bdad0563265'),\n ObjectId('6560d4e114097bdad0563266'),\n ObjectId('6560d4e114097bdad0563267'),\n ObjectId('6560d4e114097bdad0563268'),\n ObjectId('6560d4e114097bdad0563269'),\n ObjectId('6560d4e114097bdad056326a'),\n ObjectId('6560d4e114097bdad056326b'),\n ObjectId('6560d4e114097bdad056326c'),\n ObjectId('6560d4e114097bdad056326d'),\n ObjectId('6560d4e114097bdad056326e'),\n ObjectId('6560d4e114097bdad056326f'),\n ObjectId('6560d4e114097bdad0563270'),\n ObjectId('6560d4e114097bdad0563271'),\n ObjectId('6560d4e114097bdad0563272'),\n ObjectId('6560d4e114097bdad0563273'),\n ObjectId('6560d4e114097bdad0563274'),\n ObjectId('6560d4e114097bdad0563275'),\n ObjectId('6560d4e114097bdad0563276'),\n ObjectId('6560d4e114097bdad0563277'),\n ObjectId('6560d4e114097bdad0563278'),\n ObjectId('6560d4e114097bdad0563279'),\n ObjectId('6560d4e114097bdad056327a'),\n ObjectId('6560d4e114097bdad056327b'),\n ObjectId('6560d4e114097bdad056327c'),\n ObjectId('6560d4e114097bdad056327d'),\n ObjectId('6560d4e114097bdad056327e'),\n ObjectId('6560d4e114097bdad056327f'),\n ObjectId('6560d4e114097bdad0563280'),\n ObjectId('6560d4e114097bdad0563281'),\n ObjectId('6560d4e114097bdad0563282'),\n ObjectId('6560d4e114097bdad0563283'),\n ObjectId('6560d4e114097bdad0563284'),\n ObjectId('6560d4e114097bdad0563285'),\n ObjectId('6560d4e114097bdad0563286'),\n ObjectId('6560d4e114097bdad0563287'),\n ObjectId('6560d4e114097bdad0563288'),\n ObjectId('6560d4e114097bdad0563289'),\n ObjectId('6560d4e114097bdad056328a'),\n ObjectId('6560d4e114097bdad056328b'),\n ObjectId('6560d4e114097bdad056328c'),\n ObjectId('6560d4e114097bdad056328d'),\n ObjectId('6560d4e114097bdad056328e'),\n ObjectId('6560d4e114097bdad056328f'),\n ObjectId('6560d4e114097bdad0563290'),\n ObjectId('6560d4e114097bdad0563291'),\n ObjectId('6560d4e114097bdad0563292'),\n ObjectId('6560d4e114097bdad0563293'),\n ObjectId('6560d4e114097bdad0563294'),\n ObjectId('6560d4e114097bdad0563295'),\n ObjectId('6560d4e114097bdad0563296'),\n ObjectId('6560d4e114097bdad0563297'),\n ObjectId('6560d4e114097bdad0563298'),\n ObjectId('6560d4e114097bdad0563299'),\n ObjectId('6560d4e114097bdad056329a'),\n ObjectId('6560d4e114097bdad056329b'),\n ObjectId('6560d4e114097bdad056329c'),\n ObjectId('6560d4e114097bdad056329d'),\n ObjectId('6560d4e114097bdad056329e'),\n ObjectId('6560d4e114097bdad056329f'),\n ObjectId('6560d4e114097bdad05632a0'),\n ObjectId('6560d4e114097bdad05632a1'),\n ObjectId('6560d4e114097bdad05632a2'),\n ObjectId('6560d4e114097bdad05632a3'),\n ObjectId('6560d4e114097bdad05632a4'),\n ObjectId('6560d4e114097bdad05632a5'),\n ObjectId('6560d4e114097bdad05632a6'),\n ObjectId('6560d4e114097bdad05632a7'),\n ObjectId('6560d4e114097bdad05632a8'),\n ObjectId('6560d4e114097bdad05632a9'),\n ObjectId('6560d4e114097bdad05632aa'),\n ObjectId('6560d4e114097bdad05632ab'),\n ObjectId('6560d4e114097bdad05632ac'),\n ObjectId('6560d4e114097bdad05632ad'),\n ObjectId('6560d4e114097bdad05632ae'),\n ObjectId('6560d4e114097bdad05632af'),\n ObjectId('6560d4e114097bdad05632b0'),\n ObjectId('6560d4e114097bdad05632b1'),\n ObjectId('6560d4e114097bdad05632b2'),\n ObjectId('6560d4e114097bdad05632b3'),\n ObjectId('6560d4e114097bdad05632b4'),\n ObjectId('6560d4e114097bdad05632b5'),\n ObjectId('6560d4e114097bdad05632b6'),\n ObjectId('6560d4e114097bdad05632b7'),\n ObjectId('6560d4e114097bdad05632b8'),\n ObjectId('6560d4e114097bdad05632b9'),\n ObjectId('6560d4e114097bdad05632ba'),\n ObjectId('6560d4e114097bdad05632bb'),\n ObjectId('6560d4e114097bdad05632bc'),\n ObjectId('6560d4e114097bdad05632bd'),\n ObjectId('6560d4e114097bdad05632be'),\n ObjectId('6560d4e114097bdad05632bf'),\n ObjectId('6560d4e114097bdad05632c0'),\n ObjectId('6560d4e114097bdad05632c1'),\n ObjectId('6560d4e114097bdad05632c2'),\n ObjectId('6560d4e114097bdad05632c3'),\n ObjectId('6560d4e114097bdad05632c4'),\n ObjectId('6560d4e114097bdad05632c5'),\n ObjectId('6560d4e114097bdad05632c6'),\n ObjectId('6560d4e114097bdad05632c7'),\n ObjectId('6560d4e114097bdad05632c8'),\n ObjectId('6560d4e114097bdad05632c9'),\n ObjectId('6560d4e114097bdad05632ca'),\n ObjectId('6560d4e114097bdad05632cb'),\n ObjectId('6560d4e114097bdad05632cc'),\n ObjectId('6560d4e114097bdad05632cd'),\n ObjectId('6560d4e114097bdad05632ce'),\n ObjectId('6560d4e114097bdad05632cf'),\n ObjectId('6560d4e114097bdad05632d0'),\n ObjectId('6560d4e114097bdad05632d1'),\n ObjectId('6560d4e114097bdad05632d2'),\n ObjectId('6560d4e114097bdad05632d3'),\n ObjectId('6560d4e114097bdad05632d4'),\n ObjectId('6560d4e114097bdad05632d5'),\n ObjectId('6560d4e114097bdad05632d6'),\n ObjectId('6560d4e114097bdad05632d7'),\n ObjectId('6560d4e114097bdad05632d8'),\n ObjectId('6560d4e114097bdad05632d9'),\n ObjectId('6560d4e114097bdad05632da'),\n ObjectId('6560d4e114097bdad05632db'),\n ObjectId('6560d4e114097bdad05632dc'),\n ObjectId('6560d4e114097bdad05632dd'),\n ObjectId('6560d4e114097bdad05632de'),\n ObjectId('6560d4e114097bdad05632df'),\n ObjectId('6560d4e114097bdad05632e0'),\n ObjectId('6560d4e114097bdad05632e1'),\n ObjectId('6560d4e114097bdad05632e2'),\n ObjectId('6560d4e114097bdad05632e3'),\n ObjectId('6560d4e114097bdad05632e4'),\n ObjectId('6560d4e114097bdad05632e5'),\n ObjectId('6560d4e114097bdad05632e6'),\n ObjectId('6560d4e114097bdad05632e7'),\n ObjectId('6560d4e114097bdad05632e8'),\n ObjectId('6560d4e114097bdad05632e9'),\n ObjectId('6560d4e114097bdad05632ea'),\n ObjectId('6560d4e114097bdad05632eb'),\n ObjectId('6560d4e114097bdad05632ec'),\n ObjectId('6560d4e114097bdad05632ed'),\n ObjectId('6560d4e114097bdad05632ee'),\n ObjectId('6560d4e114097bdad05632ef'),\n ObjectId('6560d4e114097bdad05632f0'),\n ObjectId('6560d4e114097bdad05632f1'),\n ObjectId('6560d4e114097bdad05632f2'),\n ObjectId('6560d4e114097bdad05632f3'),\n ObjectId('6560d4e114097bdad05632f4'),\n ObjectId('6560d4e114097bdad05632f5'),\n ObjectId('6560d4e114097bdad05632f6'),\n ObjectId('6560d4e114097bdad05632f7'),\n ObjectId('6560d4e114097bdad05632f8'),\n ObjectId('6560d4e114097bdad05632f9'),\n ObjectId('6560d4e114097bdad05632fa'),\n ObjectId('6560d4e114097bdad05632fb'),\n ObjectId('6560d4e114097bdad05632fc'),\n ObjectId('6560d4e114097bdad05632fd'),\n ObjectId('6560d4e114097bdad05632fe'),\n ObjectId('6560d4e114097bdad05632ff'),\n ObjectId('6560d4e114097bdad0563300'),\n ObjectId('6560d4e114097bdad0563301'),\n ObjectId('6560d4e114097bdad0563302'),\n ObjectId('6560d4e114097bdad0563303'),\n ObjectId('6560d4e114097bdad0563304'),\n ObjectId('6560d4e114097bdad0563305'),\n ObjectId('6560d4e114097bdad0563306'),\n ObjectId('6560d4e114097bdad0563307'),\n ObjectId('6560d4e114097bdad0563308'),\n ObjectId('6560d4e114097bdad0563309'),\n ObjectId('6560d4e114097bdad056330a'),\n ObjectId('6560d4e114097bdad056330b'),\n ObjectId('6560d4e114097bdad056330c'),\n ObjectId('6560d4e114097bdad056330d'),\n ObjectId('6560d4e114097bdad056330e'),\n ObjectId('6560d4e114097bdad056330f'),\n ObjectId('6560d4e114097bdad0563310'),\n ObjectId('6560d4e114097bdad0563311'),\n ObjectId('6560d4e114097bdad0563312'),\n ObjectId('6560d4e114097bdad0563313'),\n ObjectId('6560d4e114097bdad0563314'),\n ObjectId('6560d4e114097bdad0563315'),\n ObjectId('6560d4e114097bdad0563316'),\n ObjectId('6560d4e114097bdad0563317'),\n ObjectId('6560d4e114097bdad0563318'),\n ObjectId('6560d4e114097bdad0563319'),\n ObjectId('6560d4e114097bdad056331a'),\n ObjectId('6560d4e114097bdad056331b'),\n ObjectId('6560d4e114097bdad056331c'),\n ObjectId('6560d4e114097bdad056331d'),\n ObjectId('6560d4e114097bdad056331e'),\n ObjectId('6560d4e114097bdad056331f'),\n ObjectId('6560d4e114097bdad0563320'),\n ObjectId('6560d4e114097bdad0563321'),\n ObjectId('6560d4e114097bdad0563322'),\n ObjectId('6560d4e114097bdad0563323'),\n ObjectId('6560d4e114097bdad0563324'),\n ObjectId('6560d4e114097bdad0563325'),\n ObjectId('6560d4e114097bdad0563326'),\n ObjectId('6560d4e114097bdad0563327'),\n ObjectId('6560d4e114097bdad0563328'),\n ObjectId('6560d4e114097bdad0563329'),\n ObjectId('6560d4e114097bdad056332a'),\n ObjectId('6560d4e114097bdad056332b'),\n ObjectId('6560d4e114097bdad056332c'),\n ObjectId('6560d4e114097bdad056332d'),\n ObjectId('6560d4e114097bdad056332e'),\n ObjectId('6560d4e114097bdad056332f'),\n ObjectId('6560d4e114097bdad0563330'),\n ObjectId('6560d4e114097bdad0563331'),\n ObjectId('6560d4e114097bdad0563332'),\n ObjectId('6560d4e114097bdad0563333'),\n ObjectId('6560d4e114097bdad0563334'),\n ObjectId('6560d4e114097bdad0563335'),\n ObjectId('6560d4e114097bdad0563336'),\n ObjectId('6560d4e114097bdad0563337'),\n ObjectId('6560d4e114097bdad0563338'),\n ObjectId('6560d4e114097bdad0563339'),\n ObjectId('6560d4e114097bdad056333a'),\n ObjectId('6560d4e114097bdad056333b'),\n ObjectId('6560d4e114097bdad056333c'),\n ObjectId('6560d4e114097bdad056333d'),\n ObjectId('6560d4e114097bdad056333e'),\n ObjectId('6560d4e114097bdad056333f'),\n ObjectId('6560d4e114097bdad0563340'),\n ObjectId('6560d4e114097bdad0563341'),\n ObjectId('6560d4e114097bdad0563342'),\n ObjectId('6560d4e114097bdad0563343'),\n ObjectId('6560d4e114097bdad0563344'),\n ObjectId('6560d4e114097bdad0563345'),\n ObjectId('6560d4e114097bdad0563346'),\n ObjectId('6560d4e114097bdad0563347'),\n ObjectId('6560d4e114097bdad0563348'),\n ObjectId('6560d4e114097bdad0563349'),\n ObjectId('6560d4e114097bdad056334a'),\n ObjectId('6560d4e114097bdad056334b'),\n ObjectId('6560d4e114097bdad056334c'),\n ObjectId('6560d4e114097bdad056334d'),\n ObjectId('6560d4e114097bdad056334e'),\n ObjectId('6560d4e114097bdad056334f'),\n ObjectId('6560d4e114097bdad0563350'),\n ObjectId('6560d4e114097bdad0563351'),\n ObjectId('6560d4e114097bdad0563352'),\n ObjectId('6560d4e114097bdad0563353'),\n ObjectId('6560d4e114097bdad0563354'),\n ObjectId('6560d4e114097bdad0563355'),\n ObjectId('6560d4e114097bdad0563356'),\n ObjectId('6560d4e114097bdad0563357'),\n ObjectId('6560d4e114097bdad0563358'),\n ObjectId('6560d4e114097bdad0563359'),\n ObjectId('6560d4e114097bdad056335a'),\n ObjectId('6560d4e114097bdad056335b'),\n ObjectId('6560d4e114097bdad056335c'),\n ObjectId('6560d4e114097bdad056335d'),\n ObjectId('6560d4e114097bdad056335e'),\n ObjectId('6560d4e114097bdad056335f'),\n ObjectId('6560d4e114097bdad0563360'),\n ObjectId('6560d4e114097bdad0563361'),\n ObjectId('6560d4e114097bdad0563362'),\n ObjectId('6560d4e114097bdad0563363'),\n ObjectId('6560d4e114097bdad0563364'),\n ObjectId('6560d4e114097bdad0563365'),\n ObjectId('6560d4e114097bdad0563366'),\n ObjectId('6560d4e114097bdad0563367'),\n ObjectId('6560d4e114097bdad0563368'),\n ObjectId('6560d4e114097bdad0563369'),\n ObjectId('6560d4e114097bdad056336a'),\n ObjectId('6560d4e114097bdad056336b'),\n ObjectId('6560d4e114097bdad056336c'),\n ObjectId('6560d4e114097bdad056336d'),\n ObjectId('6560d4e114097bdad056336e'),\n ObjectId('6560d4e114097bdad056336f'),\n ObjectId('6560d4e114097bdad0563370'),\n ObjectId('6560d4e114097bdad0563371'),\n ObjectId('6560d4e114097bdad0563372'),\n ObjectId('6560d4e114097bdad0563373'),\n ObjectId('6560d4e114097bdad0563374'),\n ObjectId('6560d4e114097bdad0563375'),\n ObjectId('6560d4e114097bdad0563376'),\n ObjectId('6560d4e114097bdad0563377'),\n ObjectId('6560d4e114097bdad0563378'),\n ObjectId('6560d4e114097bdad0563379'),\n ObjectId('6560d4e114097bdad056337a'),\n ObjectId('6560d4e114097bdad056337b'),\n ObjectId('6560d4e114097bdad056337c'),\n ObjectId('6560d4e114097bdad056337d'),\n ObjectId('6560d4e114097bdad056337e'),\n ObjectId('6560d4e114097bdad056337f'),\n ObjectId('6560d4e114097bdad0563380'),\n ObjectId('6560d4e114097bdad0563381'),\n ObjectId('6560d4e114097bdad0563382'),\n ObjectId('6560d4e114097bdad0563383'),\n ObjectId('6560d4e114097bdad0563384'),\n ObjectId('6560d4e114097bdad0563385'),\n ObjectId('6560d4e114097bdad0563386'),\n ObjectId('6560d4e114097bdad0563387'),\n ObjectId('6560d4e114097bdad0563388'),\n ObjectId('6560d4e114097bdad0563389'),\n ObjectId('6560d4e114097bdad056338a'),\n ObjectId('6560d4e114097bdad056338b'),\n ObjectId('6560d4e114097bdad056338c'),\n ObjectId('6560d4e114097bdad056338d'),\n ObjectId('6560d4e114097bdad056338e'),\n ObjectId('6560d4e114097bdad056338f'),\n ObjectId('6560d4e114097bdad0563390'),\n ObjectId('6560d4e114097bdad0563391'),\n ObjectId('6560d4e114097bdad0563392'),\n ObjectId('6560d4e114097bdad0563393'),\n ObjectId('6560d4e114097bdad0563394'),\n ObjectId('6560d4e114097bdad0563395'),\n ObjectId('6560d4e114097bdad0563396'),\n ObjectId('6560d4e114097bdad0563397'),\n ObjectId('6560d4e114097bdad0563398'),\n ObjectId('6560d4e114097bdad0563399'),\n ObjectId('6560d4e114097bdad056339a'),\n ObjectId('6560d4e114097bdad056339b'),\n ObjectId('6560d4e114097bdad056339c'),\n ObjectId('6560d4e114097bdad056339d'),\n ObjectId('6560d4e114097bdad056339e'),\n ObjectId('6560d4e114097bdad056339f'),\n ObjectId('6560d4e114097bdad05633a0'),\n ObjectId('6560d4e114097bdad05633a1'),\n ObjectId('6560d4e114097bdad05633a2'),\n ObjectId('6560d4e114097bdad05633a3'),\n ObjectId('6560d4e114097bdad05633a4'),\n ObjectId('6560d4e114097bdad05633a5'),\n ObjectId('6560d4e114097bdad05633a6'),\n ObjectId('6560d4e114097bdad05633a7'),\n ObjectId('6560d4e114097bdad05633a8'),\n ObjectId('6560d4e114097bdad05633a9'),\n ObjectId('6560d4e114097bdad05633aa'),\n ObjectId('6560d4e114097bdad05633ab'),\n ObjectId('6560d4e114097bdad05633ac'),\n ObjectId('6560d4e114097bdad05633ad'),\n ObjectId('6560d4e114097bdad05633ae'),\n ObjectId('6560d4e114097bdad05633af'),\n ObjectId('6560d4e114097bdad05633b0'),\n ObjectId('6560d4e114097bdad05633b1'),\n ObjectId('6560d4e114097bdad05633b2'),\n ObjectId('6560d4e114097bdad05633b3'),\n ObjectId('6560d4e114097bdad05633b4'),\n ObjectId('6560d4e114097bdad05633b5'),\n ObjectId('6560d4e114097bdad05633b6'),\n ObjectId('6560d4e114097bdad05633b7'),\n ObjectId('6560d4e114097bdad05633b8'),\n ObjectId('6560d4e114097bdad05633b9'),\n ObjectId('6560d4e114097bdad05633ba'),\n ObjectId('6560d4e114097bdad05633bb'),\n ObjectId('6560d4e114097bdad05633bc'),\n ObjectId('6560d4e114097bdad05633bd'),\n ObjectId('6560d4e114097bdad05633be'),\n ObjectId('6560d4e114097bdad05633bf'),\n ObjectId('6560d4e114097bdad05633c0'),\n ObjectId('6560d4e114097bdad05633c1'),\n ObjectId('6560d4e114097bdad05633c2'),\n ObjectId('6560d4e114097bdad05633c3'),\n ObjectId('6560d4e114097bdad05633c4'),\n ObjectId('6560d4e114097bdad05633c5'),\n ObjectId('6560d4e114097bdad05633c6'),\n ObjectId('6560d4e114097bdad05633c7'),\n ObjectId('6560d4e114097bdad05633c8'),\n ObjectId('6560d4e114097bdad05633c9'),\n ObjectId('6560d4e114097bdad05633ca'),\n ObjectId('6560d4e114097bdad05633cb'),\n ObjectId('6560d4e114097bdad05633cc'),\n ObjectId('6560d4e114097bdad05633cd'),\n ObjectId('6560d4e114097bdad05633ce'),\n ObjectId('6560d4e114097bdad05633cf'),\n ObjectId('6560d4e114097bdad05633d0'),\n ObjectId('6560d4e114097bdad05633d1'),\n ObjectId('6560d4e114097bdad05633d2'),\n ObjectId('6560d4e114097bdad05633d3'),\n ObjectId('6560d4e114097bdad05633d4'),\n ObjectId('6560d4e114097bdad05633d5'),\n ObjectId('6560d4e114097bdad05633d6'),\n ObjectId('6560d4e114097bdad05633d7'),\n ObjectId('6560d4e114097bdad05633d8'),\n ObjectId('6560d4e114097bdad05633d9'),\n ObjectId('6560d4e114097bdad05633da'),\n ObjectId('6560d4e114097bdad05633db'),\n ObjectId('6560d4e114097bdad05633dc'),\n ObjectId('6560d4e114097bdad05633dd'),\n ObjectId('6560d4e114097bdad05633de'),\n ObjectId('6560d4e114097bdad05633df'),\n ObjectId('6560d4e114097bdad05633e0'),\n ObjectId('6560d4e114097bdad05633e1'),\n ObjectId('6560d4e114097bdad05633e2'),\n ObjectId('6560d4e114097bdad05633e3'),\n ObjectId('6560d4e114097bdad05633e4'),\n ObjectId('6560d4e114097bdad05633e5'),\n ObjectId('6560d4e114097bdad05633e6'),\n ObjectId('6560d4e114097bdad05633e7'),\n ObjectId('6560d4e114097bdad05633e8'),\n ObjectId('6560d4e114097bdad05633e9'),\n ObjectId('6560d4e114097bdad05633ea'),\n ObjectId('6560d4e114097bdad05633eb'),\n ObjectId('6560d4e114097bdad05633ec'),\n ObjectId('6560d4e114097bdad05633ed'),\n ObjectId('6560d4e114097bdad05633ee'),\n ObjectId('6560d4e114097bdad05633ef'),\n ObjectId('6560d4e114097bdad05633f0'),\n ObjectId('6560d4e114097bdad05633f1'),\n ObjectId('6560d4e114097bdad05633f2'),\n ObjectId('6560d4e114097bdad05633f3'),\n ObjectId('6560d4e114097bdad05633f4'),\n ObjectId('6560d4e114097bdad05633f5'),\n ObjectId('6560d4e114097bdad05633f6'),\n ObjectId('6560d4e114097bdad05633f7'),\n ObjectId('6560d4e114097bdad05633f8'),\n ObjectId('6560d4e114097bdad05633f9'),\n ObjectId('6560d4e114097bdad05633fa'),\n ObjectId('6560d4e114097bdad05633fb'),\n ObjectId('6560d4e114097bdad05633fc'),\n ObjectId('6560d4e114097bdad05633fd'),\n ObjectId('6560d4e114097bdad05633fe'),\n ObjectId('6560d4e114097bdad05633ff'),\n ObjectId('6560d4e114097bdad0563400'),\n ObjectId('6560d4e114097bdad0563401'),\n ObjectId('6560d4e114097bdad0563402'),\n ObjectId('6560d4e114097bdad0563403'),\n ObjectId('6560d4e114097bdad0563404'),\n ObjectId('6560d4e114097bdad0563405'),\n ObjectId('6560d4e114097bdad0563406'),\n ObjectId('6560d4e114097bdad0563407'),\n ObjectId('6560d4e114097bdad0563408'),\n ObjectId('6560d4e114097bdad0563409'),\n ObjectId('6560d4e114097bdad056340a'),\n ObjectId('6560d4e114097bdad056340b'),\n ObjectId('6560d4e114097bdad056340c'),\n ObjectId('6560d4e114097bdad056340d'),\n ObjectId('6560d4e114097bdad056340e'),\n ObjectId('6560d4e114097bdad056340f'),\n ObjectId('6560d4e114097bdad0563410'),\n ObjectId('6560d4e114097bdad0563411'),\n ObjectId('6560d4e114097bdad0563412'),\n ObjectId('6560d4e114097bdad0563413'),\n ObjectId('6560d4e114097bdad0563414'),\n ObjectId('6560d4e114097bdad0563415'),\n ObjectId('6560d4e114097bdad0563416'),\n ObjectId('6560d4e114097bdad0563417'),\n ObjectId('6560d4e114097bdad0563418'),\n ObjectId('6560d4e114097bdad0563419'),\n ObjectId('6560d4e114097bdad056341a'),\n ObjectId('6560d4e114097bdad056341b'),\n ObjectId('6560d4e114097bdad056341c'),\n ObjectId('6560d4e114097bdad056341d'),\n ObjectId('6560d4e114097bdad056341e'),\n ObjectId('6560d4e114097bdad056341f'),\n ObjectId('6560d4e114097bdad0563420'),\n ObjectId('6560d4e114097bdad0563421'),\n ObjectId('6560d4e114097bdad0563422'),\n ObjectId('6560d4e114097bdad0563423'),\n ObjectId('6560d4e114097bdad0563424'),\n ObjectId('6560d4e114097bdad0563425'),\n ObjectId('6560d4e114097bdad0563426'),\n ObjectId('6560d4e114097bdad0563427'),\n ObjectId('6560d4e114097bdad0563428'),\n ObjectId('6560d4e114097bdad0563429'),\n ObjectId('6560d4e114097bdad056342a'),\n ObjectId('6560d4e114097bdad056342b'),\n ObjectId('6560d4e114097bdad056342c'),\n ObjectId('6560d4e114097bdad056342d'),\n ObjectId('6560d4e114097bdad056342e'),\n ObjectId('6560d4e114097bdad056342f'),\n ObjectId('6560d4e114097bdad0563430'),\n ObjectId('6560d4e114097bdad0563431'),\n ObjectId('6560d4e114097bdad0563432'),\n ObjectId('6560d4e114097bdad0563433'),\n ObjectId('6560d4e114097bdad0563434'),\n ObjectId('6560d4e114097bdad0563435'),\n ObjectId('6560d4e114097bdad0563436'),\n ObjectId('6560d4e114097bdad0563437'),\n ObjectId('6560d4e114097bdad0563438'),\n ObjectId('6560d4e114097bdad0563439'),\n ObjectId('6560d4e114097bdad056343a'),\n ObjectId('6560d4e114097bdad056343b'),\n ObjectId('6560d4e114097bdad056343c'),\n ObjectId('6560d4e114097bdad056343d'),\n ObjectId('6560d4e114097bdad056343e'),\n ObjectId('6560d4e114097bdad056343f'),\n ObjectId('6560d4e114097bdad0563440'),\n ObjectId('6560d4e114097bdad0563441'),\n ObjectId('6560d4e114097bdad0563442'),\n ObjectId('6560d4e114097bdad0563443'),\n ObjectId('6560d4e114097bdad0563444'),\n ObjectId('6560d4e114097bdad0563445'),\n ObjectId('6560d4e114097bdad0563446'),\n ObjectId('6560d4e114097bdad0563447'),\n ObjectId('6560d4e114097bdad0563448'),\n ObjectId('6560d4e114097bdad0563449'),\n ObjectId('6560d4e114097bdad056344a'),\n ObjectId('6560d4e114097bdad056344b'),\n ObjectId('6560d4e114097bdad056344c'),\n ObjectId('6560d4e114097bdad056344d'),\n ObjectId('6560d4e114097bdad056344e'),\n ObjectId('6560d4e114097bdad056344f'),\n ObjectId('6560d4e114097bdad0563450'),\n ObjectId('6560d4e114097bdad0563451'),\n ObjectId('6560d4e114097bdad0563452'),\n ObjectId('6560d4e114097bdad0563453'),\n ObjectId('6560d4e114097bdad0563454'),\n ObjectId('6560d4e114097bdad0563455'),\n ObjectId('6560d4e114097bdad0563456'),\n ObjectId('6560d4e114097bdad0563457'),\n ObjectId('6560d4e114097bdad0563458'),\n ObjectId('6560d4e114097bdad0563459'),\n ObjectId('6560d4e114097bdad056345a'),\n ObjectId('6560d4e114097bdad056345b'),\n ObjectId('6560d4e114097bdad056345c'),\n ObjectId('6560d4e114097bdad056345d'),\n ObjectId('6560d4e114097bdad056345e'),\n ObjectId('6560d4e114097bdad056345f'),\n ObjectId('6560d4e114097bdad0563460'),\n ObjectId('6560d4e114097bdad0563461'),\n ObjectId('6560d4e114097bdad0563462'),\n ObjectId('6560d4e114097bdad0563463'),\n ObjectId('6560d4e114097bdad0563464'),\n ObjectId('6560d4e114097bdad0563465'),\n ObjectId('6560d4e114097bdad0563466'),\n ObjectId('6560d4e114097bdad0563467'),\n ObjectId('6560d4e114097bdad0563468'),\n ObjectId('6560d4e114097bdad0563469'),\n ObjectId('6560d4e114097bdad056346a'),\n ObjectId('6560d4e114097bdad056346b'),\n ObjectId('6560d4e114097bdad056346c'),\n ObjectId('6560d4e114097bdad056346d'),\n ObjectId('6560d4e114097bdad056346e'),\n ObjectId('6560d4e114097bdad056346f'),\n ObjectId('6560d4e114097bdad0563470'),\n ObjectId('6560d4e114097bdad0563471'),\n ObjectId('6560d4e114097bdad0563472'),\n ObjectId('6560d4e114097bdad0563473'),\n ObjectId('6560d4e114097bdad0563474'),\n ObjectId('6560d4e114097bdad0563475'),\n ObjectId('6560d4e114097bdad0563476'),\n ObjectId('6560d4e114097bdad0563477'),\n ObjectId('6560d4e114097bdad0563478'),\n ObjectId('6560d4e114097bdad0563479'),\n ObjectId('6560d4e114097bdad056347a'),\n ObjectId('6560d4e114097bdad056347b'),\n ObjectId('6560d4e114097bdad056347c'),\n ObjectId('6560d4e114097bdad056347d'),\n ObjectId('6560d4e114097bdad056347e'),\n ObjectId('6560d4e114097bdad056347f'),\n ObjectId('6560d4e114097bdad0563480'),\n ObjectId('6560d4e114097bdad0563481'),\n ObjectId('6560d4e114097bdad0563482'),\n ObjectId('6560d4e114097bdad0563483'),\n ObjectId('6560d4e114097bdad0563484'),\n ObjectId('6560d4e114097bdad0563485'),\n ObjectId('6560d4e114097bdad0563486'),\n ObjectId('6560d4e114097bdad0563487'),\n ObjectId('6560d4e114097bdad0563488'),\n ObjectId('6560d4e114097bdad0563489'),\n ObjectId('6560d4e114097bdad056348a'),\n ObjectId('6560d4e114097bdad056348b'),\n ObjectId('6560d4e114097bdad056348c'),\n ObjectId('6560d4e114097bdad056348d'),\n ObjectId('6560d4e114097bdad056348e'),\n ObjectId('6560d4e114097bdad056348f'),\n ObjectId('6560d4e114097bdad0563490'),\n ObjectId('6560d4e114097bdad0563491'),\n ObjectId('6560d4e114097bdad0563492'),\n ObjectId('6560d4e114097bdad0563493'),\n ObjectId('6560d4e114097bdad0563494'),\n ObjectId('6560d4e114097bdad0563495'),\n ObjectId('6560d4e114097bdad0563496'),\n ObjectId('6560d4e114097bdad0563497'),\n ObjectId('6560d4e114097bdad0563498'),\n ObjectId('6560d4e114097bdad0563499'),\n ObjectId('6560d4e114097bdad056349a'),\n ObjectId('6560d4e114097bdad056349b'),\n ObjectId('6560d4e114097bdad056349c'),\n ObjectId('6560d4e114097bdad056349d'),\n ObjectId('6560d4e114097bdad056349e'),\n ObjectId('6560d4e114097bdad056349f'),\n ObjectId('6560d4e114097bdad05634a0'),\n ObjectId('6560d4e114097bdad05634a1'),\n ObjectId('6560d4e114097bdad05634a2'),\n ObjectId('6560d4e114097bdad05634a3'),\n ObjectId('6560d4e114097bdad05634a4'),\n ObjectId('6560d4e114097bdad05634a5'),\n ObjectId('6560d4e114097bdad05634a6'),\n ObjectId('6560d4e114097bdad05634a7'),\n ObjectId('6560d4e114097bdad05634a8'),\n ObjectId('6560d4e114097bdad05634a9'),\n ObjectId('6560d4e114097bdad05634aa'),\n ObjectId('6560d4e114097bdad05634ab'),\n ObjectId('6560d4e114097bdad05634ac'),\n ObjectId('6560d4e114097bdad05634ad'),\n ObjectId('6560d4e114097bdad05634ae'),\n ObjectId('6560d4e114097bdad05634af'),\n ObjectId('6560d4e114097bdad05634b0'),\n ObjectId('6560d4e114097bdad05634b1'),\n ObjectId('6560d4e114097bdad05634b2'),\n ObjectId('6560d4e114097bdad05634b3'),\n ObjectId('6560d4e114097bdad05634b4'),\n ObjectId('6560d4e114097bdad05634b5'),\n ObjectId('6560d4e114097bdad05634b6'),\n ObjectId('6560d4e114097bdad05634b7'),\n ObjectId('6560d4e114097bdad05634b8'),\n ObjectId('6560d4e114097bdad05634b9'),\n ObjectId('6560d4e114097bdad05634ba'),\n ObjectId('6560d4e114097bdad05634bb'),\n ObjectId('6560d4e114097bdad05634bc'),\n ObjectId('6560d4e114097bdad05634bd'),\n ObjectId('6560d4e114097bdad05634be'),\n ObjectId('6560d4e114097bdad05634bf'),\n ObjectId('6560d4e114097bdad05634c0'),\n ObjectId('6560d4e114097bdad05634c1'),\n ObjectId('6560d4e114097bdad05634c2'),\n ObjectId('6560d4e114097bdad05634c3'),\n ObjectId('6560d4e114097bdad05634c4'),\n ObjectId('6560d4e114097bdad05634c5'),\n ObjectId('6560d4e114097bdad05634c6'),\n ObjectId('6560d4e114097bdad05634c7'),\n ObjectId('6560d4e114097bdad05634c8'),\n ObjectId('6560d4e114097bdad05634c9'),\n ObjectId('6560d4e114097bdad05634ca'),\n ObjectId('6560d4e114097bdad05634cb'),\n ObjectId('6560d4e114097bdad05634cc'),\n ObjectId('6560d4e114097bdad05634cd'),\n ObjectId('6560d4e114097bdad05634ce'),\n ObjectId('6560d4e114097bdad05634cf'),\n ObjectId('6560d4e114097bdad05634d0'),\n ObjectId('6560d4e114097bdad05634d1'),\n ObjectId('6560d4e114097bdad05634d2'),\n ObjectId('6560d4e114097bdad05634d3'),\n ObjectId('6560d4e114097bdad05634d4'),\n ObjectId('6560d4e114097bdad05634d5'),\n ObjectId('6560d4e114097bdad05634d6'),\n ObjectId('6560d4e114097bdad05634d7'),\n ObjectId('6560d4e114097bdad05634d8'),\n ObjectId('6560d4e114097bdad05634d9'),\n ObjectId('6560d4e114097bdad05634da'),\n ObjectId('6560d4e114097bdad05634db'),\n ObjectId('6560d4e114097bdad05634dc'),\n ObjectId('6560d4e114097bdad05634dd'),\n ObjectId('6560d4e114097bdad05634de'),\n ObjectId('6560d4e114097bdad05634df'),\n ObjectId('6560d4e114097bdad05634e0'),\n ObjectId('6560d4e114097bdad05634e1'),\n ObjectId('6560d4e114097bdad05634e2'),\n ObjectId('6560d4e114097bdad05634e3'),\n ObjectId('6560d4e114097bdad05634e4'),\n ObjectId('6560d4e114097bdad05634e5'),\n ObjectId('6560d4e114097bdad05634e6'),\n ObjectId('6560d4e114097bdad05634e7'),\n ObjectId('6560d4e114097bdad05634e8'),\n ObjectId('6560d4e114097bdad05634e9'),\n ObjectId('6560d4e114097bdad05634ea'),\n ObjectId('6560d4e114097bdad05634eb'),\n ObjectId('6560d4e114097bdad05634ec'),\n ObjectId('6560d4e114097bdad05634ed'),\n ObjectId('6560d4e114097bdad05634ee'),\n ObjectId('6560d4e114097bdad05634ef'),\n ObjectId('6560d4e114097bdad05634f0'),\n ObjectId('6560d4e114097bdad05634f1'),\n ObjectId('6560d4e114097bdad05634f2'),\n ObjectId('6560d4e114097bdad05634f3'),\n ObjectId('6560d4e114097bdad05634f4'),\n ObjectId('6560d4e114097bdad05634f5'),\n ObjectId('6560d4e114097bdad05634f6'),\n ObjectId('6560d4e114097bdad05634f7'),\n ObjectId('6560d4e114097bdad05634f8'),\n ObjectId('6560d4e114097bdad05634f9'),\n ObjectId('6560d4e114097bdad05634fa'),\n ObjectId('6560d4e114097bdad05634fb'),\n ObjectId('6560d4e114097bdad05634fc'),\n ObjectId('6560d4e114097bdad05634fd'),\n ObjectId('6560d4e114097bdad05634fe'),\n ObjectId('6560d4e114097bdad05634ff'),\n ObjectId('6560d4e114097bdad0563500'),\n ObjectId('6560d4e114097bdad0563501'),\n ObjectId('6560d4e114097bdad0563502'),\n ObjectId('6560d4e114097bdad0563503'),\n ObjectId('6560d4e114097bdad0563504'),\n ObjectId('6560d4e114097bdad0563505'),\n ObjectId('6560d4e114097bdad0563506'),\n ObjectId('6560d4e114097bdad0563507'),\n ObjectId('6560d4e114097bdad0563508'),\n ObjectId('6560d4e114097bdad0563509'),\n ObjectId('6560d4e114097bdad056350a'),\n ObjectId('6560d4e114097bdad056350b'),\n ObjectId('6560d4e114097bdad056350c'),\n ObjectId('6560d4e114097bdad056350d'),\n ObjectId('6560d4e114097bdad056350e'),\n ObjectId('6560d4e114097bdad056350f'),\n ObjectId('6560d4e114097bdad0563510'),\n ObjectId('6560d4e114097bdad0563511'),\n ObjectId('6560d4e114097bdad0563512'),\n ObjectId('6560d4e114097bdad0563513'),\n ObjectId('6560d4e114097bdad0563514'),\n ObjectId('6560d4e114097bdad0563515'),\n ObjectId('6560d4e114097bdad0563516'),\n ObjectId('6560d4e114097bdad0563517'),\n ObjectId('6560d4e114097bdad0563518'),\n ObjectId('6560d4e114097bdad0563519'),\n ObjectId('6560d4e114097bdad056351a'),\n ObjectId('6560d4e114097bdad056351b'),\n ObjectId('6560d4e114097bdad056351c'),\n ObjectId('6560d4e114097bdad056351d'),\n ObjectId('6560d4e114097bdad056351e'),\n ObjectId('6560d4e114097bdad056351f'),\n ObjectId('6560d4e114097bdad0563520'),\n ObjectId('6560d4e114097bdad0563521'),\n ObjectId('6560d4e114097bdad0563522'),\n ObjectId('6560d4e114097bdad0563523'),\n ObjectId('6560d4e114097bdad0563524'),\n ObjectId('6560d4e114097bdad0563525'),\n ObjectId('6560d4e114097bdad0563526'),\n ObjectId('6560d4e114097bdad0563527'),\n ObjectId('6560d4e114097bdad0563528'),\n ObjectId('6560d4e114097bdad0563529'),\n ObjectId('6560d4e114097bdad056352a'),\n ObjectId('6560d4e114097bdad056352b'),\n ObjectId('6560d4e114097bdad056352c'),\n ObjectId('6560d4e114097bdad056352d'),\n ObjectId('6560d4e114097bdad056352e'),\n ObjectId('6560d4e114097bdad056352f'),\n ObjectId('6560d4e114097bdad0563530'),\n ObjectId('6560d4e114097bdad0563531'),\n ObjectId('6560d4e114097bdad0563532'),\n ObjectId('6560d4e114097bdad0563533'),\n ObjectId('6560d4e114097bdad0563534'),\n ObjectId('6560d4e114097bdad0563535'),\n ObjectId('6560d4e114097bdad0563536'),\n ObjectId('6560d4e114097bdad0563537'),\n ObjectId('6560d4e114097bdad0563538'),\n ObjectId('6560d4e114097bdad0563539'),\n ObjectId('6560d4e114097bdad056353a'),\n ObjectId('6560d4e114097bdad056353b'),\n ObjectId('6560d4e114097bdad056353c'),\n ObjectId('6560d4e114097bdad056353d'),\n ObjectId('6560d4e114097bdad056353e'),\n ObjectId('6560d4e114097bdad056353f'),\n ObjectId('6560d4e114097bdad0563540'),\n ObjectId('6560d4e114097bdad0563541'),\n ObjectId('6560d4e114097bdad0563542'),\n ObjectId('6560d4e114097bdad0563543'),\n ObjectId('6560d4e114097bdad0563544'),\n ObjectId('6560d4e114097bdad0563545'),\n ObjectId('6560d4e114097bdad0563546'),\n ObjectId('6560d4e114097bdad0563547'),\n ObjectId('6560d4e114097bdad0563548'),\n ObjectId('6560d4e114097bdad0563549'),\n ObjectId('6560d4e114097bdad056354a'),\n ObjectId('6560d4e114097bdad056354b'),\n ObjectId('6560d4e114097bdad056354c'),\n ObjectId('6560d4e114097bdad056354d'),\n ObjectId('6560d4e114097bdad056354e'),\n ObjectId('6560d4e114097bdad056354f'),\n ObjectId('6560d4e114097bdad0563550'),\n ObjectId('6560d4e114097bdad0563551'),\n ObjectId('6560d4e114097bdad0563552'),\n ObjectId('6560d4e114097bdad0563553'),\n ObjectId('6560d4e114097bdad0563554'),\n ObjectId('6560d4e114097bdad0563555'),\n ObjectId('6560d4e114097bdad0563556'),\n ObjectId('6560d4e114097bdad0563557'),\n ObjectId('6560d4e114097bdad0563558'),\n ObjectId('6560d4e114097bdad0563559'),\n ObjectId('6560d4e114097bdad056355a'),\n ObjectId('6560d4e114097bdad056355b'),\n ObjectId('6560d4e114097bdad056355c'),\n ObjectId('6560d4e114097bdad056355d'),\n ObjectId('6560d4e114097bdad056355e'),\n ObjectId('6560d4e114097bdad056355f'),\n ObjectId('6560d4e114097bdad0563560'),\n ObjectId('6560d4e114097bdad0563561'),\n ObjectId('6560d4e114097bdad0563562'),\n ObjectId('6560d4e114097bdad0563563'),\n ObjectId('6560d4e114097bdad0563564'),\n ObjectId('6560d4e114097bdad0563565'),\n ObjectId('6560d4e114097bdad0563566'),\n ObjectId('6560d4e114097bdad0563567'),\n ObjectId('6560d4e114097bdad0563568'),\n ObjectId('6560d4e114097bdad0563569'),\n ObjectId('6560d4e114097bdad056356a'),\n ObjectId('6560d4e114097bdad056356b'),\n ObjectId('6560d4e114097bdad056356c'),\n ObjectId('6560d4e114097bdad056356d'),\n ObjectId('6560d4e114097bdad056356e'),\n ObjectId('6560d4e114097bdad056356f'),\n ObjectId('6560d4e114097bdad0563570'),\n ObjectId('6560d4e114097bdad0563571'),\n ObjectId('6560d4e114097bdad0563572'),\n ObjectId('6560d4e114097bdad0563573'),\n ObjectId('6560d4e114097bdad0563574'),\n ObjectId('6560d4e114097bdad0563575'),\n ObjectId('6560d4e114097bdad0563576'),\n ObjectId('6560d4e114097bdad0563577'),\n ObjectId('6560d4e114097bdad0563578'),\n ObjectId('6560d4e114097bdad0563579'),\n ObjectId('6560d4e114097bdad056357a'),\n ObjectId('6560d4e114097bdad056357b'),\n ObjectId('6560d4e114097bdad056357c'),\n ObjectId('6560d4e114097bdad056357d'),\n ObjectId('6560d4e114097bdad056357e'),\n ObjectId('6560d4e114097bdad056357f'),\n ObjectId('6560d4e114097bdad0563580'),\n ObjectId('6560d4e114097bdad0563581'),\n ObjectId('6560d4e114097bdad0563582'),\n ObjectId('6560d4e114097bdad0563583'),\n ObjectId('6560d4e114097bdad0563584'),\n ObjectId('6560d4e114097bdad0563585'),\n ObjectId('6560d4e114097bdad0563586'),\n ObjectId('6560d4e114097bdad0563587'),\n ObjectId('6560d4e114097bdad0563588'),\n ObjectId('6560d4e114097bdad0563589'),\n ObjectId('6560d4e114097bdad056358a'),\n ObjectId('6560d4e114097bdad056358b'),\n ObjectId('6560d4e114097bdad056358c'),\n ObjectId('6560d4e114097bdad056358d'),\n ObjectId('6560d4e114097bdad056358e'),\n ObjectId('6560d4e114097bdad056358f'),\n ObjectId('6560d4e114097bdad0563590'),\n ObjectId('6560d4e114097bdad0563591'),\n ObjectId('6560d4e114097bdad0563592'),\n ObjectId('6560d4e114097bdad0563593'),\n ObjectId('6560d4e114097bdad0563594'),\n ObjectId('6560d4e114097bdad0563595'),\n ObjectId('6560d4e114097bdad0563596'),\n ObjectId('6560d4e114097bdad0563597'),\n ObjectId('6560d4e114097bdad0563598'),\n ObjectId('6560d4e114097bdad0563599'),\n ObjectId('6560d4e114097bdad056359a'),\n ObjectId('6560d4e114097bdad056359b'),\n ObjectId('6560d4e114097bdad056359c'),\n ObjectId('6560d4e114097bdad056359d')],\n TaskWorkflow(database=, G=))" + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torchvision\n", + "from superduperdb.ext.pillow import pil_image\n", + "from superduperdb import Document\n", + "from superduperdb.backends.mongodb import Collection\n", + "\n", + "import random\n", + "\n", + "# Load MNIST images as Python objects using the Python Imaging Library.\n", + "mnist_data = list(torchvision.datasets.MNIST(root='./data', download=True))\n", + "document_list = [Document({'img': pil_image(x[0]), 'class': x[1]}) for x in mnist_data]\n", + "\n", + "# Shuffle the data and select a subset of 1000 documents\n", + "random.shuffle(document_list)\n", + "data = document_list[:1000]\n", + "\n", + "# Insert the selected data into the mnist_collection\n", + "db.execute(\n", + " mnist_collection.insert_many(data[:-100]), # Insert all but the last 100 documents\n", + " encoders=(pil_image,) # Encode images using the Pillow library.\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c5341135", + "metadata": {}, + "source": [ + "Now that the images and their classes are inserted into the database, we can query the data in its original format. Particularly, we can use the `PIL.Image` instances to inspect the data." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "a36f9c3b", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:49.397496082Z", + "start_time": "2023-11-24T16:52:49.393955875Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA1UlEQVR4AdWQLQ7CQBCFpwSDqGhAIQgKRTAVSBQbHMcgXIADoNpwEjSGSmzrcZCCQDQIAjjyBroJ222661CM2Zf55ufNEv1tDBN+88Bq31s+AMbIBkXKuIczpF2Teik4nlIAjE24Yo4aRMxsQvcMyYhtnQInN2c+42U6Eghz5mWMi15ZU9JpE7lBixyVKL0CWWeyk3eWOhXvZbjmZp5WSHPZFfftkPxttJB3Wsaq8YyNkkSF2yK1L5QFakb1kpbSsR76LVrjoMurO2901LCqmon845/jA7n0WJvNBiE1AAAAAElFTkSuQmCC", + "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APn+t7w54L8ReLXkXQ9LluxF999yoi+25iBn2zmsvUdPutJ1K506+i8q6tpGilj3BtrA4IyCQfwqrUtvF591FDnb5jhc+mTivoS5XXNV+Idh4B0OyksfC2hyQ/bcAqs6Ltdt7D+8OAO5OT7ecfG+WOT4sasEXaUWFW46nylOf1rz2lBKkEEgjkEV6t4b+L/i7VNc8N6NealGlp/aNsk8yoEkmTzFBDt6Y64xnvnmuf8Ai/I8vxX19pE2MJUUD2EaAH8QAfxriKKfFLJBMksTskiMGR1OCpHIINWdV1W+1vU59S1K4NxeTkNJKQAWIAHQADoBVOv/2Q==" + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get and display one of the images\n", + "r = db.execute(mnist_collection.find_one())\n", + "r.unpack()['img']" + ] + }, + { + "cell_type": "markdown", + "id": "1413d4c5", + "metadata": {}, + "source": [ + "## Build Model" + ] + }, + { + "cell_type": "markdown", + "id": "68fde8bb", + "metadata": {}, + "source": [ + "Next, we create our machine learning model. SuperDuperDB supports various frameworks out of the box, and in this case, we are using PyTorch, which is well-suited for computer vision tasks. In this example, we combine torch with torchvision.\n", + "\n", + "We create `postprocess` and `preprocess` functions to handle the communication with the SuperDuperDB `Datalayer`, and then wrap model, preprocessing and postprocessing to create a native SuperDuperDB handler.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "cfb425e1", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:49.439286553Z", + "start_time": "2023-11-24T16:52:49.398458108Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "([],\n TorchModel(identifier='lenet5', object=, flatten=False, output_schema=None, encoder=None, preprocess= serializer=dill>, postprocess= serializer=dill>, collate_fn=None, metrics=None, predict_method=None, model_to_device_method='to', batch_predict=False, takes_context=False, train_X=None, train_y=None, training_select=None, metric_values={}, training_configuration=None, model_update_kwargs={}, serializer='dill', device='cpu', preferred_devices=('cpu',), version=0, is_batch=None, num_directions=2, optimizer_state=None, forward_method='__call__', train_forward_method='__call__'))" + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "\n", + "class LeNet5(torch.nn.Module):\n", + " def __init__(self, num_classes):\n", + " super().__init__()\n", + " self.layer1 = torch.nn.Sequential(\n", + " torch.nn.Conv2d(1, 6, kernel_size=5, stride=1, padding=0),\n", + " torch.nn.BatchNorm2d(6),\n", + " torch.nn.ReLU(),\n", + " torch.nn.MaxPool2d(kernel_size=2, stride=2))\n", + " self.layer2 = torch.nn.Sequential(\n", + " torch.nn.Conv2d(6, 16, kernel_size=5, stride=1, padding=0),\n", + " torch.nn.BatchNorm2d(16),\n", + " torch.nn.ReLU(),\n", + " torch.nn.MaxPool2d(kernel_size=2, stride=2))\n", + " self.fc = torch.nn.Linear(400, 120)\n", + " self.relu = torch.nn.ReLU()\n", + " self.fc1 = torch.nn.Linear(120, 84)\n", + " self.relu1 = torch.nn.ReLU()\n", + " self.fc2 = torch.nn.Linear(84, num_classes)\n", + "\n", + " def forward(self, x):\n", + " out = self.layer1(x)\n", + " out = self.layer2(out)\n", + " out = out.reshape(out.size(0), -1)\n", + " out = self.fc(out)\n", + " out = self.relu(out)\n", + " out = self.fc1(out)\n", + " out = self.relu1(out)\n", + " out = self.fc2(out)\n", + " return out\n", + "\n", + " \n", + "def postprocess(x):\n", + " return int(x.topk(1)[1].item())\n", + "\n", + "\n", + "def preprocess(x):\n", + " return torchvision.transforms.Compose([\n", + " torchvision.transforms.Resize((32, 32)),\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize(mean=(0.1307,), std=(0.3081,))]\n", + " )(x)\n", + "\n", + "\n", + "# Create and insert a SuperDuperDB model into the database\n", + "model = superduper(LeNet5(10), preprocess=preprocess, postprocess=postprocess, preferred_devices=('cpu',))\n", + "db.add(model)" + ] + }, + { + "cell_type": "markdown", + "id": "dcf0457e", + "metadata": {}, + "source": [ + "## Train Model\n", + "\n", + "Now we are ready to \"train\" or \"fit\" the model. Trainable models in SuperDuperDB come with a sklearn-like `.fit` method. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e7c610c1", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:49.769778141Z", + "start_time": "2023-11-24T16:52:49.438957733Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:49.41\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m894 \u001B[0m | \u001B[34m\u001B[1mmodel/lenet5/0 already exists - doing nothing\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.44\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 0; objective: 2.3723697662353516; \u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 44/44 [00:00<00:00, 701.65it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:49.52\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: VALID; iteration: 0; my_valid/acc: 0.11363636363636363; objective: 2.310549592971802; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.55\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 1; objective: 2.3386154174804688; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.55\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 2; objective: 2.1943118572235107; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.56\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 3; objective: 2.279111623764038; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.57\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 4; objective: 2.1872403621673584; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.57\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 5; objective: 2.2938268184661865; \u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 44/44 [00:00<00:00, 706.50it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:49.65\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: VALID; iteration: 5; my_valid/acc: 0.2727272727272727; objective: 2.2436946392059327; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.67\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 6; objective: 2.2607929706573486; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.68\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 7; objective: 2.1825790405273438; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.68\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 8; objective: 2.1962833404541016; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.68\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 9; objective: 2.0930800437927246; \u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:49.68\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: TRAIN; iteration: 10; objective: 1.9310920238494873; \u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 44/44 [00:00<00:00, 918.26it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:49.74\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m210 \u001B[0m | \u001B[1mfold: VALID; iteration: 10; my_valid/acc: 0.22727272727272727; objective: 2.1574175357818604; \u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "from torch.nn.functional import cross_entropy\n", + "\n", + "from superduperdb import Metric\n", + "from superduperdb import Dataset\n", + "from superduperdb.ext.torch.model import TorchTrainerConfiguration\n", + "\n", + "# Fit the model to the training data\n", + "job = model.fit(\n", + " X='img', # Feature matrix used as input data \n", + " y='class', # Target variable for training\n", + " db=db, # Database used for data retrieval\n", + " select=mnist_collection.find(), # Select the dataset\n", + " configuration=TorchTrainerConfiguration(\n", + " identifier='my_configuration',\n", + " objective=cross_entropy,\n", + " loader_kwargs={'batch_size': 10},\n", + " max_iterations=10,\n", + " validation_interval=5,\n", + " ),\n", + " metrics=[Metric(identifier='acc', object=lambda x, y: sum([xx == yy for xx, yy in zip(x, y)]) / len(x))],\n", + " validation_sets=[\n", + " Dataset(\n", + " identifier='my_valid',\n", + " select=Collection('mnist').find({'_fold': 'valid'}),\n", + " )\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Monitoring Training Efficiency\n", + "You can monitor the training efficiency with visualization tools like Matplotlib:" + ], + "metadata": { + "collapsed": false + }, + "id": "fdf5cccb2fe0b97b" + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "200d3be1", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:50.000089503Z", + "start_time": "2023-11-24T16:52:49.770191267Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:49.90\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m362 \u001B[0m | \u001B[34m\u001B[1mto will be overriden with `to`\u001B[0m\n" + ] + }, + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGhCAYAAABCse9yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABT8klEQVR4nO3deVxU9f4/8NfMAAOyIzIsDoy4b4AKjJBbt0k0Ky019Zqgt+Uuahllar8EvVagUlrhV2/eSlpM07LMClMULUFRcN83FlE2F4Z9mTm/P8gprqIMAmcYXs/HYx7Jh885vD8dYF6c95kzEkEQBBARERGZMKnYBRARERHdDwMLERERmTwGFiIiIjJ5DCxERERk8hhYiIiIyOQxsBAREZHJY2AhIiIik8fAQkRERCaPgYWIiIhMHgMLERERmbwmBZZVq1ZBpVLB2toaarUaaWlpDc5du3Ythg4dCmdnZzg7O0Oj0dwxv7S0FLNmzULnzp1hY2ODPn36YM2aNU0pjYiIiMyQ0YFl48aNiIyMRHR0NDIyMuDv74+wsDAUFBTcdX5ycjKmTJmC3bt3IzU1FUqlEiNHjkRubq5hTmRkJBITE/HFF1/g9OnTmDNnDmbNmoWtW7c2fWVERERkNiTGvvmhWq1GUFAQ4uPjAQB6vR5KpRKzZ8/G/Pnz77u9TqeDs7Mz4uPjER4eDgDo168fJk2ahIULFxrmDRo0CKNHj8Zbb73VqLr0ej2uXr0Ke3t7SCQSY5ZEREREIhEEASUlJfD09IRU2vB5FAtjdlpdXY309HQsWLDAMCaVSqHRaJCamtqofZSXl6OmpgYuLi6GsdDQUGzduhV/+9vf4OnpieTkZJw7dw4rVqxocD9VVVWoqqoyfJybm4s+ffoYsxwiIiIyETk5OejcuXODnzcqsBQVFUGn00GhUNQbVygUOHPmTKP2MW/ePHh6ekKj0RjGPvzwQ7z44ovo3LkzLCwsIJVKsXbtWgwbNqzB/cTExGDx4sV3jOfk5MDBwaGRKyIiIiIxabVaKJVK2Nvb33OeUYHlQcXGxmLDhg1ITk6GtbW1YfzDDz/E/v37sXXrVvj4+GDv3r2YOXPmHcHmzxYsWIDIyEjDx7cX7ODgwMBCRETUxtzvcg6jAourqytkMhny8/Prjefn58Pd3f2e28bFxSE2NhY7d+6En5+fYbyiogJvvPEGtmzZgjFjxgAA/Pz8cOTIEcTFxTUYWORyOeRyuTHlExERURtl1KuErKysMGjQICQlJRnG9Ho9kpKSEBIS0uB2y5Ytw5IlS5CYmIjAwMB6n6upqUFNTc0dF9rIZDLo9XpjyiMiIiIzZXRLKDIyEhEREQgMDERwcDBWrlyJsrIyzJgxAwAQHh4OLy8vxMTEAACWLl2KqKgorF+/HiqVCnl5eQAAOzs72NnZwcHBAcOHD8fcuXNhY2MDHx8f7NmzB5999hnee++9ZlwqERERtVVGB5ZJkyahsLAQUVFRyMvLQ0BAABITEw0X4mZnZ9c7W7J69WpUV1djwoQJ9fYTHR2NRYsWAQA2bNiABQsWYOrUqbhx4wZ8fHzw9ttv4x//+McDLI2IiIjMhdH3YTFVWq0Wjo6OKC4u5kW3REREbURjn7/5XkJERERk8hhYiIiIyOQxsBAREZHJY2AhIiIik8fAQkRERCaPgYWIiIhMHgMLERERmTwGFiIyadrKGqzafQF7zxXCTG4bRURN0Krv1kxEZAxBEDB301FsP1n3hqu+nWwREaLC+EGdYSfnry+i9oRnWIjIZG09ehXbT+bDQiqBndwClwrLEL31JAa/k4RFW0/iUmGp2CUSUSthYCEik1RQUonorScBALP/0h3733gEi5/sC99OtiitqsW6lEz85d09CP8kDbvO5EOvZ7uIyJzxvYSIyOQIgoAXP0/HjlP56OvpgO9mPgRLWd3fV3q9gN8uFCEhJRO7zhbg9m8wn44dMG2wDyYGKuFoYyli9URkjMY+fzOwEJHJ2XL4Cl7ZeBSWMgl+mD0Evdzv/jOddb0Mn6dmYeOhHJRU1gIAOljJ8NQAL0wPVaG7wr41yyaiJmBgIaI2KV9biUff2wNtZS3mhvXEzIe73Xeb8upabDmci4SUTJzL/+O6ltCuHRERqoKmtwIyqaQlyyaiJmJgIaI2RxAEPJdwCLvOFMCvsyO+/WcoLGSNv9ROEASkXrqOhJRM7DiVj9uXtXg52WBaiA8mBynh1MGqhaonoqZgYCGiNmdz+hW8tukorGRSbHtpCHo8QEvnys1yfLE/GxsOZuNWeQ0AQG4hxbgAL0SEqtDHk78niEwBAwsRtSnXiiswcsVelFTWYt6oXvjniK7Nst/KGh22HrmKdSmZOHVNaxgPVrlg+kMqjOyjMOosDhE1LwYWImozBEHA9E8PYs+5QgQonbD5HyHNHiIEQcChrJtYl5KJxBN50P3eL/JwtMZUtTemBHujo528Wb8mEd0fAwsRtRkbD2Zj3jfHYWUhxU8vDUU3N7sW/Xp5xZX48kAWvkrLRlFpNQDASibF4/4emB6qgl9npxb9+kT0BwYWImoTcm9VYNSKvSipqsUbj/XCi8OapxXUGFW1Ovx47BoSUjJx9EqxYXyAtxOmh6owup8HrCzYLiJqSQwsRGTyBEFA+Cdp+PV8EQZ6O2HTP0JFe/nx4eybSEjJxI/Hr6FGV/drsZO9HFPV3vir2htu9tai1EVk7hhYiMjkrT+QjTe2HIfcQoqfXx4K304t2wpqjIKSSnx1IAdfHshCQUkVAMBSJsHofh6ICFVhoLcTJBLe04WouTCwEJFJy7lRjlEr96KsWoc3x/TG80N9xS6pnupaPRJP5iEhJRPpWTcN4/29HBERqsLjfh6wtpSJWCGReWBgISKTpdcLePbjA0i5eB1BKmdseDHEpO9EeyK3GOtSMrH16FVU1+oBAB1trTA5WIlnB/vAw9FG5AqJ2i4GFiIyWZ/vz8LC707A2lKKxJeHQeVqK3ZJjXK9tAobDubgi/1ZuFZcCQCQSSUI66vA9NAuCFI5s11EZCQGFiIySdnXyzHq/b0or9Zh0RN9MP2hLmKXZLRanR47TuVjXUomDly+YRjv7eGA6aE+GBvgxXYRUSMxsBCRydHrBUxZux8HLt+AuosLvnphMKQm3ApqjNPXtPgsNRNbDueisqauXeTUwRKTgpR4Vu0DpUsHkSskMm0MLERkctbtu4xFP5xCBysZEl8eBu+O5vNkfqu8Gl8fysFnqVm4crMCACCVAI/0VmB6qAqhXTuyXUR0FwwsRGRSMovKMPr9X1FRo8OSsX0xLUQldkktQqcXsOtMARJSMvHbhSLDeHc3O0SEqvDUAC/Yyi1ErJDItDCwEJHJ0OsFTPooFQczbyK0a0d88Zy6zbeCGuNCQQkSUrLwTcYVlFfrAAD21hZ4JlCJ8BAf+HRsGxcbE7UkBhYiMhkf/3YZS7adgq2VDIlzhrW76zq0lTXYfOgKPkvNROb1cgCARAI83NMN4SE+GNa9U7sIcER309jn7ya9ScaqVaugUqlgbW0NtVqNtLS0BueuXbsWQ4cOhbOzM5ydnaHRaO46//Tp03jyySfh6OgIW1tbBAUFITs7uynlEZEJuVRYimWJZwAAb4zp3e7CCgA4WFvib0O6YNerI/DpjCCM6NkJggDsOlOA6Z8ehOa9Pfh032WUVNaIXSqRyTI6sGzcuBGRkZGIjo5GRkYG/P39ERYWhoKCgrvOT05OxpQpU7B7926kpqZCqVRi5MiRyM3NNcy5ePEihgwZgl69eiE5ORnHjh3DwoULYW3N9+4gast0egFzNx9DVa0eQ7q54q/B3mKXJCqpVIKHe7ph3Yxg7H5tBGY8pIK93AKXisqw+IdTGPxOEqK+P4ELBaVil0pkcoxuCanVagQFBSE+Ph4AoNfroVQqMXv2bMyfP/++2+t0Ojg7OyM+Ph7h4eEAgMmTJ8PS0hKff/55E5ZQhy0hItOzdu8lvP3TadjJLbD9lWHwcuIdYf9XaVUttmRcQUJqVr2gMrS7K6aHqjCip5tJ3wWY6EG1SEuouroa6enp0Gg0f+xAKoVGo0Fqamqj9lFeXo6amhq4uLgAqAs8P/74I3r06IGwsDC4ublBrVbju+++u+d+qqqqoNVq6z2IyHRcKCjF8l/OAgAWPt6bYaUBdnILTAtRYccrw/DFc2poeisgkQC/ni/CcwmH8HBcMv776yUUV7BdRO2bUYGlqKgIOp0OCoWi3rhCoUBeXl6j9jFv3jx4enoaQk9BQQFKS0sRGxuLUaNG4ZdffsFTTz2Fp59+Gnv27GlwPzExMXB0dDQ8lEqlMUshohZUq9Pj1U1HUV2rx/AenfBMIH8+70cikWBId1f8NyIQe+c+jBeH+cLB2gLZN8rx1o+nMfidJLyx5TjO5pWIXSqRKFr1ZgCxsbHYsGEDkpOTDden6PV1d4YcO3YsXnnlFQBAQEAAUlJSsGbNGgwfPvyu+1qwYAEiIyMNH2u1WoYWIhOx9tfLOJpzC/bWFogd3583TDOS0qUD3nisN17R9MB3R3KRkJKJM3klWH8gG+sPZGOwrwumh3aBprcbLGRNeu0EUZtjVGBxdXWFTCZDfn5+vfH8/Hy4u7vfc9u4uDjExsZi586d8PPzq7dPCwsL9OnTp9783r1747fffmtwf3K5HHK53JjyiagVnM8vwYod5wAAUY/34TsZPwAbKxmmBHtjcpASBy7fQEJKJn45lY/9l25g/6Ub8HKywbODfTA5SAlnWyuxyyVqUUZFcysrKwwaNAhJSUmGMb1ej6SkJISEhDS43bJly7BkyRIkJiYiMDDwjn0GBQXh7Nmz9cbPnTsHHx8fY8ojIpEZWkE6Pf7Syw0TBnUWuySzIJFIMNi3I1Y/Owi/vv4w/jWiK5w7WCL3VgWWJp7B4JgkvL75KE5eLRa7VKIWY3RLKDIyEhEREQgMDERwcDBWrlyJsrIyzJgxAwAQHh4OLy8vxMTEAACWLl2KqKgorF+/HiqVynCti52dHezs7AAAc+fOxaRJkzBs2DA8/PDDSExMxA8//IDk5ORmWiYRtYb/7L2EY1eK4WBtgZin2QpqCZ5ONnh9VC+89Eh3/HD0KhJSM3EiV4uvD13B14euIEjljIhQFcL6usOS7SIyI0260218fDyWL1+OvLw8BAQE4IMPPoBarQYAjBgxAiqVCuvWrQMAqFQqZGVl3bGP6OhoLFq0yPDxJ598gpiYGFy5cgU9e/bE4sWLMXbs2EbXxJc1E4nrTJ4WT3z4G2p0At57xh9PD+TZldYgCAIysm9iXUoWfj5+DbX6ul/pCgc5pqp9MCXYG53s2T4n08Vb8xNRq6nR6fHU/+3DiVwtNL0VWBs+iGdXRJCvrcSXv1+YW1RaBQCwkknxuJ8HIkJV8Fc6iVsg0V0wsBBRq/kg6Tze23EOTh0s8csrw+Bmz7tUi6mqVoefj+dhXUomjuTcMowHKJ0wPVSFx/p7wMqC7SIyDQwsRNQqTl4txtj4fajVC3h/cgDGBniJXRL9ydGcW0hIycS2Y9dQrau7jYSrnRx/VXtjqtobCgeGSxIXAwsRtbjqWj3GrtqH09e0GNXXHaufHchWkIkqLKnChrRsfHEgC/naunaRhVSCUf3cMeMhFQZ6O/PYkSgYWIioxb234xw+SDoP5w6W+OWV4by4sw2o0emx/WQeElIycTDzpmG8n5cDIkJUeMLfE9aWMhErpPaGgYWIWtSJ3GKMW1XXCor/6wA87ucpdklkpBO5xfgsNRPfH7mKqtq6dpGLrRUmBynx7GAfePL9n6gVMLAQUYupqtVhbPw+nMkrwZj+Hlg1daDYJdEDuFlWjQ0Hc/DF/izk3qoAAMikEozso0BEqArqLi5sF1GLYWAhohYTt/0s4ndfQEdbK/zyyjB0tGMryBzU6vTYeboACSmZSL103TDey90eEaEqjAvwgo0V20XUvBhYiKhFHLtyC0/9Xwp0egGrpw7E6P4eYpdELeBsXgkSUjOxJSMXFTU6AICjjSUmBSkxbbAPlC4dRK6QzAUDCxE1u6paHR7/4DecLyjFE/6e+HDKALFLohZWXF6DTek5+Cw1C9k3ygEAEgnwSC8Fpoeq8FC3jmwX0QNhYCGiZrc08QxWJ1+Eq50cO14ZxncIbkd0egHJZwuwLiUTv54vMox3c7NDRIgPnh7YGbZyo9+ejoiBhYia1+Hsmxi/OgV6Afho2iCM7OsudkkkkgsFpfg8NROb06+grLquXWQvt8CEwM4ID1Ghi6utyBVSW8LAQkTNprJGhzEf/IqLhWV4aoAXVkwKELskMgEllTX4Jv0KPkvNwqWiMsP4iJ6dEBGqwvDunSCVsl1E98bAQkTN5p2fTuOjvZfQyb6uFeTUga0g+oNeL2Dv+UIkpGQi+Vwhbj+rdHG1xbTBPpgQ2BkO1pbiFkkmi4GFiJpFetYNTFiTCkEA/hseCE0fhdglkQnLLCrD5/uz8PWhHJRU1gIAOljJMH5gZ0SE+qCbm73IFZKpYWAhogdWUa3DYx/8istFZRg/sDPefcZf7JKojSirqsWWw7lISMnE+YJSw/iQbq6ICFXhL73cIGO7iMDAInY5RGZhybZT+Pi3y1A4yPHLK8PhaMPT+mQcQRCQevE61qVkYufpfOh/f8ZRuthg2mAfPBOoZIuxnWNgIaIHknb5BiZ9VNcK+nRGEB7u6SZ2SdTG5dwoxxf7s7DhYA6KK2oAANaWUjw1wAsRoSr0cufv7vaIgYWImqy8uhaPvf8rMq+X45nAzlg2ga0gaj4V1TpsPZqLdSlZOH1NaxhXd3HB9FAVHu2jgIVMKmKF1JoYWIioyRZtPYl1KZnwcLTG9leG8RUe1CIEQcDBzJtISMlE4sk86H7vF3k6WmPqYB9MCfaGC29OaPYYWIioSfZfuo7JH+0HAHz2t2AM69FJ5IqoPbhWXIEv92fjq7RsXC+rBgBYWUjxpL8npoeq0M/LUeQKqaUwsBCR0cqqajHq/b3IuVGBKcHeiHm6v9glUTtTWaPDtmPXkJCSieO5xYbxQT7OiAhVYXQ/d1iyXWRWGvv8zTd+ICKDpYlnkHOjAl5ONnjjsV5il0PtkLWlDBMGdcb4gV7IyL6FhJRM/HT8GtKzbiI96ybc7OWYqvbBFLUSbvbWYpdLrYhnWIgIAJByoQh//e8BAMAXz6kxpLuryBUR1SnQVuLLA9lYn5aNwpIqAIClTIIx/T0QEarCAG9nkSukB8GWEBE1WmlVLcJW7EXurQo8O9gbb41jK4hMT3WtHj+fqGsXZWTfMoz7d3ZERKgKY/w8ILeQiVcgNQkDCxE12htbjmP9gWx0drbB9jnDYCtnt5hM27Ert7AuJRPbjl5DtU4PAHC1s8KUYG9MVfvA3ZHtoraCgYWIGuXX84WY9nEaAOCrFwYjpGtHkSsiaryi0ipsSMvGF/uzkaetBABYSCUI6+eO6aEqBPo4QyLhWwCYMgYWIrqvksoahK3Yi6vFlYgI8cHisf3ELomoSWp0evxyMh8JKZlIy7xhGO/j4YDpoSo8GeAJa0u2i0wRAwsR3df8b45hw8EceLt0QOKcoehgxVYQtX2nrmqRkJKJ747koqq2rl3k3MESk4K8MS3EB15ONiJXSH/GwEJE95R8tgDTPz0IiQTY+GIIgru4iF0SUbO6WVaNjYdy8HlqFnJvVQAApBLg0T4KRISqEOLbke0iE8DAQkQNKq6oawXlaSsx4yEVop/oK3ZJRC1Gpxew83Rduyjl4nXDeE+FPcJDffDUAC+eXRQRAwsRNWjupqPYlH4Fqo4d8PPLw2Bjxd4+tQ/n8kuQkJKJbzNyUVGjAwA4WFvgmUAlwkNU8O7YQeQK25/GPn836f7Gq1atgkqlgrW1NdRqNdLS0hqcu3btWgwdOhTOzs5wdnaGRqO55/x//OMfkEgkWLlyZVNKI6L72HUmH5vSr0AiAeIm+jOsULvSQ2GPt5/qj/1vPII3x/SGT8cO0FbW4r+/XcbwuN14bt1B7D1XCDP5W96sGB1YNm7ciMjISERHRyMjIwP+/v4ICwtDQUHBXecnJydjypQp2L17N1JTU6FUKjFy5Ejk5ubeMXfLli3Yv38/PD09jV8JEd1XcXkN5n9zHADw/JAuCFTxuhVqnxxtLPH8UF/sfnUEPpkeiGE9OkEQgKQzBQj/JA2PvLcHCSmZKK2qFbtU+p3RLSG1Wo2goCDEx8cDAPR6PZRKJWbPno358+ffd3udTgdnZ2fEx8cjPDzcMJ6bmwu1Wo3t27djzJgxmDNnDubMmdPoutgSIrq/yI1H8O3hXPh2ssVPLw3lyzyJ/uRiYSk+T83C5vQrhqBiJ7fAhEGdER7iA99OdiJXaJ5apCVUXV2N9PR0aDSaP3YglUKj0SA1NbVR+ygvL0dNTQ1cXP74y06v12PatGmYO3cu+vblxX9ELWHHqXx8ezgX0t9bQQwrRPV17WSHRU/2xf43HsHiJ/vCt5MtSqtqsS4lE395dw/CP0nDrjP50OvZLhKDUZdFFxUVQafTQaFQ1BtXKBQ4c+ZMo/Yxb948eHp61gs9S5cuhYWFBV566aVG11JVVYWqqirDx1qtttHbErU3N8uq8caWulbQC8N8MZBvFkfUIDu5BSJCVZg22Ae/XShCQkomdp0twN5zhdh7rhA+HTtg2mAfTAxUwtHGUuxy241WfR1XbGwsNmzYgOTkZFhb173PQ3p6Ot5//31kZGQY9Xr4mJgYLF68uKVKJTIri344icKSKnRzs8Mrmh5il0PUJkilEgzr0QnDenRC1vUyfJ6aha8P5SDrejne+vE03ttxDk8N8ML0UBW6K+zFLtfsGdUScnV1hUwmQ35+fr3x/Px8uLu733PbuLg4xMbG4pdffoGfn59h/Ndff0VBQQG8vb1hYWEBCwsLZGVl4dVXX4VKpWpwfwsWLEBxcbHhkZOTY8xSiNqNxBPX8P2Rq5BJJXiXrSCiJvHpaIs3H++D/W88gref6oceCjuUV+vw5YFsPLpiL/66dj+2n8yDju2iFtOki26Dg4Px4YcfAqi7/sTb2xuzZs1q8KLbZcuW4e2338b27dsxePDgep+7fv06rl27Vm8sLCwM06ZNw4wZM9CzZ89G1cWLbonudKOsGiNX7EFRaTX+NaIrXh/VS+ySiMyCIAhIvXQdCSmZ2HEqH7dzipeTDaaF+GBSoBLOtlbiFtlGNPb52+iWUGRkJCIiIhAYGIjg4GCsXLkSZWVlmDFjBgAgPDwcXl5eiImJAVB3fUpUVBTWr18PlUqFvLw8AICdnR3s7OzQsWNHdOxY/91hLS0t4e7u3uiwQkR3F/X9CRSVVqOHwg4va7qLXQ6R2ZBIJAjt6orQrq64crMcX+zPxoaD2ci9VYHYn89gxY5zGBfghYhQFfp48o/o5mB0YJk0aRIKCwsRFRWFvLw8BAQEIDEx0XAhbnZ2NqTSPzpNq1evRnV1NSZMmFBvP9HR0Vi0aNGDVU9EDfrx2DVsO3bt91ZQAOQWbAURtYTOzh0wf3QvzNF0x9YjV7EuJROnrmmx8VAONh7KQbDKBRGhKozsq4ClrEn3ayXw1vxEZqmotAojV+zFjbJqvPSXbogcybOVRK1FEAQcyrqJdSmZSDzxx3Ut7g7WeHawNyYHe8PVTi5ylaaD7yVE1E4JgoB/fZmBn0/koZe7PbbOGgIrC/5VRySGvOJKfHkgC1+lZaOotBoAYCWT4nF/D0wPVcGvs5O4BZoABhaiduqHo1cx+6vDsJBK8P2sh9DX01HskojavapaHX48dg0JKZk4eqXYMD7A2wnTQ1UY3c+j3f5hwcBC1A4VlFRi5Iq9uFVegzma7pjDe64QmZzD2TeRkJKJH49fQ42u7im4k70cfw32xlS1N9wcrEWusHUxsBC1M4Ig4MXP07HjVD76eDjg+1kP8QI/IhNWUFKJrw7k4MsDWSgoqbtzu6VMgtH9PBARqsJAbyejbqjaVjGwELUz3x3OxZyNR2Apk2DrrCHo7cGfA6K2oLpWj8STeUhIyUR61k3DeH8vR0SEqvC4n4dZ3/CRgYWoHSnQVuLRFXtRXFGDVx/tgdmP8J4rRG3RidxirEvJxNajV1FdqwcAuNhaYUqwEs8O9oGHo43IFTY/BhaidkIQBLzw2SHsPF2A/l6O+PZfoWwFEbVx10ursOFgDr7Yn4VrxZUAAJlUgrC+CkSEqBDcxcVs2kUMLETtxDfpV/DqpqOwkknxw+wh6OnON2EjMhe1Oj12nMrHupRMHLh8wzDe28MBESE+GBvgBRurtt0uYmAhagfyiivx6Io9KKmsxeujeuJfI7qJXRIRtZDT17T4LDUTWw7norKmrl3k1MESkwLr2kVKlw4iV9g0DCxEZk4QBMxYdxDJZwvhr3TCN/8IgQVbQURm71Z5Nb4+lIPPUrNw5WYFAEAqAR7prcD0UBVCu3ZsU+0iBhYiM/f1oRy8vvkYrCyk+OmlIejmxlYQUXui0wvYdaYACSmZ+O1CkWG8u5sdwkNVeHqAF2zlRr9lYKtjYCEyY1dvVSBsxV6UVNViwehe+PvwrmKXREQiulBQgoSULHyTcQXl1ToAgL21BSYOUiI8xAcqV1uRK2wYAwuRmRIEAeGfpOHX80UY4O2Ezf8IhUzadk7/ElHL0VbWYPOhK/gsNROZ18sBABIJMKJHJ0SEqjCseydITez3BQMLkZn6Ki0bC749DrmFFD+9PBRdO9mJXRIRmRi9XsCe84VISMlE8tlCw3gXV1uEh/hgwqDOsLe2FLHCPzCwEJmhKzfLEbZiL8qqdXhzTG88P9RX7JKIyMRdLirDZ6mZ2HzoCkqqagEAtlYyjB/UGeEhKnRzE/ePHgYWIjMjCAKe/fgA9l24jkAfZ2z8ewhbQUTUaKVVtdiScQUJqVm4UFBqGB/a3RURISo83MtNlN8pDCxEZuaL/Vl487sTsLaU4ueXh6GLCV9ER0SmSxAE7LtwHetSMpF0Jh+3U4C3SwdMG+yDZwKVcOzQeu0iBhYiM5JzoxxhK/eivFqH6Cf6YMZDXcQuiYjMQM6Ncny+PwsbD+aguKIGAGBjKcO4AV6YHqpqlTtnM7AQmQm9XsBf/7sf+y/dQHAXF2x4YbDJXeVPRG1bRbUO3x3JRUJKJs7klRjGB/u6YHqoCpreiha7MSUDC5GZ+Cw1E1Hfn4SNpQzb5wyDd8e2efttIjJ9giDgwOUbSEjJxC+n8qHT10UELycbTB3sjb8Ge8Opg1Wzfs3GPn+b/i3wiNqxrOtliPnpDABgwWO9GFaIqEVJJBIM9u2Iwb4dcfVWBb7Yn4Wv0rKRe6sCyxLP4i+93Jo9sDQWAwuRidLrBczddAwVNTqE+HbEs2ofsUsionbE08kGr4/qhZce6Y4fjl7FsSvF6OUuXgeDgYXIRK1LyURa5g3YWsmwbIIfr1shIlFYW8owMVCJiYFKUevgW7sSmaDLRWVYtv12K6h3m33beCKi5sLAQmRidHoBczcdRWWNHkO6uWKq2lvskoiIRMfAQmRiPt13GYeybsJOboHY8f0hkbAVRETEwEJkQi4UlGL59rMAgDfH9EZnZ7aCiIgABhYik6HTC5i7+SiqavUY1qMTJgWJe4EbEZEpYWAhMhH//fUSDmffgr3cArFPsxVERPRnDCxEJuB8fgne3XEOALDwiT7wdLIRuSIiItPCwEIkslqdHq9tOorqWj0e7tkJEwd1FrskIiKTw8BCJLL/7L2Eo1eK4WBtgZin/dgKIiK6iyYFllWrVkGlUsHa2hpqtRppaWkNzl27di2GDh0KZ2dnODs7Q6PR1JtfU1ODefPmoX///rC1tYWnpyfCw8Nx9erVppRG1KaczSvB+zvPAwCin+gLd0drkSsiIjJNRgeWjRs3IjIyEtHR0cjIyIC/vz/CwsJQUFBw1/nJycmYMmUKdu/ejdTUVCiVSowcORK5ubkAgPLycmRkZGDhwoXIyMjAt99+i7Nnz+LJJ598sJURmbia260gnR6a3m54eqCX2CUREZksiSAIgjEbqNVqBAUFIT4+HgCg1+uhVCoxe/ZszJ8//77b63Q6ODs7Iz4+HuHh4Xedc/DgQQQHByMrKwve3o27y2dj356ayFR8mHQe7+44B0cbS+x4ZRjcHHh2hYjan8Y+fxt1hqW6uhrp6enQaDR/7EAqhUajQWpqaqP2UV5ejpqaGri4uDQ4p7i4GBKJBE5OTg3OqaqqglarrfcgaitOXdXig111raB/j+3LsEJEdB9GBZaioiLodDooFIp64wqFAnl5eY3ax7x58+Dp6Vkv9PxZZWUl5s2bhylTptwzacXExMDR0dHwUCp5ky1qG263gmp0Akb2UeBJf0+xSyIiMnmt+iqh2NhYbNiwAVu2bIG19Z1/UdbU1OCZZ56BIAhYvXr1Pfe1YMECFBcXGx45OTktVTZRs1q1+wJOXdPCuYMl3n6KN4gjImoMC2Mmu7q6QiaTIT8/v954fn4+3N3d77ltXFwcYmNjsXPnTvj5+d3x+dthJSsrC7t27brvdShyuRxyudyY8olEdyK3GPG7LgAA/j22HzrZ83uYiKgxjDrDYmVlhUGDBiEpKckwptfrkZSUhJCQkAa3W7ZsGZYsWYLExEQEBgbe8fnbYeX8+fPYuXMnOnbsaExZRG1CdW1dK6hWL+Cx/u543M9D7JKIiNoMo86wAEBkZCQiIiIQGBiI4OBgrFy5EmVlZZgxYwYAIDw8HF5eXoiJiQEALF26FFFRUVi/fj1UKpXhWhc7OzvY2dmhpqYGEyZMQEZGBrZt2wadTmeY4+LiAisrq+ZaK5GoPtx1HmfyStDR1gpLxvZjK4iIyAhGB5ZJkyahsLAQUVFRyMvLQ0BAABITEw0X4mZnZ0Mq/ePEzerVq1FdXY0JEybU2090dDQWLVqE3NxcbN26FQAQEBBQb87u3bsxYsQIY0skMjnHrxTj/5IvAgCWjOuHjnZsBRERGcPo+7CYKt6HhUxVVa0OT3z4G87ll+JxPw/E/3Wg2CUREZmMFrkPCxEZ7/2d53EuvxSudlb499h+YpdDRNQmMbAQtaAjObewZk9dK+itcf3hYstrsoiImoKBhaiFVNbo8Nqmo9ALwNgAT4zqd++X/hMRUcMYWIhayIqd53ChoBSd7OVY9ERfscshImrTGFiIWkB61k2s3XsJAPDOU/3hzFYQEdEDYWAhamaVNTrM/b0V9PRALzzaR3H/jYiI6J4YWIiaWdz2s7hUVAaFgxzRj7MVRETUHBhYiJrRocwb+HjfZQBA7NN+cOxgKXJFRETmgYGFqJlUVNe9KkgQgImDOuPhXm5il0REZDYYWIiaybLtZ5B5vRwejtZ48/E+YpdDRGRWGFiImsGBS9fx6b5MAEDseD842rAVRETUnBhYiB5QeXUt5m4+BgCYHKTE8B6dRK6IiMj8MLAQPaClP59B9o1yeDpa4/+N6S12OUREZomBhegBpFwsQkJqFgBg2QR/2FuzFURE1BIYWIiaqLSqFq//3gqaqvbGkO6uIldERGS+GFiImijmp9O4crMCnZ1tsOAxtoKIiFoSAwtRE/x2vghfHsgGACyb4Ac7uYXIFRERmTcGFiIjlVTWYN43da2g8BAfhHZlK4iIqKUxsBAZ6Z2fTiP3VgW8XTpg3qheYpdDRNQuMLAQGWHPuUJ8lZYDAFg+wQ+2bAUREbUKBhaiRtJW1mD+762g6aEqqH07ilwREVH7wcBC1EhvbTuFa8WVUHXsgNdH9RS7HCKidoWBhagRdp8pwNeHrkAiAZZP9EcHK7aCiIhaEwML0X0Ul9dg/rd1raDnHuqCIJWLyBUREbU/DCxE9/HvbaeQr62Cr6stXgtjK4iISAwMLET3sPNUPr7JuALp760ga0uZ2CUREbVLDCxEDbhVXo0FW44DAF4Y6otBPs4iV0RE1H4xsBA1YNHWkygsqULXTrZ45dEeYpdDRNSuMbAQ3cX2k3n47shVSCXAu88EsBVERCQyBhai/3GjrBr/7/dW0N+Hd0WA0kncgoiIiIGF6H9Fbz2JotJq9FDYYY6mu9jlEBERmhhYVq1aBZVKBWtra6jVaqSlpTU4d+3atRg6dCicnZ3h7OwMjUZzx3xBEBAVFQUPDw/Y2NhAo9Hg/PnzTSmN6IH8dPwafjh6FTKpBHET/SG3YCuIiMgUGB1YNm7ciMjISERHRyMjIwP+/v4ICwtDQUHBXecnJydjypQp2L17N1JTU6FUKjFy5Ejk5uYa5ixbtgwffPAB1qxZgwMHDsDW1hZhYWGorKxs+sqIjHS9tAoLvzsBAPjXiK7w6+wkbkFERGQgEQRBMGYDtVqNoKAgxMfHAwD0ej2USiVmz56N+fPn33d7nU4HZ2dnxMfHIzw8HIIgwNPTE6+++ipee+01AEBxcTEUCgXWrVuHyZMnN6ourVYLR0dHFBcXw8HBwZglEQEAZn6ZgR+PX0Mvd3tsnTUEVhbsmBIRtbTGPn8b9Ru5uroa6enp0Gg0f+xAKoVGo0Fqamqj9lFeXo6amhq4uNTd3vzy5cvIy8urt09HR0eo1ep77rOqqgparbbeg6ipth27ih+PX4PF760ghhUiItNi1G/loqIi6HQ6KBSKeuMKhQJ5eXmN2se8efPg6elpCCi3tzN2nzExMXB0dDQ8lEqlMUshMigs+aMVNPPhbujn5ShyRURE9L9a9c/I2NhYbNiwAVu2bIG1tfUD7WvBggUoLi42PHJycpqpSmpPBEHAm98dx83yGvTxcMDMh7uJXRIREd2FhTGTXV1dIZPJkJ+fX288Pz8f7u7u99w2Li4OsbGx2LlzJ/z8/Azjt7fLz8+Hh4dHvX0GBAQ0uD+5XA65XG5M+UR32Hr0KrafzIeljK0gIiJTZtRvZysrKwwaNAhJSUmGMb1ej6SkJISEhDS43bJly7BkyRIkJiYiMDCw3ue6dOkCd3f3evvUarU4cODAPfdJ9KAKtJWI+v4kAGD2X7qjjycv1iYiMlVGnWEBgMjISERERCAwMBDBwcFYuXIlysrKMGPGDABAeHg4vLy8EBMTAwBYunQpoqKisH79eqhUKsN1KXZ2drCzs4NEIsGcOXPw1ltvoXv37ujSpQsWLlwIT09PjBs3rvlWSvQngiDgjS3HUVxRg35eDvjniK5il0RERPdgdGCZNGkSCgsLERUVhby8PAQEBCAxMdFw0Wx2djak0j9O3KxevRrV1dWYMGFCvf1ER0dj0aJFAIDXX38dZWVlePHFF3Hr1i0MGTIEiYmJD3ydC1FDthzOxc7TBbCUSfDuxABYytgKIiIyZUbfh8VU8T4s1Fj52ko8+t4eaCtrMTesJy+0JSISUYvch4WorRMEAQu+PQ5tZS38Ozvi78N8xS6JiIgagYGF2pXN6Vew60wBrGRSxE30hwVbQUREbQJ/W1O7ca24Av/+4RQAIHJkD3RX2ItcERERNRYDC7ULgiBg3jfHUVJViwHeTnhhKFtBRERtCQMLtQtfH8rB3nOFkFvUtYJkUonYJRERkREYWMjs5d6qwJJtpwEAr43sia6d7ESuiIiIjMXAQmZNEATM23wMpVW1GOTjjL8N6SJ2SURE1AQMLGTW1qdl47cLRbC2lGL5BD+2goiI2igGFjJbOTfK8c6Pda2g18N6wZetICKiNouBhcySXi9g3jfHUFatQ7DKBdNDVWKXRERED4CBhczSlweykHLxOmwsZVg2wQ9StoKIiNo0BhYyO9nXy/HOT2cAAPNH94LK1VbkioiI6EExsJBZ0esFvLb5KCpqdBjs64Jpg33ELomIiJoBAwuZlc9SM5F2+QY6WMmwfII/W0FERGaCgYXMRmZRGWIT61pBCx7rDaVLB5ErIiKi5sLAQmZBpxfw2qajqKzR46FuHTE12FvskoiIqBkxsJBZ+HTfZRzKuglbKxmWjuergoiIzA0DC7V5FwtLsXz7WQDAm4/3QWdntoKIiMwNAwu1aTq9gLmbjqKqVo+h3V0xOUgpdklERNQCGFioTfv4t0vIyL4Fe7kFlo73g0TCVhARkTliYKE260JBCeJ+OQcAWPh4H3g62YhcERERtRQGFmqTanV6vLrpGKpr9RjRsxMmBnYWuyQiImpBDCzUJq399TKO5tyCvbUFYp9mK4iIyNwxsFCbcy6/BCt21LWCop/oC3dHa5ErIiKilsbAQm1KjU6PV78+imqdHo/0csP4gV5il0RERK2AgYXalP/suYjjucVwtLHEO0/3ZyuIiKidYGChNuNMnhbvJ50HACx+si8UDmwFERG1Fwws1CbcbgXV6AQ82keBsQGeYpdEREStiIGF2oT/230RJ69q4dTBEm8/1Y+tICKidoaBhUzeyavF+HBXXSvo32P7wc2erSAiovaGgYVMWnVtXSuoVi9gdD93POHnIXZJREQkgiYFllWrVkGlUsHa2hpqtRppaWkNzj158iTGjx8PlUoFiUSClStX3jFHp9Nh4cKF6NKlC2xsbNC1a1csWbIEgiA0pTwyI/G7L+BMXglcbK2wZBxbQURE7ZXRgWXjxo2IjIxEdHQ0MjIy4O/vj7CwMBQUFNx1fnl5OXx9fREbGwt3d/e7zlm6dClWr16N+Ph4nD59GkuXLsWyZcvw4YcfGlsemZETucVYtfsCAGDJ2H5wtZOLXBEREYnF6MDy3nvv4YUXXsCMGTPQp08frFmzBh06dMAnn3xy1/lBQUFYvnw5Jk+eDLn87k84KSkpGDt2LMaMGQOVSoUJEyZg5MiR9zxzQ+atqlaHV78+Cp1ewBg/D4xhK4iIqF0zKrBUV1cjPT0dGo3mjx1IpdBoNEhNTW1yEaGhoUhKSsK5c3W3Wz969Ch+++03jB49usFtqqqqoNVq6z3IfHyQdB5n80vgameFJWP7iV0OERGJzMKYyUVFRdDpdFAoFPXGFQoFzpw50+Qi5s+fD61Wi169ekEmk0Gn0+Htt9/G1KlTG9wmJiYGixcvbvLXJNN1NOcW1uy5BAB4a1x/uNhaiVwRERGJzSReJfT111/jyy+/xPr165GRkYGEhATExcUhISGhwW0WLFiA4uJiwyMnJ6cVK6aWUlmjw2ub6lpBT/p7YlS/u1/3RERE7YtRZ1hcXV0hk8mQn59fbzw/P7/BC2obY+7cuZg/fz4mT54MAOjfvz+ysrIQExODiIiIu24jl8sbvCaG2q6VO8/jfEEpXO3kWPxkX7HLISIiE2HUGRYrKysMGjQISUlJhjG9Xo+kpCSEhIQ0uYjy8nJIpfVLkclk0Ov1Td4ntT0Z2Tfx0d6LAIB3nuoHZ7aCiIjod0adYQGAyMhIREREIDAwEMHBwVi5ciXKysowY8YMAEB4eDi8vLwQExMDoO5C3VOnThn+nZubiyNHjsDOzg7dunUDADzxxBN4++234e3tjb59++Lw4cN477338Le//a251kkm7nYrSC8ATw/wwsi+bAUREdEfjA4skyZNQmFhIaKiopCXl4eAgAAkJiYaLsTNzs6ud7bk6tWrGDBggOHjuLg4xMXFYfjw4UhOTgYAfPjhh1i4cCH+9a9/oaCgAJ6envj73/+OqKioB1wetRXv7TiHS4VlcLOXI/oJtoKIiKg+iWAmt5PVarVwdHREcXExHBwcxC6HjJCedQMT1qRCEICPIwLxSG/F/TciIiKz0Njnb5N4lRC1XxXVOry26RgEAZgwqDPDChER3RUDC4lq+fazuFxUBncHayx8vI/Y5RARkYliYCHRpF2+gU9TLgMAYsf3h6ONpcgVERGRqWJgIVGUV9di7uajEARgUqASI3q6iV0SERGZMAYWEsWyxLPIul4OT0dr/L/He4tdDhERmTgGFmp1qRevY11KJgBg6QQ/OFizFURERPfGwEKtqqyqFq9/cxQA8Fe1N4Z27yRyRURE1BYwsFCriv35DHJuVMDLyQZvPMZWEBERNQ4DC7WafReK8Pn+LADAsgl+sJMbfaNlIiJqpxhYqFWUVNbg9c3HAADTBvvgoW6uIldERERtCQMLtYp3fjqD3FsVULrYYP7oXmKXQ0REbQwDC7W4vecK8VVaNgBg+QR/2LIVRERERmJgoRalrazB/G/qWkHTQ1UY7NtR5IqIiKgtYmChFvX2ttO4WlwJn44d8PqonmKXQ0REbRQDC7WY3WcLsPFQDiSSulZQByu2goiIqGkYWKhFFFfUYME3xwEAf3uoC4K7uIhcERERtWUMLNQilmw7hTxtJbq42uK1kWwFERHRg2FgoWaXdDofm9OvQCIB4ib6wcZKJnZJRETUxjGwULO6VV6NBd/WtYJeGOqLQT5sBRER0YNjYKFmtfiHUygoqULXTraIfLSH2OUQEZGZYGChZvPLyTxsOZwLqQSIm+gPa0u2goiIqHkwsFCzuFlWjTe2nAAAvDisKwZ4O4tcERERmRMGFmoW0VtPoqi0Ct3d7DBH013scoiIyMwwsNADSzxxDVuPXoVMKmEriIiIWgQDCz2Q66VV+H+/t4L+Obwr/JVO4hZERERmiYGFHkjU1pO4XlaNngp7zH6km9jlEBGRmWJgoSbbduwqfjx2DTKpBO8+4w+5BVtBRETUMhhYqEmKSqsQ9f1JAMDMh7uhn5ejyBUREZE5Y2AhowmCgIXfncCNsmr09nDArIfZCiIiopbFwEJG++HYNfx8Ig8WUgnenegPKwt+GxERUcviMw0ZpaCkElHf170qaPZfuqOPp4PIFRERUXvQpMCyatUqqFQqWFtbQ61WIy0trcG5J0+exPjx46FSqSCRSLBy5cq7zsvNzcWzzz6Ljh07wsbGBv3798ehQ4eaUh61EEEQ8P+2nMCt8hr09XTAvx7uKnZJRETUThgdWDZu3IjIyEhER0cjIyMD/v7+CAsLQ0FBwV3nl5eXw9fXF7GxsXB3d7/rnJs3b+Khhx6CpaUlfv75Z5w6dQrvvvsunJ15e3dT8t2RXOw4lQ9LWd2rgixlPEFHREStQyIIgmDMBmq1GkFBQYiPjwcA6PV6KJVKzJ49G/Pnz7/ntiqVCnPmzMGcOXPqjc+fPx/79u3Dr7/+alz1f6LVauHo6Iji4mI4OLBN0dzytZUYuWIviitqMDesJ2byQlsiImoGjX3+NupP5OrqaqSnp0Oj0fyxA6kUGo0GqampTS5269atCAwMxMSJE+Hm5oYBAwZg7dq1Td4fNS9BEPDGt8dRXFEDv86O+PswX7FLIiKidsaowFJUVASdTgeFQlFvXKFQIC8vr8lFXLp0CatXr0b37t2xfft2/POf/8RLL72EhISEBrepqqqCVqut96CW8U1GLpLOFMBKJkXcRH9YsBVEREStzELsAoC6tlJgYCDeeecdAMCAAQNw4sQJrFmzBhEREXfdJiYmBosXL27NMtulvOJKLP6h7gZxrzzaAz0U9iJXRERE7ZFRfyq7urpCJpMhPz+/3nh+fn6DF9Q2hoeHB/r06VNvrHfv3sjOzm5wmwULFqC4uNjwyMnJafLXp7sTBAHzvz2GkspaBCid8MLQLmKXRERE7ZRRgcXKygqDBg1CUlKSYUyv1yMpKQkhISFNLuKhhx7C2bNn642dO3cOPj4+DW4jl8vh4OBQ70HNa9OhK0g+WwgrC7aCiIhIXEa3hCIjIxEREYHAwEAEBwdj5cqVKCsrw4wZMwAA4eHh8PLyQkxMDIC6C3VPnTpl+Hdubi6OHDkCOzs7dOtW90qTV155BaGhoXjnnXfwzDPPIC0tDR999BE++uij5lonGSn3VgWWbKs7bq+N7IFubnYiV0RERO2Z0YFl0qRJKCwsRFRUFPLy8hAQEIDExETDhbjZ2dmQSv/4S/zq1asYMGCA4eO4uDjExcVh+PDhSE5OBgAEBQVhy5YtWLBgAf7973+jS5cuWLlyJaZOnfqAy6OmEAQB8785hpKqWgz0dsJzQ/iqICIiEpfR92ExVbwPS/NZfyAbb2w5DrmFFD+/PBS+nXh2hYiIWkaL3IeFzN+Vm+V4+8e6VtDro3oxrBARkUlgYCEDvV7A65uPoaxahyCVM2aEqsQuiYiICAADC/3Jl2nZSLl4HdaWUiyf4A+pVCJ2SURERAAYWOh3OTfKEfPTaQDA/FG9oHK1FbkiIiKiPzCwEPR6AXM3H0V5tQ7qLi4ID1GJXRIREVE9DCyEz/dnYf+lG+hgJWMriIiITBIDSzuXWVSG2J/PAAAWjO4F744dRK6IiIjoTgws7djtVlBFjQ6hXTtiqrrht0IgIiISEwNLO/ZpSiYOZt6ErZUMS8f7sRVEREQmi4GlnbpUWIrl2+taQf9vTB8oXdgKIiIi08XA0g7p9ALmbj6Gyho9hnZ3xZRgpdglERER3RMDSzv0yW+XkZ51E3ZyC8SO94NEwlYQERGZNgaWduZCQSmW/3IWALDw8d7wcrIRuSIiIqL7Y2BpR3R6Aa9tOorqWj2G9+iEZwLZCiIioraBgaUdWfvrJRzJuQV7awvEju/PVhAREbUZDCztxPn8Erz3yzkAQNTjfeDhyFYQERG1HQws7UCtTo9XNx1FtU6Pv/Ryw4RBncUuiYiIyCgMLO3Af/ZewrErxXCwtkDM02wFERFR28PAYubO5GmxcmddK2jx2L5QOFiLXBEREZHxGFjMWI1Oj9c2HUWNToCmtwLjArzELomIiKhJGFjM2OrkiziRq4VTB0u883Q/toKIiKjNYmAxU6euavFB0nkAwOIn+8LNnq0gIiJquxhYzFB1bV0rqFYvYFRfdzzp7yl2SURERA+EgcUMrdp9AaeuaeHcwRJLxrEVREREbR8Di5k5kVuMVbsvAACWjOuHTvZykSsiIiJ6cAwsZuTPraAx/T3wuB9bQUREZB4YWMzIh7vO40xeCTraWuHfY/uKXQ4REVGzYWAxE8eu3ML/JV8EALw1rh862rEVRERE5oOBxQxU1erw6tdHodMLeMLfE6P7e4hdEhERUbNiYDEDK3eex/mCUrjayfHvJ9kKIiIi88PA0sYdzr6J/+ypawW981Q/ONtaiVwRERFR82NgacMqa3R4bdNR6AXgqQFeGNnXXeySiIiIWkSTAsuqVaugUqlgbW0NtVqNtLS0BueePHkS48ePh0qlgkQiwcqVK++579jYWEgkEsyZM6cppbUrK3acw8XCMnSylyP6iT5il0NERNRijA4sGzduRGRkJKKjo5GRkQF/f3+EhYWhoKDgrvPLy8vh6+uL2NhYuLvf+wzAwYMH8Z///Ad+fn7GltXupGfdwEe/XgIAxDzVH04d2AoiIiLzZXRgee+99/DCCy9gxowZ6NOnD9asWYMOHTrgk08+uev8oKAgLF++HJMnT4Zc3vBLbUtLSzF16lSsXbsWzs7OxpbVrlTW6DB30zEIAjB+YGdo+ijELomIiKhFGRVYqqurkZ6eDo1G88cOpFJoNBqkpqY+UCEzZ87EmDFj6u37XqqqqqDVaus92ou47WdxqagMCgc5otgKIiKidsCowFJUVASdTgeFov5f9AqFAnl5eU0uYsOGDcjIyEBMTEyjt4mJiYGjo6PhoVQqm/z125KDmTfw8b7LAIDY8X5wtLEUuSIiIqKWJ/qrhHJycvDyyy/jyy+/hLW1daO3W7BgAYqLiw2PnJycFqzSNJRX12LupqMQBOCZwM54uKeb2CURERG1CgtjJru6ukImkyE/P7/eeH5+/n0vqG1Ieno6CgoKMHDgQMOYTqfD3r17ER8fj6qqKshksju2k8vl97wmxhwtSzyLzOvl8HC0xpuPsxVERETth1FnWKysrDBo0CAkJSUZxvR6PZKSkhASEtKkAh555BEcP34cR44cMTwCAwMxdepUHDly5K5hpT3af+k61qVkAgCWjveDgzVbQURE1H4YdYYFACIjIxEREYHAwEAEBwdj5cqVKCsrw4wZMwAA4eHh8PLyMlyPUl1djVOnThn+nZubiyNHjsDOzg7dunWDvb09+vXrV+9r2NraomPHjneMt1dlVbV4ffMxAMCUYG8M69FJ5IqIiIhal9GBZdKkSSgsLERUVBTy8vIQEBCAxMREw4W42dnZkEr/OHFz9epVDBgwwPBxXFwc4uLiMHz4cCQnJz/4CtqBpYlnkH2jHF5ONnjjsV5il0NERNTqJIIgCGIX0Ry0Wi0cHR1RXFwMBwcHsctpNikXivDX/x4AAHzxnBpDuruKXBEREVHzaezzt+ivEqKGlVbVYu7vraBnB3szrBARUbvFwGLCYn46jdxbFejsbIMFo3uLXQ4REZFoGFhM1K/nC/HlgWwAwPIJ/rCVG325ERERkdlgYDFBJZU1mPd7KygixAchXTuKXBEREZG4GFhM0Ns/nsbV4kp4u3TAvNF8VRAREREDi4nZc64QGw7mQCIB4ib6o4MVW0FEREQMLCakuOKPVtCM0C4I7uIickVERESmgYHFhLy17RTytJXo4mqLuWE9xS6HiIjIZDCwmIhdZ/KxKf0KJBJg+QQ/2FjxPZSIiIhuY2AxAcXlNZj/zXEAwPNDuiBQxVYQERHRnzGwmIDF206ioKQKvp1s8epItoKIiIj+FwOLyHacyse3GbmQ/v6qIGtLtoKIiIj+FwOLiG6WVeONLXWtoBeG+WKgt7PIFREREZkmBhYRLfrhJApLqtDNzQ6vaHqIXQ4REZHJYmARSeKJPHx/5CpkUgneZSuIiIjonhhYRHCjrBpvflfXCvrHcF/4K53ELYiIiMjEMbCIIOr7EygqrUZPhT1eeqS72OUQERGZPAaWVvbT8WvYduwaZFIJ4ib6Q27BVhAREdH9MLC0oqLSKrz53QkAwMwRXdG/s6PIFREREbUNDCytRBAELPzuBG6UVaOXuz1m/YWtICIiosZiYGkl245dw88n8mAhleDdZ/xhZcH/9URERI3FZ81WUFBSiYXf17WCZv2lG/p6shVERERkDAaWFiYIAt7ccgK3ymvQx8MBMx/uJnZJREREbQ4DSwvbevQqfjmVD0tZXSvIUsb/5URERMbis2cLKtBWIur7kwCAlx/pjt4eDiJXRERE1DYxsLQQQRDwxpbjKK6oQX8vR/xjeFexSyIiImqzGFhayLcZudh5ugBWMiniJvrDgq0gIiKiJuOzaAvIK67E4h/qWkFzHu2Onu72IldERETUtjGwNDNBELDg22PQVtbCX+mEF4f6il0SERFRm8fA0sw2pV/B7rOFsLKQ4t2JfmwFERERNQM+mzajq7cqsOSHUwCAVx/tgW5ubAURERE1hyYFllWrVkGlUsHa2hpqtRppaWkNzj158iTGjx8PlUoFiUSClStX3jEnJiYGQUFBsLe3h5ubG8aNG4ezZ882pTTRCIKA+d8eR0lVLQZ4O+F5toKIiIiajdGBZePGjYiMjER0dDQyMjLg7++PsLAwFBQU3HV+eXk5fH19ERsbC3d397vO2bNnD2bOnIn9+/djx44dqKmpwciRI1FWVmZseaLZeDAHe88VQm5R96ogmVQidklERERmQyIIgmDMBmq1GkFBQYiPjwcA6PV6KJVKzJ49G/Pnz7/ntiqVCnPmzMGcOXPuOa+wsBBubm7Ys2cPhg0b1qi6tFotHB0dUVxcDAeH1r1B25Wb5Ri18leUVtXizTG9eXaFiIiokRr7/G3UGZbq6mqkp6dDo9H8sQOpFBqNBqmpqU2v9n8UFxcDAFxcXBqcU1VVBa1WW+8hBkEQMO+bYyitqkWgjzNmPNRFlDqIiIjMmVGBpaioCDqdDgqFot64QqFAXl5esxSk1+sxZ84cPPTQQ+jXr1+D82JiYuDo6Gh4KJXKZvn6xvryQDb2XbgOa0splrMVRERE1CJM7lVCM2fOxIkTJ7Bhw4Z7zluwYAGKi4sNj5ycnFaq8A85N8rxzk+nAQDzRvVCF1fbVq+BiIioPbAwZrKrqytkMhny8/Prjefn5zd4Qa0xZs2ahW3btmHv3r3o3LnzPefK5XLI5fIH/ppNpdcLeH3zMZRX6xDcxQURISrRaiEiIjJ3Rp1hsbKywqBBg5CUlGQY0+v1SEpKQkhISJOLEAQBs2bNwpYtW7Br1y506WL614F8cSALqZeuw8ZShrgJ/pCyFURERNRijDrDAgCRkZGIiIhAYGAggoODsXLlSpSVlWHGjBkAgPDwcHh5eSEmJgZA3YW6p06dMvw7NzcXR44cgZ2dHbp16wagrg20fv16fP/997C3tzdcD+Po6AgbG5tmWWhzyrpehpifzgAAFjzWC94dO4hcERERkXkz+mXNABAfH4/ly5cjLy8PAQEB+OCDD6BWqwEAI0aMgEqlwrp16wAAmZmZdz1jMnz4cCQnJ9cVIbn72YlPP/0U06dPb1RNrfWyZr1ewOS1+5F2+QZCfDviy+fVPLtCRETURI19/m5SYDFFrRVYPt13GYt/OAVbKxkS5wyD0oVnV4iIiJqqRe7D0t5dLirD0sTbraDeDCtERESthIGlkXR6AXM3HUVljR5Durliqtpb7JKIiIjaDQaWRvp032UcyroJO7kFYsf3b/C6GyIiImp+DCyNcLGwFMu317179JtjeqOzM1tBRERErYmB5T50egGvbTqKqlo9hvXohElB4rwFABERUXvGwHIf//31Eg5n34K93AJL2QoiIiISBQPLPeRrK/HujnMAgIVP9IGHo+ndxI6IiKg9MPpOt+2JwsEa//fXgdh+Mg8TB937vY2IiIio5TCw3IemjwKaPgqxyyAiImrX2BIiIiIik8fAQkRERCaPgYWIiIhMHgMLERERmTwGFiIiIjJ5DCxERERk8hhYiIiIyOQxsBAREZHJY2AhIiIik8fAQkRERCaPgYWIiIhMHgMLERERmTwGFiIiIjJ5ZvNuzYIgAAC0Wq3IlRAREVFj3X7evv083hCzCSwlJSUAAKVSKXIlREREZKySkhI4Ojo2+HmJcL9I00bo9XpcvXoV9vb2kEgkzbZfrVYLpVKJnJwcODg4NNt+TYm5r5Hra/vMfY1cX9tn7mtsyfUJgoCSkhJ4enpCKm34ShWzOcMilUrRuXPnFtu/g4ODWX4T/pm5r5Hra/vMfY1cX9tn7mtsqfXd68zKbbzoloiIiEweAwsRERGZPAaW+5DL5YiOjoZcLhe7lBZj7mvk+to+c18j19f2mfsaTWF9ZnPRLREREZkvnmEhIiIik8fAQkRERCaPgYWIiIhMHgMLERERmbx2GVhWrVoFlUoFa2trqNVqpKWl3XP+pk2b0KtXL1hbW6N///746aef6n1eEARERUXBw8MDNjY20Gg0OH/+fEsu4Z6MWd/atWsxdOhQODs7w9nZGRqN5o7506dPh0QiqfcYNWpUSy+jQcasb926dXfUbm1tXW+OqR0/wLg1jhgx4o41SiQSjBkzxjDHlI7h3r178cQTT8DT0xMSiQTffffdfbdJTk7GwIEDIZfL0a1bN6xbt+6OOcb+XLcUY9f37bff4tFHH0WnTp3g4OCAkJAQbN++vd6cRYsW3XH8evXq1YKruDdj15icnHzX79G8vLx689rqMbzbz5dEIkHfvn0Nc0zpGMbExCAoKAj29vZwc3PDuHHjcPbs2ftuJ/ZzYbsLLBs3bkRkZCSio6ORkZEBf39/hIWFoaCg4K7zU1JSMGXKFDz33HM4fPgwxo0bh3HjxuHEiROGOcuWLcMHH3yANWvW4MCBA7C1tUVYWBgqKytba1kGxq4vOTkZU6ZMwe7du5GamgqlUomRI0ciNze33rxRo0bh2rVrhsdXX33VGsu5g7HrA+ruzPjn2rOysup93pSOH2D8Gr/99tt66ztx4gRkMhkmTpxYb56pHMOysjL4+/tj1apVjZp/+fJljBkzBg8//DCOHDmCOXPm4Pnnn6/3pN6U74uWYuz69u7di0cffRQ//fQT0tPT8fDDD+OJJ57A4cOH683r27dvveP322+/tUT5jWLsGm87e/ZsvTW4ubkZPteWj+H7779fb105OTlwcXG542fQVI7hnj17MHPmTOzfvx87duxATU0NRo4cibKysga3MYnnQqGdCQ4OFmbOnGn4WKfTCZ6enkJMTMxd5z/zzDPCmDFj6o2p1Wrh73//uyAIgqDX6wV3d3dh+fLlhs/funVLkMvlwldffdUCK7g3Y9f3v2prawV7e3shISHBMBYRESGMHTu2uUttEmPX9+mnnwqOjo4N7s/Ujp8gPPgxXLFihWBvby+UlpYaxkzpGP4ZAGHLli33nPP6668Lffv2rTc2adIkISwszPDxg/4/aymNWd/d9OnTR1i8eLHh4+joaMHf37/5CmtGjVnj7t27BQDCzZs3G5xjTsdwy5YtgkQiETIzMw1jpnwMCwoKBADCnj17GpxjCs+F7eoMS3V1NdLT06HRaAxjUqkUGo0Gqampd90mNTW13nwACAsLM8y/fPky8vLy6s1xdHSEWq1ucJ8tpSnr+1/l5eWoqamBi4tLvfHk5GS4ubmhZ8+e+Oc//4nr1683a+2N0dT1lZaWwsfHB0qlEmPHjsXJkycNnzOl4wc0zzH8+OOPMXnyZNja2tYbN4Vj2BT3+xlsjv9npkSv16OkpOSOn8Hz58/D09MTvr6+mDp1KrKzs0WqsOkCAgLg4eGBRx99FPv27TOMm9sx/Pjjj6HRaODj41Nv3FSPYXFxMQDc8T33Z6bwXNiuAktRURF0Oh0UCkW9cYVCcUcv9ba8vLx7zr/9X2P22VKasr7/NW/ePHh6etb7phs1ahQ+++wzJCUlYenSpdizZw9Gjx4NnU7XrPXfT1PW17NnT3zyySf4/vvv8cUXX0Cv1yM0NBRXrlwBYFrHD3jwY5iWloYTJ07g+eefrzduKsewKRr6GdRqtaioqGiW73tTEhcXh9LSUjzzzDOGMbVajXXr1iExMRGrV6/G5cuXMXToUJSUlIhYaeN5eHhgzZo1+Oabb/DNN99AqVRixIgRyMjIANA8v7tMxdWrV/Hzzz/f8TNoqsdQr9djzpw5eOihh9CvX78G55nCc6HZvFszPbjY2Fhs2LABycnJ9S5MnTx5suHf/fv3h5+fH7p27Yrk5GQ88sgjYpTaaCEhIQgJCTF8HBoait69e+M///kPlixZImJlLePjjz9G//79ERwcXG+8LR/D9mT9+vVYvHgxvv/++3rXd4wePdrwbz8/P6jVavj4+ODrr7/Gc889J0apRunZsyd69uxp+Dg0NBQXL17EihUr8Pnnn4tYWfNLSEiAk5MTxo0bV2/cVI/hzJkzceLECVGviWqsdnWGxdXVFTKZDPn5+fXG8/Pz4e7uftdt3N3d7zn/9n+N2WdLacr6bouLi0NsbCx++eUX+Pn53XOur68vXF1dceHChQeu2RgPsr7bLC0tMWDAAEPtpnT8gAdbY1lZGTZs2NCoX35iHcOmaOhn0MHBATY2Ns3yfWEKNmzYgOeffx5ff/31Hafe/5eTkxN69OjRJo5fQ4KDgw31m8sxFAQBn3zyCaZNmwYrK6t7zjWFYzhr1ixs27YNu3fvRufOne851xSeC9tVYLGyssKgQYOQlJRkGNPr9UhKSqr3V/ifhYSE1JsPADt27DDM79KlC9zd3evN0Wq1OHDgQIP7bClNWR9Qd2X3kiVLkJiYiMDAwPt+nStXruD69evw8PBolrobq6nr+zOdTofjx48bajel4wc82Bo3bdqEqqoqPPvss/f9OmIdw6a4389gc3xfiO2rr77CjBkz8NVXX9V7OXpDSktLcfHixTZx/Bpy5MgRQ/3mcAyBulffXLhwoVF/NIh5DAVBwKxZs7Blyxbs2rULXbp0ue82JvFc2CyX7rYhGzZsEORyubBu3Trh1KlTwosvvig4OTkJeXl5giAIwrRp04T58+cb5u/bt0+wsLAQ4uLihNOnTwvR0dGCpaWlcPz4ccOc2NhYwcnJSfj++++FY8eOCWPHjhW6dOkiVFRUmPz6YmNjBSsrK2Hz5s3CtWvXDI+SkhJBEAShpKREeO2114TU1FTh8uXLws6dO4WBAwcK3bt3FyorK01+fYsXLxa2b98uXLx4UUhPTxcmT54sWFtbCydPnjTMMaXjJwjGr/G2IUOGCJMmTbpj3NSOYUlJiXD48GHh8OHDAgDhvffeEw4fPixkZWUJgiAI8+fPF6ZNm2aYf+nSJaFDhw7C3LlzhdOnTwurVq0SZDKZkJiYaJhzv/9npry+L7/8UrCwsBBWrVpV72fw1q1bhjmvvvqqkJycLFy+fFnYt2+foNFoBFdXV6GgoKDV1ycIxq9xxYoVwnfffSecP39eOH78uPDyyy8LUqlU2Llzp2FOWz6Gtz377LOCWq2+6z5N6Rj+85//FBwdHYXk5OR633Pl5eWGOab4XNjuAosgCMKHH34oeHt7C1ZWVkJwcLCwf/9+w+eGDx8uRERE1Jv/9ddfCz169BCsrKyEvn37Cj/++GO9z+v1emHhwoWCQqEQ5HK58Mgjjwhnz55tjaXclTHr8/HxEQDc8YiOjhYEQRDKy8uFkSNHCp06dRIsLS0FHx8f4YUXXhDll8htxqxvzpw5hrkKhUJ47LHHhIyMjHr7M7XjJwjGf4+eOXNGACD88ssvd+zL1I7h7Ze4/u/j9poiIiKE4cOH37FNQECAYGVlJfj6+gqffvrpHfu91/+z1mTs+oYPH37P+YJQ9zJuDw8PwcrKSvDy8hImTZokXLhwoXUX9ifGrnHp0qVC165dBWtra8HFxUUYMWKEsGvXrjv221aPoSDUvYTXxsZG+Oijj+66T1M6hndbG4B6P1em+Fwo+b14IiIiIpPVrq5hISIioraJgYWIiIhMHgMLERERmTwGFiIiIjJ5DCxERERk8hhYiIiIyOQxsBAREZHJY2AhIiIik8fAQkRERCaPgYWIiIhMHgMLERERmTwGFiIiIjJ5/x9wOAHmb5FweQAAAABJRU5ErkJggg==" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from matplotlib import pyplot as plt\n", + "\n", + "# Load the model from the database\n", + "model = db.load('model', model.identifier)\n", + "\n", + "# Plot the accuracy values\n", + "plt.plot(model.metric_values['my_valid/acc'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0199b952", + "metadata": {}, + "source": [ + "\n", + "## On-the-fly Predictions\n", + "Once the model is trained, you can use it to continuously predict on new data as it arrives. This is set up by enabling a `listener` for the database (without loading all the data client-side). The listen toggle activates the model to make predictions on incoming data changes.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f0e53249", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:51.956782111Z", + "start_time": "2023-11-24T16:52:50.001555855Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:50.00\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m212 \u001B[0m | \u001B[1mAdding model lenet5 to db\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:50.00\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m894 \u001B[0m | \u001B[34m\u001B[1mmodel/lenet5/0 already exists - doing nothing\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:50.02\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m894 \u001B[0m | \u001B[34m\u001B[1mmodel/lenet5/0 already exists - doing nothing\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:50.02\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m212 \u001B[0m | \u001B[1mAdding model lenet5 to db\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:50.02\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m894 \u001B[0m | \u001B[34m\u001B[1mmodel/lenet5/0 already exists - doing nothing\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "900it [00:00, 45727.22it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:50.04\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 0/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 667.41it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:50.23\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 1/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 803.16it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:50.40\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 2/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 638.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:50.61\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 3/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 792.55it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:50.80\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 4/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 701.26it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:51.02\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 5/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 742.83it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:51.23\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 6/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 734.73it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:51.46\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 7/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 726.97it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:51.69\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 8/9\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 776.37it/s]\n" + ] + }, + { + "data": { + "text/plain": "([None],\n Listener(key='img', model=TorchModel(identifier='lenet5', object=, flatten=False, output_schema=None, encoder=None, preprocess= serializer=dill>, postprocess= serializer=dill>, collate_fn=None, metrics=[Metric(identifier='acc', object=, version=None)], predict_method=None, model_to_device_method='to', batch_predict=False, takes_context=False, train_X='img', train_y='class', training_select=, metric_values={'my_valid/acc': [0.11363636363636363, 0.2727272727272727, 0.22727272727272727], 'objective': [2.310549592971802, 2.2436946392059327, 2.1574175357818604]}, training_configuration=TorchTrainerConfiguration(identifier='my_configuration', kwargs=None, version=None, objective=, loader_kwargs={'batch_size': 10}, max_iterations=10, no_improve_then_stop=5, splitter=None, download=False, validation_interval=5, listen='objective', optimizer_cls= serializer=pickle>, optimizer_kwargs={}, target_preprocessors=None), model_update_kwargs={}, serializer='dill', device='cpu', preferred_devices=['cpu'], version=0, is_batch=None, num_directions=2, optimizer_state=, forward_method='__call__', train_forward_method='__call__'), select=, active=True, identifier='lenet5/img', predict_kwargs={'in_memory': True, 'max_chunk_size': 100}, version=0))" + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.predict(\n", + " X='img', # Input feature \n", + " db=db, # Database used for data retrieval\n", + " select=mnist_collection.find(), # Select the dataset\n", + " listen=True, # Continuous predictions on incoming data \n", + " max_chunk_size=100, # Number of predictions to return at once\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "7daae786", + "metadata": {}, + "source": [ + "We can see that predictions are available in `_outputs.img.lenet5`." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "bc71a143", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:51.958950594Z", + "start_time": "2023-11-24T16:52:51.957300020Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "{'img': ,\n 'class': 4,\n '_fold': 'valid',\n '_id': ObjectId('6560d4e114097bdad056323a'),\n '_outputs': {'img': {'lenet5': {'0': 6}}}}" + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r = db.execute(mnist_collection.find_one({'_fold': 'valid'}))\n", + "r.unpack()" + ] + }, + { + "cell_type": "markdown", + "id": "7a78a2a1", + "metadata": {}, + "source": [ + "## Verification\n", + "\n", + "The models \"activated\" can be seen here:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "outputs": [ + { + "data": { + "text/plain": "['lenet5/img']" + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.show('listener')" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-24T16:52:51.976593661Z", + "start_time": "2023-11-24T16:52:51.959401380Z" + } + }, + "id": "2a5308f4a158c931" + }, + { + "cell_type": "markdown", + "source": [ + "We can verify that the model is activated, by inserting the rest of the data:" + ], + "metadata": { + "collapsed": false + }, + "id": "dee36a804224cbb6" + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c1aa56d0", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:52.266257885Z", + "start_time": "2023-11-24T16:52:51.961648261Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:51.96\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m737 \u001B[0m | \u001B[34m\u001B[1mBuilding task workflow graph. Query:\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.96\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.backends.local.compute\u001B[0m:\u001B[36m32 \u001B[0m | \u001B[1mSubmitting job. function:\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.96\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.misc.download\u001B[0m:\u001B[36m338 \u001B[0m | \u001B[34m\u001B[1m{'cls': 'MongoCompoundSelect', 'dict': {'table_or_collection': {'cls': 'Collection', 'dict': {'identifier': 'mnist'}, 'module': 'superduperdb.backends.mongodb.query'}, 'pre_like': None, 'post_like': None, 'query_linker': {'cls': 'MongoQueryLinker', 'dict': {'table_or_collection': {'cls': 'Collection', 'dict': {'identifier': 'mnist'}, 'module': 'superduperdb.backends.mongodb.query'}, 'members': [{'cls': 'Find', 'dict': {'name': 'find', 'type': , 'args': [{}, {}], 'kwargs': {}, 'output_fields': None}, 'module': 'superduperdb.backends.mongodb.query'}]}, 'module': 'superduperdb.backends.mongodb.query'}}, 'module': 'superduperdb.backends.mongodb.query'}\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.96\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.misc.download\u001B[0m:\u001B[36m339 \u001B[0m | \u001B[34m\u001B[1m[ObjectId('6560d4e314097bdad05635a2'), ObjectId('6560d4e314097bdad05635a3'), ObjectId('6560d4e314097bdad05635a4'), ObjectId('6560d4e314097bdad05635a5'), ObjectId('6560d4e314097bdad05635a6'), ObjectId('6560d4e314097bdad05635a7'), ObjectId('6560d4e314097bdad05635a8'), ObjectId('6560d4e314097bdad05635a9'), ObjectId('6560d4e314097bdad05635aa'), ObjectId('6560d4e314097bdad05635ab'), ObjectId('6560d4e314097bdad05635ac'), ObjectId('6560d4e314097bdad05635ad'), ObjectId('6560d4e314097bdad05635ae'), ObjectId('6560d4e314097bdad05635af'), ObjectId('6560d4e314097bdad05635b0'), ObjectId('6560d4e314097bdad05635b1'), ObjectId('6560d4e314097bdad05635b2'), ObjectId('6560d4e314097bdad05635b3'), ObjectId('6560d4e314097bdad05635b4'), ObjectId('6560d4e314097bdad05635b5'), ObjectId('6560d4e314097bdad05635b6'), ObjectId('6560d4e314097bdad05635b7'), ObjectId('6560d4e314097bdad05635b8'), ObjectId('6560d4e314097bdad05635b9'), ObjectId('6560d4e314097bdad05635ba'), ObjectId('6560d4e314097bdad05635bb'), ObjectId('6560d4e314097bdad05635bc'), ObjectId('6560d4e314097bdad05635bd'), ObjectId('6560d4e314097bdad05635be'), ObjectId('6560d4e314097bdad05635bf'), ObjectId('6560d4e314097bdad05635c0'), ObjectId('6560d4e314097bdad05635c1'), ObjectId('6560d4e314097bdad05635c2'), ObjectId('6560d4e314097bdad05635c3'), ObjectId('6560d4e314097bdad05635c4'), ObjectId('6560d4e314097bdad05635c5'), ObjectId('6560d4e314097bdad05635c6'), ObjectId('6560d4e314097bdad05635c7'), ObjectId('6560d4e314097bdad05635c8'), ObjectId('6560d4e314097bdad05635c9'), ObjectId('6560d4e314097bdad05635ca'), ObjectId('6560d4e314097bdad05635cb'), ObjectId('6560d4e314097bdad05635cc'), ObjectId('6560d4e314097bdad05635cd'), ObjectId('6560d4e314097bdad05635ce'), ObjectId('6560d4e314097bdad05635cf'), ObjectId('6560d4e314097bdad05635d0'), ObjectId('6560d4e314097bdad05635d1'), ObjectId('6560d4e314097bdad05635d2'), ObjectId('6560d4e314097bdad05635d3'), ObjectId('6560d4e314097bdad05635d4'), ObjectId('6560d4e314097bdad05635d5'), ObjectId('6560d4e314097bdad05635d6'), ObjectId('6560d4e314097bdad05635d7'), ObjectId('6560d4e314097bdad05635d8'), ObjectId('6560d4e314097bdad05635d9'), ObjectId('6560d4e314097bdad05635da'), ObjectId('6560d4e314097bdad05635db'), ObjectId('6560d4e314097bdad05635dc'), ObjectId('6560d4e314097bdad05635dd'), ObjectId('6560d4e314097bdad05635de'), ObjectId('6560d4e314097bdad05635df'), ObjectId('6560d4e314097bdad05635e0'), ObjectId('6560d4e314097bdad05635e1'), ObjectId('6560d4e314097bdad05635e2'), ObjectId('6560d4e314097bdad05635e3'), ObjectId('6560d4e314097bdad05635e4'), ObjectId('6560d4e314097bdad05635e5'), ObjectId('6560d4e314097bdad05635e6'), ObjectId('6560d4e314097bdad05635e7'), ObjectId('6560d4e314097bdad05635e8'), ObjectId('6560d4e314097bdad05635e9'), ObjectId('6560d4e314097bdad05635ea'), ObjectId('6560d4e314097bdad05635eb'), ObjectId('6560d4e314097bdad05635ec'), ObjectId('6560d4e314097bdad05635ed'), ObjectId('6560d4e314097bdad05635ee'), ObjectId('6560d4e314097bdad05635ef'), ObjectId('6560d4e314097bdad05635f0'), ObjectId('6560d4e314097bdad05635f1'), ObjectId('6560d4e314097bdad05635f2'), ObjectId('6560d4e314097bdad05635f3'), ObjectId('6560d4e314097bdad05635f4'), ObjectId('6560d4e314097bdad05635f5'), ObjectId('6560d4e314097bdad05635f6'), ObjectId('6560d4e314097bdad05635f7'), ObjectId('6560d4e314097bdad05635f8'), ObjectId('6560d4e314097bdad05635f9'), ObjectId('6560d4e314097bdad05635fa'), ObjectId('6560d4e314097bdad05635fb'), ObjectId('6560d4e314097bdad05635fc'), ObjectId('6560d4e314097bdad05635fd'), ObjectId('6560d4e314097bdad05635fe'), ObjectId('6560d4e314097bdad05635ff'), ObjectId('6560d4e314097bdad0563600'), ObjectId('6560d4e314097bdad0563601'), ObjectId('6560d4e314097bdad0563602'), ObjectId('6560d4e314097bdad0563603'), ObjectId('6560d4e314097bdad0563604'), ObjectId('6560d4e314097bdad0563605')]\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.99\u001B[0m| \u001B[32m\u001B[1mSUCCESS \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.backends.local.compute\u001B[0m:\u001B[36m38 \u001B[0m | \u001B[32m\u001B[1mJob submitted. function: future:99082e9e-a32f-41d1-9311-ed266c8f74a1\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.99\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m362 \u001B[0m | \u001B[34m\u001B[1mto will be overriden with `to`\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.99\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.backends.local.compute\u001B[0m:\u001B[36m32 \u001B[0m | \u001B[1mSubmitting job. function:\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.99\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.ext.torch.model\u001B[0m:\u001B[36m362 \u001B[0m | \u001B[34m\u001B[1mto will be overriden with `to`\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.99\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m212 \u001B[0m | \u001B[1mAdding model lenet5 to db\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.99\u001B[0m| \u001B[34m\u001B[1mDEBUG \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.base.datalayer\u001B[0m:\u001B[36m894 \u001B[0m | \u001B[34m\u001B[1mmodel/lenet5/0 already exists - doing nothing\u001B[0m\n", + "\u001B[32m 2023-Nov-24 18:52:51.99\u001B[0m| \u001B[1mINFO \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.components.model\u001B[0m:\u001B[36m364 \u001B[0m | \u001B[1mComputing chunk 0/1\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [00:00<00:00, 730.62it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[32m 2023-Nov-24 18:52:52.26\u001B[0m| \u001B[32m\u001B[1mSUCCESS \u001B[0m | \u001B[36mmoumia \u001B[0m| \u001B[36m0ef02e21-b98c-4fc7-99f8-e7d308835403\u001B[0m| \u001B[36msuperduperdb.backends.local.compute\u001B[0m:\u001B[36m38 \u001B[0m | \u001B[32m\u001B[1mJob submitted. function: future:cf9f14ab-2e82-4e01-bdfc-cb29f7c8a659\u001B[0m\n" + ] + }, + { + "data": { + "text/plain": "([ObjectId('6560d4e314097bdad05635a2'),\n ObjectId('6560d4e314097bdad05635a3'),\n ObjectId('6560d4e314097bdad05635a4'),\n ObjectId('6560d4e314097bdad05635a5'),\n ObjectId('6560d4e314097bdad05635a6'),\n ObjectId('6560d4e314097bdad05635a7'),\n ObjectId('6560d4e314097bdad05635a8'),\n ObjectId('6560d4e314097bdad05635a9'),\n ObjectId('6560d4e314097bdad05635aa'),\n ObjectId('6560d4e314097bdad05635ab'),\n ObjectId('6560d4e314097bdad05635ac'),\n ObjectId('6560d4e314097bdad05635ad'),\n ObjectId('6560d4e314097bdad05635ae'),\n ObjectId('6560d4e314097bdad05635af'),\n ObjectId('6560d4e314097bdad05635b0'),\n ObjectId('6560d4e314097bdad05635b1'),\n ObjectId('6560d4e314097bdad05635b2'),\n ObjectId('6560d4e314097bdad05635b3'),\n ObjectId('6560d4e314097bdad05635b4'),\n ObjectId('6560d4e314097bdad05635b5'),\n ObjectId('6560d4e314097bdad05635b6'),\n ObjectId('6560d4e314097bdad05635b7'),\n ObjectId('6560d4e314097bdad05635b8'),\n ObjectId('6560d4e314097bdad05635b9'),\n ObjectId('6560d4e314097bdad05635ba'),\n ObjectId('6560d4e314097bdad05635bb'),\n ObjectId('6560d4e314097bdad05635bc'),\n ObjectId('6560d4e314097bdad05635bd'),\n ObjectId('6560d4e314097bdad05635be'),\n ObjectId('6560d4e314097bdad05635bf'),\n ObjectId('6560d4e314097bdad05635c0'),\n ObjectId('6560d4e314097bdad05635c1'),\n ObjectId('6560d4e314097bdad05635c2'),\n ObjectId('6560d4e314097bdad05635c3'),\n ObjectId('6560d4e314097bdad05635c4'),\n ObjectId('6560d4e314097bdad05635c5'),\n ObjectId('6560d4e314097bdad05635c6'),\n ObjectId('6560d4e314097bdad05635c7'),\n ObjectId('6560d4e314097bdad05635c8'),\n ObjectId('6560d4e314097bdad05635c9'),\n ObjectId('6560d4e314097bdad05635ca'),\n ObjectId('6560d4e314097bdad05635cb'),\n ObjectId('6560d4e314097bdad05635cc'),\n ObjectId('6560d4e314097bdad05635cd'),\n ObjectId('6560d4e314097bdad05635ce'),\n ObjectId('6560d4e314097bdad05635cf'),\n ObjectId('6560d4e314097bdad05635d0'),\n ObjectId('6560d4e314097bdad05635d1'),\n ObjectId('6560d4e314097bdad05635d2'),\n ObjectId('6560d4e314097bdad05635d3'),\n ObjectId('6560d4e314097bdad05635d4'),\n ObjectId('6560d4e314097bdad05635d5'),\n ObjectId('6560d4e314097bdad05635d6'),\n ObjectId('6560d4e314097bdad05635d7'),\n ObjectId('6560d4e314097bdad05635d8'),\n ObjectId('6560d4e314097bdad05635d9'),\n ObjectId('6560d4e314097bdad05635da'),\n ObjectId('6560d4e314097bdad05635db'),\n ObjectId('6560d4e314097bdad05635dc'),\n ObjectId('6560d4e314097bdad05635dd'),\n ObjectId('6560d4e314097bdad05635de'),\n ObjectId('6560d4e314097bdad05635df'),\n ObjectId('6560d4e314097bdad05635e0'),\n ObjectId('6560d4e314097bdad05635e1'),\n ObjectId('6560d4e314097bdad05635e2'),\n ObjectId('6560d4e314097bdad05635e3'),\n ObjectId('6560d4e314097bdad05635e4'),\n ObjectId('6560d4e314097bdad05635e5'),\n ObjectId('6560d4e314097bdad05635e6'),\n ObjectId('6560d4e314097bdad05635e7'),\n ObjectId('6560d4e314097bdad05635e8'),\n ObjectId('6560d4e314097bdad05635e9'),\n ObjectId('6560d4e314097bdad05635ea'),\n ObjectId('6560d4e314097bdad05635eb'),\n ObjectId('6560d4e314097bdad05635ec'),\n ObjectId('6560d4e314097bdad05635ed'),\n ObjectId('6560d4e314097bdad05635ee'),\n ObjectId('6560d4e314097bdad05635ef'),\n ObjectId('6560d4e314097bdad05635f0'),\n ObjectId('6560d4e314097bdad05635f1'),\n ObjectId('6560d4e314097bdad05635f2'),\n ObjectId('6560d4e314097bdad05635f3'),\n ObjectId('6560d4e314097bdad05635f4'),\n ObjectId('6560d4e314097bdad05635f5'),\n ObjectId('6560d4e314097bdad05635f6'),\n ObjectId('6560d4e314097bdad05635f7'),\n ObjectId('6560d4e314097bdad05635f8'),\n ObjectId('6560d4e314097bdad05635f9'),\n ObjectId('6560d4e314097bdad05635fa'),\n ObjectId('6560d4e314097bdad05635fb'),\n ObjectId('6560d4e314097bdad05635fc'),\n ObjectId('6560d4e314097bdad05635fd'),\n ObjectId('6560d4e314097bdad05635fe'),\n ObjectId('6560d4e314097bdad05635ff'),\n ObjectId('6560d4e314097bdad0563600'),\n ObjectId('6560d4e314097bdad0563601'),\n ObjectId('6560d4e314097bdad0563602'),\n ObjectId('6560d4e314097bdad0563603'),\n ObjectId('6560d4e314097bdad0563604'),\n ObjectId('6560d4e314097bdad0563605')],\n TaskWorkflow(database=, G=))" + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "for r in data[-100:]:\n", + " r['update'] = True\n", + "\n", + "db.execute(mnist_collection.insert_many(data[-100:]))" + ] + }, + { + "cell_type": "markdown", + "id": "9eb48a30", + "metadata": {}, + "source": [ + "You can see that the inserted data, are now also populated with predictions:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "d8161983", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-24T16:52:52.272363694Z", + "start_time": "2023-11-24T16:52:52.270566528Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "{'img': {'lenet5': {'0': 6}}}" + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.execute(mnist_collection.find_one({'update': True}))['_outputs']" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/multimodal_image_search_clip.ipynb b/examples/multimodal_image_search_clip.ipynb new file mode 100644 index 0000000000..68d7d6885d --- /dev/null +++ b/examples/multimodal_image_search_clip.ipynb @@ -0,0 +1,352 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "238520e0", + "metadata": {}, + "source": [ + "# Multimodal Search Using CLIP" + ] + }, + { + "cell_type": "markdown", + "id": "a3590f0e", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "This notebook showcases the capabilities of SuperDuperDB for performing multimodal searches using the `VectorIndex`. SuperDuperDB's flexibility enables users and developers to integrate various models into the system and use them for vectorizing diverse queries during search and inference. In this demonstration, we leverage the [CLIP multimodal architecture](https://openai.com/research/clip)." + ] + }, + { + "cell_type": "markdown", + "id": "40272d6a2681c8e8", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "## Prerequisites\n", + "\n", + "Before diving into the implementation, ensure that you have the necessary libraries installed by running the following commands:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ebe1497", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install superduperdb\n", + "!pip install ipython openai-clip\n", + "!pip install -U datasets" + ] + }, + { + "cell_type": "markdown", + "id": "b2f94ae8", + "metadata": {}, + "source": [ + "## Connect to datastore \n", + "\n", + "First, we need to establish a connection to a MongoDB datastore via SuperDuperDB. You can configure the `MongoDB_URI` based on your specific setup. \n", + "Here are some examples of MongoDB URIs:\n", + "\n", + "* For testing (default connection): `mongomock://test`\n", + "* Local MongoDB instance: `mongodb://localhost:27017`\n", + "* MongoDB with authentication: `mongodb://superduper:superduper@mongodb:27017/documents`\n", + "* MongoDB Atlas: `mongodb+srv://:@/`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b5ef986", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from superduperdb import superduper\n", + "from superduperdb.backends.mongodb import Collection\n", + "\n", + "mongodb_uri = os.getenv(\"MONGODB_URI\", \"mongomock://test\")\n", + "db = superduper(mongodb_uri, artifact_store='filesystem://./models/')\n", + "\n", + "# Super-Duper your Database!\n", + "db = superduper(mongodb_uri, artifact_store='filesystem://.data')\n", + "\n", + "collection = Collection('multimodal')" + ] + }, + { + "cell_type": "markdown", + "id": "6cd6d6b0", + "metadata": {}, + "source": [ + "## Load Dataset \n", + "\n", + "To make this notebook easily executable and interactive, we'll work with a sub-sample of the [Tiny-Imagenet dataset](https://paperswithcode.com/dataset/tiny-imagenet). The processes demonstrated here can be applied to larger datasets with higher resolution images as well. For such use-cases, however, it's advisable to use a machine with a GPU, otherwise they'll be some significant thumb twiddling to do.\n", + "\n", + "To insert images into the database, we utilize the `Encoder`-`Document` framework, which allows saving Python class instances as blobs in the `Datalayer` and retrieving them as Python objects. To this end, SuperDuperDB contains pre-configured support for `PIL.Image` instances. This simplifies the integration of Python AI models with the datalayer. It's also possible to create your own encoders.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f0f14fb-8e79-4bc6-88af-1a800aecb8db", + "metadata": {}, + "outputs": [], + "source": [ + "!curl -O https://superduperdb-public.s3.eu-west-1.amazonaws.com/coco_sample.zip\n", + "!unzip coco_sample.zip" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e41e6faa-6b83-46d8-ab37-6de6fd346ee7", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Document\n", + "from superduperdb.ext.pillow import pil_image as i\n", + "import glob\n", + "import random\n", + "\n", + "images = glob.glob('images_small/*.jpg')\n", + "documents = [Document({'image': i(uri=f'file://{img}')}) for img in images][:500]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b3a63bf-9e1f-4266-823a-7a2208937e01", + "metadata": {}, + "outputs": [], + "source": [ + "documents[1]" + ] + }, + { + "cell_type": "markdown", + "id": "c9c7e282", + "metadata": {}, + "source": [ + "The wrapped python dictionaries may be inserted directly to the `Datalayer`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c32a91a5", + "metadata": {}, + "outputs": [], + "source": [ + "db.execute(collection.insert_many(documents), encoders=(i,))" + ] + }, + { + "cell_type": "markdown", + "id": "10d37264", + "metadata": {}, + "source": [ + "You can verify that the images are correctly stored as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7282a0fe", + "metadata": {}, + "outputs": [], + "source": [ + "x = db.execute(imagenet_collection.find_one()).unpack()['image']\n", + "display(x.resize((300, 300 * int(x.size[1] / x.size[0]))))" + ] + }, + { + "cell_type": "markdown", + "id": "dab27b50", + "metadata": {}, + "source": [ + "## Build Models\n", + "We now can wrap the CLIP model, to ready it for multimodal search. It involves 2 components:\n", + "\n", + "Now, let's prepare the CLIP model for multimodal search, which involves two components: `text encoding` and `visual encoding`. After installing both components, you can perform searches using both images and text to find matching items:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "916792d3", + "metadata": {}, + "outputs": [], + "source": [ + "import clip\n", + "from superduperdb import vector\n", + "from superduperdb.ext.torch import TorchModel\n", + "\n", + "# Load the CLIP model\n", + "model, preprocess = clip.load(\"RN50\", device='cpu')\n", + "\n", + "# Define a vector\n", + "e = vector(shape=(1024,))\n", + "\n", + "# Create a TorchModel for text encoding\n", + "text_model = TorchModel(\n", + " identifier='clip_text',\n", + " object=model,\n", + " preprocess=lambda x: clip.tokenize(x)[0],\n", + " postprocess=lambda x: x.tolist(),\n", + " encoder=e,\n", + " forward_method='encode_text', \n", + ")\n", + "\n", + "# Create a TorchModel for visual encoding\n", + "visual_model = TorchModel(\n", + " identifier='clip_image',\n", + " object=model.visual, \n", + " preprocess=preprocess,\n", + " postprocess=lambda x: x.tolist(),\n", + " encoder=e,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b716bcb2", + "metadata": {}, + "source": [ + "## Create a Vector-Search Index\n", + "\n", + "Let's create the index for vector-based searching. We'll register both models with the index simultaneously, but specify that the `visual_model` will be responsible for creating the vectors in the database (`indexing_listener`). The `compatible_listener` specifies how an alternative model can be used to search the vectors, enabling multimodal search with models expecting different types of indexes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4e0302c", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import VectorIndex\n", + "from superduperdb import Listener\n", + "\n", + "# Create a VectorIndex and add it to the database\n", + "db.add(\n", + " VectorIndex(\n", + " 'my-index',\n", + " indexing_listener=Listener(\n", + " model=visual_model,\n", + " key='image',\n", + " select=collection.find(),\n", + " predict_kwargs={'batch_size': 10},\n", + " ),\n", + " compatible_listener=Listener(\n", + " model=text_model,\n", + " key='text',\n", + " active=False,\n", + " select=None,\n", + " )\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "18971a6d", + "metadata": {}, + "source": [ + "## Search Images Using Text\n", + "\n", + "Now we can demonstrate searching for images using text queries:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab994b5e", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display\n", + "from superduperdb import Document\n", + "\n", + "query_string = 'sports'\n", + "\n", + "out = db.execute(\n", + " collection.like(Document({'text': query_string}), vector_index='my-index', n=3).find({})\n", + ")\n", + "\n", + "# Display the images from the search results\n", + "for r in search_results:\n", + " x = r['image'].x\n", + " display(x.resize((300, int(300 * x.size[1] / x.size[0]))))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5e3ac22-044f-4675-976a-68ff9b59efe9", + "metadata": {}, + "outputs": [], + "source": [ + "img = db.execute(collection.find_one({}))['image']\n", + "img.x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8569e4f-74f2-4ee5-9674-7829b2fcc62b", + "metadata": {}, + "outputs": [], + "source": [ + "cur = db.execute(\n", + " collection.like(Document({'image': img}), vector_index='my-index', n=3).find({})\n", + ")\n", + "\n", + "for r in cur:\n", + " x = r['image'].x\n", + " display(x.resize((300, int(300 * x.size[1] / x.size[0]))))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "806a445f1dfacd90", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/question_the_docs.ipynb b/examples/question_the_docs.ipynb new file mode 100644 index 0000000000..1ad4c65b07 --- /dev/null +++ b/examples/question_the_docs.ipynb @@ -0,0 +1,450 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c042ddbb-c2c9-46ed-b36c-c965c0d7ff5b", + "metadata": {}, + "source": [ + "# Building Q&A Assistant Using Mongo and OpenAI" + ] + }, + { + "cell_type": "markdown", + "id": "7e6fbce6-fec9-47af-8701-99721eedec50", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "This notebook is designed to demonstrate how to implement a document Question-and-Answer (Q&A) task using SuperDuperDB in conjunction with OpenAI and MongoDB. It provides a step-by-step guide and explanation of each component involved in the process.\n" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Prerequisites\n", + "\n", + "Before diving into the implementation, ensure that you have the necessary libraries installed by running the following commands:" + ], + "metadata": { + "collapsed": false + }, + "id": "f98f1c7ae8e02278" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6858da67-597d-4d98-ae4a-41003bb569f4", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install superduperdb\n", + "!pip install ipython openai==0.27.6" + ] + }, + { + "cell_type": "markdown", + "id": "e3befb73", + "metadata": {}, + "source": [ + "Additionally, ensure that you have set your openai API key as an environment variable. You can uncomment the following code and add your API key:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5bcdade-f988-4464-bfcf-806245031bb3", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "#os.environ['OPENAI_API_KEY'] = 'sk-...'\n", + "\n", + "if 'OPENAI_API_KEY' not in os.environ:\n", + " raise Exception('Environment variable \"OPENAI_API_KEY\" not set')" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Connect to datastore \n", + "\n", + "First, we need to establish a connection to a MongoDB datastore via SuperDuperDB. You can configure the `MongoDB_URI` based on your specific setup. \n", + "Here are some examples of MongoDB URIs:\n", + "\n", + "* For testing (default connection): `mongomock://test`\n", + "* Local MongoDB instance: `mongodb://localhost:27017`\n", + "* MongoDB with authentication: `mongodb://superduper:superduper@mongodb:27017/documents`\n", + "* MongoDB Atlas: `mongodb+srv://:@/`" + ], + "metadata": { + "collapsed": false + }, + "id": "85c1a0f7572c43ba" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f42c42cc-af6a-4712-a993-d9c921693819", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import superduper\n", + "from superduperdb.backends.mongodb import Collection\n", + "import os\n", + "\n", + "mongodb_uri = os.getenv(\"MONGODB_URI\",\"mongomock://test\")\n", + "db = superduper(mongodb_uri)\n", + "\n", + "collection = Collection('questiondocs')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ce857b0-738c-4d7f-bee0-f709c6fc5ddf", + "metadata": {}, + "outputs": [], + "source": [ + "db.metadata" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Load Dataset \n", + "\n", + "In this example we use the internal textual data from the `superduperdb` project's API documentation. The goal is to create a chatbot that can provide information about the project. You can either load the data from your local project or use the provided data. \n", + "\n", + "If you have the SuperDuperDB project locally and want to load the latest version of the API, uncomment the following cell:" + ], + "metadata": { + "collapsed": false + }, + "id": "737497f7d5032bf" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d72a2a52-964f-456e-88b6-040965f5ed1e", + "metadata": {}, + "outputs": [], + "source": [ + "# import glob\n", + "\n", + "# ROOT = '../docs/hr/content/docs/'\n", + "\n", + "# STRIDE = 3 # stride in numbers of lines\n", + "# WINDOW = 25 # length of window in numbers of lines\n", + "\n", + "# files = sorted(glob.glob(f'{ROOT}/*.md') + glob.glob(f'{ROOT}/*.mdx'))\n", + "\n", + "# content = sum([open(file).read().split('\\n') for file in files], [])\n", + "# chunks = ['\\n'.join(content[i: i + WINDOW]) for i in range(0, len(content), STRIDE)]" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Otherwise, you can load the data from an external source. The chunks of text contain code snippets and explanations, which will be used to build the document Q&A chatbot. " + ], + "metadata": { + "collapsed": false + }, + "id": "c9803aef243ad58c" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a20bb184-d45b-4647-b3c3-7043db9a3239", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import *\n", + "\n", + "Markdown(chunks[20])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e587e284-0876-4464-a977-ac97a9070787", + "metadata": {}, + "outputs": [], + "source": [ + "!curl -O https://superduperdb-public.s3.eu-west-1.amazonaws.com/superduperdb_docs.json\n", + "\n", + "import json\n", + "from IPython.display import Markdown\n", + "\n", + "with open('superduperdb_docs.json') as f:\n", + " chunks = json.load(f)" + ] + }, + { + "cell_type": "markdown", + "id": "4f8c4636-88c6-42a4-b471-41be7c20680f", + "metadata": {}, + "source": [ + "You can see that the chunks of text contain bits of code, and explanations, \n", + "which can become useful in building a document Q&A chatbot." + ] + }, + { + "cell_type": "markdown", + "id": "0370732b-0c55-4672-b6be-0830f9a3a755", + "metadata": {}, + "source": [ + "As usual we insert the data. The `Document` wrapper allows `superduperdb` to handle records with special data types such as images,\n", + "video, and custom data-types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7208ef2-c035-43b9-a624-ade42a06ed09", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Document\n", + "\n", + "db.execute(collection.insert_many([Document({'txt': chunk}) for chunk in chunks]))" + ] + }, + { + "cell_type": "markdown", + "id": "4b299b6f-37ae-46d7-b064-7d368d98d68a", + "metadata": {}, + "source": [ + "## Create a Vector-Search Index\n", + "\n", + "To enable question-answering over your documents, we need to setup a standard `superduperdb` vector-search index using `openai` (although there are many options\n", + "here: `torch`, `sentence_transformers`, `transformers`, ...)" + ] + }, + { + "cell_type": "markdown", + "id": "7930b9a1-1483-4106-873c-d85a3920c64e", + "metadata": {}, + "source": [ + "A `Model` is a wrapper around a self-built or ecosystem model, such as `torch`, `transformers`, `openai`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56905f2e-485e-4179-8585-34eac26c0751", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb.ext.openai import OpenAIEmbedding\n", + "\n", + "model = OpenAIEmbedding(model='text-embedding-ada-002')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bb05a78-263e-4e6f-b429-8e51dbb932b8", + "metadata": {}, + "outputs": [], + "source": [ + "model.predict('This is a test', one=True)" + ] + }, + { + "cell_type": "markdown", + "id": "4331b81b-c257-4353-aab4-8f601bef78de", + "metadata": {}, + "source": [ + "A `Listener` \"deploys\" a `Model` to \"listen\" to incoming data, and compute outputs, which are saved in the database, via `db`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1625dab-6438-494b-b74d-efb58bfc8610", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Listener\n", + "\n", + "listener = Listener(model=model, key='txt', select=collection.find())" + ] + }, + { + "cell_type": "markdown", + "id": "591dad80-3788-441b-96db-a5bf23a16979", + "metadata": {}, + "source": [ + "A `VectorIndex` wraps a `Listener`, making its outputs searchable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1aa132d0-e6a2-46f6-9eb8-13fbce90ff11", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import VectorIndex\n", + "\n", + "db.add(\n", + " VectorIndex(identifier='my-index', indexing_listener=listener)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fde5b17-9d71-4535-aaf6-85f4fa9910e4", + "metadata": {}, + "outputs": [], + "source": [ + "db.execute(collection.find_one())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92948823-0d18-4e1b-b103-f226d6b09e52", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb.backends.mongodb import Collection\n", + "from superduperdb import Document as D\n", + "from IPython.display import *\n", + "\n", + "query = 'Code snippet how to create a `VectorIndex` with a torchvision model'\n", + "\n", + "result = db.execute(\n", + " collection\n", + " .like(D({'txt': query}), vector_index='my-index', n=5)\n", + " .find()\n", + ")\n", + "\n", + "display(Markdown('---'))\n", + "\n", + "for r in result:\n", + " display(Markdown(r['txt']))\n", + " display(Markdown('---'))" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Create a Chat-Completion Component\n", + "\n", + "In this step, a chat-completion component is created and added to the system. This component is essential for the Q&A functionality:" + ], + "metadata": { + "collapsed": false + }, + "id": "e0922a0dc623d7bf" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abfa4df6-73ac-4d46-8047-011648e24958", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb.ext.openai import OpenAIChatCompletion\n", + "\n", + "chat = OpenAIChatCompletion(\n", + " model='gpt-3.5-turbo',\n", + " prompt=(\n", + " 'Use the following description and code-snippets aboout SuperDuperDB to answer this question about SuperDuperDB\\n'\n", + " 'Do not use any other information you might have learned about other python packages\\n'\n", + " 'Only base your answer on the code-snippets retrieved\\n'\n", + " '{context}\\n\\n'\n", + " 'Here\\'s the question:\\n'\n", + " ),\n", + ")\n", + "\n", + "db.add(chat)\n", + "\n", + "print(db.show('model'))" + ] + }, + { + "cell_type": "markdown", + "id": "696ac7bb-eaaf-4bec-9561-603b3c98a736", + "metadata": {}, + "source": [ + "## Ask Questions to Your Docs\n", + "\n", + "Finally, you can ask questions about the documents. You can target specific queries and use the power of MongoDB for vector-search and filtering rules. Here's an example of asking a question:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc4a0f6c-9e24-47aa-bc73-7cc4507e94ff", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Document\n", + "from IPython.display import Markdown\n", + "\n", + "# Define the search parameters\n", + "search_term = 'Can you give me a code-snippet to set up a `VectorIndex`?'\n", + "num_results = 5\n", + "\n", + "output, context = db.predict(\n", + " model_name='gpt-3.5-turbo',\n", + " input=search_term,\n", + " context_select=(\n", + " collection\n", + " .like(Document({'txt': search_term}), vector_index='my-index', n=num_results)\n", + " .find()\n", + " ),\n", + " context_key='txt',\n", + ")\n", + "\n", + "Markdown(output.content)" + ] + }, + { + "cell_type": "markdown", + "id": "b3d1fe16-78d7-4c8d-9991-1086cc9e51bb", + "metadata": {}, + "source": [ + "Reset the demo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab688589-5180-4a78-8fc3-8d3ddaf11e37", + "metadata": {}, + "outputs": [], + "source": [ + "db.remove('vector_index', 'my-index', force=True)\n", + "db.remove('listener', 'text-embedding-ada-002/txt', force=True)\n", + "db.remove('model', 'text-embedding-ada-002', force=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/sql-example.ipynb b/examples/sql-example.ipynb new file mode 100644 index 0000000000..c1dad4d701 --- /dev/null +++ b/examples/sql-example.ipynb @@ -0,0 +1,339 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f0e29fef", + "metadata": {}, + "source": [ + "# End-2-end example using SQL databases\n", + "\n", + "SuperDuperDB allows users to connect to a MongoDB database, or any one of a range of SQL databases, i.e. from this selection:\n", + "\n", + "- MongoDB\n", + "- PostgreSQL\n", + "- SQLite\n", + "- DuckDB\n", + "- BigQuery\n", + "- ClickHouse\n", + "- DataFusion\n", + "- Druid\n", + "- Impala\n", + "- MSSQL\n", + "- MySQL\n", + "- Oracle\n", + "- pandas\n", + "- Polars\n", + "- PySpark\n", + "- Snowflake\n", + "- Trino\n", + "\n", + "In this example we show case how to implement multimodal vector-search with DuckDB.\n", + "This is a simple extension of multimodal vector-search with MongoDB, which is \n", + "just slightly easier to set-up (see [here](https://docs.superduperdb.com/docs/use_cases/items/multimodal_image_search_clip)).\n", + "Everything we do here applies equally to any of the above supported SQL databases, as well as to tabular data formats on disk, such as `pandas`." + ] + }, + { + "cell_type": "markdown", + "id": "ff1db9c6", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "Before working on this use-case, make sure that you've installed the software requirements:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38d752ab", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install superduperdb[demo]" + ] + }, + { + "cell_type": "markdown", + "id": "dfde8264", + "metadata": {}, + "source": [ + "## Connect to datastore\n", + "\n", + "The first step in any `superduperdb` workflow is to connect to your datastore.\n", + "In order to connect to a different datastore, add a different `URI`, e.g. `postgres://...`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8e7ef91-9eda-4fbd-b34f-b49b5411fc47", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from superduperdb import superduper\n", + "\n", + "os.makedirs('.superduperdb', exist_ok=True)\n", + "db = superduper('duckdb://.superduperdb/test.ddb')" + ] + }, + { + "cell_type": "markdown", + "id": "b8794451", + "metadata": {}, + "source": [ + "## Load dataset\n", + "\n", + "Now, Once connected, add some data to the datastore:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9d2b073-38b0-4d29-aa65-e568f19e7852", + "metadata": {}, + "outputs": [], + "source": [ + "!curl -O https://superduperdb-public.s3.eu-west-1.amazonaws.com/coco_sample.zip\n", + "!curl -O https://superduperdb-public.s3.eu-west-1.amazonaws.com/captions_tiny.json\n", + "!unzip coco_sample.zip\n", + "!mkdir -p data/coco\n", + "!mv images_small data/coco/images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec5d36d3-7e74-4c87-92c2-ed1586330858", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import pandas\n", + "import PIL.Image\n", + "\n", + "with open(captions_tiny.json') as f:\n", + " data = json.load(f)[:500]\n", + " \n", + "data = pandas.DataFrame([\n", + " {\n", + " 'image': r['image']['_content']['path'], \n", + " 'captions': r['captions']\n", + " } for r in data \n", + "])\n", + "data['id'] = pandas.Series(data.index).apply(str)\n", + "images_df = data[['id', 'image']]\n", + "\n", + "images_df['image'] = images_df['image'].apply(PIL.Image.open)\n", + "captions_df = data[['id', 'captions']].explode('captions')" + ] + }, + { + "cell_type": "markdown", + "id": "b43cd7d2", + "metadata": {}, + "source": [ + "## Define schema\n", + "\n", + "This use-case requires a table with images, and a table with text. \n", + "SuperDuperDB extends standard SQL functionality, by allowing developers to define\n", + "their own data-types via the `Encoder` abstraction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d9483f3-78c5-47df-9fa2-4cd070282791", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb.backends.ibis.query import Table\n", + "from superduperdb.backends.ibis.field_types import dtype\n", + "from superduperdb.ext.pillow import pil_image\n", + "from superduperdb import Schema\n", + "\n", + "captions = Table(\n", + " 'captions', \n", + " primary_id='id',\n", + " schema=Schema(\n", + " 'captions-schema',\n", + " fields={'id': dtype(str), 'captions': dtype(str)},\n", + " )\n", + ")\n", + "\n", + "images = Table(\n", + " 'images', \n", + " primary_id='id',\n", + " schema=Schema(\n", + " 'images-schema',\n", + " fields={'id': dtype(str), 'image': pil_image},\n", + " )\n", + ")\n", + "\n", + "db.add(captions)\n", + "db.add(images)" + ] + }, + { + "cell_type": "markdown", + "id": "115b2c14", + "metadata": {}, + "source": [ + "## Add data to the datastore" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93cce29b-dd04-47d2-bdfc-fe3780e06ddb", + "metadata": {}, + "outputs": [], + "source": [ + "_ = db.execute(images.insert(images_df))\n", + "_ = db.execute(captions.insert(captions_df))" + ] + }, + { + "cell_type": "markdown", + "id": "def10282", + "metadata": {}, + "source": [ + "## Build SuperDuperDB `Model` instances\n", + "\n", + "This use-case uses the `superduperdb.ext.torch` extension. \n", + "Both models used, output `torch` tensors, which are encoded with `tensor`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d87ed43d-6f90-46c1-8851-6050ae21a051", + "metadata": {}, + "outputs": [], + "source": [ + "import clip\n", + "import torch\n", + "from superduperdb.ext.torch import TorchModel, tensor\n", + "\n", + "# Load the CLIP model\n", + "model, preprocess = clip.load(\"RN50\", device='cpu')\n", + "\n", + "# Define a tensor type\n", + "t = tensor(torch.float, shape=(1024,))\n", + "\n", + "# Create a TorchModel for text encoding\n", + "text_model = TorchModel(\n", + " identifier='clip_text',\n", + " object=model,\n", + " preprocess=lambda x: clip.tokenize(x)[0],\n", + " encoder=t,\n", + " forward_method='encode_text', \n", + ")\n", + "\n", + "# Create a TorchModel for visual encoding\n", + "visual_model = TorchModel(\n", + " identifier='clip_image',\n", + " object=model.visual, \n", + " preprocess=preprocess,\n", + " encoder=t,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "89c5c236", + "metadata": {}, + "source": [ + "## Create a Vector-Search Index\n", + "\n", + "Let's define a mult-modal search index on the basis of the models imported above.\n", + "The `visual_model` is applied to the images, to make the `images` table searchable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb8aef2c-484f-41a9-9956-d80b9c58eaa8", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import VectorIndex, Listener\n", + "\n", + "db.add(\n", + " VectorIndex(\n", + " 'my-index',\n", + " indexing_listener=Listener(\n", + " model=visual_model,\n", + " key='image',\n", + " select=images,\n", + " ),\n", + " compatible_listener=Listener(\n", + " model=text_model,\n", + " key='captions',\n", + " active=False,\n", + " select=None,\n", + " )\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "4d8b9e84", + "metadata": {}, + "source": [ + "## Search Images Using Text\n", + "\n", + "Now we can demonstrate searching for images using text queries:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1731a574-921a-4cba-a65c-26bff9fb9c8c", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Document\n", + "\n", + "res = db.execute(\n", + " images\n", + " .like(Document({'captions': 'dog catches frisbee'}), vector_index='my-index', n=10)\n", + " .limit(10)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72522031-0af8-452a-bbd1-b27dede55154", + "metadata": {}, + "outputs": [], + "source": [ + "res[3]['image'].x" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/transfer_learning.ipynb b/examples/transfer_learning.ipynb new file mode 100644 index 0000000000..1aeea0dcbd --- /dev/null +++ b/examples/transfer_learning.ipynb @@ -0,0 +1,267 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Transfer Learning with Sentence Transformers and Scikit-Learn" + ], + "metadata": { + "collapsed": false + }, + "id": "fe6fd0ab0e1ad844" + }, + { + "cell_type": "markdown", + "source": [ + "## Introduction\n", + "\n", + "In this notebook, we will explore the process of transfer learning using SuperDuperDB. We will demonstrate how to connect to a MongoDB datastore, load a dataset, create a SuperDuperDB model based on Sentence Transformers, train a downstream model using Scikit-Learn, and apply the trained model to the database. Transfer learning is a powerful technique that can be used in various applications, such as vector search and downstream learning tasks." + ], + "metadata": { + "collapsed": false + }, + "id": "8dcde44d942793ff" + }, + { + "cell_type": "markdown", + "source": [ + "## Prerequisites\n", + "\n", + "Before diving into the implementation, ensure that you have the necessary libraries installed by running the following commands:" + ], + "metadata": { + "collapsed": false + }, + "id": "1809feca8a8dca5a" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "!pip install superduperdb\n", + "!pip install ipython numpy datasets sentence-transformers" + ], + "metadata": { + "collapsed": false + }, + "id": "94f3219ad932a327" + }, + { + "cell_type": "markdown", + "id": "6bc151f6", + "metadata": {}, + "source": [ + "## Connect to datastore " + ] + }, + { + "cell_type": "markdown", + "source": [ + "First, we need to establish a connection to a MongoDB datastore via SuperDuperDB. You can configure the `MongoDB_URI` based on your specific setup. \n", + "Here are some examples of MongoDB URIs:\n", + "\n", + "* For testing (default connection): `mongomock://test`\n", + "* Local MongoDB instance: `mongodb://localhost:27017`\n", + "* MongoDB with authentication: `mongodb://superduper:superduper@mongodb:27017/documents`\n", + "* MongoDB Atlas: `mongodb+srv://:@/`" + ], + "metadata": { + "collapsed": false + }, + "id": "5379007991707d17" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44f8ef76", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import superduper\n", + "from superduperdb.backends.mongodb import Collection\n", + "import os\n", + "\n", + "mongodb_uri = os.getenv(\"MONGODB_URI\",\"mongomock://test\")\n", + "db = superduper(mongodb_uri)\n", + "\n", + "collection = Collection('transfer')" + ] + }, + { + "cell_type": "markdown", + "id": "97fede97", + "metadata": {}, + "source": [ + "## Load Dataset\n", + "\n", + "Transfer learning can be applied to any data that can be processed with SuperDuperDB models.\n", + "For our example, we will use a labeled textual dataset with sentiment analysis. We'll load a subset of the IMDb dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8bb65106", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy\n", + "from datasets import load_dataset\n", + "from superduperdb import Document as D\n", + "\n", + "data = load_dataset(\"imdb\")\n", + "\n", + "N_DATAPOINTS = 500 # Increase for higher quality\n", + "\n", + "train_data = [\n", + " D({'_fold': 'train', **data['train'][int(i)]}) \n", + " for i in numpy.random.permutation(len(data['train']))\n", + "][:N_DATAPOINTS]\n", + "\n", + "valid_data = [\n", + " D({'_fold': 'valid', **data['test'][int(i)]}) \n", + " for i in numpy.random.permutation(len(data['test']))\n", + "][:N_DATAPOINTS // 10]\n", + "\n", + "db.execute(collection.insert_many(train_data))" + ] + }, + { + "cell_type": "markdown", + "id": "00a92214", + "metadata": {}, + "source": [ + "## Run Model\n", + "\n", + "We'll create a SuperDuperDB model based on the `sentence_transformers` library. This demonstrates that you don't necessarily need a native SuperDuperDB integration with a model library to leverage its power. We configure the `Model wrapper` to work with the `SentenceTransformer class`. After configuration, we can link the model to a collection and daemonize the model with the `listen=True` keyword." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fef91c74", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Model\n", + "import sentence_transformers\n", + "from superduperdb.ext.numpy import array\n", + "\n", + "m = Model(\n", + " identifier='all-MiniLM-L6-v2',\n", + " object=sentence_transformers.SentenceTransformer('all-MiniLM-L6-v2'),\n", + " encoder=array('float32', shape=(384,)),\n", + " predict_method='encode',\n", + " batch_predict=True,\n", + ")\n", + "\n", + "m.predict(\n", + " X='text',\n", + " db=db,\n", + " select=collection.find(),\n", + " listen=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "68fefc17", + "metadata": {}, + "source": [ + "## Train Downstream Model\n", + "Now that we've created and added the model that computes features for the `\"text\"`, we can train a downstream model using Scikit-Learn." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c2faeeb", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.svm import SVC\n", + "\n", + "model = superduper(\n", + " SVC(gamma='scale', class_weight='balanced', C=100, verbose=True),\n", + " postprocess=lambda x: int(x)\n", + ")\n", + "\n", + "model.fit(\n", + " X='text',\n", + " y='label',\n", + " db=db,\n", + " select=collection.find().featurize({'text': 'all-MiniLM-L6-v2'}),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "d1e1f164", + "metadata": {}, + "source": [ + "## Run Downstream Model\n", + "\n", + "With the model trained, we can now apply it to the database. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eee16436", + "metadata": {}, + "outputs": [], + "source": [ + "model.predict(\n", + " X='text',\n", + " db=db,\n", + " select=collection.find().featurize({'text': 'all-MiniLM-L6-v2'}),\n", + " listen=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "67b156c1", + "metadata": {}, + "source": [ + "## Verification\n", + "\n", + "To verify that the process has worked, we can sample a few records to inspect the sanity of the predictions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76958a1e", + "metadata": {}, + "outputs": [], + "source": [ + "r = next(db.execute(collection.aggregate([{'$sample': {'size': 1}}])))\n", + "print(r['text'][:100])\n", + "print(r['_outputs']['text']['svc'])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vector_search.ipynb b/examples/vector_search.ipynb new file mode 100644 index 0000000000..cf4172a6e3 --- /dev/null +++ b/examples/vector_search.ipynb @@ -0,0 +1,324 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0d352545-a8c6-45ad-8359-c9b6edd2b7d2", + "metadata": {}, + "source": [ + "# Vector-search with SuperDuperDB\n", + "\n", + "## Introduction\n", + "This notebook provides a detailed guide on performing vector search using SuperDuperDB. Vector search is a powerful technique for searching and retrieving documents based on their similarity to a query vector. In this guide, we will demonstrate how to set up SuperDuperDB for vector search and use it to search a dataset of documents." + ] + }, + { + "cell_type": "markdown", + "id": "f283b5675bea4619", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "## Prerequisites\n", + "\n", + "Before diving into the implementation, ensure that you have the necessary libraries installed by running the following commands:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1f9e69e-75f4-42f9-a48d-b1f68f02646d", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install superduperdb\n", + "!pip install ipython" + ] + }, + { + "cell_type": "markdown", + "id": "f79d7ef8-46eb-4210-8d96-a09648314e37", + "metadata": {}, + "source": [ + "Additionally, ensure that you have set your openai API key as an environment variable. You can uncomment the following code and add your API key:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1c8e68c-045f-44b8-bfbf-4c9dff5cf30c", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "#os.environ['OPENAI_API_KEY'] = 'sk-...'\n", + "\n", + "if 'OPENAI_API_KEY' not in os.environ:\n", + " raise Exception('You need to set an OpenAI key as environment variable: \"export OPEN_API_KEY=sk-...\"')" + ] + }, + { + "cell_type": "markdown", + "id": "4db1c2b4-e0b3-420f-ba1c-bd49655bff2b", + "metadata": {}, + "source": [ + "## Connect to datastore \n", + "\n", + "First, we need to establish a connection to a MongoDB datastore via SuperDuperDB. You can configure the `MongoDB_URI` based on your specific setup. \n", + "Here are some examples of MongoDB URIs:\n", + "\n", + "* For testing (default connection): `mongomock://test`\n", + "* Local MongoDB instance: `mongodb://localhost:27017`\n", + "* MongoDB with authentication: `mongodb://superduper:superduper@mongodb:27017/documents`\n", + "* MongoDB Atlas: `mongodb+srv://:@/`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e097557-7c50-4442-9e38-1df8a9d8f211", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import superduper\n", + "from superduperdb.backends.mongodb import Collection\n", + "import os\n", + "\n", + "mongodb_uri = os.getenv(\"MONGODB_URI\",\"mongomock://test\")\n", + "db = superduper(mongodb_uri, artifact_store='filesystem://./data/')\n", + "\n", + "doc_collection = Collection('documents')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f41b3a35-760e-49aa-8387-6a5efb990ea5", + "metadata": {}, + "outputs": [], + "source": [ + "db.metadata" + ] + }, + { + "cell_type": "markdown", + "id": "6bb5ee2b-f0bb-4660-961d-fdf98833f33d", + "metadata": {}, + "source": [ + "## Load Dataset \n", + "\n", + "We have prepared a dataset, which is the inline documentation of the pymongo API. Let's load this dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "049b4122-b2c9-4ca5-be3c-df788912ce34", + "metadata": {}, + "outputs": [], + "source": [ + "!curl -O https://superduperdb-public.s3.eu-west-1.amazonaws.com/pymongo.json\n", + "\n", + "import json\n", + "\n", + "with open('pymongo.json') as f:\n", + " data = json.load(f)" + ] + }, + { + "cell_type": "markdown", + "id": "420ef3662c07d91e", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "As usual, we insert the data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "468ec3dc-fa1f-4c23-b569-456b8900b72c", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Document\n", + "\n", + "db.execute(doc_collection.insert_many([Document(r) for r in data]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aba4aa66-2aec-4986-b263-c510788bf478", + "metadata": {}, + "outputs": [], + "source": [ + "db.execute(Collection('documents').find_one())" + ] + }, + { + "cell_type": "markdown", + "id": "e7f78f2d-86c0-463f-8eb1-630cd65d48ef", + "metadata": {}, + "source": [ + "## Create Vectors\n", + "\n", + "In the remainder of the notebook, you can choose between using the `openai` or `sentence_transformers` libraries to perform vector search. After instantiating the model wrappers, the rest of the notebook remains identical.\n", + "\n", + "For OpenAI vectors:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0a30873-b7fc-4ec5-ace9-f3d4ca01bab2", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb.ext.openai.model import OpenAIEmbedding\n", + "\n", + "model = OpenAIEmbedding(model='text-embedding-ada-002')" + ] + }, + { + "cell_type": "markdown", + "id": "7e8d1d264dd7ba1b", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "For Sentence-Transformers vectors, uncomment the following section:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a14c5c5f-c770-4a94-884c-3705f1d0a627", + "metadata": {}, + "outputs": [], + "source": [ + "#import sentence_transformers\n", + "#from superduperdb import Model, vector\n", + "\n", + "#model = Model(\n", + "# identifier='all-MiniLM-L6-v2', \n", + "# object=sentence_transformers.SentenceTransformer('all-MiniLM-L6-v2'),\n", + "# encoder=vector(shape=(384,)),\n", + "# predict_method='encode', # Specify the prediction method\n", + "# postprocess=lambda x: x.tolist(), # Define postprocessing function\n", + "# batch_predict=True, # Generate predictions for a set of observations all at once \n", + "#)" + ] + }, + { + "cell_type": "markdown", + "id": "b546308d-45f2-4605-8778-7aca46fe3c7c", + "metadata": {}, + "source": [ + "## Index Vectors\n", + "\n", + "Now we can configure the Atlas vector-search index. This command saves and sets up a model to `listen` to a particular subfield (or the whole document) for new text, converts it on the fly to vectors, and then indexes these vectors using Atlas vector-search." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46ce7a59-cdd2-46e5-a218-77ef73df7a95", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Listener, VectorIndex\n", + "\n", + "db.add(\n", + " VectorIndex(\n", + " identifier=f'pymongo-docs-{model.identifier}',\n", + " indexing_listener=Listener(\n", + " select=doc_collection.find(),\n", + " key='value',\n", + " model=model,\n", + " predict_kwargs={'max_chunk_size': 1000},\n", + " ),\n", + " )\n", + ")\n", + "\n", + "db.show('vector_index')" + ] + }, + { + "cell_type": "raw", + "id": "8aea59ff7e8ef67c", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "## Perform Vector Search\n", + "\n", + "Now that the index is set up, we can use it in a query. SuperDuperDB provides some syntactic sugar for the `aggregate` search pipelines, which can be helpful. It also handles all the conversion of inputs to vectors under the hood." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0184fb1-10ae-4488-9e93-c56b5fcd9ac2", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Document\n", + "from IPython.display import *\n", + "\n", + "# Define the search parameters\n", + "search_term = 'Query the database'\n", + "num_results = 5\n", + "\n", + "# Execute the query\n", + "result = db.execute(doc_collection\n", + " .like(Document({'value': search_term}), vector_index=f'pymongo-docs-{model.identifier}', n=num_results)\n", + " .find()\n", + ")\n", + "\n", + "# Display a horizontal line\n", + "display(Markdown('---'))\n", + "\n", + "# Iterate through the query results and display them\n", + "for r in result:\n", + " display(Markdown(f'### `{r[\"parent\"] + \".\" if r[\"parent\"] else \"\"}{r[\"res\"]}`'))\n", + " display(Markdown(r['value']))\n", + " display(Markdown('---'))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/video_search.ipynb b/examples/video_search.ipynb new file mode 100644 index 0000000000..4ae081080d --- /dev/null +++ b/examples/video_search.ipynb @@ -0,0 +1,399 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a58575c6-59c4-4289-869f-f5a1ac7e021c", + "metadata": {}, + "source": [ + "# Search within videos with text\n", + "\n", + "## Introduction\n", + "This notebook outlines the process of searching for specific textual information within videos and retrieving relevant video segments. To accomplish this, we utilize various libraries and techniques, such as:\n", + "* clip: A library for vision and language understanding.\n", + "* PIL: Python Imaging Library for image processing.\n", + "* torch: The PyTorch library for deep learning." + ] + }, + { + "cell_type": "markdown", + "id": "6eec562900dd0cff", + "metadata": { + "collapsed": false + }, + "source": [ + "## Prerequisites\n", + "\n", + "Before diving into the implementation, ensure that you have the necessary libraries installed by running the following commands:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab56c57e-fa04-43dd-9670-ade9b5c6d4ac", + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install superduperdb\n", + "!pip install ipython opencv-python pillow openai-clip" + ] + }, + { + "cell_type": "markdown", + "id": "f559fff0-df68-473a-94a2-afe39e4d5577", + "metadata": {}, + "source": [ + "## Connect to datastore \n", + "\n", + "First, we need to establish a connection to a MongoDB datastore via SuperDuperDB. You can configure the `MongoDB_URI` based on your specific setup. \n", + "Here are some examples of MongoDB URIs:\n", + "\n", + "* For testing (default connection): `mongomock://test`\n", + "* Local MongoDB instance: `mongodb://localhost:27017`\n", + "* MongoDB with authentication: `mongodb://superduper:superduper@mongodb:27017/documents`\n", + "* MongoDB Atlas: `mongodb+srv://:@/`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99de0e3d-8918-4fc4-a45b-0a58b70793c6", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import superduper\n", + "from superduperdb.backends.mongodb import Collection\n", + "from superduperdb import CFG\n", + "import os\n", + "\n", + "CFG.downloads.hybrid = True\n", + "CFG.downloads.root = './'\n", + "\n", + "mongodb_uri = os.getenv(\"MONGODB_URI\",\"mongomock://test\")\n", + "db = superduper(mongodb_uri, artifact_store='filesystem://./data/')\n", + "\n", + "video_collection = Collection('videos')" + ] + }, + { + "cell_type": "markdown", + "id": "1e53ce4113115246", + "metadata": { + "collapsed": false + }, + "source": [ + "## Load Dataset\n", + "\n", + "We'll begin by configuring a video encoder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ebac4921-5c83-4ba7-b793-67f5f90d42ec", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Encoder\n", + "\n", + "vid_enc = Encoder(\n", + " identifier='video_on_file',\n", + " load_hybrid=False,\n", + ")\n", + "\n", + "db.add(vid_enc)" + ] + }, + { + "cell_type": "markdown", + "id": "bf1cef0e-21ac-4291-b2c8-41065717ee67", + "metadata": {}, + "source": [ + "Now, let's retrieve a sample video from the internet and insert it into our collection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee6335cb-960d-4239-be6e-501d52b88026", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb.base.document import Document\n", + "\n", + "db.execute(video_collection.insert_one(\n", + " Document({'video': vid_enc(uri='https://superduperdb-public.s3.eu-west-1.amazonaws.com/animals_excerpt.mp4')})\n", + " )\n", + ")\n", + "\n", + "# Display the list of videos in the collection\n", + "list(db.execute(Collection('videos').find()))" + ] + }, + { + "cell_type": "markdown", + "id": "441fe6d6a9dee06b", + "metadata": { + "collapsed": false + }, + "source": [ + "## Register Encoders\n", + "\n", + "Next, we'll create encoders for processing videos and extracting frames. This encoder will help us convert videos into individual frames." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2af2d178-9ff2-496d-8293-e5aee3f12a19", + "metadata": {}, + "outputs": [], + "source": [ + "import cv2\n", + "import tqdm\n", + "from PIL import Image\n", + "from superduperdb.ext.pillow import pil_image\n", + "from superduperdb import Model, Schema\n", + "\n", + "\n", + "def video2images(video_file):\n", + " sample_freq = 10\n", + " cap = cv2.VideoCapture(video_file)\n", + "\n", + " frame_count = 0\n", + "\n", + " fps = cap.get(cv2.CAP_PROP_FPS)\n", + " print(fps)\n", + " extracted_frames = []\n", + " progress = tqdm.tqdm()\n", + "\n", + " while True:\n", + " ret, frame = cap.read()\n", + " if not ret:\n", + " break\n", + " current_timestamp = frame_count // fps\n", + " \n", + " if frame_count % sample_freq == 0:\n", + " extracted_frames.append({\n", + " 'image': Image.fromarray(frame[:,:,::-1]),\n", + " 'current_timestamp': current_timestamp,\n", + " })\n", + " frame_count += 1 \n", + " progress.update(1)\n", + " \n", + " cap.release()\n", + " cv2.destroyAllWindows()\n", + " return extracted_frames\n", + "\n", + "\n", + "video2images = Model(\n", + " identifier='video2images',\n", + " object=video2images,\n", + " flatten=True,\n", + " model_update_kwargs={'document_embedded': False},\n", + " output_schema=Schema(identifier='myschema', fields={'image': pil_image})\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "19a28dbe-dcec-4c6b-bd2d-72dbd48daf39", + "metadata": {}, + "source": [ + "We'll also set up a listener to continuously download video URLs and save the best frames into another collection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a30d093b-03d3-4bdb-aa8b-46ff974d1995", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Listener\n", + "\n", + "db.add(\n", + " Listener(\n", + " model=video2images,\n", + " select=video_collection.find(),\n", + " key='video',\n", + " )\n", + ")\n", + "\n", + "db.execute(Collection('_outputs.video.video2images').find_one()).unpack()['_outputs']['video']['video2images']['image']" + ] + }, + { + "cell_type": "markdown", + "id": "8ef3c353-fcc4-4f23-892b-c8a3796f952c", + "metadata": {}, + "source": [ + "## Create CLIP model\n", + "Now, we'll create a model for the CLIP (Contrastive Language-Image Pre-training) model, which will be used for visual and textual analysis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7329ea-75d1-4275-b754-1a977e76161a", + "metadata": {}, + "outputs": [], + "source": [ + "import clip\n", + "from superduperdb import vector\n", + "from superduperdb.ext.torch import TorchModel\n", + "\n", + "model, preprocess = clip.load(\"RN50\", device='cpu')\n", + "t = vector(shape=(1024,))\n", + "\n", + "visual_model = TorchModel(\n", + " identifier='clip_image',\n", + " preprocess=preprocess,\n", + " object=model.visual,\n", + " encoder=t,\n", + " postprocess=lambda x: x.tolist(),\n", + ")\n", + "\n", + "text_model = TorchModel(\n", + " identifier='clip_text',\n", + " object=model,\n", + " preprocess=lambda x: clip.tokenize(x)[0],\n", + " forward_method='encode_text',\n", + " encoder=t,\n", + " device='cpu',\n", + " preferred_devices=None,\n", + " postprocess=lambda x: x.tolist(),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "dfa470d9-35d0-4c53-a5d0-afba5456320a", + "metadata": {}, + "source": [ + "## Create VectorIndex\n", + "\n", + "We will set up a VectorIndex to index and search the video frames based on both visual and textual content. This involves creating an indexing listener for visual data and a compatible listener for textual data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "475d27c7-81e0-47ae-a02b-b1df4332002c", + "metadata": {}, + "outputs": [], + "source": [ + "from superduperdb import Listener, VectorIndex\n", + "from superduperdb.backends.mongodb import Collection\n", + "\n", + "db.add(\n", + " VectorIndex(\n", + " identifier='video_search_index',\n", + " indexing_listener=Listener(\n", + " model=visual_model,\n", + " key='_outputs.video.video2images.image',\n", + " select=Collection('_outputs.video.video2images').find(),\n", + " ),\n", + " compatible_listener=Listener(\n", + " model=text_model,\n", + " key='text',\n", + " select=None,\n", + " active=False\n", + " )\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "aeb7df08-c1ec-45db-8716-79db58ad6502", + "metadata": {}, + "source": [ + "## Query a text against saved frames." + ] + }, + { + "cell_type": "markdown", + "id": "95c48d0c-4f7a-4c32-a2e3-3f8d8985733a", + "metadata": {}, + "source": [ + "Now, let's search for something that happened during the video:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31ba463f-97ae-4f83-890e-c852f9818e63", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the search parameters\n", + "search_term = 'Some ducks'\n", + "num_results = 1\n", + "\n", + "\n", + "r = next(db.execute(\n", + " Collection('_outputs.video.video2images').like(Document({'text': search_term}), vector_index='video_search_index', n=num_results).find()\n", + "))\n", + "\n", + "search_timestamp = r['_outputs']['video']['video2images']['current_timestamp']\n", + "\n", + "# Get the back reference to the original video\n", + "video = db.execute(Collection('videos').find_one({'_id': r['_source']}))" + ] + }, + { + "cell_type": "markdown", + "id": "78fc11ff-dafc-4525-88a5-327ed547b89e", + "metadata": {}, + "source": [ + "## Start the video from the resultant timestamp:\n", + "\n", + "Finally, we can display and play the video starting from the timestamp where the searched text is found." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eeda6711-15a4-465e-903d-ed0a1d0db672", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display, HTML\n", + "\n", + "video_html = f\"\"\"\n", + "\n", + "\n", + "\"\"\"\n", + "\n", + "display(HTML(video_html))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/voice_memos.ipynb b/examples/voice_memos.ipynb new file mode 100644 index 0000000000..2b50791743 --- /dev/null +++ b/examples/voice_memos.ipynb @@ -0,0 +1,371 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ae374483", + "metadata": {}, + "source": [ + "# Cataloguing voice-memos for a self managed personal assistant" + ] + }, + { + "cell_type": "markdown", + "id": "cc4fa500665eccb9", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "Discover the magic of SuperDuperDB as we seamlessly integrate models across different data modalities, such as audio and text. Experience the creation of highly sophisticated data-based applications with minimal boilerplate code.\n", + "\n", + "### Objectives:\n", + "\n", + "1. Maintain a database of audio recordings\n", + "2. Index the content of these audio recordings\n", + "3. Search and interrogate the content of these audio recordings\n", + "\n", + "### Our approach involves:\n", + "\n", + "* Utilizing a transformers model by Facebook's AI team to transcribe audio to text.\n", + "* Employing an OpenAI vectorization model to index the transcribed text.\n", + "* Harnessing OpenAI ChatGPT model in conjunction with relevant recordings to query the audio database." + ] + }, + { + "cell_type": "markdown", + "id": "ecf9f0ec45cb1f3", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "Before diving into the implementation, ensure that you have the necessary libraries installed by running the following commands:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dce1a857", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "!pip install superduperdb\n", + "!pip install transformers soundfile torchaudio librosa openai\n", + "!pip install -U datasets" + ] + }, + { + "cell_type": "markdown", + "id": "0d02e472-8395-435c-b46d-6a5158ef67fb", + "metadata": { + "tags": [] + }, + "source": [ + "Additionally, ensure that you have set your openai API key as an environment variable. You can uncomment the following code and add your API key:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94262bf76c630b10", + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "#os.environ['OPENAI_API_KEY'] = 'sk-XXXX'\n", + "\n", + "if 'OPENAI_API_KEY' not in os.environ:\n", + " raise Exception('Environment variable \"OPENAI_API_KEY\" not set')" + ] + }, + { + "cell_type": "markdown", + "id": "32971b8afdf76fe5", + "metadata": {}, + "source": [ + "## Connect to datastore \n", + "\n", + "First, we need to establish a connection to a MongoDB datastore via SuperDuperDB. You can configure the `MongoDB_URI` based on your specific setup. \n", + "Here are some examples of MongoDB URIs:\n", + "\n", + "* For testing (default connection): `mongomock://test`\n", + "* Local MongoDB instance: `mongodb://localhost:27017`\n", + "* MongoDB with authentication: `mongodb://superduper:superduper@mongodb:27017/documents`\n", + "* MongoDB Atlas: `mongodb+srv://:@/`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b84da3f2ef58e401", + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "from superduperdb import superduper\n", + "from superduperdb.backends.mongodb import Collection\n", + "import os\n", + "\n", + "mongodb_uri = os.getenv(\"MONGODB_URI\",\"mongomock://test\")\n", + "db = superduper(mongodb_uri)\n", + "\n", + "# Create a collection for Voice memos\n", + "voice_collection = Collection('voice-memos')" + ] + }, + { + "cell_type": "markdown", + "id": "d13d051e8f5f6f", + "metadata": {}, + "source": [ + "\n", + "## Load Dataset\n", + "\n", + "In this example se use `LibriSpeech` as our voice recording dataset. It is a corpus of approximately 1000 hours of read English speech. The same functionality could be accomplised using any audio, in particular audio hosted on the web, or in an `s3` bucket. For instance, if you have a repository of audio of conference calls, or memos, this may be indexed in the same way. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10ab7114", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "from superduperdb.ext.numpy import array\n", + "from superduperdb import Document\n", + "\n", + "data = load_dataset(\"hf-internal-testing/librispeech_asr_demo\", \"clean\", split=\"validation\")\n", + "\n", + "# Using an `Encoder`, we may add the audio data directly to a MongoDB collection:\n", + "enc = array('float64', shape=(None,))\n", + "\n", + "db.add(enc)\n", + "\n", + "db.execute(voice_collection.insert_many([\n", + " Document({'audio': enc(r['audio']['array'])}) for r in data\n", + "]))" + ] + }, + { + "cell_type": "markdown", + "id": "721f31f4626881e0", + "metadata": {}, + "source": [ + "## Install Pre-Trained Model (LibreSpeech) into Database\n", + "\n", + "Apply a pretrained `transformers` model to the data: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "222284f7", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from transformers import Speech2TextProcessor, Speech2TextForConditionalGeneration\n", + "from superduperdb.ext.transformers import Pipeline\n", + "\n", + "model = Speech2TextForConditionalGeneration.from_pretrained(\"facebook/s2t-small-librispeech-asr\")\n", + "processor = Speech2TextProcessor.from_pretrained(\"facebook/s2t-small-librispeech-asr\")\n", + "\n", + "SAMPLING_RATE = 16000\n", + "\n", + "transcriber = Pipeline(\n", + " identifier='transcription',\n", + " object=model,\n", + " preprocess=processor,\n", + " preprocess_kwargs={'sampling_rate': SAMPLING_RATE, 'return_tensors': 'pt', 'padding': True},\n", + " postprocess=lambda x: processor.batch_decode(x, skip_special_tokens=True),\n", + " predict_method='generate',\n", + " preprocess_type='other',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ed83a8b084844292", + "metadata": {}, + "source": [ + "# Run Predictions on All Recordings in the Collection\n", + "Apply the `Pipeline` to all audio recordings:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "573dccc4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "transcriber.predict(X='audio', db=db, select=voice_collection.find(), max_chunk_size=10)" + ] + }, + { + "cell_type": "markdown", + "id": "64a6cbd8e3d429d9", + "metadata": {}, + "source": [ + "## Ask Questions to Your Voice Assistant\n", + "\n", + "Ask questions to your voice assistant, targeting specific queries and utilizing the power of MongoDB for vector-search and filtering rules:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3aedc03c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from superduperdb import VectorIndex, Listener\n", + "from superduperdb.ext.openai import OpenAIEmbedding\n", + "\n", + "db.add(\n", + " VectorIndex(\n", + " identifier='my-index',\n", + " indexing_listener=Listener(\n", + " model=OpenAIEmbedding(model='text-embedding-ada-002'),\n", + " key='_outputs.audio.transcription',\n", + " select=voice_collection.find(),\n", + " ),\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2f92b56f", + "metadata": {}, + "source": [ + "Let's confirm this has worked, by searching for the `royal cavern`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d2e3e56", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Define the search parameters\n", + "search_term = 'royal cavern'\n", + "num_results = 2\n", + "\n", + "list(db.execute(\n", + " voice_collection.like(\n", + " {'_outputs.audio.transcription': search_term},\n", + " n=num_results,\n", + " vector_index='my-index',\n", + " ).find({}, {'_outputs.audio.transcription': 1})\n", + "))" + ] + }, + { + "cell_type": "markdown", + "id": "6068514b31268846", + "metadata": {}, + "source": [ + "## Enrich it with Chat-Completion \n", + "\n", + "Connect the previous steps with the gpt-3.5.turbo, a chat-completion model on OpenAI. The plan is to seed the completions with the most relevant audio recordings, as judged by their textual transcriptions. These transcriptions are retrieved using the previously configured `VectorIndex`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99e206af", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from superduperdb.ext.openai import OpenAIChatCompletion\n", + "\n", + "chat = OpenAIChatCompletion(\n", + " model='gpt-3.5-turbo',\n", + " prompt=(\n", + " 'Use the following facts to answer this question\\n'\n", + " '{context}\\n\\n'\n", + " 'Here\\'s the question:\\n'\n", + " ),\n", + ")\n", + "\n", + "db.add(chat)\n", + "\n", + "print(db.show('model'))" + ] + }, + { + "cell_type": "markdown", + "id": "9cb623c4", + "metadata": {}, + "source": [ + "## Full Voice-Assistant Experience\n", + "\n", + "Test the full model by asking a question about a specific fact mentioned in the audio recordings. The model will retrieve the most relevant recordings and use them to formulate its answer:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60d7f0af-6305-4c8c-be65-4b75ec7dbf50", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from superduperdb import Document\n", + "\n", + "q = 'Is anything really Greek?'\n", + "\n", + "print(db.predict(\n", + " model_name='gpt-3.5-turbo',\n", + " input=q,\n", + " context_select=voice_collection.like(\n", + " Document({'_outputs.audio.transcription': q}), vector_index='my-index'\n", + " ).find(),\n", + " context_key='_outputs.audio.transcription',\n", + ")[0].content)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}