Templates for documentation

docs/hr/content/build_use_cases.py

@@ -0,0 +1,59 @@
+import json
+import os
+import re
+
+
+def process_snippet(nb, tabs):
+    to_delete = []
+    for i, cell in enumerate(nb['cells']):
+        if tabs != '*':
+            match = re.match('^#[ ]+<tab: ([^>]+)>', cell['source'][0])
+            if match:
+                tab = match.groups()[0]
+                if tab not in tabs:
+                    to_delete.append(i)
+                    continue
+        if cell['cell_type'] == 'markdown':
+            for j, line in enumerate(cell['source']):
+                line = re.sub('^####', '##### ', line)
+                line = re.sub('^###', '#### ', line)
+                line = re.sub('^## ', '### ', line)
+                line = re.sub('^# ', '## ', line)
+                nb['cells'][i]['source'][j] = line
+    nb['cells'] = [cell for i, cell in enumerate(nb['cells']) if i not in to_delete]
+    return nb
+
+
+def build_use_case(path):
+    with open(path) as f:
+        nb = json.load(f)
+    built_nb = {k: v for k, v in nb.items() if k != 'cells'}
+    built_nb['cells'] = []
+
+    for cell in nb['cells']:
+        if (
+            cell['cell_type'] == 'raw' 
+            and cell['source'] 
+            and cell['source'][0].startswith('<snippet:')
+        ):
+            snippet, tabs = re.match(
+                '^<snippet: ([a-z0-9_\-]+): ([a-zA-Z0-9_\-\,\*]+)>$',
+                cell['source'][0].strip(),
+            ).groups()
+            with open(f'docs/reusable_snippets/{snippet}.ipynb') as f:
+                snippet_nb = json.load(f)
+            snippet_nb = process_snippet(snippet_nb, tabs)
+            built_nb['cells'].extend(snippet_nb['cells'])
+        else:
+            built_nb['cells'].append(cell)
+    return built_nb
+
+file = '_multimodal_vector_search.ipynb'
+
+
+for file in os.listdir('./use_cases'):
+    if not file.startswith('_'):
+        continue
+    built = build_use_case(f'./use_cases/{file}')
+    with open(f'./use_cases/{file[1:]}', 'w') as f:
+        json.dump(built, f)

docs/hr/content/docs/fundamentals/design.md

@@ -4,13 +4,6 @@ sidebar_position: 2
 
 # Design
 
-## Architecture
-Here is a detailed view of the `superduperdb` architecture:
-
-![architecture_detailed](/img/architecture_detailed.png)
-
-## Schematic
-
 Here is a schematic of the `superduperdb` design.
 
 ![](/img/light.png)

docs/hr/content/docs/intro.md

@@ -4,6 +4,8 @@ sidebar_position: 1
 
 # Welcome to SuperDuperDB!
 
+Testing
+
 ## What is SuperDuperDB?
 
 SuperDuperDB is an open-source framework for integrating your database with AI models, APIs, and vector search engines, providing streaming inference and scalable training/fine-tuning.

docs/hr/content/docs/reusable_snippets/build_and_train_classifier.ipynb

@@ -0,0 +1,92 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "c2da0ab6-8fc0-41fc-b8c9-0f8a127d9e8d",
+   "metadata": {},
+   "source": [
+    "<!-- TABS -->\n",
+    "# Build and train classifier"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d3b94fca-3a0b-433f-88cf-aab5b71b8596",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: sklearn>\n",
+    "\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bc3a91a1-0f6c-42f4-b83f-77bc563eeeff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: torch>\n",
+    "\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a65f96f9-d6ca-4008-9540-cee4706ec0c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: torchvision>\n",
+    "\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b8f9919c-ee0a-4123-a592-7d84d1bec3d0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: transformers>\n",
+    "\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f53c67e9-db8a-4d3b-8c39-ec62075191ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db.add(model)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/hr/content/docs/reusable_snippets/build_and_train_classifier.md

@@ -0,0 +1,40 @@
+---
+sidebar_label: Build and train classifier
+---
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+<!-- TABS -->
+# Build and train classifier
+
+
+<Tabs>
+    <TabItem value="sklearn" label="sklearn" default>
+        ```python
+        
+        ...        
+        ```
+    </TabItem>
+    <TabItem value="torch" label="torch" default>
+        ```python
+        
+        ...        
+        ```
+    </TabItem>
+    <TabItem value="torchvision" label="torchvision" default>
+        ```python
+        
+        ...        
+        ```
+    </TabItem>
+    <TabItem value="transformers" label="transformers" default>
+        ```python
+        
+        ...        
+        ```
+    </TabItem>
+</Tabs>
+```python
+db.add(model)
+```
+

docs/hr/content/docs/reusable_snippets/build_image_embedding_model.ipynb

@@ -0,0 +1,80 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "db56aac9-cac5-49c7-99a5-012cd2ab86c5",
+   "metadata": {},
+   "source": [
+    "<!-- TABS -->\n",
+    "# Build image embedding model\n",
+    "\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ec250b81-e9d2-46bd-929c-c71358b5fac6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: torchvision>\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63b72f37-041e-4b4f-bc43-6cf2259ccd48",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: CLIP-multimodal>\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "591f5a19-51f7-4a5c-be82-cf629298e1d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: OpenAI>\n",
+    "..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c9226610-33d9-445d-9352-3073f229d8f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# <tab: Other>\n",
+    "..."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/hr/content/docs/reusable_snippets/build_image_embedding_model.md

@@ -0,0 +1,34 @@
+---
+sidebar_label: Build image embedding model
+---
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+<!-- TABS -->
+# Build image embedding model
+
+...
+
+
+<Tabs>
+    <TabItem value="torchvision" label="torchvision" default>
+        ```python
+        ...        
+        ```
+    </TabItem>
+    <TabItem value="CLIP-multimodal" label="CLIP-multimodal" default>
+        ```python
+        ...        
+        ```
+    </TabItem>
+    <TabItem value="OpenAI" label="OpenAI" default>
+        ```python
+        ...        
+        ```
+    </TabItem>
+    <TabItem value="Other" label="Other" default>
+        ```python
+        ...        
+        ```
+    </TabItem>
+</Tabs>