Stratechery RAG demo

.prettierrc

@@ -0,0 +1,6 @@
+{
+  "semi": true,
+  "tabWidth": 2,
+  "useTabs": false,
+  "singleQuote": false
+}

README.md

@@ -1,21 +1,30 @@
 [![Try](https://img.shields.io/badge/try_it-here-blue)](https://anthropic.dailybots.ai)
 [![Deploy](https://img.shields.io/badge/Deploy_to_Vercel-black?style=flat&logo=Vercel&logoColor=white)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fdaily-demos%2Fdaily-bots-web-demo&env=DAILY_BOTS_URL,DAILY_API_KEY,NEXT_PUBLIC_BASE_URL&project-name=daily-bots-demo&repository-name=daily-bots-web-demo)
 
-
 <img src="public/icon.png" width="120px">
 
+# Daily Bots RAG demo
 
-# Daily Bots Function Calling Demo with Anthropic
-
-Example NextJS app that demonstrates core capabilities of [Daily Bots](https://bots.daily.co). 
+Example NextJS app that demonstrates core capabilities of [Daily Bots](https://bots.daily.co).
 
 ## Other demos
 
 - [Multi-model](https://github.com/daily-demos/daily-bots-web-demo/) - Main demo showcase.
 - [Vision](https://github.com/daily-demos/daily-bots-web-demo/tree/khk/vision-for-launch) - Anthropic, describe webcam.
+- [Function calling](https://github.com/daily-demos/daily-bots-web-demo/tree/cb/function-calling) - Anthropic with function calling
 
 ## Getting started
 
+### Prerequisites
+
+1. Create an OpenAI developer account at https://platform.openai.com and copy your OpenAI API key.
+2. Create a Pinecone account at https://login.pinecone.io.
+3. Create a new Pinecone project. This project will contain your vector DB, which will store your embeddings.
+
+- Create index
+- Set up your index by model > select `text-embedding-3-small`
+- Select Capacity mode > Serverless > AWS > Region. Take note of your region, you'll use this below.
+
 ### Configure your local environment
 
 ```shell
@@ -28,10 +37,16 @@ cp env.example .env.local
 
 `DAILY_API_KEY` your Daily API key obtained by registering at https://bots.daily.co.
 
+`OPENAI_API_KEY` your OpenAI API key.
+
+`PINECONE_API_KEY` your Pinecone API key.
+
+`PINECONE_ENVIRONMENT` your Pinecone index's region that you set up in Prerequisites. This should be a value like `us-east-1` or similar.
+
 ### Install dependencies
 
 ```shell
-yarn 
+yarn
 ```
 
 ### Run the project
@@ -57,17 +72,28 @@ All Voice Client configuration can be found in the [rtvi.config.ts](/rtvi.config
 
 ### API routes
 
-This project one three server-side route:
+This project has two server-side routes:
 
-- [api/route.ts](app/api/route.ts)
+- [api/route.ts](app/api/route.ts): Used to start your Daily Bot
+- [api/rag/route.ts](app/api/rag/route.ts): Used to query your vector DB
 
-The routes project a secure way to pass any required secrets or configuration directly to the Daily Bots API. Your `NEXT_PUBLIC_BASE_URL` must point to your `/api` route and passed to the `VoiceClient`. 
+The routes project a secure way to pass any required secrets or configuration directly to the Daily Bots API. Your `NEXT_PUBLIC_BASE_URL` must point to your `/api` route and passed to the `VoiceClient`.
 
-The routes are passed a `config` array and `services` map, which can be passed to the Daily Bots REST API, or modified securely. 
+The routes are passed a `config` array and `services` map, which can be passed to the Daily Bots REST API, or modified securely.
 
 Daily Bots `https://api.daily.co/v1/bots/start` has some required properties, which you can read more about [here](https://docs.dailybots.ai/api-reference/endpoint/startBot). You must set:
 
 - `bot_profile`
 - `max_duration`
 - `config`
 - `services`
+
+### RAG details
+
+In the system message, located in [rtvi.config.ts](/rtvi.config.ts), you can see that the LLM has a single function call configured. This function call enables the LLM to query the vector DB when it requires supplementary information to respond to the user. You'll find the RAG query specifics in:
+
+- [app/page.tsx](app/page.tsx), which is setting up the Daily Bot with access to the function call
+- [api/rag/route.ts](app/api/rag/route.ts), which is the server-side route to query the vector DB
+- [rag_query.ts](utils/rag_query.ts), which is a utility function with the core RAG querying logic
+
+The data in the vector DB was created from the raw Stratechery articles. These articles where semantically chunked to create token efficient divisions of the articles. A key to a great conversational app is low latency interactions. The semantic chunks help to provide sufficient information to the LLM after a single RAG query, which helps the interaction remain low latency. You can see the time to first byte (TTFB) measurements along with the token use and links to source articles in the demo app—a drawer will pop out with this information after your first turn speaking to the LLM.

app/api/rag/route.ts

@@ -0,0 +1,72 @@
+import { NextResponse } from "next/server";
+
+import {
+  generateResponse,
+  parseDateQuery,
+  query_similar_content,
+} from "@/utils/rag_query";
+
+export async function POST(request: Request) {
+  const { query } = await request.json();
+
+  try {
+    const startTime = performance.now();
+
+    const dateFilter = parseDateQuery(query);
+
+    const querySimilarContentStartTime = performance.now();
+    const ragResults = await query_similar_content(
+      query,
+      5,
+      dateFilter || undefined
+    );
+    const querySimilarContentTime =
+      performance.now() - querySimilarContentStartTime;
+
+    const generateResponseStartTime = performance.now();
+    const { response: llmResponse, tokenUsage } = await generateResponse(
+      query,
+      ragResults
+    );
+    const generateResponseTime = performance.now() - generateResponseStartTime;
+
+    const totalRAGTime = performance.now() - startTime;
+
+    const uniqueLinksSet = new Set();
+
+    const links = ragResults
+      .map((result) => {
+        const file = result.metadata.file_name;
+        const title = result.metadata.title.replace(/\s*-\s*Chunk\s*\d+$/, "");
+        const url = `https://stratechery.com/${file.split("_")[0]}/${file
+          .split("_")[1]
+          .replace(".json", "")}/`;
+
+        const linkIdentifier = `${title}|${url}`;
+
+        if (!uniqueLinksSet.has(linkIdentifier)) {
+          uniqueLinksSet.add(linkIdentifier);
+          return { title, url };
+        }
+
+        return null;
+      })
+      .filter(Boolean);
+
+    const ragStats = {
+      querySimilarContentTime,
+      generateResponseTime,
+      totalRAGTime,
+      links,
+      tokenUsage,
+    };
+
+    return NextResponse.json({ ragResults, llmResponse, ragStats });
+  } catch (error) {
+    console.error("RAG query error:", error);
+    return NextResponse.json(
+      { error: "Failed to process query", details: (error as Error).message },
+      { status: 500 }
+    );
+  }
+}

app/api/route.ts

@@ -13,9 +13,10 @@ export async function POST(request: Request) {
   const payload = {
     bot_profile: defaultBotProfile,
     max_duration: defaultMaxDuration,
+    api_keys: { openai: process.env.OPENAI_API_KEY },
     services,
     config: [...config],
-  }; 
+  };
 
   const req = await fetch(process.env.DAILY_BOTS_URL, {
     method: "POST",

app/page.tsx

@@ -12,48 +12,100 @@ import Header from "@/components/Header";
 import Splash from "@/components/Splash";
 import {
   BOT_READY_TIMEOUT,
-  defaultConfig,
   defaultServices,
+  getDefaultConfig,
 } from "@/rtvi.config";
 
 export default function Home() {
   const [showSplash, setShowSplash] = useState(true);
-  const [fetchingWeather, setFetchingWeather] = useState(false);
+  const [fetchingRAG, setFetchingRAG] = useState(false);
+  const [ragStats, setRagStats] = useState<any>(null);
   const voiceClientRef = useRef<DailyVoiceClient | null>(null);
 
+  const updateRAGStats = (stats: any) => {
+    setRagStats(stats);
+  };
+
   useEffect(() => {
     if (!showSplash || voiceClientRef.current) {
       return;
     }
 
+    const currentDate = new Date().toLocaleDateString("en-US", {
+      year: "numeric",
+      month: "long",
+      day: "numeric",
+    });
+
+    // Get the config with the current date
+    const config = getDefaultConfig(currentDate);
+
     const voiceClient = new DailyVoiceClient({
       baseUrl: process.env.NEXT_PUBLIC_BASE_URL || "/api",
       services: defaultServices,
-      config: defaultConfig,
+      config: config,
       timeout: BOT_READY_TIMEOUT,
     });
 
     const llmHelper = new LLMHelper({
       callbacks: {
         onLLMFunctionCall: (fn) => {
-          setFetchingWeather(true);
+          setFetchingRAG(true);
         },
       },
     });
     voiceClient.registerHelper("llm", llmHelper);
 
     llmHelper.handleFunctionCall(async (fn: FunctionCallParams) => {
       const args = fn.arguments as any;
-      if (fn.functionName === "get_weather" && args.location) {
-        const response = await fetch(
-          `/api/weather?location=${encodeURIComponent(args.location)}`
-        );
-        const json = await response.json();
-        setFetchingWeather(false);
-        return json;
-      } else {
-        setFetchingWeather(false);
-        return { error: "couldn't fetch weather" };
+      try {
+        if (fn.functionName === "get_rag_context" && args.query) {
+          console.log("get_rag_context", args.query);
+          setFetchingRAG(true);
+
+          const response = await fetch("/api/rag", {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+            },
+            body: JSON.stringify({ query: args.query }),
+          });
+
+          if (!response.ok) {
+            setFetchingRAG(false);
+            throw new Error("Failed to fetch RAG context");
+          }
+
+          const data = await response.json();
+          setFetchingRAG(false);
+
+          const { ragStats } = data;
+
+          updateRAGStats(ragStats);
+
+          const formattedContext = `
+            Relevant Context:
+            ${data.ragResults
+              .map(
+                (result: any) =>
+                  `Title: ${result.metadata.title}
+                   Content: ${result.metadata.content}`
+              )
+              .join("\n\n")}
+
+            AI Response:
+            ${data.llmResponse}
+          `;
+
+          return { context: formattedContext };
+        } else {
+          setFetchingRAG(false);
+          return { error: "Invalid function call or missing query" };
+        }
+      } catch (error) {
+        console.error("Error fetching RAG context:", error);
+        setFetchingRAG(false);
+        return { error: "Couldn't fetch RAG context" };
       }
     });
 
@@ -71,7 +123,7 @@ export default function Home() {
           <main>
             <Header />
             <div id="app">
-              <App fetchingWeather={fetchingWeather} />
+              <App fetchingRAG={fetchingRAG} ragStats={ragStats} />
             </div>
           </main>
           <aside id="tray" />

components/App.tsx

@@ -23,7 +23,13 @@ const status_text = {
   connecting: "Connecting...",
 };
 
-export default function App({ fetchingWeather }: { fetchingWeather: boolean }) {
+export default function App({
+  fetchingRAG,
+  ragStats,
+}: {
+  fetchingRAG: boolean;
+  ragStats: any;
+}) {
   const voiceClient = useVoiceClient()!;
   const transportState = useVoiceClientTransportState();
 
@@ -106,7 +112,8 @@ export default function App({ fetchingWeather }: { fetchingWeather: boolean }) {
   if (appState === "connected") {
     return (
       <Session
-        fetchingWeather={fetchingWeather}
+        fetchingRAG={fetchingRAG}
+        ragStats={ragStats}
         state={transportState}
         onLeave={() => leave()}
         startAudioOff={startAudioOff}

components/Session/Agent/index.tsx

@@ -12,9 +12,9 @@ import styles from "./styles.module.css";
 export const Agent: React.FC<{
   isReady: boolean;
   statsAggregator: StatsAggregator;
-  fetchingWeather: boolean;
+  fetchingRAG: boolean;
 }> = memo(
-  ({ isReady, statsAggregator, fetchingWeather = false }) => {
+  ({ isReady, statsAggregator, fetchingRAG = false }) => {
     const [hasStarted, setHasStarted] = useState<boolean>(false);
     const [botStatus, setBotStatus] = useState<
       "initializing" | "connected" | "disconnected"
@@ -69,19 +69,14 @@ export const Agent: React.FC<{
             </span>
           ) : (
             <>
-              {fetchingWeather && (
-                <span className={styles.functionCalling}>
-                  <Loader2 size={32} className="animate-spin" />
-                </span>
-              )}
-              <WaveForm />
+              <WaveForm isThinking={fetchingRAG} scanningSpeed={1} />
             </>
           )}
         </div>
       </div>
     );
   },
-  (p, n) => p.isReady === n.isReady && p.fetchingWeather === n.fetchingWeather
+  (p, n) => p.isReady === n.isReady && p.fetchingRAG === n.fetchingRAG
 );
 Agent.displayName = "Agent";