diff --git a/screenpipe-app-tauri/app/page.tsx b/screenpipe-app-tauri/app/page.tsx index 41fa8117..18b9c832 100644 --- a/screenpipe-app-tauri/app/page.tsx +++ b/screenpipe-app-tauri/app/page.tsx @@ -63,10 +63,10 @@ export default function Home() { ) : settings.aiUrl ? ( <> -

- where pixels become magic -

- + +
+ +
) : ( // void; + isLoading: boolean; +} + +export function NaturalLanguageInput({ + onSubmit, + isLoading, +}: NaturalLanguageInputProps) { + const [query, setQuery] = useState(""); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + onSubmit(query); + }; + + const handleUseCaseClick = (text: string) => { + setQuery(text); + }; + + return ( +
+
+ {useCases.map((useCase) => ( + + handleUseCaseClick(useCase.text)} + > + +

{useCase.text}

+ + + + ))} +
+
+ setQuery(e.target.value)} + className="w-full" + /> + +
+
+ ); +} diff --git a/screenpipe-app-tauri/components/recording-settings.tsx b/screenpipe-app-tauri/components/recording-settings.tsx index feb3e15b..bdb1a167 100644 --- a/screenpipe-app-tauri/components/recording-settings.tsx +++ b/screenpipe-app-tauri/components/recording-settings.tsx @@ -83,9 +83,7 @@ export function RecordingSettings({ const [isUpdating, setIsUpdating] = useState(false); const { health } = useHealthCheck(); const isDisabled = health?.status_code === 500; - console.log("localSettings", localSettings); - console.log("settings", settings); - console.log("availableMonitors", availableMonitors); + useEffect(() => { const loadDevices = async () => { try { diff --git a/screenpipe-app-tauri/components/search-chat.tsx b/screenpipe-app-tauri/components/search-chat.tsx index 1af62ef0..3e8be3f2 100644 --- a/screenpipe-app-tauri/components/search-chat.tsx +++ b/screenpipe-app-tauri/components/search-chat.tsx @@ -33,6 +33,8 @@ import { Send, X, Square, + ArrowDown, + ArrowUp, } from "lucide-react"; import { useToast } from "./ui/use-toast"; import posthog from "posthog-js"; @@ -40,6 +42,8 @@ import { AnimatePresence, motion } from "framer-motion"; import { useSettings } from "@/lib/hooks/use-settings"; import { convertToCoreMessages, generateId, Message, streamText } from "ai"; import { OpenAI } from "openai"; +import { zodResponseFormat } from "openai/helpers/zod"; +import { z } from "zod"; import { ChatMessage } from "./chat-message-v2"; import { spinner } from "./spinner"; import { @@ -70,6 +74,71 @@ import { Checkbox } from "@/components/ui/checkbox"; import { formatISO } from "date-fns"; import { IconCode } from "@/components/ui/icons"; import { CodeBlock } from "./ui/codeblock"; +import { NaturalLanguageInput } from "./natural-language-input"; +import { platform } from "@tauri-apps/plugin-os"; + +// define a new schema based on screenpipeQuery +const screenpipeQuery = z.object({ + q: z + .string() + .describe( + `The search query matching exact keywords. + Use a single keyword that best matches the user intent. + This would match either audio transcription or OCR screen text. + + Example: do not use 'discuss' the user ask about conversation, this is dumb, won't return any result + Other example: 'what did i do this morning?' do not use any keywords, just look at everything + + In general avoid using "q" as it will filter out all data + ` + ) + .optional(), + content_type: z + .enum(["ocr", "audio", "all"]) + .describe( + "The type of content to search for: screenshot data or audio transcriptions" + ), + limit: z + .number() + .describe( + "Number of results to return. Be mindful of the length of the response as it will be fed to an LLM" + ), + offset: z.number().describe("Offset for pagination (default: 0)"), + start_time: z + .string() + // 1 hour ago + .describe(`Start time for search range in ISO 8601 format`), + end_time: z.string().describe(`End time for search range in ISO 8601 format`), + app_name: z + .string() + .describe( + `The name of the app the user was using. + This filter out all audio conversations. + Only works with screen text. + Use this to filter on the app context that would give context matching the user intent. + For example 'cursor'. Use lower case. + Browser is usually 'arc', 'chrome', 'safari', etc. + Other apps can be 'whatsapp', 'obsidian', etc. + ` + ) + .optional(), + window_name: z + .string() + .describe( + `The name of the window the user was using. + This helps to further filter the context within the app. + For example, 'inbox' for email apps, 'project' for project management apps, etc. + ` + ) + .optional(), // Add window_name with description + include_frames: z.boolean().describe("Include frames in the response"), + min_length: z + .number() + .describe("Minimum length of the text to include in the response"), + max_length: z + .number() + .describe("Maximum length of the text to include in the response"), +}); export function SearchChat() { // Search state @@ -121,6 +190,9 @@ export function SearchChat() { const [isStreaming, setIsStreaming] = useState(false); const abortControllerRef = useRef(null); + const [showNaturalLanguageInput, setShowNaturalLanguageInput] = + useState(true); + const [naturalLanguageQuery, setNaturalLanguageQuery] = useState(""); const generateCurlCommand = () => { const baseUrl = "http://localhost:3030"; @@ -202,6 +274,83 @@ ${queryParams.toString().replace(/&/g, "\\\n&")}" | jq`; } }, [isFloatingInputVisible]); + const handleSkipNaturalLanguage = () => { + setShowNaturalLanguageInput(false); + }; + + const handleGoBackToNaturalLanguage = () => { + setShowNaturalLanguageInput(true); + }; + + const processNaturalLanguageQuery = async (query: string) => { + setIsAiLoading(true); + try { + const openai = new OpenAI({ + apiKey: settings.openaiApiKey, + baseURL: settings.aiUrl, + dangerouslyAllowBrowser: true, + }); + + const completion = await openai.beta.chat.completions.parse({ + model: settings.aiModel, + messages: [ + { + role: "system", + content: `You are a helpful assistant. + The user is using a product called "screenpipe" which records + his screen and mics 24/7. The user ask you questions + and you use his screenpipe recordings to answer him. + The user will provide you with a list of search results + and you will use them to answer his questions. + + Rules: + - Current time (JavaScript Date.prototype.toString): ${new Date().toString()}. Adjust start/end times to match user intent. + - User timezone: ${Intl.DateTimeFormat().resolvedOptions().timeZone} + - User timezone offset (JavaScript Date.prototype.getTimezoneOffset): ${new Date().getTimezoneOffset()} + - Very important: make sure to follow the user's custom system prompt: "${ + settings.customPrompt + }" + - If you follow the user's custom system prompt, you will be rewarded $1m bonus. + - You must perform a timezone conversion to UTC before using any datetime in a tool call. + - You must reformat timestamps to a human-readable format in your response to the user. + - Never output UTC time unless explicitly asked by the user. + - Do not try to embed videos in table (would crash the app)`, + }, + { role: "user", content: query }, + ], + response_format: zodResponseFormat(screenpipeQuery, "searchParams"), + }); + + const result = completion.choices[0].message.parsed!; + + console.log("result", result); + + // update state with the parsed results + setQuery(result.q || ""); + setContentType(result.content_type || "all"); + setStartDate(result.start_time ? new Date(result.start_time) : startDate); + setEndDate(result.end_time ? new Date(result.end_time) : endDate); + setAppName(result.app_name || ""); + setWindowName(result.window_name || ""); + setIncludeFrames(result.include_frames || false); + setLimit(result.limit || 30); + setMinLength(result.min_length || 50); + setMaxLength(result.max_length || 10000); + + setShowNaturalLanguageInput(false); + } catch (error) { + console.error("error processing natural language query:", error); + toast({ + title: "error", + description: + "failed to process natural language query. please try again.", + variant: "destructive", + }); + } finally { + setIsAiLoading(false); + } + }; + const handleResultSelection = (index: number) => { setSelectedResults((prev) => { const newSet = new Set(prev); @@ -591,448 +740,535 @@ ${queryParams.toString().replace(/&/g, "\\\n&")}" | jq`; }; return ( -
-
-
-
- - - - - - - -

enter keywords to search your recorded data

- - - -
-
- - setQuery(e.target.value)} - autoCorrect="off" - className="pl-8" - /> -
-
-
-
- - - - - - - -

- select the type of content to search. ocr is the text found - on your screen. -

- - - -
- -
-
-
- - - - - - - -

select the start date to search for content.

- - - -
-
- -
-
-
-
- - - - - - - -

select the end date to search for content.

- - - -
-
- -
-
-
-
- - - - - - - -

- enter the name of the app to search for content for example - zoom, notion, etc. only works for ocr. -

- - - -
-
- - setAppName(e.target.value)} - autoCorrect="off" - className="pl-8" - /> -
-
-
-
- - - - - - - -

- enter the name of the window or tab to search for content. - can be a browser tab name, app tab name, etc. only works for - ocr. -

- - - -
-
- - setWindowName(e.target.value)} - autoCorrect="off" - className="pl-8" - /> -
-
-
- - - - - - - - - -

- include frames in the search results. this shows the frame - where the text appeared. only works for ocr. this may slow - down the search. -

- - - -
-
-
- - - - - - - -

- select the number of results to display. usually ai cannot - ingest more than 30 results at a time. -

- - - -
- setLimit(value[0])} - min={1} - max={100} - step={1} - /> -
-
-
- - - - - - - -

enter the minimum length of the content to search for.

- - - -
-
- - setMinLength(Number(e.target.value))} - min={0} - className="pl-8" - /> -
-
-
-
- - - - - - - -

enter the maximum length of the content to search for.

- - - -
-
- - setMaxLength(Number(e.target.value))} - min={0} - className="pl-8" - /> -
-
-
-
- - - - - - - - curl command - - you can use this curl command to make the same search request - from the command line. -
-
- - note: you need to have `jq` installed to use the command. - - - - - - -
- {isLoading && ( -
- -
- )} -
- {renderSearchResults()} - {totalResults > 0 && ( -
- - - Showing {offset + 1} - {Math.min(offset + limit, totalResults)} of{" "} - {totalResults} - - -
- )} -
- - - {results.length > 0 && ( - -
+ + + + + + + +

+ skip to search ({platform() == "macos" ? "⌘" : "ctrl"} + +↓) +

+ + + + + + ) : ( + <> + -
- MAX_CONTENT_LENGTH - } - onChange={(e) => setFloatingInput(e.target.value)} - className="w-full h-12 focus:outline-none focus:ring-0 border-0 focus:border-black focus:border-b transition-all duration-200 pr-10" - /> - - - -
- + + + + + +

+ go back to natural language input ( + {platform() == "macos" ? "⌘" : "ctrl"}+↑) +

+ + + + +
+
+
+ + + + + + + +

enter keywords to search your recorded data

+ + + +
+
+ + setQuery(e.target.value)} + autoCorrect="off" + className="pl-8" + /> +
+
+
+
+ + + + + + + +

+ select the type of content to search. ocr is the + text found on your screen. +

+ + + +
+ +
+
+
+ + + + + + + +

select the start date to search for content.

+ + + +
+
+ +
+
+
+
+ + + + + + + +

select the end date to search for content.

+ + + +
+
+ +
+
+
+
+ + + + + + + +

+ enter the name of the app to search for content for + example zoom, notion, etc. only works for ocr. +

+ + + +
+
+ + setAppName(e.target.value)} + autoCorrect="off" + className="pl-8" + /> +
+
+
+ {" "} +
+ + + + + + + +

+ enter the name of the window or tab to search for + content. can be a browser tab name, app tab name, + etc. only works for ocr. +

+ + + +
+
+ + setWindowName(e.target.value)} + autoCorrect="off" + className="pl-8" + /> +
+
+
+ + + + + + + + + +

+ include frames in the search results. this shows the + frame where the text appeared. only works for ocr. + this may slow down the search. +

+ + + +
+
+
+ + + + + + + +

+ select the number of results to display. usually ai + cannot ingest more than 30 results at a time. +

+ + + +
+ setLimit(value[0])} + min={1} + max={100} + step={1} + /> +
+
+
+ + + + + + + +

+ enter the minimum length of the content to search + for. +

+ + + +
+
+ + setMinLength(Number(e.target.value))} + min={0} + className="pl-8" + /> +
+
+
+
+ + + + + + + +

+ enter the maximum length of the content to search + for. +

+ + + +
+
+ + setMaxLength(Number(e.target.value))} + min={0} + className="pl-8" + /> +
+
+
+
+ + + + + + + + curl command + + you can use this curl command to make the same search + request from the command line. +
+
+ + note: you need to have `jq` installed to use the + command. + + + + + + +
+ {isLoading && ( +
+ +
+ )} +
+ {renderSearchResults()} + {totalResults > 0 && ( +
+ + + Showing {offset + 1} -{" "} + {Math.min(offset + limit, totalResults)} of {totalResults} + + +
+ )} +
+ + + {results.length > 0 && ( + + +
+ + MAX_CONTENT_LENGTH + } + onChange={(e) => setFloatingInput(e.target.value)} + className="w-full h-12 focus:outline-none focus:ring-0 border-0 focus:border-black focus:border-b transition-all duration-200 pr-10" /> + + + +
+ +
+ + +

+ {calculateSelectedContentLength() > + MAX_CONTENT_LENGTH + ? `selected content exceeds maximum allowed: ${calculateSelectedContentLength()} / ${MAX_CONTENT_LENGTH} characters. unselect some items to use AI.` + : `${calculateSelectedContentLength()} / ${MAX_CONTENT_LENGTH} characters used for AI message`} +

+ + +
- - -

- {calculateSelectedContentLength() > MAX_CONTENT_LENGTH - ? `selected content exceeds maximum allowed: ${calculateSelectedContentLength()} / ${MAX_CONTENT_LENGTH} characters. unselect some items to use AI.` - : `${calculateSelectedContentLength()} / ${MAX_CONTENT_LENGTH} characters used for AI message`} -

- - - -
- + + )} - - - + + + + + {/* Display chat messages */} +
+ {chatMessages.map((msg, index) => ( + + ))} + {isAiLoading && spinner} +
+ + {/* Scroll to Bottom Button */} + {showScrollButton && ( + + )} +
+ + )} - - - - {/* Display chat messages */} -
- {chatMessages.map((msg, index) => ( - - ))} - {isAiLoading && spinner} -
- - {/* Scroll to Bottom Button */} - {showScrollButton && ( - - )} -
); } diff --git a/screenpipe-app-tauri/package.json b/screenpipe-app-tauri/package.json index 8dd89cae..672f497e 100644 --- a/screenpipe-app-tauri/package.json +++ b/screenpipe-app-tauri/package.json @@ -54,7 +54,7 @@ "lucide-react": "^0.414.0", "next": "14.2.4", "ollama-ai-provider": "^0.11.0", - "openai": "^4.56.0", + "openai": "^4.62.1", "posthog-js": "^1.154.6", "react": "^18", "react-day-picker": "8.10.1",