Feat: Email Validation

app/__tests__/dashboard.test.ts

@@ -1,7 +1,7 @@
 import { shortest } from "@antiwork/shortest";
 import { clerk, clerkSetup } from "@clerk/testing/playwright";
 
-let frontendUrl = process.env.SHORTEST_TEST_BASE_URL ?? "http://localhost:3000";
+let frontendUrl = process.env.PLAYWRIGHT_TEST_BASE_URL ?? "http://localhost:3000";
 
 shortest.beforeAll(async ({ page }) => {
   await clerkSetup({
@@ -18,4 +18,4 @@ shortest.beforeAll(async ({ page }) => {
   await page.goto(frontendUrl + "/dashboard");
 });
 
-shortest("Verify that the user can access the /dashboard page");
+shortest("Verify that the user can access the /dashboard page");

packages/shortest/CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.2] - 2024-12-26
+
+### Added  
+- Added support for Mailosaur email validation
+- Added email rendering feature in the browser
+- Added sleep_milliseconds tool to add delays in the test execution
+- Added more robust error handling for Mailosaur email validation
+
 ## [0.1.1] - 2024-12-24
 
 ### Fixed

packages/shortest/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antiwork/shortest",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "AI-powered natural language end-to-end testing framework",
   "type": "module",
   "main": "./dist/index.js",
@@ -29,36 +29,38 @@
     "build:types": "tsc --emitDeclarationOnly --outDir dist/types && cp index.d.ts dist/",
     "build:js": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.js --external:esbuild --external:punycode --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv",
     "build:cjs": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs --external:esbuild --external:punycode --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv",
-    "build:cli": "esbuild src/cli/bin.ts src/cli/setup.ts --bundle --platform=node --format=esm --outdir=dist/cli --metafile=dist/meta-cli.json --external:fsevents --external:chokidar --external:glob --external:esbuild --external:events --external:path --external:fs --external:util --external:stream --external:os --external:assert --external:url --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv --external:otplib --external:picocolors --external:punycode",
+    "build:cli": "esbuild src/cli/bin.ts src/cli/setup.ts --bundle --platform=node --format=esm --outdir=dist/cli --metafile=dist/meta-cli.json --external:fsevents --external:chokidar --external:glob --external:esbuild --external:events --external:path --external:fs --external:util --external:stream --external:os --external:assert --external:url --external:playwright --external:@anthropic-ai/sdk --external:expect --external:dotenv --external:otplib --external:picocolors --external:punycode --external:https --external:http --external:net --external:tls --external:crypto --external:mailosaur",
     "dev": "pnpm build:types --watch",
     "test:ai": "tsx tests/test-ai.ts",
     "test:browser": "tsx tests/test-browser.ts",
     "test:coordinates": "tsx tests/test-coordinates.ts",
     "test:github": "tsx tests/test-github.ts",
     "test:assertion": "tsx tests/test-assertion.ts",
-    "test:keyboard": "tsx tests/test-keyboard.ts"
+    "test:keyboard": "tsx tests/test-keyboard.ts",
+    "test:email": "tsx tests/test-email.ts"
   },
   "dependencies": {
-    "glob": "^10.3.10",
     "chromium-bidi": "^0.5.2",
+    "glob": "^10.3.10",
     "otplib": "^12.0.1",
     "picocolors": "^1.0.0"
   },
   "devDependencies": {
-    "tsx": "^4.7.1",
-    "typescript": "~5.6.2",
     "@types/jest": "^29.5.12",
-    "@types/node": "^20.11.24"
+    "@types/node": "^20.11.24",
+    "tsx": "^4.7.1",
+    "typescript": "~5.6.2"
   },
   "engines": {
     "node": ">=18"
   },
   "peerDependencies": {
-    "playwright": "^1.48.2",
-    "esbuild": "^0.20.1",
     "@anthropic-ai/sdk": "0.32.0",
+    "mailosaur": "^8.7.0",
+    "dotenv": "^16.4.5",
+    "esbuild": "^0.20.1",
     "expect": "^29.7.0",
-    "dotenv": "^16.4.5"
+    "playwright": "^1.48.2"
   },
   "author": "Antiwork",
   "license": "MIT",

packages/shortest/src/ai/prompts/index.ts

@@ -11,9 +11,10 @@ Expect: 1. Test case to be generated within at least 20 seconds [HAS_CALLBACK]
 IMPORTANT GLOBAL RULES:
 
 1. **Waiting for Conditions**:
-   - Some steps will require waiting before proceeding to the next action.
-   - This waiting can be based on a time delay (e.g., seconds or minutes) or waiting for an element to become visible or clickable.
-   - If the specified condition is not met after the allotted time, the test should be considered failed.
+   - When you are specifically asked to wait for a condition, use the "sleep" tool. Otherwise, we internally handle proper waiting for elements to load.
+   - For time-based waits:
+     - Use { action: "sleep", duration: X } where X is milliseconds
+   - Always wait for the tool to finish before proceeding to the next action. You will recieve a message to continue with your next action once the wait is over. Then validate the condition is met.
 
 2. **Tool Usage**:
    - You may need to use provided tools to perform certain actions (e.g., clicking, navigating, or running callbacks).
@@ -42,6 +43,14 @@ IMPORTANT GLOBAL RULES:
    - All expectations listed in the test instructions must be fulfilled.
    - If any expectation is not met, the test case must be marked as failed.
 
+8. **Testing Email**:
+   - If you need to test a condition that involves seeing the contents of an email, use the "check_email" tool.
+   - For email validation, you MUST always use 'Click' and 'Mouse' action instead of using keyboard shortcuts.
+   - This tool will grab the latest email from the email address given to you and will render it in a new tab for you to see.
+   - Once you are done with validating the email, navigate back to the original tab.
+   - You MUST pass the email address that is given to you to the tool as a parameter otherwise it will fail.
+   - If no email address is given to you for this test, you should fail the test.
+
 Your task is to:
 1. Execute browser actions to validate test cases
 2. Use provided browser tools to interact with the page

packages/shortest/src/ai/tools/index.ts

@@ -30,6 +30,42 @@ export const AITools = [
       required: ["action", "username", "password"],
     },
   },
+  {
+    name: "check_email",
+    description: "Fetch, decode and render received email in browser",
+    input_schema: {
+      type: "object",
+      properties: {
+        action: {
+          type: "string",
+          enum: ["check_email"],
+          description: "Render the received email in a new tab"
+        }
+      },
+      required: ["action", "email"]
+    }
+  },
+  {
+    name: "sleep",
+    description: "Pause test execution for specified duration",
+    input_schema: {
+      type: "object",
+      properties: {
+        action: {
+          type: "string",
+          enum: ["sleep"],
+          description: "The action to perform"
+        },
+        duration: {
+          type: "number",
+          description: "Duration to sleep in milliseconds (e.g. 5000 for 5 seconds)",
+          minimum: 0,
+          maximum: 60000
+        }
+      },
+      required: ["action", "duration"]
+    }
+  },
   {
     name: "run_callback",
     description: "Run callback function for current test step",