Having trouble using other models in node.js example. #81
Comments
|
Hmm seems like you're still hitting openais endpoint? Since this is js you'll have to do some things manually that we usually do for you in the python sdk wrapper / patch. I'll take a look at your provided example as soon as I get a chance, don't see anything obviously wrong though. In the meantime, this might help if you don't mind code diving: https://github.com/datastax/astra-assistants-api/blob/main/client/astra_assistants/patch.py#L580 |
|
@phact So the var to change the url in openai is actually baseURL or the .env var OPENAI_BASE_URL. It seems like your example was going to OpenAI the entire time. But when I update the URL I'm getting a new error. I'm going to try just using the CURL method next. I'll post my update here. BadRequestError: 400 status code (no body) |
|
Thanks! It's entirely possible that the js example may be wrong or outdated. Maybe sticking a breakpoint inside the SDK might give us some clues. I may be able to have a go at it later tonight. |
|
@phact I got it working via Curl and yes the baseUrl just needs updated in the example. Do you have an array of the supported models? Here is some JS to test all of the different models if it helps. I'd love to get a list of current models so I can know what models I can test with. import * as dotenv from 'dotenv';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
// Configure dotenv with the correct path
const __dirname = dirname(fileURLToPath(import.meta.url));
dotenv.config({ path: join(__dirname, '../.env') });
// Define constants and environment variables
const baseUrl = process.env.OPENAI_BASE_URL || 'https://open-assistant-ai.astra.datastax.com/v1';
const astraApiToken = process.env.ASTRA_DB_APPLICATION_TOKEN;
// Define model keys and API mappings
const modelApiKeys = {
'openai': {
apiKey: process.env.OPENAI_API_KEY,
models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4']
},
'anthropic': {
apiKey: process.env.ANTHROPIC_API_KEY,
models: ['anthropic.claude-v2']
},
'groq': {
apiKey: process.env.GROQ_API_KEY,
models: ['groq/llama3-8b-8192']
},
'gemini': {
apiKey: process.env.GEMINI_API_KEY,
models: ['gemini-model-1', 'gemini-model-2']
},
'perplexity': {
apiKey: process.env.PERPLEXITYAI_API_KEY,
models: ['perplexity/llama-3.1-sonar-small-128k-online', 'perplexity/llama-3.1-sonar-huge-128k-online']
},
'cohere': {
apiKey: process.env.COHERE_API_KEY,
models: ['cohere-model-1', 'cohere-model-2']
}
};
// Test prompt to use for all models
const testPrompt = "What is 2+2? Reply with just the number.";
async function testModel(model) {
// Find the API key for this model
let apiKey;
for (const { apiKey: key, models } of Object.values(modelApiKeys)) {
if (models.includes(model)) {
apiKey = key;
break;
}
}
if (!apiKey) {
return { model, error: "API key not found" };
}
try {
const response = await fetch(`${baseUrl}/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'astra-api-token': astraApiToken,
'Authorization': `Bearer ${apiKey}`,
'OpenAI-Beta': 'assistants=v2'
},
body: JSON.stringify({
model: model,
messages: [
{ role: 'system', content: 'You are a helpful assistant.' },
{ role: 'user', content: testPrompt }
]
})
});
const data = await response.json();
if (!response.ok) {
return {
model,
status: response.status,
error: data
};
}
return {
model,
status: response.status,
message: data.choices[0].message.content
};
} catch (error) {
return {
model,
error: error.message
};
}
}
async function testAllModels() {
console.log('Testing all models with prompt:', testPrompt);
console.log('-----------------------------------');
// Get all unique models
const allModels = Object.values(modelApiKeys)
.flatMap(provider => provider.models);
// Test each model
const results = await Promise.all(allModels.map(testModel));
// Display results in a formatted way
results.forEach(result => {
console.log(`\nModel: ${result.model}`);
if (result.error) {
console.log('Error:', result.error);
} else {
console.log('Status:', result.status);
console.log('Response:', result.message);
}
console.log('-----------------------------------');
});
}
// Run the tests
testAllModels().catch(error => {
console.error('Test execution failed:', error);
}); |
|
Everything supported by litellm, I've been pulling them from here: |
|
Thank You! FYI. It looks like the latest version of openai's node library doesn't support overriding the headers. https://github.com/openai/openai-node/blob/master/src/index.ts For now I'll use this example to stream. https://github.com/openai/openai-node/blob/master/examples/stream-to-client-browser.ts |
|
Here's' some sample code that will test each of the models. import * as dotenv from 'dotenv';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import fetch from 'node-fetch';
// Configure dotenv with the correct path
const __dirname = dirname(fileURLToPath(import.meta.url));
dotenv.config({ path: join(__dirname, '../.env') });
const LITELLM_MODELS_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/model_prices_and_context_window.json';
const baseUrl = process.env.BASE_URL || 'https://open-assistant-ai.astra.datastax.com/v1';
// Define API key mapping
const providerApiKeys = {
'openai': process.env.OPENAI_API_KEY,
'anthropic': process.env.ANTHROPIC_API_KEY,
'groq': process.env.GROQ_API_KEY,
'gemini': process.env.GEMINI_API_KEY,
'perplexity': process.env.PERPLEXITY_API_KEY,
'cohere': process.env.COHERE_API_KEY,
};
async function fetchModelData() {
try {
const response = await fetch(LITELLM_MODELS_URL);
const data = await response.json();
return data;
} catch (error) {
console.error('Error fetching model data:', error);
throw error;
}
}
async function testModel(model, modelData) {
const modelInfo = modelData[model];
if (!modelInfo) return null;
const provider = modelInfo.litellm_provider;
const apiKey = providerApiKeys[provider];
if (!apiKey) return null;
try {
const response = await fetch(`${baseUrl}/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'astra-api-token': process.env.ASTRA_DB_APPLICATION_TOKEN,
'Authorization': `Bearer ${apiKey}`
},
body: JSON.stringify({
model: model,
messages: [
{ role: 'system', content: 'You are a helpful assistant.' },
{ role: 'user', content: 'What is 2+2? Reply with just the number.' }
]
})
});
const data = await response.json();
if (!response.ok) return null;
return {
model,
provider,
maxTokens: modelInfo.max_tokens,
inputCost: modelInfo.input_cost_per_token,
outputCost: modelInfo.output_cost_per_token
};
} catch (error) {
return null;
}
}
async function main() {
try {
console.log('Testing models...');
const modelData = await fetchModelData();
// Filter for chat models with matching providers
const eligibleModels = Object.entries(modelData)
.filter(([_, value]) =>
value.mode === 'chat' &&
value.litellm_provider &&
providerApiKeys[value.litellm_provider]
)
.map(([key]) => key);
// Test all models and collect working ones
const results = await Promise.all(
eligibleModels.map(model => testModel(model, modelData))
);
// Filter out null results and format for output
const workingModels = results
.filter(result => result !== null)
.reduce((acc, model) => {
acc[model.provider] = acc[model.provider] || {
apiKey: `process.env.${model.provider.toUpperCase()}_API_KEY`,
models: []
};
acc[model.provider].models.push(model.model);
return acc;
}, {});
// Output in a format ready to paste into code
console.log('\nWorking models configuration:');
console.log('const modelApiKeys = ' +
JSON.stringify(workingModels, null, 2)
.replace(/"process\.env\./g, 'process.env.')
.replace(/"/g, "'")
);
} catch (error) {
console.error('Execution error:', error);
}
}
main().catch(console.error); |
|
Thanks! Hmm, I wonder if providing a custom fetch is the way to mess with headers. https://github.com/openai/openai-node/tree/master?tab=readme-ov-file#customizing-the-fetch-client |
|
You motivated me to make a new astra-assistants patch library for node so that you can use the full SDK seamlessly. Check out this new example: |
|
Checking in, are you all set? |
I'm trying to use other models besides openai in your example code but I'm just getting a response back "invalid model". I've only tried with anthropic and perplexity so haven't set any of the other models supported in the array but those two models don't appear to be working. Note: It works fine if I use OpenAI models.
Here is some sample javascript code I'm using.
The text was updated successfully, but these errors were encountered: