import { StreamingTextResponse } from 'ai';
import { ChatMessage, MessageContent, OpenAI, TogetherLLM } from 'llamaindex';
import { NextRequest, NextResponse } from 'next/server';
import { createChatEngine } from './engine';
import { LlamaIndexStream } from './llamaindex-stream';
export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
const convertMessageContent = (
textMessage: string,
imageUrl: string | undefined
): MessageContent => {
if (!imageUrl) return textMessage;
return [
{
type: 'text',
text: textMessage,
},
{
type: 'image_url',
image_url: {
url: imageUrl,
},
},
];
};
export async function POST(request: NextRequest) {
try {
const body = await request.json();
const { messages, data }: { messages: ChatMessage[]; data: any } = body;
const userMessage = messages.pop();
if (!messages || !userMessage || userMessage.role !== 'user') {
return NextResponse.json(
{
error:
'messages are required in the request body and the last message must be from the user',
},
{ status: 400 }
);
}
const llm = new TogetherLLM({
model: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
maxTokens: 512,
apiKey: process.env.TOGETHER_API_KEY,
});
const chatEngine = await createChatEngine(llm);
// Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
const userMessageContent = convertMessageContent(
userMessage.content,
data?.imageUrl
);
// Calling LlamaIndex's ChatEngine to get a streamed response
const response = await chatEngine.chat({
message: userMessageContent,
chatHistory: messages,
stream: true,
});
// Transform LlamaIndex stream to Vercel/AI format
const { stream, data: streamData } = LlamaIndexStream(response, {
parserOptions: {
image_url: data?.imageUrl,
},
});
// Return a StreamingTextResponse, which can be consumed by the Vercel/AI client
return new StreamingTextResponse(stream, {}, streamData);
} catch (error) {
console.error('[LlamaIndex]', error);
return NextResponse.json(
{
error: (error as Error).message,
},
{
status: 500,
}
);
}
}