Add images support for Claude 3 Vision (mckaywrigley#1562)

* Add images support for Claude 3 Vision * Fix condition logic for image_url field in api route --------- Co-authored-by: Mckay Wrigley <[email protected]>
bolechen · Mar 23, 2024 · 8602e2a · 8602e2a
1 parent 063001c
commit 8602e2a
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 37 deletions.
diff --git a/app/api/chat/anthropic/route.ts b/app/api/chat/anthropic/route.ts
@@ -1,65 +1,111 @@
-import { CHAT_SETTING_LIMITS } from "@/lib/chat-setting-limits";
-import { checkApiKey, getServerProfile } from "@/lib/server/server-chat-helpers";
-import { ChatSettings } from "@/types";
-import Anthropic from "@anthropic-ai/sdk";
-import { AnthropicStream, StreamingTextResponse } from "ai";
-import { NextRequest, NextResponse } from "next/server";
+import { CHAT_SETTING_LIMITS } from "@/lib/chat-setting-limits"
+import { checkApiKey, getServerProfile } from "@/lib/server/server-chat-helpers"
+import { getBase64FromDataURL, getMediaTypeFromDataURL } from "@/lib/utils"
+import { ChatSettings } from "@/types"
+import Anthropic from "@anthropic-ai/sdk"
+import { AnthropicStream, StreamingTextResponse } from "ai"
+import { NextRequest, NextResponse } from "next/server"
 
-export const runtime = "edge";
+export const runtime = "edge"
 
 export async function POST(request: NextRequest) {
-  const json = await request.json();
+  const json = await request.json()
   const { chatSettings, messages } = json as {
-    chatSettings: ChatSettings;
-    messages: any[];
-  };
+    chatSettings: ChatSettings
+    messages: any[]
+  }
 
   try {
-    const profile = await getServerProfile();
-    checkApiKey(profile.anthropic_api_key, "Anthropic");
-    let ANTHROPIC_FORMATTED_MESSAGES: any = messages.slice(1);
+    const profile = await getServerProfile()
+
+    checkApiKey(profile.anthropic_api_key, "Anthropic")
+
+    let ANTHROPIC_FORMATTED_MESSAGES: any = messages.slice(1)
+
+    ANTHROPIC_FORMATTED_MESSAGES = ANTHROPIC_FORMATTED_MESSAGES?.map(
+      (message: any) => {
+        const messageContent =
+          typeof message?.content === "string"
+            ? [message.content]
+            : message?.content
+
+        return {
+          ...message,
+          content: messageContent.map((content: any) => {
+            if (typeof content === "string") {
+              // Handle the case where content is a string
+              return { type: "text", text: content }
+            } else if (
+              content?.type === "image_url" &&
+              content?.image_url?.length
+            ) {
+              return {
+                type: "image",
+                source: {
+                  type: "base64",
+                  media_type: getMediaTypeFromDataURL(content.image_url),
+                  data: getBase64FromDataURL(content.image_url)
+                }
+              }
+            } else {
+              return content
+            }
+          })
+        }
+      }
+    )
 
     const anthropic = new Anthropic({
-      apiKey: profile.anthropic_api_key || "",
-    });
+      apiKey: profile.anthropic_api_key || ""
+    })
 
     try {
       const response = await anthropic.messages.create({
         model: chatSettings.model,
         messages: ANTHROPIC_FORMATTED_MESSAGES,
         temperature: chatSettings.temperature,
         system: messages[0].content,
-        max_tokens: CHAT_SETTING_LIMITS[chatSettings.model].MAX_TOKEN_OUTPUT_LENGTH,
-        stream: true,
-      });
+        max_tokens:
+          CHAT_SETTING_LIMITS[chatSettings.model].MAX_TOKEN_OUTPUT_LENGTH,
+        stream: true
+      })
 
       try {
-        const stream = AnthropicStream(response);
-        return new StreamingTextResponse(stream);
+        const stream = AnthropicStream(response)
+        return new StreamingTextResponse(stream)
       } catch (error: any) {
-        console.error("Error parsing Anthropic API response:", error);
+        console.error("Error parsing Anthropic API response:", error)
         return new NextResponse(
-          JSON.stringify({ message: "An error occurred while parsing the Anthropic API response" }),
+          JSON.stringify({
+            message:
+              "An error occurred while parsing the Anthropic API response"
+          }),
           { status: 500 }
-        );
+        )
       }
     } catch (error: any) {
-      console.error("Error calling Anthropic API:", error);
+      console.error("Error calling Anthropic API:", error)
       return new NextResponse(
-        JSON.stringify({ message: "An error occurred while calling the Anthropic API" }),
+        JSON.stringify({
+          message: "An error occurred while calling the Anthropic API"
+        }),
         { status: 500 }
-      );
+      )
     }
   } catch (error: any) {
-    let errorMessage = error.message || "An unexpected error occurred";
-    const errorCode = error.status || 500;
+    let errorMessage = error.message || "An unexpected error occurred"
+    const errorCode = error.status || 500
+
     if (errorMessage.toLowerCase().includes("api key not found")) {
-      errorMessage = "Anthropic API Key not found. Please set it in your profile settings.";
+      errorMessage =
+        "Anthropic API Key not found. Please set it in your profile settings."
     } else if (errorCode === 401) {
-      errorMessage = "Anthropic API Key is incorrect. Please fix it in your profile settings.";
+      errorMessage =
+        "Anthropic API Key is incorrect. Please fix it in your profile settings."
     }
+
     return new NextResponse(JSON.stringify({ message: errorMessage }), {
-      status: errorCode,
-    });
+      status: errorCode
+    })
   }
 }
diff --git a/components/chat/chat-input.tsx b/components/chat/chat-input.tsx
@@ -11,6 +11,7 @@ import {
 import Image from "next/image"
 import { FC, useContext, useEffect, useRef, useState } from "react"
 import { useTranslation } from "react-i18next"
+import { toast } from "sonner"
 import { Input } from "../ui/input"
 import { TextareaAutosize } from "../ui/textarea-autosize"
 import { ChatCommandInput } from "./chat-command-input"
@@ -19,7 +20,6 @@ import { useChatHandler } from "./chat-hooks/use-chat-handler"
 import { useChatHistoryHandler } from "./chat-hooks/use-chat-history"
 import { usePromptAndCommand } from "./chat-hooks/use-prompt-and-command"
 import { useSelectFileHandler } from "./chat-hooks/use-select-file-handler"
-import { toast } from "sonner"
 
 interface ChatInputProps {}
 

diff --git a/lib/models/llm/anthropic-llm-list.ts b/lib/models/llm/anthropic-llm-list.ts
@@ -32,7 +32,7 @@ const CLAUDE_3_HAIKU: LLM = {
   provider: "anthropic",
   hostedId: "claude-3-haiku-20240307",
   platformLink: ANTHROPIC_PLATFORM_LINK,
-  imageInput: false
+  imageInput: true
 }
 
 // Claude 3 Sonnet (UPDATED 03/04/24)
@@ -42,7 +42,7 @@ const CLAUDE_3_SONNET: LLM = {
   provider: "anthropic",
   hostedId: "claude-3-sonnet-20240229",
   platformLink: ANTHROPIC_PLATFORM_LINK,
-  imageInput: false
+  imageInput: true
 }
 
 // Claude 3 Opus (UPDATED 03/04/24)
@@ -52,7 +52,7 @@ const CLAUDE_3_OPUS: LLM = {
   provider: "anthropic",
   hostedId: "claude-3-opus-20240229",
   platformLink: ANTHROPIC_PLATFORM_LINK,
-  imageInput: false
+  imageInput: true
 }
 
 export const ANTHROPIC_LLM_LIST: LLM[] = [

diff --git a/lib/utils.ts b/lib/utils.ts
@@ -13,3 +13,13 @@ export function formatDate(input: string | number | Date): string {
     year: "numeric"
   })
 }
+
+export function getMediaTypeFromDataURL(dataURL: string): string | null {
+  const matches = dataURL.match(/^data:([A-Za-z-+\/]+);base64/)
+  return matches ? matches[1] : null
+}
+
+export function getBase64FromDataURL(dataURL: string): string | null {
+  const matches = dataURL.match(/^data:[A-Za-z-+\/]+;base64,(.*)$/)
+  return matches ? matches[1] : null
+}