Add experimental support for Claude

.env

@@ -11,6 +11,7 @@ RENDERING_ENGINE="INFERENCE_API"
 # - INFERENCE_API
 # - OPENAI
 # - GROQ
+# - ANTHROPIC
 LLM_ENGINE="INFERENCE_API"
 
 # set this to control the number of pages
@@ -46,19 +47,22 @@ AUTH_VIDEOCHAIN_API_TOKEN=
 # Groq.com key: available for the LLM engine
 AUTH_GROQ_API_KEY=
 
+# Anthropic.com key: available for the LLM engine
+AUTH_ANTHROPIC_API_KEY=
+
 # ------------- RENDERING API CONFIG --------------
 
-# If you decided to use Replicate for the RENDERING engine
+# If you decide to use Replicate for the RENDERING engine
 RENDERING_REPLICATE_API_MODEL="stabilityai/sdxl"
 RENDERING_REPLICATE_API_MODEL_VERSION="da77bc59ee60423279fd632efb4795ab731d9e3ca9705ef3341091fb989b7eaf"
 
-# If you decided to use a private Hugging Face Inference Endpoint for the RENDERING engine
+# If you decide to use a private Hugging Face Inference Endpoint for the RENDERING engine
 RENDERING_HF_INFERENCE_ENDPOINT_URL="https://XXXXXXXXXX.endpoints.huggingface.cloud"
 
-# If you decided to use a Hugging Face Inference API model for the RENDERING engine
+# If you decide to use a Hugging Face Inference API model for the RENDERING engine
 RENDERING_HF_INFERENCE_API_BASE_MODEL="stabilityai/stable-diffusion-xl-base-1.0"
 
-# If you decided to use a Hugging Face Inference API model for the RENDERING engine
+# If you decide to use a Hugging Face Inference API model for the RENDERING engine
 RENDERING_HF_INFERENCE_API_REFINER_MODEL="stabilityai/stable-diffusion-xl-refiner-1.0"
 
 # If your model returns a different file type (eg. jpg or webp) change it here
@@ -66,7 +70,7 @@ RENDERING_HF_INFERENCE_API_FILE_TYPE="image/png"
 
 # An experimental RENDERING engine (sorry it is not very documented yet, so you can use one of the other engines)
 RENDERING_VIDEOCHAIN_API_URL="http://localhost:7860"
-
+you decide
 RENDERING_OPENAI_API_BASE_URL="https://api.openai.com/v1"
 RENDERING_OPENAI_API_MODEL="dall-e-3"
 
@@ -74,14 +78,18 @@ RENDERING_OPENAI_API_MODEL="dall-e-3"
 
 LLM_GROQ_API_MODEL="mixtral-8x7b-32768"
 
-# If you decided to use OpenAI for the LLM engine
+# If you decide to use OpenAI for the LLM engine
 LLM_OPENAI_API_BASE_URL="https://api.openai.com/v1"
 LLM_OPENAI_API_MODEL="gpt-4"
 
-# If you decided to use a private Hugging Face Inference Endpoint for the LLM engine
+# If you decide to use Anthropic (eg. Claude) for the LLM engine
+# https://docs.anthropic.com/claude/docs/models-overview
+LLM_ANTHROPIC_API_MODEL="claude-3-opus-20240229"
+
+# If you decide to use a private Hugging Face Inference Endpoint for the LLM engine
 LLM_HF_INFERENCE_ENDPOINT_URL=""
 
-# If you decided to use a Hugging Face Inference API model for the LLM engine
+# If you decide to use a Hugging Face Inference API model for the LLM engine
 # LLM_HF_INFERENCE_API_MODEL="HuggingFaceH4/zephyr-7b-beta"
 LLM_HF_INFERENCE_API_MODEL="HuggingFaceH4/zephyr-7b-beta"
 

README.md

@@ -31,13 +31,14 @@ it requires various components to run for the frontend, backend, LLM, SDXL etc.
 If you try to duplicate the project, open the `.env` you will see it requires some variables.
 
 Provider config:
-- `LLM_ENGINE`: can be one of: "INFERENCE_API", "INFERENCE_ENDPOINT", "OPENAI", or "GROQ"
+- `LLM_ENGINE`: can be one of `INFERENCE_API`, `INFERENCE_ENDPOINT`, `OPENAI`, `GROQ`, `ANTHROPIC`
 - `RENDERING_ENGINE`: can be one of: "INFERENCE_API", "INFERENCE_ENDPOINT", "REPLICATE", "VIDEOCHAIN", "OPENAI" for now, unless you code your custom solution
 
 Auth config:
 - `AUTH_HF_API_TOKEN`:  if you decide to use Hugging Face for the LLM engine (inference api model or a custom inference endpoint)
 - `AUTH_OPENAI_API_KEY`: to use OpenAI for the LLM engine
 - `AUTH_GROQ_API_KEY`: to use Groq for the LLM engine
+- `AUTH_ANTHROPIC_API_KEY`: to use Anthropic (Claude) for the LLM engine
 - `AUTH_VIDEOCHAIN_API_TOKEN`: secret token to access the VideoChain API server
 - `AUTH_REPLICATE_API_TOKEN`: in case you want to use Replicate.com
 
@@ -56,6 +57,7 @@ Language model config (depending on the LLM engine you decide to use):
 - `LLM_OPENAI_API_BASE_URL`: "https://api.openai.com/v1"
 - `LLM_OPENAI_API_MODEL`: "gpt-4"
 - `LLM_GROQ_API_MODEL`: "mixtral-8x7b-32768"
+- `LLM_ANTHROPIC_API_MODEL`: "claude-3-opus-20240229"
 
 In addition, there are some community sharing variables that you can just ignore.
 Those variables are not required to run the AI Comic Factory on your own website or computer
@@ -76,7 +78,7 @@ To customise a variable locally, you should create a `.env.local`
 
 Currently the AI Comic Factory uses [zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) through an [Inference Endpoint](https://huggingface.co/docs/inference-endpoints/index).
 
-You have three options:
+You have multiple options:
 
 ### Option 1: Use an Inference API model
 
@@ -134,8 +136,17 @@ LLM_GROQ_API_MODEL="mixtral-8x7b-32768"
 
 AUTH_GROQ_API_KEY="Your own GROQ API Key"
 ```
+### Option 5: (new, experimental) use Anthropic (Claude)
 
-### Option 5: Fork and modify the code to use a different LLM system
+```bash
+LLM_ENGINE="ANTHROPIC"
+
+LLM_ANTHROPIC_API_MODEL="claude-3-opus-20240229"
+
+AUTH_ANTHROPIC_API_KEY="Your own ANTHROPIC API Key"
+```
+
+### Option 6: Fork and modify the code to use a different LLM system
 
 Another option could be to disable the LLM completely and replace it with another LLM protocol and/or provider (eg. Claude, Replicate), or a human-generated story instead (by returning mock or static data).
 

package-lock.json

@@ -1,13 +1,14 @@
 {
   "name": "@jbilcke/comic-factory",
-  "version": "0.0.0",
+  "version": "1.2.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@jbilcke/comic-factory",
-      "version": "0.0.0",
+      "version": "1.2.0",
       "dependencies": {
+        "@anthropic-ai/sdk": "^0.19.1",
         "@huggingface/hub": "^0.14.2",
         "@huggingface/inference": "^2.6.1",
         "@radix-ui/react-accordion": "^1.1.2",
@@ -93,6 +94,30 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/@anthropic-ai/sdk": {
+      "version": "0.19.1",
+      "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.19.1.tgz",
+      "integrity": "sha512-u9i8yN8wAr/ujaXSRjfYXiYzhCk2mdUG6G9y5IAKEAPJHwFTrEyf76Z4V1LqqFbDBlZqm0tkoMMpU8tmp65ocA==",
+      "dependencies": {
+        "@types/node": "^18.11.18",
+        "@types/node-fetch": "^2.6.4",
+        "abort-controller": "^3.0.0",
+        "agentkeepalive": "^4.2.1",
+        "digest-fetch": "^1.3.0",
+        "form-data-encoder": "1.7.2",
+        "formdata-node": "^4.3.2",
+        "node-fetch": "^2.6.7",
+        "web-streams-polyfill": "^3.2.1"
+      }
+    },
+    "node_modules/@anthropic-ai/sdk/node_modules/@types/node": {
+      "version": "18.19.28",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.28.tgz",
+      "integrity": "sha512-J5cOGD9n4x3YGgVuaND6khm5x07MMdAKkRyXnjVR6KFhLMNh2yONGiP7Z+4+tBOt5mK+GvDTiacTOVGGpqiecw==",
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
     "node_modules/@babel/runtime": {
       "version": "7.24.1",
       "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.1.tgz",

package.json

@@ -9,6 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@anthropic-ai/sdk": "^0.19.1",
     "@huggingface/hub": "^0.14.2",
     "@huggingface/inference": "^2.6.1",
     "@radix-ui/react-accordion": "^1.1.2",

src/app/queries/predict.ts

@@ -4,10 +4,12 @@ import { LLMEngine } from "@/types"
 import { predict as predictWithHuggingFace } from "./predictWithHuggingFace"
 import { predict as predictWithOpenAI } from "./predictWithOpenAI"
 import { predict as predictWithGroq } from "./predictWithGroq"
+import { predict as predictWithAnthropic } from "./predictWithAnthropic"
 
 const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
 
 export const predict =
   llmEngine === "GROQ" ? predictWithGroq :
+  llmEngine === "ANTHROPIC" ? predictWithAnthropic :
   llmEngine === "OPENAI" ? predictWithOpenAI :
   predictWithHuggingFace

src/app/queries/predictNextPanels.ts

@@ -1,12 +1,11 @@
-
-import { predict } from "./predict"
-import { Preset } from "../engine/presets"
 import { GeneratedPanel } from "@/types"
 import { cleanJson } from "@/lib/cleanJson"
-import { createZephyrPrompt } from "@/lib/createZephyrPrompt"
 import { dirtyGeneratedPanelCleaner } from "@/lib/dirtyGeneratedPanelCleaner"
 import { dirtyGeneratedPanelsParser } from "@/lib/dirtyGeneratedPanelsParser"
 import { sleep } from "@/lib/sleep"
+
+import { Preset } from "../engine/presets"
+import { predict } from "./predict"
 import { getSystemPrompt } from "./getSystemPrompt"
 import { getUserPrompt } from "./getUserPrompt"
 

src/app/queries/predictWithAnthropic.ts

@@ -0,0 +1,41 @@
+"use server"
+
+import Anthropic from '@anthropic-ai/sdk';
+import { MessageParam } from '@anthropic-ai/sdk/resources';
+
+export async function predict({
+  systemPrompt,
+  userPrompt,
+  nbMaxNewTokens,
+}: {
+  systemPrompt: string
+  userPrompt: string
+  nbMaxNewTokens: number
+}): Promise<string> {
+  const anthropicApiKey = `${process.env.AUTH_ANTHROPIC_API_KEY || ""}`
+  const anthropicApiModel = `${process.env.LLM_ANTHROPIC_API_MODEL || "claude-3-opus-20240229"}`
+  
+  const anthropic = new Anthropic({
+    apiKey: anthropicApiKey,
+  })
+
+  const messages: MessageParam[] = [
+    { role: "user", content: userPrompt },
+  ]
+
+  try {
+    const res = await anthropic.messages.create({
+      messages: messages,
+      // stream: false,
+      system: systemPrompt,
+      model: anthropicApiModel,
+      // temperature: 0.8,
+      max_tokens: nbMaxNewTokens,
+    })
+
+    return res.content[0]?.text || ""
+  } catch (err) {
+    console.error(`error during generation: ${err}`)
+    return ""
+  }
+}

src/types.ts

@@ -101,6 +101,7 @@ export type LLMEngine =
   | "OPENAI"
   | "REPLICATE"
   | "GROQ"
+  | "ANTHROPIC"
 
   export type RenderingEngine =
   | "VIDEOCHAIN"