Merge pull request #66 from mst-mkt/fix/ai_#65

AIまわりの改善

apps/backend/.dev.vars.example

@@ -1,3 +1,3 @@
 FRONTEND_BASE_URL = http://localhost:5173
 OPENAI_API_KEY =
-OPENAI_BASE_URL =
+OPENAI_BASE_URL = https://gateway.ai.cloudflare.com/v1/aff21982f7963fd8f2d9d373bd3b0f6f/chefcam/openai

apps/backend/package.json

@@ -8,12 +8,11 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@ai-sdk/openai": "^0.0.55",
     "@hono/zod-validator": "^0.2.2",
-    "@langchain/core": "^0.2.31",
-    "@langchain/openai": "^0.1.3",
+    "ai": "^3.3.26",
     "cheerio": "1.0.0-rc.12",
     "hono": "^4.5.11",
-    "langchain": "^0.2.17",
     "zod": "^3.23.8"
   },
   "devDependencies": {

apps/backend/src/factory.ts

@@ -1,7 +1,7 @@
-import type { ChatOpenAI } from '@langchain/openai'
+import type { LanguageModel } from 'ai'
 import { cors } from 'hono/cors'
 import { createFactory } from 'hono/factory'
-import { aiMiddleware } from './middlewares/ai'
+import { aiModelMiddleware } from './middlewares/aiModel'
 
 export type BindingsType = {
   FRONTEND_BASE_URL: string | undefined
@@ -10,7 +10,7 @@ export type BindingsType = {
 }
 
 type VariablesType = {
-  ai: ChatOpenAI
+  model: LanguageModel
 }
 
 type HonoConfigType = {
@@ -34,7 +34,7 @@ const honoFactory = createFactory<HonoConfigType>({
           },
         }),
       )
-      .use('*', aiMiddleware)
+      .use('*', aiModelMiddleware)
   },
 })
 

apps/backend/src/features/upload/imageToFoods.ts

@@ -1,34 +1,31 @@
-import { HumanMessage } from '@langchain/core/messages'
-import type { ChatOpenAI } from '@langchain/openai'
+import { type LanguageModel, generateObject } from 'ai'
 import { z } from 'zod'
 import { fileToBase64 } from '../../utils/fileToBase64'
 
-const ingredientsSchema = z.object({
-  data: z.array(z.string()).describe('食材のリスト'),
-})
-
-export const imageToFoods = async (ai: ChatOpenAI, file: File) => {
+export const imageToFoods = async (model: LanguageModel, file: File) => {
   const imageUrl = await fileToBase64(file)
 
-  const structuredLlm = ai.withStructuredOutput(ingredientsSchema, {
-    name: 'food_detection',
-  })
-
-  const message = new HumanMessage({
-    content: [
+  const { object } = await generateObject({
+    model,
+    messages: [
       {
-        type: 'text',
-        text: `# 以下のステップに従ってください、ステップが全て完了するまで出力は決定しないでください。:
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: `# 以下のステップに従ってください、ステップが全て完了するまで出力は決定しないでください。:
               ## ステップ1: この画像に含まれている食材を日本語かつ短い単語で具体的にリストアップしてください。:
               ## ステップ2: リストアップした食材から以下の基準でフィルタリングしてください：
                  - 料理の材料として直接使用できるものだけを含めます。
                  - 曖昧な表現（例：「冷凍食品」）は避け、具体的な食材名を使用してください。
               ## ステップ3: 食材のリストを提出してください。`,
+          },
+          { type: 'image', image: imageUrl },
+        ],
       },
-      { type: 'image_url', image_url: { url: imageUrl } },
     ],
+    schema: z.object({ data: z.array(z.string()).describe('食材のリスト') }),
   })
 
-  const { data } = await structuredLlm.invoke([message])
-  return data
+  return object.data
 }

apps/backend/src/middlewares/ai.ts → apps/backend/src/middlewares/aiModel.ts

@@ -1,7 +1,7 @@
-import { ChatOpenAI } from '@langchain/openai'
+import { createOpenAI } from '@ai-sdk/openai'
 import { createMiddleware } from 'hono/factory'
 
-export const aiMiddleware = createMiddleware(async (c, next) => {
+export const aiModelMiddleware = createMiddleware(async (c, next) => {
   const { OPENAI_API_KEY, OPENAI_BASE_URL } = c.env
   if (OPENAI_API_KEY === undefined) {
     throw new Error('OPENAI_API_KEY is not defined')
@@ -11,16 +11,14 @@ export const aiMiddleware = createMiddleware(async (c, next) => {
   }
   const requestUrl = new URL(OPENAI_BASE_URL).toString()
 
-  const ai = new ChatOpenAI({
-    model: 'gpt-4o',
+  const openai = createOpenAI({
     apiKey: OPENAI_API_KEY,
-    configuration: {
-      baseURL: requestUrl,
-    },
-    temperature: 0,
+    baseURL: requestUrl,
   })
 
-  c.set('ai', ai)
+  const model = openai('gpt-4o-mini')
+
+  c.set('model', model)
 
   await next()
 })

apps/backend/src/routers/upload.ts

@@ -22,7 +22,7 @@ const uploadRouter = honoFactory
     }
 
     try {
-      const foods = await imageToFoods(c.var.ai, file)
+      const foods = await imageToFoods(c.var.model, file)
       return c.json({ foods }, 200)
     } catch (error) {
       return c.json({ error }, 500)