diff --git a/lib/langchain/assistant/llm/adapters/openai.rb b/lib/langchain/assistant/llm/adapters/openai.rb index 67b734bbb..2d03f76d1 100644 --- a/lib/langchain/assistant/llm/adapters/openai.rb +++ b/lib/langchain/assistant/llm/adapters/openai.rb @@ -34,12 +34,19 @@ def build_chat_params( # Build a OpenAI message # # @param role [String] The role of the message - # @param content [String] The content of the message + # @param content [String | Array] The content of the message # @param image_url [String] The image URL # @param tool_calls [Array] The tool calls # @param tool_call_id [String] The tool call ID # @return [Messages::OpenAIMessage] The OpenAI message def build_message(role:, content: nil, image_url: nil, tool_calls: [], tool_call_id: nil) + if content.is_a?(Array) + content.each do |c| + content = c[:text] if c[:type] == "text" + image_url = c[:image_url][:url] if c[:type] == "image_url" + end + end + Messages::OpenAIMessage.new(role: role, content: content, image_url: image_url, tool_calls: tool_calls, tool_call_id: tool_call_id) end diff --git a/spec/langchain/assistant/llm/adapters/openai_spec.rb b/spec/langchain/assistant/llm/adapters/openai_spec.rb index c56c14d57..c140c9011 100644 --- a/spec/langchain/assistant/llm/adapters/openai_spec.rb +++ b/spec/langchain/assistant/llm/adapters/openai_spec.rb @@ -31,4 +31,33 @@ expect(subject.tool_role).to eq("tool") end end + + describe "#build_message" do + context "when content is a string" do + it "returns the OpenAI message" do + expect( + subject.build_message( + role: "user", + content: "Hello", + image_url: "https://example.com/image.png", + tool_calls: [{"id" => "tool_call_id", "function" => {"name" => "langchain_tool_calculator__execute", "arguments" => "{\"a\": 1, \"b\": 2}"}}], + tool_call_id: "tool_call_id" + ) + ).to have_attributes(role: "user", content: "Hello", image_url: "https://example.com/image.png", tool_calls: [{"id" => "tool_call_id", "function" => {"name" => "langchain_tool_calculator__execute", "arguments" => "{\"a\": 1, \"b\": 2}"}}], tool_call_id: "tool_call_id") + end + end + + context "when content is an array" do + it "returns the OpenAI message" do + expect( + subject.build_message( + role: "user", + content: [{type: "text", text: "Hello"}, {type: "image_url", image_url: {url: "https://example.com/image.png"}}], + tool_calls: [{"id" => "tool_call_id", "function" => {"name" => "langchain_tool_calculator__execute", "arguments" => "{\"a\": 1, \"b\": 2}"}}], + tool_call_id: "tool_call_id" + ) + ).to have_attributes(role: "user", content: "Hello", image_url: "https://example.com/image.png", tool_calls: [{"id" => "tool_call_id", "function" => {"name" => "langchain_tool_calculator__execute", "arguments" => "{\"a\": 1, \"b\": 2}"}}], tool_call_id: "tool_call_id") + end + end + end end