From 8da53cbee0b8522840cecf2a80e163635e8ad830 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Fri, 6 Dec 2024 17:07:04 -0500 Subject: [PATCH 1/3] added documentation about sambanova integration --- docs/blog/posts/open_source.md | 60 ++++++++++++++++++++++ docs/examples/index.md | 35 ++++++------- docs/examples/sambanova.md | 68 +++++++++++++++++++++++++ docs/index.md | 36 +++++++++++++ docs/integrations/index.md | 1 + docs/integrations/sambanova.md | 60 ++++++++++++++++++++++ examples/sambanova/sambanova_example.py | 39 ++++++++++++++ mkdocs.yml | 4 ++ 8 files changed, 286 insertions(+), 17 deletions(-) create mode 100644 docs/examples/sambanova.md create mode 100644 docs/integrations/sambanova.md create mode 100644 examples/sambanova/sambanova_example.py diff --git a/docs/blog/posts/open_source.md b/docs/blog/posts/open_source.md index ec5fee1db..c714d947a 100644 --- a/docs/blog/posts/open_source.md +++ b/docs/blog/posts/open_source.md @@ -182,6 +182,66 @@ print(user) #> name='jason' age=25 ``` +### Sambanova + +[SambaNova Cloud](https://cloud.sambanova.ai/) and [SambaStudio](https://docs.sambanova.ai/sambastudio/latest/sambastudio-intro.html) are the two main Samabanova Systems platforms for using generative and powerful Large Language Models (LLMs) and more! [SambaNova Cloud](https://cloud.sambanova.ai/) is a high-performance inference service that delivers rapid and precise results. Likewise, [SambaStudio](https://docs.sambanova.ai/sambastudio/latest/sambastudio-intro.html) is a rich, GUI-based platform that provides the functionality to train, deploy, and manage models. Check the docs and decide whether SambaNova Cloud or SambaStudio are the best fit for you. Here is the way to use them: + +```bash +export SAMBANOVA_API_KEY="your-sambanova-cloud-api-key" +export SAMBANOVA_URL="sambanova-cloud-url" + +export SAMBASTUDIO_API_KEY="your-sambastudio-api-key" +export SAMBASTUDIO_URL="sambastudio-url" +``` + +```python +import os +from pydantic import BaseModel + +import openai +import instructor + + +# SambaNova Cloud client +sncloud_client = openai.OpenAI( + api_key=os.environ.get("SAMBANOVA_API_KEY"), + base_url=os.getenv("SAMBANOVA_URL"), +) + +# SambaStudio client +sambastudio_client = openai.OpenAI( + api_key=os.environ.get("SAMBASTUDIO_API_KEY"), + base_url=os.getenv("SAMBASTUDIO_URL"), +) + +# By default, the patch function will patch the ChatCompletion.create and ChatCompletion.create methods +# to support the response_model parameter +# You can use either SambaNova Cloud or SambaStudio as your prefered LLM client +client = instructor.from_sambanova(sncloud_client, mode=instructor.Mode.TOOLS) +# client = instructor.from_sambanova(sambastudio_client, mode=instructor.Mode.TOOLS) + + +# Now, we can use the response_model parameter using only a base model +# rather than having to use the OpenAISchema class +class UserExtract(BaseModel): + name: str + age: int + + +user: UserExtract = client.chat.completions.create( + model="llama3-405b", + response_model=UserExtract, + messages=[ + {"role": "user", "content": "Extract jason is 25 years old"}, + ], +) + +assert isinstance(user, UserExtract), "Should be instance of UserExtract" + +print(user) +#> name='jason' age=25 +``` + ### Together AI Together AI, when combined with Instructor, offers a seamless experience for developers looking to leverage structured outputs in their applications. For more details, refer to our [Together AI documentation](../../integrations/together.md) and explore the [patching guide](../../concepts/patching.md) to enhance your applications. diff --git a/docs/examples/index.md b/docs/examples/index.md index 1324cfe09..4b55f4ef0 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -27,23 +27,24 @@ Welcome to our collection of cookbooks showcasing the power of structured output 16. [LLM-Based Document Segmentation](document_segmentation.md): Intelligently divide long documents into meaningful sections. 17. [Cost Optimization with OpenAI's Batch API](batch_job_oai.md): Reduce API costs by processing multiple requests efficiently. 18. [Groq Cloud API Integration](groq.md): Leverage Groq's high-performance AI inference platform. -19. [Mistral and Mixtral Model Usage](mistral.md): Implement state-of-the-art open-source language models in your projects. -20. [Multi-Modal AI with Gemini](multi_modal_gemini.md): Process and analyze text, images, and other data types simultaneously. -21. [IBM watsonx.ai Integration](watsonx.md): Utilize IBM's enterprise AI platform for advanced language processing tasks. -22. [Receipt Information Extraction with GPT-4 Vision](extracting_receipts.md): Extract structured data from receipt images using advanced AI vision capabilities. -23. [Slide Content Extraction with GPT-4 Vision](extract_slides.md): Convert presentation slide images into structured, analyzable text data. -24. [Few-Shot Learning with Examples](examples.md): Improve AI model performance by providing contextual examples in prompts. -25. [Local Classification without API](local_classification.md): Perform text classification tasks locally without relying on external API calls. -26. [Action Items Extraction](action_items.md): Extract structured action items and tasks from text content. -27. [Batch Classification with LangSmith](batch_classification_langsmith.md): Efficiently classify content in batches using LangSmith integration. -28. [Contact Information Extraction](extract_contact_info.md): Extract structured contact details from unstructured text. -29. [Knowledge Graph Building](building_knowledge_graph.md): Create and manipulate knowledge graphs from textual data. -30. [Multiple Classification Tasks](multiple_classification.md): Handle multiple classification categories simultaneously. -31. [Pandas DataFrame Integration](pandas_df.md): Work with structured data using Pandas DataFrames. -32. [Partial Response Streaming](partial_streaming.md): Stream partial results for real-time processing. -33. [Single Classification Tasks](single_classification.md): Implement focused single-category classification. -34. [Table Extraction from Images](tables_from_vision.md): Convert visual tables into structured data formats. -35. [YouTube Clip Analysis](youtube_clips.md): Extract and analyze information from YouTube video clips. +19. [SambaNova Cloud API Integration](sambanova.md): Generate fast, low-cost, and structured AI responses. +20. [Mistral and Mixtral Model Usage](mistral.md): Implement state-of-the-art open-source language models in your projects. +21. [Multi-Modal AI with Gemini](multi_modal_gemini.md): Process and analyze text, images, and other data types simultaneously. +22. [IBM watsonx.ai Integration](watsonx.md): Utilize IBM's enterprise AI platform for advanced language processing tasks. +23. [Receipt Information Extraction with GPT-4 Vision](extracting_receipts.md): Extract structured data from receipt images using advanced AI vision capabilities. +24. [Slide Content Extraction with GPT-4 Vision](extract_slides.md): Convert presentation slide images into structured, analyzable text data. +25. [Few-Shot Learning with Examples](examples.md): Improve AI model performance by providing contextual examples in prompts. +26. [Local Classification without API](local_classification.md): Perform text classification tasks locally without relying on external API calls. +27. [Action Items Extraction](action_items.md): Extract structured action items and tasks from text content. +28. [Batch Classification with LangSmith](batch_classification_langsmith.md): Efficiently classify content in batches using LangSmith integration. +29. [Contact Information Extraction](extract_contact_info.md): Extract structured contact details from unstructured text. +30. [Knowledge Graph Building](building_knowledge_graph.md): Create and manipulate knowledge graphs from textual data. +31. [Multiple Classification Tasks](multiple_classification.md): Handle multiple classification categories simultaneously. +32. [Pandas DataFrame Integration](pandas_df.md): Work with structured data using Pandas DataFrames. +33. [Partial Response Streaming](partial_streaming.md): Stream partial results for real-time processing. +34. [Single Classification Tasks](single_classification.md): Implement focused single-category classification. +35. [Table Extraction from Images](tables_from_vision.md): Convert visual tables into structured data formats. +36. [YouTube Clip Analysis](youtube_clips.md): Extract and analyze information from YouTube video clips. ## Subscribe to our Newsletter for Updates and Tips diff --git a/docs/examples/sambanova.md b/docs/examples/sambanova.md new file mode 100644 index 000000000..b5900400a --- /dev/null +++ b/docs/examples/sambanova.md @@ -0,0 +1,68 @@ +--- +title: Using SambaNova for Inference: Setup and Example +description: Learn how to use SambaNova Cloud for inference with the Llama3-405B model, including API setup and a practical Python example. +--- + +# Structured Outputs using SambaNova Cloud +Instead of using openai or antrophic you can now also use sambanova for inference by using from_sambanova. + +The examples are using llama3-405b model. + +## SambaNova Cloud API +To use SambaNova Cloud you need to obtain a SambaNova Cloud API key. +Goto [SambaNova Cloud](https://cloud.sambanova.ai/) and login. Select APIs from the left menu and then create an account. After that, go again to APIs and generate your API key. + +## Use example +Some pip packages need to be installed to use the example: +``` +pip install instructor pydantic openai +``` +You need to export the SambaNova Cloud API key and URL: +``` +export SAMBANOVA_API_KEY="your-sambanova-cloud-api-key" +export SAMBANOVA_URL="sambanova-cloud-url" +``` + +An example: +```python +import os +from pydantic import BaseModel, Field +from typing import List +from openai import OpenAI +import instructor + + +class Character(BaseModel): + name: str + fact: List[str] = Field(..., description="A list of facts about the subject") + + +client = OpenAI( + api_key=os.getenv("SAMBANOVA_API_KEY"), + base_url=os.getenv("SAMBANOVA_URL"), +) + +client = instructor.from_sambanova(client, mode=instructor.Mode.TOOLS) + +resp = client.chat.completions.create( + model="llama3-405b", + messages=[ + { + "role": "user", + "content": "Tell me about the company SambaNova", + } + ], + response_model=Character, +) +print(resp.model_dump_json(indent=2)) +""" +{ + "name": "SambaNova", + "fact": [ + "SambaNova is a company that specializes in artificial intelligence and machine learning.", + "They are known for their work in natural language processing and computer vision.", + "SambaNova has received significant funding from investors and has partnered with several major companies to develop and implement their technology." + ] +} +""" +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 8357a03d0..8f1123cd4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -325,6 +325,42 @@ If you want to see all the integrations, check out the [integrations guide](./in [See more :material-arrow-right:](./integrations/groq.md){: .md-button } +=== "Sambanova" + ```bash + pip install "instructor[sambanova]" + + export SAMBANOVA_API_KEY="your-sambanova-cloud-api-key" + export SAMBANOVA_URL="sambanova-cloud-url" + ``` + + ```python + import instructor + from openai import OpenAI + from pydantic import BaseModel + + client = instructor.from_sambanova( + OpenAI( + api_key=os.environ.get("SAMBANOVA_API_KEY"), + base_url=os.getenv("SAMBANOVA_URL") + ) + ) + + class ExtractUser(BaseModel): + name: str + age: int + + resp = client.chat.completions.create( + model="llama3-405b", + response_model=ExtractUser, + messages=[{"role": "user", "content": "Extract Jason is 25 years old."}], + ) + + assert resp.name == "Jason" + assert resp.age == 25 + ``` + + [See more :material-arrow-right:](./integrations/sambanova.md){: .md-button } + === "Litellm" ```bash pip install "instructor[litellm]" diff --git a/docs/integrations/index.md b/docs/integrations/index.md index ff4336505..27e54a8e2 100644 --- a/docs/integrations/index.md +++ b/docs/integrations/index.md @@ -25,6 +25,7 @@ Instructor supports a wide range of AI model providers, each with their own capa - [Vertex AI](./vertex.md) - Google Cloud's AI platform - [Cohere](./cohere.md) - Command-R and other Cohere models - [Groq](./groq.md) - High-performance inference platform +- [SambaNova](./sambanova.md) - Fast, low-cost, and structured AI responses. - [Mistral](./mistral.md) - Mistral's hosted models - [Fireworks](./fireworks.md) - High-performance model inference - [Cerebras](./cerebras.md) - Llama-3-70B and other Open Source Models at blazing fast inference speeds diff --git a/docs/integrations/sambanova.md b/docs/integrations/sambanova.md new file mode 100644 index 000000000..e50777096 --- /dev/null +++ b/docs/integrations/sambanova.md @@ -0,0 +1,60 @@ +--- +title: Structured Outputs with SambaNova AI and Pydantic +description: Learn how to use SambaNova Cloud for structured outputs with Pydantic in Python and enhance API interactions. +--- + +# Structured Outputs with SambaNova Cloud + +This guide demonstrates how to use SambaNova Cloud with Instructor to generate structured outputs. You'll learn how to use SambaNova Cloud's LLM models to create type-safe responses. + +you'll need to sign up for an account and get an API key. You can do that [here](https://cloud.sambanova.ai/). + +```bash +export SAMBANOVA_API_KEY="your-sambanova-cloud-api-key" +export SAMBANOVA_URL="sambanova-cloud-url" +pip install "instructor openai pydantic" +``` + +## SambaNova Cloud + +SambaNova Cloud supports structured outputs with multiple models like the big `llama3-405b` model. + +### Sync Example + +```python +import os +from openai import OpenAI +import instructor +from pydantic import BaseModel + +# Initialize with API key +client = OpenAI( + api_key=os.environ.get("SAMBANOVA_API_KEY"), + base_url=os.getenv("SAMBANOVA_URL"), +) + +# Enable instructor patches for proxy OpenAI client +client = instructor.from_sambanova(client) + + +class User(BaseModel): + name: str + age: int + + +# Create structured output +user = client.chat.completions.create( + model="llama3-405b", + messages=[ + {"role": "user", "content": "Extract: Jason is 25 years old"}, + ], + response_model=User, +) + +print(user) +# > User(name='Jason', age=25) +``` + +### Async Example + +Not supported yet. Available soon! diff --git a/examples/sambanova/sambanova_example.py b/examples/sambanova/sambanova_example.py new file mode 100644 index 000000000..5e2a4e17b --- /dev/null +++ b/examples/sambanova/sambanova_example.py @@ -0,0 +1,39 @@ +import os +from pydantic import BaseModel, Field +from openai import OpenAI +import instructor + + +class Character(BaseModel): + name: str + fact: list[str] = Field(..., description="A list of facts about the subject") + + +client = OpenAI( + api_key=os.environ.get("SAMBANOVA_API_KEY"), + base_url=os.getenv("SAMBANOVA_URL"), +) + +client = instructor.from_sambanova(client, mode=instructor.Mode.TOOLS) + +resp = client.chat.completions.create( + model="llama3-405b", + messages=[ + { + "role": "user", + "content": "Tell me about the company SambaNova", + } + ], + response_model=Character, +) +print(resp.model_dump_json(indent=2)) +""" +{ + "name": "SambaNova", + "fact": [ + "SambaNova is a company that specializes in artificial intelligence and machine learning.", + "They are known for their work in natural language processing and computer vision.", + "SambaNova has received significant funding from investors and has partnered with several major companies to develop and implement their technology." + ] +} +""" diff --git a/mkdocs.yml b/mkdocs.yml index 75d5251e3..249bb25a8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -163,6 +163,7 @@ nav: - "Structured Output with watsonx.ai": 'examples/watsonx.md' - "OpenAI Batch Jobs with Instructor": 'examples/batch_job_oai.md' - "Structured Outputs with Groq": 'examples/groq.md' + - "Structured Outputs with Sambanova": 'examples/sambanova.md' - "Structured Outputs with Mistral": 'examples/mistral.md' - "Action Items Extraction": 'examples/action_items.md' - "Batch Classification with LangSmith": 'examples/batch_classification_langsmith.md' @@ -217,6 +218,7 @@ nav: - Fireworks: 'integrations/fireworks.md' - Gemini: 'integrations/google.md' - Groq: 'integrations/groq.md' + - Sambanova: 'integrations/sambanova.md' - LiteLLM: 'integrations/litellm.md' - llama-cpp-python: 'integrations/llama-cpp-python.md' - Mistral: 'integrations/mistral.md' @@ -317,6 +319,7 @@ plugins: hub/fireworks.md: integrations/fireworks.md hub/google.md: integrations/google.md hub/groq.md: integrations/groq.md + hub/sambanova.md: integrations/sambanova.md hub/litellm.md: integrations/litellm.md hub/mistral.md: integrations/mistral.md hub/openai.md: integrations/openai.md @@ -334,6 +337,7 @@ plugins: 'hub/clients/cohere.md': 'integrations/cohere.md' 'hub/clients/fireworks.md': 'integrations/fireworks.md' 'hub/clients/groq.md': 'integrations/groq.md' + 'hub/clients/sambanova.md': 'integrations/sambanova.md' 'hub/clients/mistral.md': 'integrations/mistral.md' 'hub/clients/openai.md': 'integrations/openai.md' 'hub/clients/together.md': 'integrations/together.md' From fab42696a64a89d65c2a2a3bb39564152cf9dc37 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Fri, 6 Dec 2024 17:09:23 -0500 Subject: [PATCH 2/3] added sambanova integration --- instructor/__init__.py | 8 +++++++- instructor/client_sambanova.py | 35 ++++++++++++++++++++++++++++++++++ instructor/utils.py | 1 + 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 instructor/client_sambanova.py diff --git a/instructor/__init__.py b/instructor/__init__.py index efd503c22..f88d0532d 100644 --- a/instructor/__init__.py +++ b/instructor/__init__.py @@ -95,4 +95,10 @@ if importlib.util.find_spec("writerai") is not None: from .client_writer import from_writer - __all__ += ["from_writer"] \ No newline at end of file + __all__ += ["from_writer"] + +if importlib.util.find_spec("openai") is not None: + print('sambanova') + from .client_sambanova import from_sambanova + + __all__ += ["from_sambanova"] \ No newline at end of file diff --git a/instructor/client_sambanova.py b/instructor/client_sambanova.py new file mode 100644 index 000000000..e89268e07 --- /dev/null +++ b/instructor/client_sambanova.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import overload, Any + +import openai +import instructor + + +@overload +def from_sambanova( + client: openai.OpenAI, + mode: instructor.Mode = instructor.Mode.TOOLS, + **kwargs: Any, +) -> instructor.Instructor: ... + +def from_sambanova( + client: openai.OpenAI, + mode: instructor.Mode = instructor.Mode.TOOLS, + **kwargs: Any, +) -> instructor.Instructor: + assert mode in { + instructor.Mode.TOOLS, + }, "Mode be one of { instructor.Mode.TOOLS }" + + assert isinstance( + client, (openai.OpenAI) + ), "Client must be an instance of openai.OpenAI" + + return instructor.Instructor( + client=client, + create=instructor.patch(create=client.chat.completions.create, mode=mode), + provider=instructor.Provider.SAMBANOVA, + mode=mode, + **kwargs, + ) diff --git a/instructor/utils.py b/instructor/utils.py index 55d746760..6672416e9 100644 --- a/instructor/utils.py +++ b/instructor/utils.py @@ -47,6 +47,7 @@ class Provider(Enum): ANYSCALE = "anyscale" TOGETHER = "together" GROQ = "groq" + SAMBANOVA = "sambanova" MISTRAL = "mistral" COHERE = "cohere" GEMINI = "gemini" From b4eb16a3205d09162e496974162a82137c94c6b1 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Fri, 6 Dec 2024 17:11:03 -0500 Subject: [PATCH 3/3] added patch to support arguments in dict format --- instructor/function_calls.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/instructor/function_calls.py b/instructor/function_calls.py index 8507c2cd6..2e840ef9c 100644 --- a/instructor/function_calls.py +++ b/instructor/function_calls.py @@ -365,6 +365,10 @@ def parse_tools( assert ( tool_call.function.name == cls.openai_schema["name"] # type: ignore[index] ), "Tool name does not match" + + if isinstance(tool_call.function.arguments, dict): + tool_call.function.arguments = json.dumps(tool_call.function.arguments) + return cls.model_validate_json( tool_call.function.arguments, # type: ignore context=validation_context,