From 3a65f1a0c290946c27751f056a2295c764187ec9 Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Thu, 20 Mar 2025 15:46:03 +0000 Subject: [PATCH 1/9] Create mistral generator and mistralai module --- garak/generators/mistral.py | 32 ++++ requirements.txt | 305 ++++++++++++++++++++++++++++++------ 2 files changed, 292 insertions(+), 45 deletions(-) create mode 100644 garak/generators/mistral.py diff --git a/garak/generators/mistral.py b/garak/generators/mistral.py new file mode 100644 index 00000000..8e4115b9 --- /dev/null +++ b/garak/generators/mistral.py @@ -0,0 +1,32 @@ +DEFAULT_CLASS = "MistralGenerator" +import os +from garak.generators.base import Generator +import garak._config as _config +from mistralai import Mistral + + +class MistralGenerator(Generator): + generator_family_name = "mistral" + fullname = "Mistral AI" + supports_multiple_generations = False + + def __init__(self, name="mistral-large-latest", config_root=_config): + self.name = name + api_key = os.getenv("MISTRAL_API_KEY") + if not api_key: + raise ValueError( + "Please define mistral api key by defining environment variable MISTRAL_API_KEY" + ) + self.client = Mistral(api_key=api_key) + + def _call_model(self, prompt, generations_this_call=1): + chat_response = self.client.chat.complete( + model=self.name, + messages=[ + { + "role": "user", + "content": prompt, + }, + ], + ) + return [chat_response.choices[0].message.content] diff --git a/requirements.txt b/requirements.txt index e91cf024..6e681482 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,50 +1,265 @@ -base2048>=0.1.3 -transformers>=4.43.0 -datasets>=2.14.6,<2.17 -colorama>=0.4.3 -tqdm>=4.64.0 -cohere>=4.5.1,<5 -openai>=1.45.0,<2 -replicate>=0.8.3 -google-api-python-client>=2.0 -backoff>=2.1.1 -rapidfuzz>=3.0.0 -jinja2>=3.1.2 -nltk>=3.8.1 -accelerate>=0.23.0 +accelerate==1.5.2 +aiohappyeyeballs==2.6.1 +aiohttp==3.11.14 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +astroid==3.3.9 +asttokens==3.0.0 +async-lru==2.0.4 +attrs==25.3.0 avidtools==0.1.2 -stdlibs>=2022.10.9 -langchain>=0.0.300 -nemollm>=0.3.0 -octoai-sdk>=0.8.0 +babel==2.16.0 +backoff==2.2.1 +base2048==0.1.3 +beautifulsoup4==4.12.3 +black==24.4.2 +bleach==6.2.0 +boto3==1.37.16 +botocore==1.37.16 +cachetools==5.5.2 +certifi==2024.8.30 +cffi==1.17.1 +charset-normalizer==3.4.1 +chevron==0.14.0 +click==8.1.8 cmd2==2.4.3 -torch>=2.1.3 -sentencepiece>=0.1.99 -markdown>=3.4.3 -numpy>=1.26.1 -zalgolib>=0.2.2 -ecoji>=0.1.1 +cohere==4.57 +colorama==0.4.6 +comm==0.2.2 +contourpy==1.3.1 +coverage==7.7.0 +cycler==0.12.1 +datasets==2.16.1 +DateTime==5.5 +debugpy==1.8.9 +decorator==5.1.1 deepl==1.17.0 -fschat>=0.2.36 -litellm>=1.41.21 -jsonpath-ng>=1.6.1 -huggingface_hub>=0.21.0 -python-magic-bin>=0.4.14; sys_platform == "win32" -python-magic>=0.4.21; sys_platform != "win32" +defusedxml==0.7.1 +dill==0.3.7 +distro==1.9.0 +ecoji==0.1.1 +eval_type_backport==0.2.2 +executing==2.1.0 +fastapi==0.115.11 +fastavro==1.10.0 +fastjsonschema==2.21.1 +filelock==3.18.0 +fonttools==4.55.3 +fqdn==1.5.1 +frozenlist==1.5.0 +fschat==0.2.36 +fsspec==2023.10.0 +-e git+https://github.com/dimensi0n/garak@cda3990bf8b6652a9a97bc1878b3f26e2c559084#egg=garak +gitdb==4.0.11 +GitPython==3.1.43 +google-api-core==2.24.2 +google-api-python-client==2.165.0 +google-auth==2.38.0 +google-auth-httplib2==0.2.0 +googleapis-common-protos==1.69.2 +greenlet==3.1.1 +h11==0.14.0 +httpcore==1.0.7 +httplib2==0.22.0 +httpx==0.28.1 +huggingface-hub==0.29.3 +idna==3.10 +importlib-metadata==6.11.0 +iniconfig==2.1.0 +ipykernel==6.29.5 +ipython==8.30.0 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +jiter==0.9.0 +jmespath==1.0.1 +joblib==1.4.2 +json5==0.10.0 +jsonpatch==1.33 +jsonpath-ng==1.7.0 +jsonpath-python==1.0.6 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +jupyter-events==0.10.0 +jupyter-lsp==2.2.5 +jupyter-server-mathjax==0.2.6 +jupyter_client==8.6.3 +jupyter_core==5.7.2 +jupyter_server==2.14.2 +jupyter_server_terminals==0.5.3 +jupyterlab==4.3.3 +jupyterlab_git==0.50.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +kiwisolver==1.4.7 +langchain==0.3.21 +langchain-core==0.3.46 +langchain-text-splitters==0.3.7 +langsmith==0.3.18 +latex2mathml==3.77.0 +litellm==1.63.12 lorem==0.1.1 -xdg-base-dirs>=6.0.1 -wn==0.9.5 -ollama>=0.4.7 -tiktoken>=0.7.0 -# tests -pytest>=8.0 -pytest-mock>=3.14.0 +Markdown==3.7 +markdown-it-py==3.0.0 +markdown2==2.5.3 +MarkupSafe==3.0.2 +matplotlib==3.9.3 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistralai==1.5.2 +mistune==3.0.2 +mpmath==1.3.0 +multidict==6.2.0 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +nbclient==0.10.1 +nbconvert==7.16.4 +nbdime==4.0.2 +nbformat==5.10.4 +nemollm==0.3.5 +nest-asyncio==1.6.0 +networkx==3.4.2 +nh3==0.2.21 +nltk==3.9.1 +notebook_shim==0.2.4 +numpy==1.26.4 +nvdlib==0.8.0 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +octoai-sdk==0.10.1 +ollama==0.4.7 +openai==1.67.0 +orjson==3.10.15 +overrides==7.7.0 +packaging==24.2 +pandas==2.2.3 +pandocfilters==1.5.1 +parso==0.8.4 +pathspec==0.12.1 +pexpect==4.9.0 +pillow==10.4.0 +platformdirs==4.3.7 +plotly==5.24.1 +pluggy==1.5.0 +ply==3.11 +prometheus_client==0.21.1 +prompt_toolkit==3.0.50 +propcache==0.3.0 +proto-plus==1.26.1 +protobuf==6.30.1 +psutil==7.0.0 +ptyprocess==0.7.0 +pure_eval==0.2.3 +pyarrow==19.0.1 +pyarrow-hotfix==0.6 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 +pycparser==2.22 +pydantic==2.10.6 +pydantic_core==2.27.2 +Pygments==2.19.1 +pylint==3.3.6 +pyparsing==3.2.1 +pyperclip==1.9.0 +pytest==8.3.5 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest_httpserver==1.1.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +python-json-logger==3.2.0 +python-magic==0.4.27 +python-multipart==0.0.20 +pytz==2025.1 +PyYAML==6.0.2 +pyzmq==26.2.0 +RapidFuzz==3.12.2 +referencing==0.36.2 +regex==2024.11.6 +replicate==1.0.4 +requests==2.32.3 +requests-futures==1.0.2 requests-mock==1.12.1 -respx>=0.21.1 -pytest-cov>=5.0.0 -pytest_httpserver>=1.1.0 -# lint -black==24.4.2 -pylint>=3.1.0 -# calibration -scipy>=1.14.0 +requests-toolbelt==1.0.0 +respx==0.22.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==13.9.4 +rpds-py==0.23.1 +rsa==4.9 +s3transfer==0.11.4 +safetensors==0.5.3 +scikit-learn==1.6.0 +scipy==1.15.2 +seaborn==0.13.2 +Send2Trash==1.8.3 +sentencepiece==0.2.0 +setuptools==75.6.0 +shortuuid==1.0.13 +six==1.17.0 +smmap==5.0.1 +sniffio==1.3.1 +soundfile==0.13.1 +soupsieve==2.6 +SQLAlchemy==2.0.39 +stack-data==0.6.3 +starlette==0.46.1 +stdlibs==2024.12.3 +svgwrite==1.4.3 +sympy==1.13.1 +tenacity==9.0.0 +terminado==0.18.1 +threadpoolctl==3.5.0 +tiktoken==0.9.0 +tinycss2==1.4.0 +tokenizers==0.21.1 +tomli==2.2.1 +tomlkit==0.13.2 +torch==2.6.0 +tornado==6.4.2 +tqdm==4.67.1 +traitlets==5.14.3 +transformers==4.49.0 +triton==3.2.0 +types-python-dateutil==2.9.0.20241206 +types-PyYAML==6.0.12.20241230 +types-requests==2.32.0.20250306 +typing-inspect==0.9.0 +typing_extensions==4.12.2 +tzdata==2025.1 +uri-template==1.3.0 +uritemplate==4.1.1 +urllib3==2.3.0 +uvicorn==0.34.0 +wavedrom==2.0.3.post3 +wcwidth==0.2.13 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wn==0.9.5 +xdg-base-dirs==6.0.2 +xxhash==3.5.0 +yarl==1.18.3 +zalgolib==0.2.2 +zipp==3.21.0 +zope.interface==7.2 +zstandard==0.23.0 From 0f971598a294b551e0859d0774dccc1d8967366d Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Sat, 22 Mar 2025 22:16:06 +0000 Subject: [PATCH 2/9] Clean requirements.txt because of pip freeze adding subpackages --- requirements.txt | 304 +++++++---------------------------------------- 1 file changed, 45 insertions(+), 259 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6e681482..551ce56b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,265 +1,51 @@ -accelerate==1.5.2 -aiohappyeyeballs==2.6.1 -aiohttp==3.11.14 -aiosignal==1.3.2 -annotated-types==0.7.0 -anyio==4.9.0 -argon2-cffi==23.1.0 -argon2-cffi-bindings==21.2.0 -arrow==1.3.0 -astroid==3.3.9 -asttokens==3.0.0 -async-lru==2.0.4 -attrs==25.3.0 +base2048>=0.1.3 +transformers>=4.43.0 +datasets>=2.14.6,<2.17 +colorama>=0.4.3 +tqdm>=4.64.0 +cohere>=4.5.1,<5 +openai>=1.45.0,<2 +replicate>=0.8.3 +google-api-python-client>=2.0 +backoff>=2.1.1 +rapidfuzz>=3.0.0 +jinja2>=3.1.2 +nltk>=3.8.1 +accelerate>=0.23.0 avidtools==0.1.2 -babel==2.16.0 -backoff==2.2.1 -base2048==0.1.3 -beautifulsoup4==4.12.3 -black==24.4.2 -bleach==6.2.0 -boto3==1.37.16 -botocore==1.37.16 -cachetools==5.5.2 -certifi==2024.8.30 -cffi==1.17.1 -charset-normalizer==3.4.1 -chevron==0.14.0 -click==8.1.8 +stdlibs>=2022.10.9 +langchain>=0.0.300 +nemollm>=0.3.0 +octoai-sdk>=0.8.0 cmd2==2.4.3 -cohere==4.57 -colorama==0.4.6 -comm==0.2.2 -contourpy==1.3.1 -coverage==7.7.0 -cycler==0.12.1 -datasets==2.16.1 -DateTime==5.5 -debugpy==1.8.9 -decorator==5.1.1 +torch>=2.1.3 +sentencepiece>=0.1.99 +markdown>=3.4.3 +numpy>=1.26.1 +zalgolib>=0.2.2 +ecoji>=0.1.1 deepl==1.17.0 -defusedxml==0.7.1 -dill==0.3.7 -distro==1.9.0 -ecoji==0.1.1 -eval_type_backport==0.2.2 -executing==2.1.0 -fastapi==0.115.11 -fastavro==1.10.0 -fastjsonschema==2.21.1 -filelock==3.18.0 -fonttools==4.55.3 -fqdn==1.5.1 -frozenlist==1.5.0 -fschat==0.2.36 -fsspec==2023.10.0 --e git+https://github.com/dimensi0n/garak@cda3990bf8b6652a9a97bc1878b3f26e2c559084#egg=garak -gitdb==4.0.11 -GitPython==3.1.43 -google-api-core==2.24.2 -google-api-python-client==2.165.0 -google-auth==2.38.0 -google-auth-httplib2==0.2.0 -googleapis-common-protos==1.69.2 -greenlet==3.1.1 -h11==0.14.0 -httpcore==1.0.7 -httplib2==0.22.0 -httpx==0.28.1 -huggingface-hub==0.29.3 -idna==3.10 -importlib-metadata==6.11.0 -iniconfig==2.1.0 -ipykernel==6.29.5 -ipython==8.30.0 -isoduration==20.11.0 -isort==6.0.1 -jedi==0.19.2 -Jinja2==3.1.6 -jiter==0.9.0 -jmespath==1.0.1 -joblib==1.4.2 -json5==0.10.0 -jsonpatch==1.33 -jsonpath-ng==1.7.0 -jsonpath-python==1.0.6 -jsonpointer==3.0.0 -jsonschema==4.23.0 -jsonschema-specifications==2024.10.1 -jupyter-events==0.10.0 -jupyter-lsp==2.2.5 -jupyter-server-mathjax==0.2.6 -jupyter_client==8.6.3 -jupyter_core==5.7.2 -jupyter_server==2.14.2 -jupyter_server_terminals==0.5.3 -jupyterlab==4.3.3 -jupyterlab_git==0.50.2 -jupyterlab_pygments==0.3.0 -jupyterlab_server==2.27.3 -kiwisolver==1.4.7 -langchain==0.3.21 -langchain-core==0.3.46 -langchain-text-splitters==0.3.7 -langsmith==0.3.18 -latex2mathml==3.77.0 -litellm==1.63.12 +fschat>=0.2.36 +litellm>=1.41.21 +jsonpath-ng>=1.6.1 +huggingface_hub>=0.21.0 +python-magic-bin>=0.4.14; sys_platform == "win32" +python-magic>=0.4.21; sys_platform != "win32" lorem==0.1.1 -Markdown==3.7 -markdown-it-py==3.0.0 -markdown2==2.5.3 -MarkupSafe==3.0.2 -matplotlib==3.9.3 -matplotlib-inline==0.1.7 -mccabe==0.7.0 -mdurl==0.1.2 +xdg-base-dirs>=6.0.1 +wn==0.9.5 +ollama>=0.4.7 +tiktoken>=0.7.0 mistralai==1.5.2 -mistune==3.0.2 -mpmath==1.3.0 -multidict==6.2.0 -multiprocess==0.70.15 -mypy-extensions==1.0.0 -nbclient==0.10.1 -nbconvert==7.16.4 -nbdime==4.0.2 -nbformat==5.10.4 -nemollm==0.3.5 -nest-asyncio==1.6.0 -networkx==3.4.2 -nh3==0.2.21 -nltk==3.9.1 -notebook_shim==0.2.4 -numpy==1.26.4 -nvdlib==0.8.0 -nvidia-cublas-cu12==12.4.5.8 -nvidia-cuda-cupti-cu12==12.4.127 -nvidia-cuda-nvrtc-cu12==12.4.127 -nvidia-cuda-runtime-cu12==12.4.127 -nvidia-cudnn-cu12==9.1.0.70 -nvidia-cufft-cu12==11.2.1.3 -nvidia-curand-cu12==10.3.5.147 -nvidia-cusolver-cu12==11.6.1.9 -nvidia-cusparse-cu12==12.3.1.170 -nvidia-cusparselt-cu12==0.6.2 -nvidia-nccl-cu12==2.21.5 -nvidia-nvjitlink-cu12==12.4.127 -nvidia-nvtx-cu12==12.4.127 -octoai-sdk==0.10.1 -ollama==0.4.7 -openai==1.67.0 -orjson==3.10.15 -overrides==7.7.0 -packaging==24.2 -pandas==2.2.3 -pandocfilters==1.5.1 -parso==0.8.4 -pathspec==0.12.1 -pexpect==4.9.0 -pillow==10.4.0 -platformdirs==4.3.7 -plotly==5.24.1 -pluggy==1.5.0 -ply==3.11 -prometheus_client==0.21.1 -prompt_toolkit==3.0.50 -propcache==0.3.0 -proto-plus==1.26.1 -protobuf==6.30.1 -psutil==7.0.0 -ptyprocess==0.7.0 -pure_eval==0.2.3 -pyarrow==19.0.1 -pyarrow-hotfix==0.6 -pyasn1==0.6.1 -pyasn1_modules==0.4.1 -pycparser==2.22 -pydantic==2.10.6 -pydantic_core==2.27.2 -Pygments==2.19.1 -pylint==3.3.6 -pyparsing==3.2.1 -pyperclip==1.9.0 -pytest==8.3.5 -pytest-cov==6.0.0 -pytest-mock==3.14.0 -pytest_httpserver==1.1.2 -python-dateutil==2.9.0.post0 -python-dotenv==1.0.1 -python-json-logger==3.2.0 -python-magic==0.4.27 -python-multipart==0.0.20 -pytz==2025.1 -PyYAML==6.0.2 -pyzmq==26.2.0 -RapidFuzz==3.12.2 -referencing==0.36.2 -regex==2024.11.6 -replicate==1.0.4 -requests==2.32.3 -requests-futures==1.0.2 +# tests +pytest>=8.0 +pytest-mock>=3.14.0 requests-mock==1.12.1 -requests-toolbelt==1.0.0 -respx==0.22.0 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rich==13.9.4 -rpds-py==0.23.1 -rsa==4.9 -s3transfer==0.11.4 -safetensors==0.5.3 -scikit-learn==1.6.0 -scipy==1.15.2 -seaborn==0.13.2 -Send2Trash==1.8.3 -sentencepiece==0.2.0 -setuptools==75.6.0 -shortuuid==1.0.13 -six==1.17.0 -smmap==5.0.1 -sniffio==1.3.1 -soundfile==0.13.1 -soupsieve==2.6 -SQLAlchemy==2.0.39 -stack-data==0.6.3 -starlette==0.46.1 -stdlibs==2024.12.3 -svgwrite==1.4.3 -sympy==1.13.1 -tenacity==9.0.0 -terminado==0.18.1 -threadpoolctl==3.5.0 -tiktoken==0.9.0 -tinycss2==1.4.0 -tokenizers==0.21.1 -tomli==2.2.1 -tomlkit==0.13.2 -torch==2.6.0 -tornado==6.4.2 -tqdm==4.67.1 -traitlets==5.14.3 -transformers==4.49.0 -triton==3.2.0 -types-python-dateutil==2.9.0.20241206 -types-PyYAML==6.0.12.20241230 -types-requests==2.32.0.20250306 -typing-inspect==0.9.0 -typing_extensions==4.12.2 -tzdata==2025.1 -uri-template==1.3.0 -uritemplate==4.1.1 -urllib3==2.3.0 -uvicorn==0.34.0 -wavedrom==2.0.3.post3 -wcwidth==0.2.13 -webcolors==24.11.1 -webencodings==0.5.1 -websocket-client==1.8.0 -Werkzeug==3.1.3 -wn==0.9.5 -xdg-base-dirs==6.0.2 -xxhash==3.5.0 -yarl==1.18.3 -zalgolib==0.2.2 -zipp==3.21.0 -zope.interface==7.2 -zstandard==0.23.0 +respx>=0.21.1 +pytest-cov>=5.0.0 +pytest_httpserver>=1.1.0 +# lint +black==24.4.2 +pylint>=3.1.0 +# calibration +scipy>=1.14.0 From ce9fc38b4c5138559f999acda652bd5a42d46311 Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Mon, 24 Mar 2025 15:06:07 +0100 Subject: [PATCH 3/9] Define ENV_VAR and DEFAULT_PARAMS before __init__ Co-authored-by: Jeffrey Martin Signed-off-by: Erwan ROUSSEL --- garak/generators/mistral.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/garak/generators/mistral.py b/garak/generators/mistral.py index 8e4115b9..86462131 100644 --- a/garak/generators/mistral.py +++ b/garak/generators/mistral.py @@ -9,15 +9,31 @@ class MistralGenerator(Generator): generator_family_name = "mistral" fullname = "Mistral AI" supports_multiple_generations = False + ENV_VAR="MISTRAL_API_KEY"` + DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | { + "name": "mistral-large-latest", + } - def __init__(self, name="mistral-large-latest", config_root=_config): - self.name = name - api_key = os.getenv("MISTRAL_API_KEY") - if not api_key: - raise ValueError( - "Please define mistral api key by defining environment variable MISTRAL_API_KEY" - ) - self.client = Mistral(api_key=api_key) + # avoid attempt to pickle the client attribute + def __getstate__(self) -> object: + self._clear_client() + return dict(self.__dict__) + + # restore the client attribute + def __setstate__(self, d) -> object: + self.__dict__.update(d) + self._load_client() + + def _load_client(self): + self.client = Mistral(api_key=self.api_key) + + def _clear_client(self): + self.client = None + + + def __init__(self, name="", **config_root=_config):** + super().__init__(config_root=config_root) + self._load_client() def _call_model(self, prompt, generations_this_call=1): chat_response = self.client.chat.complete( From bfd57097c788daa58d56891d6cc3a0f8b12d6794 Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Mon, 24 Mar 2025 15:02:31 +0000 Subject: [PATCH 4/9] Add backoff for rate limit exceptions and fix typo --- garak/generators/mistral.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/garak/generators/mistral.py b/garak/generators/mistral.py index 86462131..e0bbadf1 100644 --- a/garak/generators/mistral.py +++ b/garak/generators/mistral.py @@ -1,16 +1,23 @@ DEFAULT_CLASS = "MistralGenerator" import os +import backoff from garak.generators.base import Generator import garak._config as _config from mistralai import Mistral +from garak import exception class MistralGenerator(Generator): + """ + Interface for public endpoints of models hosted in Mistral La Plateforme (console.mistral.ai). + Expects API key in MISTRAL_API_TOKEN environment variable. + """ + generator_family_name = "mistral" fullname = "Mistral AI" supports_multiple_generations = False - ENV_VAR="MISTRAL_API_KEY"` - DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | { + ENV_VAR = "MISTRAL_API_KEY" + DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | { "name": "mistral-large-latest", } @@ -30,12 +37,16 @@ def _load_client(self): def _clear_client(self): self.client = None - - def __init__(self, name="", **config_root=_config):** - super().__init__(config_root=config_root) + def __init__(self, name="", config_root=_config): + super().__init__(name, config_root) + if self.api_key is not None: + # ensure the token is in the expected runtime env var + os.environ[self.ENV_VAR] = self.api_key self._load_client() + @backoff.on_exception(backoff.fibo, exception.RateLimitHit, max_value=70) def _call_model(self, prompt, generations_this_call=1): + print(self.name) chat_response = self.client.chat.complete( model=self.name, messages=[ From 7d4c23371ce0fe05eae505ef0de3e9de957e6779 Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Mon, 24 Mar 2025 16:30:50 +0000 Subject: [PATCH 5/9] Add mockup test for mistral generator --- tests/generators/test_mistral.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/generators/test_mistral.py diff --git a/tests/generators/test_mistral.py b/tests/generators/test_mistral.py new file mode 100644 index 00000000..4f572fe5 --- /dev/null +++ b/tests/generators/test_mistral.py @@ -0,0 +1,30 @@ +import os +import pytest +from garak.generators.mistral import MistralGenerator + +DEFAULT_DEPLOYMENT_NAME = "mistral-small-latest" + +@pytest.fixture +def set_fake_env(request) -> None: + stored_env = os.getenv(MistralGenerator.ENV_VAR, None) + + def restore_env(): + if stored_env is not None: + os.environ[MistralGenerator.ENV_VAR] = stored_env + else: + del os.environ[MistralGenerator.ENV_VAR] + + os.environ[MistralGenerator.ENV_VAR] = os.path.abspath(__file__) + + request.addfinalizer(restore_env) + +@pytest.mark.skipif( + os.getenv(MistralGenerator.ENV_VAR, None) is None, + reason=f"OpenAI API key is not set in {MistralGenerator.ENV_VAR}", +) +def test_mistral_chat(): + generator = MistralGenerator(name=DEFAULT_DEPLOYMENT_NAME) + assert generator.name == DEFAULT_DEPLOYMENT_NAME + output = generator.generate("Hello Mistral!") + assert len(output) == 1 # expect 1 generation by default + print("test passed!") From 7f3cddab753262b8d7615837cf946beabaa2ab3c Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Mon, 24 Mar 2025 17:06:18 +0000 Subject: [PATCH 6/9] Add empty doc file, remove unused test code and add mistralai to pyproject --- docs/source/garak.generators.mistral.rst | 6 ++++++ docs/source/generators.rst | 1 + pyproject.toml | 3 ++- tests/generators/test_mistral.py | 16 +--------------- 4 files changed, 10 insertions(+), 16 deletions(-) create mode 100644 docs/source/garak.generators.mistral.rst diff --git a/docs/source/garak.generators.mistral.rst b/docs/source/garak.generators.mistral.rst new file mode 100644 index 00000000..ff4559a2 --- /dev/null +++ b/docs/source/garak.generators.mistral.rst @@ -0,0 +1,6 @@ +garak.generators.mistral + +.. automodule:: garak.generators.mistral + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/generators.rst b/docs/source/generators.rst index f74d9153..51952670 100644 --- a/docs/source/generators.rst +++ b/docs/source/generators.rst @@ -20,6 +20,7 @@ For a detailed oversight into how a generator operates, see :ref:`garak.generato garak.generators.langchain garak.generators.langchain_serve garak.generators.litellm + garak.generators.mistral garak.generators.octo garak.generators.ollama garak.generators.openai diff --git a/pyproject.toml b/pyproject.toml index ba0106e6..cc706be9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,7 +91,8 @@ dependencies = [ "xdg-base-dirs>=6.0.1", "wn==0.9.5", "ollama>=0.4.7", - "tiktoken>=0.7.0" + "tiktoken>=0.7.0", + "mistralai==1.5.2" ] [project.optional-dependencies] diff --git a/tests/generators/test_mistral.py b/tests/generators/test_mistral.py index 4f572fe5..b4620f2d 100644 --- a/tests/generators/test_mistral.py +++ b/tests/generators/test_mistral.py @@ -4,23 +4,9 @@ DEFAULT_DEPLOYMENT_NAME = "mistral-small-latest" -@pytest.fixture -def set_fake_env(request) -> None: - stored_env = os.getenv(MistralGenerator.ENV_VAR, None) - - def restore_env(): - if stored_env is not None: - os.environ[MistralGenerator.ENV_VAR] = stored_env - else: - del os.environ[MistralGenerator.ENV_VAR] - - os.environ[MistralGenerator.ENV_VAR] = os.path.abspath(__file__) - - request.addfinalizer(restore_env) - @pytest.mark.skipif( os.getenv(MistralGenerator.ENV_VAR, None) is None, - reason=f"OpenAI API key is not set in {MistralGenerator.ENV_VAR}", + reason=f"Mistral API key is not set in {MistralGenerator.ENV_VAR}", ) def test_mistral_chat(): generator = MistralGenerator(name=DEFAULT_DEPLOYMENT_NAME) From fc99bd671f64dd17b47be767de0e3541908fdfd8 Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Mon, 24 Mar 2025 21:48:24 +0000 Subject: [PATCH 7/9] Add mock prompt test --- tests/generators/test_mistral.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/generators/test_mistral.py b/tests/generators/test_mistral.py index b4620f2d..f5157cea 100644 --- a/tests/generators/test_mistral.py +++ b/tests/generators/test_mistral.py @@ -1,9 +1,28 @@ import os import pytest +from unittest.mock import patch from garak.generators.mistral import MistralGenerator DEFAULT_DEPLOYMENT_NAME = "mistral-small-latest" +@patch.dict(os.environ, {"MISTRAL_API_KEY": "fake_api_key"}) +@patch("garak.generators.mistral.MistralGenerator.generate") +def test_mistral_generator(mock_generate): + # Définir le retour simulé + mock_generate.return_value = ["Mocked response"] + + # Initialiser le générateur + generator = MistralGenerator() + + # Appeler la méthode générer + output = generator.generate("Test prompt") + + # Vérifier que la fonction a bien été appelée + mock_generate.assert_called_once_with("Test prompt") + + # Vérifier le résultat + assert output == ["Mocked response"] + @pytest.mark.skipif( os.getenv(MistralGenerator.ENV_VAR, None) is None, reason=f"Mistral API key is not set in {MistralGenerator.ENV_VAR}", From 320f4a2877a55046734827ba8a29e77aa4d7001f Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Tue, 25 Mar 2025 13:37:53 +0000 Subject: [PATCH 8/9] Use mockx decorator with mistral.json mock file instead of patch decorator --- tests/generators/conftest.py | 6 +++++ tests/generators/mistral.json | 26 ++++++++++++++++++++++ tests/generators/test_mistral.py | 38 ++++++++++++++++++++------------ 3 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 tests/generators/mistral.json diff --git a/tests/generators/conftest.py b/tests/generators/conftest.py index 52d89c16..0a7fe6da 100644 --- a/tests/generators/conftest.py +++ b/tests/generators/conftest.py @@ -24,3 +24,9 @@ def watsonx_compat_mocks(): """Mock responses for watsonx.ai based endpoints""" with open(pathlib.Path(__file__).parents[0] / "watsonx.json") as mock_watsonx: return json.load(mock_watsonx) + +@pytest.fixture +def mistral_compat_mocks(): + """Mock responses for OpenAI compatible endpoints""" + with open(pathlib.Path(__file__).parents[0] / "mistral.json") as mock_mistral: + return json.load(mock_mistral) \ No newline at end of file diff --git a/tests/generators/mistral.json b/tests/generators/mistral.json new file mode 100644 index 00000000..d4146fbf --- /dev/null +++ b/tests/generators/mistral.json @@ -0,0 +1,26 @@ +{ + "mistralai_generation": { + "code": 200, + "json": { + "id": "cmpl-e5cc70bb28c444948073e77776eb30ef", + "object": "chat.completion", + "model": "mistral-small-latest", + "created": 1742909709, + "choices": [ + { + "message": { + "role": "assistant", + "content": "Ceci est une génération de test. :)" + }, + "finish_reason": "stop", + "index": 0 + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 32, + "total_tokens": 37 + } + } + } +} \ No newline at end of file diff --git a/tests/generators/test_mistral.py b/tests/generators/test_mistral.py index f5157cea..dbd1356f 100644 --- a/tests/generators/test_mistral.py +++ b/tests/generators/test_mistral.py @@ -1,27 +1,37 @@ import os import pytest +import httpx from unittest.mock import patch from garak.generators.mistral import MistralGenerator DEFAULT_DEPLOYMENT_NAME = "mistral-small-latest" -@patch.dict(os.environ, {"MISTRAL_API_KEY": "fake_api_key"}) -@patch("garak.generators.mistral.MistralGenerator.generate") -def test_mistral_generator(mock_generate): - # Définir le retour simulé - mock_generate.return_value = ["Mocked response"] +@pytest.fixture +def set_fake_env(request) -> None: + stored_env = os.getenv(MistralGenerator.ENV_VAR, None) - # Initialiser le générateur - generator = MistralGenerator() + def restore_env(): + if stored_env is not None: + os.environ[MistralGenerator.ENV_VAR] = stored_env + else: + del os.environ[MistralGenerator.ENV_VAR] - # Appeler la méthode générer - output = generator.generate("Test prompt") + os.environ[MistralGenerator.ENV_VAR] = os.path.abspath(__file__) + request.addfinalizer(restore_env) - # Vérifier que la fonction a bien été appelée - mock_generate.assert_called_once_with("Test prompt") - - # Vérifier le résultat - assert output == ["Mocked response"] +@pytest.mark.usefixtures("set_fake_env") +@pytest.mark.respx(base_url="https://api.mistral.ai/v1") +def test_mistral_generator(respx_mock, mistral_compat_mocks): + mock_response = mistral_compat_mocks["mistralai_generation"] + extended_request = "chat/completions" + respx_mock.post(extended_request).mock( + return_value=httpx.Response(mock_response["code"], json=mock_response["json"]) + ) + generator = MistralGenerator(name=DEFAULT_DEPLOYMENT_NAME) + assert generator.name == DEFAULT_DEPLOYMENT_NAME + output = generator.generate("Hello Mistral!") + assert len(output) == 1 # expect 1 generation by default + print("test passed!") @pytest.mark.skipif( os.getenv(MistralGenerator.ENV_VAR, None) is None, From 89bb454c935f3376de4945f932ee2344eed43856 Mon Sep 17 00:00:00 2001 From: Erwan ROUSSEL Date: Wed, 26 Mar 2025 09:04:16 +0000 Subject: [PATCH 9/9] Remove useless api key enforcement & change backoff exception type Signed-off-by: Erwan ROUSSEL --- garak/generators/mistral.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/garak/generators/mistral.py b/garak/generators/mistral.py index e0bbadf1..1948179c 100644 --- a/garak/generators/mistral.py +++ b/garak/generators/mistral.py @@ -3,7 +3,7 @@ import backoff from garak.generators.base import Generator import garak._config as _config -from mistralai import Mistral +from mistralai import Mistral, models from garak import exception @@ -39,12 +39,9 @@ def _clear_client(self): def __init__(self, name="", config_root=_config): super().__init__(name, config_root) - if self.api_key is not None: - # ensure the token is in the expected runtime env var - os.environ[self.ENV_VAR] = self.api_key self._load_client() - @backoff.on_exception(backoff.fibo, exception.RateLimitHit, max_value=70) + @backoff.on_exception(backoff.fibo, models.SDKError, max_value=70) def _call_model(self, prompt, generations_this_call=1): print(self.name) chat_response = self.client.chat.complete(