From b10dc28ac0f30b0907be2c0747c1890d4d0ba034 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Thu, 30 Jan 2025 12:35:22 +0200 Subject: [PATCH] fix(testing+linting): add nox lint+format directives (#123) * fix(testing+linting): add nox lint+format directives This change introduces new nox directives: * blacken: `nox -s blacken` * format: `nox -s format` to apply formatting to files * lint: `nox -s lint` to flag linting issues * integration: to run integration tests * unit: to run unit tests locally which are the basis to enable scalable development and continuous testing as I prepare to bring in Approximate Nearest Neighors (ANN) functionality into this package. * Rebased from main and re-ran nox -s format/lint --- noxfile.py | 109 +++++++++++++++++- pyproject.toml | 7 ++ src/langchain_google_spanner/graph_qa.py | 1 - .../graph_retriever.py | 6 +- src/langchain_google_spanner/loader.py | 1 - tests/integration/test_spanner_graph_qa.py | 1 - tests/integration/test_spanner_loader.py | 12 +- .../integration/test_spanner_vector_store.py | 4 +- 8 files changed, 121 insertions(+), 20 deletions(-) diff --git a/noxfile.py b/noxfile.py index 2ad8aec..d524d41 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,23 +20,28 @@ import pathlib import shutil from pathlib import Path -from typing import Optional import nox DEFAULT_PYTHON_VERSION = "3.10" CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() +LINT_PATHS = ["src", "tests", "noxfile.py"] + nox.options.sessions = [ - "docs", + "blacken", "docfx", + "docs", + "format", + "lint", + "unit", ] # Error if a python version is missing nox.options.error_on_missing_interpreters = True -@nox.session(python="3.10") +@nox.session(python=DEFAULT_PYTHON_VERSION) def docs(session): """Build the docs for this library.""" @@ -71,7 +76,7 @@ def docs(session): ) -@nox.session(python="3.10") +@nox.session(python=DEFAULT_PYTHON_VERSION) def docfx(session): """Build the docfx yaml files for this library.""" @@ -115,3 +120,99 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or + sufficiently serious code quality issues. + """ + session.install(".[lint]") + session.run( + "black", + "--check", + *LINT_PATHS, + ) + session.run("flake8", "src", "tests") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including an RST check).""" + session.install("docutils", "pygments") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + session.install(".[lint]") + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + session.install(".[lint]") + # Sort imports in strict alphabetical order. + session.run( + "isort", + *LINT_PATHS, + ) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def unit(session): + install_unittest_dependencies(session) + session.run( + "py.test", + "--quiet", + os.path.join("tests", "unit"), + *session.posargs, + ) + + +def install_unittest_dependencies(session, *constraints): + session.install(".[test]") + session.run( + "pip", + "install", + "--no-compile", # To ensure no byte recompliation which is usually super slow + "-q", + "--disable-pip-version-check", # Avoid the slow version check + ".", + "-r", + "requirements.txt", + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def integration(session): + install_integrationtest_dependencies(session) + session.run( + "py.test", + "--quiet", + os.path.join("tests", "integration"), + *session.posargs, + ) + + +def install_integrationtest_dependencies(session): + session.install(".[test]") + session.run( + "pip", + "install", + "--no-compile", # To ensure no byte recompliation which is usually super slow + "-q", + "--disable-pip-version-check", # Avoid the slow version check + ".", + "-r", + "requirements.txt", + ) diff --git a/pyproject.toml b/pyproject.toml index 597a6f3..0d3f6a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,9 +35,16 @@ Repository = "https://github.com/googleapis/langchain-google-spanner-python.git" Changelog = "https://github.com/googleapis/langchain-google-spanner-python/blob/main/CHANGELOG.md" [project.optional-dependencies] +lint = [ + "black[jupyter]==24.8.0", + "flake8==6.1.0", + "isort==5.13.2", +] + test = [ "black[jupyter]==24.8.0", "bs4==0.0.2", + "flake8==6.1.0", "isort==5.13.2", "mypy==1.11.2", "pytest==8.3.3", diff --git a/src/langchain_google_spanner/graph_qa.py b/src/langchain_google_spanner/graph_qa.py index f071773..36251f4 100644 --- a/src/langchain_google_spanner/graph_qa.py +++ b/src/langchain_google_spanner/graph_qa.py @@ -288,7 +288,6 @@ def _call( inputs: Dict[str, Any], run_manager: Optional[CallbackManagerForChainRun] = None, ) -> Dict[str, str]: - intermediate_steps: List = [] """Generate gql statement, uses it to look up in db and answer question.""" diff --git a/src/langchain_google_spanner/graph_retriever.py b/src/langchain_google_spanner/graph_retriever.py index d746eb0..0e3b59e 100644 --- a/src/langchain_google_spanner/graph_retriever.py +++ b/src/langchain_google_spanner/graph_retriever.py @@ -16,11 +16,7 @@ from typing import Any, List, Optional from langchain.schema.retriever import BaseRetriever -from langchain_community.graphs.graph_document import GraphDocument -from langchain_core.callbacks import ( - CallbackManagerForChainRun, - CallbackManagerForRetrieverRun, -) +from langchain_core.callbacks import CallbackManagerForRetrieverRun from langchain_core.documents import Document from langchain_core.embeddings import Embeddings from langchain_core.example_selectors import SemanticSimilarityExampleSelector diff --git a/src/langchain_google_spanner/loader.py b/src/langchain_google_spanner/loader.py index d71de11..5bc8286 100644 --- a/src/langchain_google_spanner/loader.py +++ b/src/langchain_google_spanner/loader.py @@ -19,7 +19,6 @@ from google.cloud.spanner import Client, KeySet # type: ignore from google.cloud.spanner_admin_database_v1.types import DatabaseDialect # type: ignore -from google.cloud.spanner_v1.data_types import JsonObject # type: ignore from langchain_community.document_loaders.base import BaseLoader from langchain_core.documents import Document diff --git a/tests/integration/test_spanner_graph_qa.py b/tests/integration/test_spanner_graph_qa.py index 8bac7b8..48758fc 100644 --- a/tests/integration/test_spanner_graph_qa.py +++ b/tests/integration/test_spanner_graph_qa.py @@ -145,7 +145,6 @@ def load_data(graph: SpannerGraphStore): class TestSpannerGraphQAChain: - @pytest.fixture(scope="module") def setup_db_load_data(self): graph = get_spanner_graph() diff --git a/tests/integration/test_spanner_loader.py b/tests/integration/test_spanner_loader.py index 85f483f..fd8f028 100644 --- a/tests/integration/test_spanner_loader.py +++ b/tests/integration/test_spanner_loader.py @@ -16,7 +16,7 @@ import uuid import pytest -from google.cloud.spanner import Client, KeySet # type: ignore +from google.cloud.spanner import Client # type: ignore from langchain_core.documents import Document from langchain_google_spanner.loader import Column, SpannerDocumentSaver, SpannerLoader @@ -363,7 +363,7 @@ def test_loader_custom_format_error(self, client): client=client, format="NOT_A_FORMAT", ) - docs = loader.load() + docs = loader.load() # noqa def test_loader_custom_content_key_error(self, client): query = f"SELECT * FROM {table_name}" @@ -375,7 +375,7 @@ def test_loader_custom_content_key_error(self, client): client=client, content_columns=["NOT_A_COLUMN"], ) - docs = loader.load() + docs = loader.load() # noqa def test_loader_custom_metadata_key_error(self, client): query = f"SELECT * FROM {table_name}" @@ -387,7 +387,7 @@ def test_loader_custom_metadata_key_error(self, client): client=client, metadata_columns=["NOT_A_COLUMN"], ) - docs = loader.load() + docs = loader.load() # noqa def test_loader_custom_json_metadata(self, client): database = client.instance(instance_id).database(google_database) @@ -792,7 +792,7 @@ def test_loader_custom_content_key_error(self, client): client=client, content_columns=["NOT_A_COLUMN"], ) - docs = loader.load() + docs = loader.load() # noqa def test_loader_custom_metadata_key_error(self, client): query = f"SELECT * FROM {table_name}" @@ -804,7 +804,7 @@ def test_loader_custom_metadata_key_error(self, client): client=client, metadata_columns=["NOT_A_COLUMN"], ) - docs = loader.load() + docs = loader.load() # noqa def test_loader_custom_json_metadata(self, client): database = client.instance(instance_id).database(pg_database) diff --git a/tests/integration/test_spanner_vector_store.py b/tests/integration/test_spanner_vector_store.py index 9a19f7b..bf4de63 100644 --- a/tests/integration/test_spanner_vector_store.py +++ b/tests/integration/test_spanner_vector_store.py @@ -301,7 +301,7 @@ def test_spanner_vector_delete_data(self, setup_database): deleted = db.delete(documents=[docs[0], docs[1]]) - assert deleted == True + assert deleted def test_spanner_vector_search_data1(self, setup_database): loader, embeddings = setup_database @@ -483,7 +483,7 @@ def test_spanner_vector_delete_data(self, setup_database): deleted = db.delete(documents=[docs[0], docs[1]]) - assert deleted == True + assert deleted def test_spanner_vector_search_data1(self, setup_database): loader, embeddings = setup_database