[PROPOSAL]: Local Hatchet for Tests (#271)

* feat: local compose stack

* try running tests against local hatchet

* fix: hack

* fix: tracer provider warnings

* fix: make sure workflows exist

* debug: test

* cleanup: unwind print statements

* fix: lint

* chore: version

* fix: remove third-party action

* fix: daemon

.github/workflows/e2e.yaml

@@ -11,6 +11,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+
+      - name: Run Hatchet Engine
+        run: docker compose up -d
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -25,14 +29,20 @@ jobs:
           virtualenvs-in-project: true
       - name: Install dependencies
         run: poetry install --no-interaction
+
+      - name: Generate Env File
+        run: |
+          cat <<EOF > .env
+            HATCHET_CLIENT_TOKEN="$(docker compose run --no-deps setup-config /hatchet/hatchet-admin token create --config /hatchet/config --tenant-id 707d0855-80ab-4e1f-a156-f1c4546cbf52 | xargs)"
+            HATCHET_CLIENT_TLS_STRATEGY=none
+          EOF
+
       - name: Set HATCHET_CLIENT_NAMESPACE
         run: |
           PYTHON_VERSION=$(python -c "import sys; print(f'py{sys.version_info.major}{sys.version_info.minor}')")
           SHORT_SHA=$(git rev-parse --short HEAD)
           echo "HATCHET_CLIENT_NAMESPACE=${PYTHON_VERSION}-${SHORT_SHA}" >> $GITHUB_ENV
       - name: Run pytest
-        env:
-          HATCHET_CLIENT_TOKEN: ${{ secrets.HATCHET_CLIENT_TOKEN }}
         run: |
           echo "Using HATCHET_CLIENT_NAMESPACE: $HATCHET_CLIENT_NAMESPACE"
           poetry run pytest -s -vvv --maxfail=5 --timeout=120 --capture=no

Caddyfile

@@ -0,0 +1,9 @@
+http://localhost:8080 {
+	handle /api/* {
+		reverse_proxy hatchet-api:8080
+	}
+
+	handle /* {
+		reverse_proxy hatchet-frontend:80
+	}
+}

compose.yml

@@ -0,0 +1,115 @@
+version: "3.8"
+services:
+  postgres:
+    image: postgres:15.6
+    command: postgres -c 'max_connections=200'
+    restart: always
+    hostname: "postgres"
+    environment:
+      - POSTGRES_USER=hatchet
+      - POSTGRES_PASSWORD=hatchet
+      - POSTGRES_DB=hatchet
+    ports:
+      - "5435:5432"
+    volumes:
+      - hatchet_postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d hatchet -U hatchet"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 10s
+  rabbitmq:
+    image: "rabbitmq:3-management"
+    hostname: "rabbitmq"
+    ports:
+      - "5673:5672" # RabbitMQ
+      - "15673:15672" # Management UI
+    environment:
+      RABBITMQ_DEFAULT_USER: "user"
+      RABBITMQ_DEFAULT_PASS: "password"
+    volumes:
+      - "hatchet_rabbitmq_data:/var/lib/rabbitmq"
+      - "hatchet_rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf" # Configuration file mount
+    healthcheck:
+      test: ["CMD", "rabbitmqctl", "status"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+  migration:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-migrate:latest
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@postgres:5432/hatchet"
+    depends_on:
+      postgres:
+        condition: service_healthy
+  setup-config:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-admin:latest
+    command: /hatchet/hatchet-admin quickstart --skip certs --generated-config-dir /hatchet/config --overwrite=false
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@postgres:5432/hatchet"
+      DATABASE_POSTGRES_PORT: "5432"
+      DATABASE_POSTGRES_HOST: "postgres"
+      SERVER_TASKQUEUE_RABBITMQ_URL: amqp://user:password@rabbitmq:5672/
+      SERVER_AUTH_COOKIE_DOMAIN: localhost:8080
+      SERVER_AUTH_COOKIE_INSECURE: "t"
+      SERVER_GRPC_BIND_ADDRESS: "0.0.0.0"
+      SERVER_GRPC_INSECURE: "t"
+      SERVER_GRPC_BROADCAST_ADDRESS: localhost:7077
+    volumes:
+      - hatchet_certs:/hatchet/certs
+      - hatchet_config:/hatchet/config
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+      rabbitmq:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+  hatchet-engine:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-engine:latest
+    command: /hatchet/hatchet-engine --config /hatchet/config
+    restart: on-failure
+    depends_on:
+      setup-config:
+        condition: service_completed_successfully
+      migration:
+        condition: service_completed_successfully
+    ports:
+      - "7077:7070"
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@postgres:5432/hatchet"
+      SERVER_GRPC_BIND_ADDRESS: "0.0.0.0"
+      SERVER_GRPC_INSECURE: "t"
+    volumes:
+      - hatchet_certs:/hatchet/certs
+      - hatchet_config:/hatchet/config
+  hatchet-api:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-api:latest
+    command: /hatchet/hatchet-api --config /hatchet/config
+    restart: on-failure
+    depends_on:
+      setup-config:
+        condition: service_completed_successfully
+      migration:
+        condition: service_completed_successfully
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@postgres:5432/hatchet"
+    volumes:
+      - hatchet_certs:/hatchet/certs
+      - hatchet_config:/hatchet/config
+  hatchet-frontend:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-frontend:latest
+  caddy:
+    image: caddy:2.7.6-alpine
+    ports:
+      - 8080:8080
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile
+
+volumes:
+  hatchet_postgres_data:
+  hatchet_rabbitmq_data:
+  hatchet_rabbitmq.conf:
+  hatchet_config:
+  hatchet_certs:

examples/api/test_api.py

@@ -1,19 +1,23 @@
 import pytest
 
 from hatchet_sdk import Hatchet
+from tests.utils import fixture_bg_worker
+
+## IMPORTANT: Worker needs to be set here to ensure at least one workflow exists
+worker = fixture_bg_worker(["poetry", "run", "concurrency_limit_rr"])
 
 
 # requires scope module or higher for shared event loop
 @pytest.mark.asyncio(scope="session")
-async def test_list_workflows(hatchet: Hatchet):
-    list = hatchet.rest.workflow_list()
+async def test_list_workflows(hatchet: Hatchet, worker):
+    workflows = hatchet.rest.workflow_list()
 
-    assert len(list.rows) != 0
+    assert len(workflows.rows) != 0
 
 
 # requires scope module or higher for shared event loop
 @pytest.mark.asyncio(scope="session")
-async def test_async_list_workflows(aiohatchet: Hatchet):
-    list = await aiohatchet.rest.aio.workflow_list()
+async def test_async_list_workflows(aiohatchet: Hatchet, worker):
+    workflows = await aiohatchet.rest.aio.workflow_list()
 
-    assert len(list.rows) != 0
+    assert len(workflows.rows) != 0

examples/async/test_async.py

@@ -7,7 +7,6 @@
 
 
 # requires scope module or higher for shared event loop
-@pytest.mark.skip(reason="Skipping this test until we can dedicate more time to debug")
 @pytest.mark.asyncio(scope="session")
 async def test_run(hatchet: Hatchet):
     run = hatchet.admin.run_workflow("AsyncWorkflow", {})

examples/on_failure/test_on_failure.py

@@ -15,16 +15,18 @@ async def test_run_timeout(hatchet: Hatchet):
     run = hatchet.admin.run_workflow("OnFailureWorkflow", {})
     try:
         await run.result()
+
         assert False, "Expected workflow to timeout"
     except Exception as e:
         assert "step1 failed" in str(e)
 
-    await asyncio.sleep(2)  # Wait for the on_failure job to finish
+    await asyncio.sleep(5)  # Wait for the on_failure job to finish
 
     job_runs = hatchet.rest.workflow_run_get(run.workflow_run_id).job_runs
     assert len(job_runs) == 2
 
     successful_job_runs = [jr for jr in job_runs if jr.status == JobRunStatus.SUCCEEDED]
     failed_job_runs = [jr for jr in job_runs if jr.status == JobRunStatus.FAILED]
+
     assert len(successful_job_runs) == 1
     assert len(failed_job_runs) == 1

hatchet_sdk/utils/tracing.py

@@ -31,12 +31,14 @@ def create_tracer(config: ClientConfig) -> Tracer:
             ),
         )
 
-        trace_provider = TracerProvider(resource=resource)
-        trace_provider.add_span_processor(processor)
+        ## If tracer provider is already set, we don't need to override it
+        if not isinstance(trace.get_tracer_provider(), TracerProvider):
+            trace_provider = TracerProvider(resource=resource)
+            trace_provider.add_span_processor(processor)
+            trace.set_tracer_provider(trace_provider)
     else:
-        trace_provider = NoOpTracerProvider()
-
-    trace.set_tracer_provider(trace_provider)
+        if not isinstance(trace.get_tracer_provider(), NoOpTracerProvider):
+            trace.set_tracer_provider(NoOpTracerProvider())
 
     return trace.get_tracer(__name__)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hatchet-sdk"
-version = "0.40.0"
+version = "0.40.1"
 description = ""
 authors = ["Alexander Belanger <[email protected]>"]
 readme = "README.md"