fixes docker-compose and 401 error

poggiolabs · Oct 3, 2024 · d316d52 · d316d52
1 parent 50c59b6
commit d316d52
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 27 deletions.
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -425,7 +425,7 @@ export async function scrapSingleUrl(
         Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`);
         break;
       }
-      if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400 || pageStatusCode == 401)) {
+      if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400)) {
         Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code ${pageStatusCode}, breaking`);
         break;
       }

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -4,30 +4,6 @@ x-common-service: &common-service
   build: apps/api
   networks:
     - backend
-  environment:
-    - REDIS_URL=${REDIS_URL:-redis://redis:6379}
-    - REDIS_RATE_LIMIT_URL=${REDIS_URL:-redis://redis:6379}
-    - PLAYWRIGHT_MICROSERVICE_URL=${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
-    - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION}
-    - PORT=${PORT:-3002}
-    - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE}
-    - OPENAI_API_KEY=${OPENAI_API_KEY}
-    - OPENAI_BASE_URL=${OPENAI_BASE_URL}
-    - MODEL_NAME=${MODEL_NAME:-gpt-4o}
-    - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
-    - LLAMAPARSE_API_KEY=${LLAMAPARSE_API_KEY}
-    - LOGTAIL_KEY=${LOGTAIL_KEY}
-    - BULL_AUTH_KEY=${BULL_AUTH_KEY}
-    - TEST_API_KEY=${TEST_API_KEY}
-    - POSTHOG_API_KEY=${POSTHOG_API_KEY}
-    - POSTHOG_HOST=${POSTHOG_HOST}
-    - SUPABASE_ANON_TOKEN=${SUPABASE_ANON_TOKEN}
-    - SUPABASE_URL=${SUPABASE_URL}
-    - SUPABASE_SERVICE_TOKEN=${SUPABASE_SERVICE_TOKEN}
-    - SCRAPING_BEE_API_KEY=${SCRAPING_BEE_API_KEY}
-    - HOST=${HOST:-0.0.0.0}
-    - SELF_HOSTED_WEBHOOK_URL=${SELF_HOSTED_WEBHOOK_URL}
-    - LOGGING_LEVEL=${LOGGING_LEVEL}
   extra_hosts:
     - "host.docker.internal:host-gateway"
 
@@ -46,7 +22,29 @@ services:
   api:
     <<: *common-service
     environment:
-      <<: *common-service.environment
+      REDIS_URL: ${REDIS_URL:-redis://redis:6379}
+      REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
+      PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
+      USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
+      PORT: ${PORT:-3002}
+      NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}
+      OPENAI_API_KEY: ${OPENAI_API_KEY}
+      OPENAI_BASE_URL: ${OPENAI_BASE_URL}
+      MODEL_NAME: ${MODEL_NAME:-gpt-4o}
+      SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
+      LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}
+      LOGTAIL_KEY: ${LOGTAIL_KEY}
+      BULL_AUTH_KEY: ${BULL_AUTH_KEY}
+      TEST_API_KEY: ${TEST_API_KEY}
+      POSTHOG_API_KEY: ${POSTHOG_API_KEY}
+      POSTHOG_HOST: ${POSTHOG_HOST}
+      SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
+      SUPABASE_URL: ${SUPABASE_URL}
+      SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
+      SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
+      HOST: ${HOST:-0.0.0.0}
+      SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+      LOGGING_LEVEL: ${LOGGING_LEVEL}
       FLY_PROCESS_GROUP: app
     depends_on:
       - redis
@@ -58,7 +56,29 @@ services:
   worker:
     <<: *common-service
     environment:
-      <<: *common-service.environment
+      REDIS_URL: ${REDIS_URL:-redis://redis:6379}
+      REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
+      PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
+      USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
+      PORT: ${PORT:-3002}
+      NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}
+      OPENAI_API_KEY: ${OPENAI_API_KEY}
+      OPENAI_BASE_URL: ${OPENAI_BASE_URL}
+      MODEL_NAME: ${MODEL_NAME:-gpt-4o}
+      SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
+      LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}
+      LOGTAIL_KEY: ${LOGTAIL_KEY}
+      BULL_AUTH_KEY: ${BULL_AUTH_KEY}
+      TEST_API_KEY: ${TEST_API_KEY}
+      POSTHOG_API_KEY: ${POSTHOG_API_KEY}
+      POSTHOG_HOST: ${POSTHOG_HOST}
+      SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
+      SUPABASE_URL: ${SUPABASE_URL}
+      SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
+      SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
+      HOST: ${HOST:-0.0.0.0}
+      SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+      LOGGING_LEVEL: ${LOGGING_LEVEL}
       FLY_PROCESS_GROUP: worker
     depends_on:
       - redis