HDXDSYS-685 Output csvs (#83)

* Output csvs for all views in db export

Co-authored-by: Alexandru Gartner <[email protected]>

.github/workflows/db_export.yaml

@@ -31,7 +31,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 
@@ -46,11 +46,9 @@ jobs:
         env:
           HDX_KEY: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
 
-      - name: Dump PostgreSQL DB
+      - name: Dump PostgreSQL Views
         run: |
-          mkdir database
-          docker exec -t postgres-container pg_dump -U postgres -Fc hapi -f hapi_db.pg_restore
-          docker cp postgres-container:/hapi_db.pg_restore database/hapi_db.pg_restore
+          ./dump_views.sh
 
       - name: Commit updated DB export
         uses: JamesIves/github-pages-deploy-action@v4

.github/workflows/run-python-tests.yaml

@@ -18,31 +18,31 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.11'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install --upgrade hatch
-    - name: Test with hatch/pytest
-      run: |
-        hatch run test:test
-    - name: Check styling
-      if: always()
-      run: |
-        hatch run lint:style
-    - name: Publish Unit Test Results
-      uses: EnricoMi/publish-unit-test-result-action@v2
-      if: always()
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        junit_files: test-results.xml
-    - name: Publish in Coveralls
-      uses: coverallsapp/github-action@v2
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        flag-name: tests
-        format: lcov
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade hatch
+      - name: Test with hatch/pytest
+        run: |
+          hatch run test:test
+      - name: Check styling
+        if: always()
+        run: |
+          hatch run lint:style
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          junit_files: test-results.xml
+      - name: Publish in Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          flag-name: tests
+          format: lcov

dump_views.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+printf -v now '%(%Y-%m-%d %H:%M:%S)T\n' -1
+echo "Started at $now"
+
+mkdir -p database/csv
+
+tables=$(docker exec -t postgres-container psql -t -U postgres -d hapi -c \
+  "select table_name from INFORMATION_SCHEMA.views WHERE table_schema = ANY (current_schemas(false))")
+
+for table in $tables
+do
+  table=${table::-1}
+  if [[ -z "$table" ]]; then
+    continue
+  fi
+
+  echo "Saving $table"
+
+  docker exec -t postgres-container psql -U postgres -d hapi -c \
+    "\copy (select * from ${table}_view) TO STDOUT (FORMAT csv, DELIMITER ',',  HEADER);" \
+    | tee \
+    >(gzip > database/csv/${table}.csv.gz) \
+    >(md5sum | awk '{print $1}' > database/csv/${table}.hash) \
+    >/dev/null
+done
+
+docker exec -t postgres-container pg_dump -U postgres -Fc hapi -f hapi_db.pg_restore
+docker cp postgres-container:/hapi_db.pg_restore database/hapi_db.pg_restore
+
+printf -v now '%(%Y-%m-%d %H:%M:%S)T\n' -1
+echo "Ended at $now"

requirements.txt

@@ -167,7 +167,7 @@ pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
-pytest==8.1.1
+pytest==8.2.0
     # via
     #   pytest-cov
     #   pytest-mock
@@ -193,7 +193,7 @@ quantulum3==0.9.1
     # via hdx-python-api
 ratelimit==2.2.1
     # via hdx-python-utilities
-regex==2024.4.16
+regex==2024.4.28
     # via hdx-python-scraper
 requests==2.31.0
     # via