Merge pull request #8 from EJOOSTEROP:wip_adding_sentry

Adding Sentry logging when env vairable specified.
EJOOSTEROP · Mar 2, 2024 · c9705ea · c9705ea
2 parents 598165f + 938d3e0
commit c9705ea
Show file tree

Hide file tree

Showing 6 changed files with 6,508 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 <!-- insertion marker -->
 ## [Unreleased]
 
+## [0.4.4] 2024-03-02
+### Added
+- Added logging of dlt job to sentry.io. Need to specify env variable
+
+### Changed
+- Switched of loading of trace data into database as this starts to slow down the jobs significantly
+
 ## [0.4.3] 2024-03-01
 ### Fixed
 - Correct missing dates with flexible start and end dates (add where clause to query)

diff --git a/README.md b/README.md
@@ -215,6 +215,9 @@ These are the technologies driving the content on [gas.aspireto.win][aspireto-ga
 
 NOTE: As of November 2023 it is possible to fully deploy this stack without breaking the bank (using free tiers of the cloud services used). Dive into our GitHub repository and the linked ones for the Google Function and Evidence notebook, where all the code awaits. 🚀
 
+### Sentry
+To enable logging iusing sentry.io specify the environment variable ```RUNTIME__SENTRY_DSN```
+
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
 NOTE: For some reason the environment variable DESTINATION__MOTHERDUCK__CREDENTIALS is oftentimes incorrectly set between runs when using the dev container. Use ```unset DESTINATION__MOTHERDUCK__CREDENTIALS``` to clear the environment variable.

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -18,7 +18,7 @@ disallow_incomplete_defs = true
 
 [tool.poetry]
 name = "ternyxmimosa"
-version = "0.4.3"
+version = "0.4.4"
 description = "A minimal modern data stack with working data pipelines in a single Docker container."
 authors = ["Erik Oosterop <[email protected]>"]
 license = "MIT"
@@ -36,6 +36,7 @@ loguru = "^0.7.2"
 pandas = "2.0.3"
 streamlit = "^1.26.0"
 dlt = {extras = ["cli", "filesystem", "motherduck", "parquet"], version = "^0.4.4"}
+sentry-sdk = "^1.40.6"
 
 [tool.poetry.group.dev.dependencies]
 pytest-cov = "^4.1.0"

diff --git a/src/mimosa/pipelines.py b/src/mimosa/pipelines.py
@@ -132,7 +132,7 @@ def run_landing_pipeline(
             row_counts = pipeline.last_trace.last_normalize_info
 
             # Load lineage and run related info into destination
-            pipeline.run([load_info], table_name="_load_info")
+            # TODO: Switch back on: pipeline.run([load_info], table_name="_load_info") # TODO: This does really slows down the job (after some iterations).
 
             # Log outcome
             logger.debug(row_counts)