From 9ea0ab5462f89abfd585e817c761983d5197e2b1 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Wed, 31 Jan 2024 18:34:44 +0900
Subject: [PATCH 01/20] add new services to docker-compose

---
 docker-compose.yml | 66 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 6d661e5..a433ea7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,3 +1,14 @@
+x-logging: &default-logging
+  driver: loki
+  options:
+    loki-url: 'http://localhost:3100/api/prom/push'
+    loki-pipeline-stages: |
+      - multiline:
+          firstline: '^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}'
+          max_wait_time: 3s
+      - regex:
+          expression: '^(?P<time>\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2},d{3}) (?P<message>(?s:.*))$$'
+
 version: '3.8'
 
 services:
@@ -20,6 +31,7 @@ services:
       - redis
     volumes:
       - ./src/app:/code/app
+    logging: *default-logging
 
   worker:
     build:
@@ -33,6 +45,7 @@ services:
       - redis
     volumes:
       - ./src/app:/code/app
+    logging: *default-logging
 
   db:
     image: postgres:13
@@ -53,6 +66,56 @@ services:
     expose:
       - "6379"
 
+  loki:
+    container_name: loki
+    image: grafana/loki:2.8.3
+    command: -config.file=/etc/loki/local-config.yaml
+    ports:
+      - "3100:3100"
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    ports:
+      - 9090:9090
+    volumes:
+      - ./dashboard/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    depends_on:
+      - loki
+      - loonshot-backend
+    logging: *default-logging
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - ./dashboard/grafana/datasources:/etc/grafana/provisioning/datasources
+      - ./dashboard/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+    depends_on:
+      - loki
+      - prometheus
+      - tempo
+    logging: *default-logging
+
+  tempo:
+    container_name: tempo
+    image: grafana/tempo:2.1.1
+    command:
+      [
+        "--target=all",
+        "--storage.trace.backend=local",
+        "--storage.trace.local.path=/var/tempo",
+        "--auth.enabled=false"
+      ]
+    ports:
+      - "14250:14250"
+    depends_on:
+      - loki
+    logging: *default-logging
   #-------- uncomment to run with pgadmin --------
   # pgadmin:
   #   container_name: pgadmin4
@@ -122,5 +185,4 @@ services:
 
 volumes:
   postgres-data:
-  redis-data:
-  #pgadmin-data:
+  redis-data: #pgadmin-data:

From 38cd3e2917e7ba8bb158e2b65aba2b57f2c14e19 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Wed, 31 Jan 2024 18:36:58 +0900
Subject: [PATCH 02/20] add new services to docker-compose

---
 docker-compose.yml | 66 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 6d661e5..774a575 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,3 +1,14 @@
+x-logging: &default-logging
+  driver: loki
+  options:
+    loki-url: 'http://localhost:3100/api/prom/push'
+    loki-pipeline-stages: |
+      - multiline:
+          firstline: '^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}'
+          max_wait_time: 3s
+      - regex:
+          expression: '^(?P<time>\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2},d{3}) (?P<message>(?s:.*))$$'
+
 version: '3.8'
 
 services:
@@ -20,6 +31,7 @@ services:
       - redis
     volumes:
       - ./src/app:/code/app
+    logging: *default-logging
 
   worker:
     build:
@@ -33,6 +45,7 @@ services:
       - redis
     volumes:
       - ./src/app:/code/app
+    logging: *default-logging
 
   db:
     image: postgres:13
@@ -53,6 +66,56 @@ services:
     expose:
       - "6379"
 
+  loki:
+    container_name: loki
+    image: grafana/loki:2.8.3
+    command: -config.file=/etc/loki/local-config.yaml
+    ports:
+      - "3100:3100"
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    ports:
+      - 9090:9090
+    volumes:
+      - ./dashboard/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    depends_on:
+      - loki
+      - web
+    logging: *default-logging
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - ./dashboard/grafana/datasources:/etc/grafana/provisioning/datasources
+      - ./dashboard/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+    depends_on:
+      - loki
+      - prometheus
+      - tempo
+    logging: *default-logging
+
+  tempo:
+    container_name: tempo
+    image: grafana/tempo:2.1.1
+    command:
+      [
+        "--target=all",
+        "--storage.trace.backend=local",
+        "--storage.trace.local.path=/var/tempo",
+        "--auth.enabled=false"
+      ]
+    ports:
+      - "14250:14250"
+    depends_on:
+      - loki
+    logging: *default-logging
   #-------- uncomment to run with pgadmin --------
   # pgadmin:
   #   container_name: pgadmin4
@@ -122,5 +185,4 @@ services:
 
 volumes:
   postgres-data:
-  redis-data:
-  #pgadmin-data:
+  redis-data: #pgadmin-data:

From 5dfffad72414d8028d71c53c48e2d3401e3161c2 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Wed, 31 Jan 2024 18:42:20 +0900
Subject: [PATCH 03/20] add init dashboard configs

---
 .../grafana/datasources/datasource-config.yml |   78 ++
 .../provisioning/dashboards/dashboard.yml     |   10 +
 .../dashboards/fastapi-observability.json     | 1166 +++++++++++++++++
 dashboard/prometheus/prometheus.yml           |   30 +
 4 files changed, 1284 insertions(+)
 create mode 100644 dashboard/grafana/datasources/datasource-config.yml
 create mode 100644 dashboard/grafana/provisioning/dashboards/dashboard.yml
 create mode 100644 dashboard/grafana/provisioning/dashboards/fastapi-observability.json
 create mode 100644 dashboard/prometheus/prometheus.yml

diff --git a/dashboard/grafana/datasources/datasource-config.yml b/dashboard/grafana/datasources/datasource-config.yml
new file mode 100644
index 0000000..00b6c8d
--- /dev/null
+++ b/dashboard/grafana/datasources/datasource-config.yml
@@ -0,0 +1,78 @@
+# config file version
+apiVersion: 1
+# list of datasources that should be deleted from the database
+# deleteDatasources:
+#   - name: Prometheus
+#     orgId: 1
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+  - uid: prometheus
+    orgId: 1
+    name: Prometheus
+    type: prometheus
+    typeName: Prometheus
+    access: proxy
+    url: http://prometheus:9090
+    password: ''
+    user: ''
+    database: ''
+    basicAuth: false
+    isDefault: true
+    jsonData:
+      exemplarTraceIdDestinations:
+        - datasourceUid: tempo
+          name: TraceID
+      httpMethod: POST
+    readOnly: false
+    editable: true
+  - uid: tempo
+    orgId: 1
+    name: Tempo
+    type: tempo
+    typeName: Tempo
+    access: proxy
+    url: http://tempo
+    password: ''
+    user: ''
+    database: ''
+    basicAuth: false
+    isDefault: false
+    jsonData:
+      nodeGraph:
+        enabled: true
+      search:
+        hide: false
+      lokiSearch:
+        datasourceUid: loki
+      tracesToLogs:
+        datasourceUid: loki
+        filterBySpanID: false
+        filterByTraceID: true
+        mapTagNamesEnabled: false
+        tags:
+          - compose_service
+    readOnly: false
+    editable: true
+  - uid: loki
+    orgId: 1
+    name: Loki
+    type: loki
+    auth_enabled: false
+    typeName: Loki
+    access: proxy
+    url: http://loki:3100
+    password: ''
+    user: ''
+    database: ''
+    basicAuth: false
+    isDefault: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: (?:trace_id)=(\w+)
+          name: TraceID
+          url: $${__value.raw}
+    readOnly: false
+    editable: true
diff --git a/dashboard/grafana/provisioning/dashboards/dashboard.yml b/dashboard/grafana/provisioning/dashboards/dashboard.yml
new file mode 100644
index 0000000..caa27cb
--- /dev/null
+++ b/dashboard/grafana/provisioning/dashboards/dashboard.yml
@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+  - name: Dashs
+    folder: 'dashboards'
+    type: file
+    disableDeletion: true
+    editable: true
+    options:
+      path: /etc/grafana/provisioning/dashboards
diff --git a/dashboard/grafana/provisioning/dashboards/fastapi-observability.json b/dashboard/grafana/provisioning/dashboards/fastapi-observability.json
new file mode 100644
index 0000000..a19e9bb
--- /dev/null
+++ b/dashboard/grafana/provisioning/dashboards/fastapi-observability.json
@@ -0,0 +1,1166 @@
+{
+    "annotations": {
+        "list": [
+            {
+                "builtIn": 1,
+                "datasource": {
+                    "type": "datasource",
+                    "uid": "grafana"
+                },
+                "enable": true,
+                "hide": true,
+                "iconColor": "rgba(0, 211, 255, 1)",
+                "name": "Annotations & Alerts",
+                "target": {
+                    "limit": 100,
+                    "matchAny": false,
+                    "tags": [],
+                    "type": "dashboard"
+                },
+                "type": "dashboard"
+            }
+        ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "gnetId": 16110,
+    "graphTooltip": 0,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+        {
+            "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            }
+                        ]
+                    },
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 6,
+                "w": 4,
+                "x": 0,
+                "y": 0
+            },
+            "id": 4,
+            "options": {
+                "colorMode": "value",
+                "graphMode": "area",
+                "justifyMode": "auto",
+                "orientation": "auto",
+                "reduceOptions": {
+                    "calcs": [
+                        "lastNotNull"
+                    ],
+                    "fields": "",
+                    "values": false
+                },
+                "textMode": "auto",
+                "wideLayout": true
+            },
+            "pluginVersion": "10.2.2",
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "sum(increase(fastapi_requests_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[$__range]))",
+                    "instant": true,
+                    "interval": "",
+                    "legendFormat": "",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Total Requests",
+            "transformations": [
+                {
+                    "id": "seriesToRows",
+                    "options": {}
+                },
+                {
+                    "id": "sortBy",
+                    "options": {
+                        "fields": {},
+                        "sort": [
+                            {
+                                "field": "Time"
+                            }
+                        ]
+                    }
+                }
+            ],
+            "type": "stat"
+        },
+        {
+            "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "thresholds"
+                    },
+                    "decimals": 0,
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            }
+                        ]
+                    },
+                    "unit": "none"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 6,
+                "w": 8,
+                "x": 4,
+                "y": 0
+            },
+            "id": 16,
+            "options": {
+                "displayMode": "lcd",
+                "minVizHeight": 10,
+                "minVizWidth": 0,
+                "namePlacement": "auto",
+                "orientation": "horizontal",
+                "reduceOptions": {
+                    "calcs": [
+                        "lastNotNull"
+                    ],
+                    "fields": "",
+                    "values": false
+                },
+                "showUnfilled": true,
+                "valueMode": "color"
+            },
+            "pluginVersion": "10.2.2",
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "increase(fastapi_requests_duration_seconds_count{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[$__range])",
+                    "instant": true,
+                    "interval": "",
+                    "legendFormat": "{{method}} {{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Requests Count",
+            "transformations": [
+                {
+                    "id": "seriesToRows",
+                    "options": {}
+                },
+                {
+                    "id": "sortBy",
+                    "options": {
+                        "fields": {},
+                        "sort": [
+                            {
+                                "field": "Time"
+                            }
+                        ]
+                    }
+                },
+                {
+                    "id": "partitionByValues",
+                    "options": {
+                        "fields": [
+                            "Metric"
+                        ]
+                    }
+                }
+            ],
+            "type": "bargauge"
+        },
+        {
+            "datasource": {
+                "type": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "continuous-GrYlRd"
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 80
+                            }
+                        ]
+                    },
+                    "unit": "s"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 12,
+                "y": 0
+            },
+            "id": 6,
+            "options": {
+                "displayMode": "lcd",
+                "minVizHeight": 10,
+                "minVizWidth": 0,
+                "namePlacement": "auto",
+                "orientation": "horizontal",
+                "reduceOptions": {
+                    "calcs": [
+                        "lastNotNull"
+                    ],
+                    "fields": "",
+                    "values": false
+                },
+                "showUnfilled": true,
+                "valueMode": "color"
+            },
+            "pluginVersion": "10.2.2",
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "fastapi_requests_duration_seconds_sum{app_name=\"$app_name\", path!~\"/metrics|/health|/\"} / fastapi_requests_duration_seconds_count{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}",
+                    "interval": "",
+                    "legendFormat": "{{method}} {{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Requests Average Duration",
+            "transformations": [
+                {
+                    "id": "spatial",
+                    "options": {
+                        "action": "prepare",
+                        "source": {
+                            "mode": "coords"
+                        }
+                    }
+                }
+            ],
+            "type": "bargauge"
+        },
+        {
+            "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "continuous-GrYlRd"
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 80
+                            }
+                        ]
+                    }
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 7,
+                "w": 7,
+                "x": 0,
+                "y": 6
+            },
+            "id": 22,
+            "options": {
+                "displayMode": "basic",
+                "minVizHeight": 10,
+                "minVizWidth": 0,
+                "namePlacement": "auto",
+                "orientation": "horizontal",
+                "reduceOptions": {
+                    "calcs": [
+                        "lastNotNull"
+                    ],
+                    "fields": "",
+                    "values": false
+                },
+                "showUnfilled": true,
+                "valueMode": "color"
+            },
+            "pluginVersion": "10.2.2",
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "sum(fastapi_exceptions_total{app_name=\"$app_name\"}) by (exception_type, path, method)",
+                    "interval": "",
+                    "legendFormat": "{{exception_type}} - {{method}}{{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "timeFrom": "24h",
+            "title": "Total Exceptions",
+            "type": "bargauge"
+        },
+        {
+            "datasource": {
+                "type": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisBorderShow": false,
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "axisSoftMax": 1,
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "area"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "red",
+                                "value": null
+                            },
+                            {
+                                "color": "green",
+                                "value": 0.8
+                            }
+                        ]
+                    },
+                    "unit": "percentunit"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 7,
+                "w": 7,
+                "x": 7,
+                "y": 6
+            },
+            "id": 18,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"2.*\", path!~\"/metrics|/health|/\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"})",
+                    "interval": "",
+                    "legendFormat": "{{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Percent of 2xx Requests",
+            "type": "timeseries"
+        },
+        {
+            "datasource": {
+                "type": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisBorderShow": false,
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "axisSoftMax": 1,
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "area"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 0.1
+                            }
+                        ]
+                    },
+                    "unit": "percentunit"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 7,
+                "w": 5,
+                "x": 14,
+                "y": 6
+            },
+            "id": 23,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"4.*\", path!~\"/metrics|/health|/\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"})",
+                    "interval": "",
+                    "legendFormat": "{{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Percent of 4xx Requests",
+            "type": "timeseries"
+        },
+        {
+            "datasource": {
+                "type": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisBorderShow": false,
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "axisSoftMax": 1,
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "area"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 0.1
+                            }
+                        ]
+                    },
+                    "unit": "percentunit"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 7,
+                "w": 5,
+                "x": 19,
+                "y": 6
+            },
+            "id": 20,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"5.*\", path!~\"/metrics|/health|/\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"})",
+                    "interval": "",
+                    "legendFormat": "{{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Percent of 5xx Requests",
+            "type": "timeseries"
+        },
+        {
+            "datasource": {
+                "type": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisBorderShow": false,
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "off"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 80
+                            }
+                        ]
+                    },
+                    "unit": "s"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 0,
+                "y": 13
+            },
+            "id": 8,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "histogram_quantile(.99,sum(rate(fastapi_requests_duration_seconds_bucket{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[1m])) by(path, le))",
+                    "interval": "",
+                    "legendFormat": "{{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "PR 99 Requests Duration",
+            "type": "timeseries"
+        },
+        {
+            "datasource": {
+                "type": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisBorderShow": false,
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "off"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 80
+                            }
+                        ]
+                    }
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 8,
+                "y": 13
+            },
+            "id": 10,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "fastapi_requests_in_progress{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}",
+                    "interval": "",
+                    "legendFormat": "{{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Request In Process",
+            "type": "timeseries"
+        },
+        {
+            "datasource": {
+                "type": "prometheus"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisBorderShow": false,
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "off"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 80
+                            }
+                        ]
+                    },
+                    "unit": "reqps"
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 8,
+                "w": 8,
+                "x": 16,
+                "y": 13
+            },
+            "id": 12,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus"
+                    },
+                    "editorMode": "code",
+                    "exemplar": true,
+                    "expr": "rate(fastapi_requests_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[1m])",
+                    "interval": "",
+                    "legendFormat": "{{path}}",
+                    "range": true,
+                    "refId": "A"
+                }
+            ],
+            "title": "Request Per Sec",
+            "type": "timeseries"
+        },
+        {
+            "datasource": {
+                "type": "loki",
+                "uid": "loki"
+            },
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisBorderShow": false,
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "bars",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "normal"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "off"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 80
+                            }
+                        ]
+                    }
+                },
+                "overrides": [
+                    {
+                        "matcher": {
+                            "id": "byName",
+                            "options": "ERROR"
+                        },
+                        "properties": [
+                            {
+                                "id": "color",
+                                "value": {
+                                    "fixedColor": "red",
+                                    "mode": "fixed"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "matcher": {
+                            "id": "byName",
+                            "options": "INFO"
+                        },
+                        "properties": [
+                            {
+                                "id": "color",
+                                "value": {
+                                    "fixedColor": "green",
+                                    "mode": "fixed"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "matcher": {
+                            "id": "byName",
+                            "options": "CRITICAL"
+                        },
+                        "properties": [
+                            {
+                                "id": "color",
+                                "value": {
+                                    "fixedColor": "purple",
+                                    "mode": "fixed"
+                                }
+                            }
+                        ]
+                    }
+                ]
+            },
+            "gridPos": {
+                "h": 9,
+                "w": 6,
+                "x": 0,
+                "y": 21
+            },
+            "id": 14,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "pluginVersion": "8.4.3",
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "loki",
+                        "uid": "loki"
+                    },
+                    "expr": "sum by(type) (rate({compose_service=~\"web-.*\"} | pattern `<date> <time> <type> <logger> <code> [<_>=<trace_id> <_>=<span_id> <_>=<service_name>] - <msg>` | type != \"\" |= \"$log_keyword\" [1m]))",
+                    "legendFormat": "{{type}}",
+                    "refId": "A"
+                }
+            ],
+            "title": "Log Type Rate",
+            "type": "timeseries"
+        },
+        {
+            "datasource": {
+                "type": "loki",
+                "uid": "loki"
+            },
+            "gridPos": {
+                "h": 9,
+                "w": 18,
+                "x": 6,
+                "y": 21
+            },
+            "id": 2,
+            "options": {
+                "dedupStrategy": "none",
+                "enableLogDetails": true,
+                "prettifyLogMessage": false,
+                "showCommonLabels": false,
+                "showLabels": false,
+                "showTime": false,
+                "sortOrder": "Descending",
+                "wrapLogMessage": true
+            },
+            "pluginVersion": "8.4.3",
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "loki",
+                        "uid": "loki"
+                    },
+                    "editorMode": "code",
+                    "expr": "{compose_service=~\"web-.*\", source=\"stderr\"} | pattern `<date> <time> <type> <logger> <code> [<_>=<trace_id> <_>=<span_id> <_>=<service_name>] - <msg>` | line_format \"{{.compose_service}}\\t{{.type}}\\t trace_id={{.trace_id}}\\t {{.msg}}\" |= \"$log_keyword\"",
+                    "hide": false,
+                    "queryType": "range",
+                    "refId": "A"
+                }
+            ],
+            "title": "Log of All FastAPI App",
+            "type": "logs"
+        }
+    ],
+    "refresh": "5s",
+    "schemaVersion": 38,
+    "tags": [],
+    "templating": {
+        "list": [
+            {
+                "current": {
+                    "selected": false,
+                    "text": "web",
+                    "value": "web"
+                },
+                "datasource": {
+                    "type": "prometheus",
+                    "uid": "prometheus"
+                },
+                "definition": "label_values(fastapi_app_info{}, app_name)",
+                "hide": 0,
+                "includeAll": false,
+                "label": "Application Name",
+                "multi": false,
+                "name": "app_name",
+                "options": [],
+                "query": {
+                    "query": "label_values(fastapi_app_info{}, app_name)",
+                    "refId": "StandardVariableQuery"
+                },
+                "refresh": 1,
+                "regex": "",
+                "skipUrlSync": false,
+                "sort": 0,
+                "type": "query"
+            },
+            {
+                "current": {
+                    "selected": false,
+                    "text": "",
+                    "value": ""
+                },
+                "description": "query with keyword",
+                "hide": 0,
+                "label": "Log Query",
+                "name": "log_keyword",
+                "options": [
+                    {
+                        "selected": true,
+                        "text": "",
+                        "value": ""
+                    }
+                ],
+                "query": "",
+                "skipUrlSync": false,
+                "type": "textbox"
+            }
+        ]
+    },
+    "time": {
+        "from": "now-24h",
+        "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "FastAPI Observability",
+    "uid": "fastapi-observability",
+    "version": 3,
+    "weekStart": ""
+}
diff --git a/dashboard/prometheus/prometheus.yml b/dashboard/prometheus/prometheus.yml
new file mode 100644
index 0000000..5853304
--- /dev/null
+++ b/dashboard/prometheus/prometheus.yml
@@ -0,0 +1,30 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - follow_redirects: true
+    enable_http2: true
+    scheme: http
+    timeout: 10s
+    api_version: v2
+    static_configs:
+    - targets: []
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  follow_redirects: true
+  enable_http2: true
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: 'web'
+  scrape_interval: 10s
+  metrics_path: /metrics
+  static_configs:
+    - targets: ['web:5000']

From e3afafb4905cbd3ec9f81bec26ed627982eece92 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Wed, 14 Feb 2024 11:15:11 +0900
Subject: [PATCH 04/20] fix container names

---
 dashboard/prometheus/prometheus.yml | 2 +-
 docker-compose.yml                  | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/dashboard/prometheus/prometheus.yml b/dashboard/prometheus/prometheus.yml
index 5853304..7037f23 100644
--- a/dashboard/prometheus/prometheus.yml
+++ b/dashboard/prometheus/prometheus.yml
@@ -22,7 +22,7 @@ scrape_configs:
   enable_http2: true
   static_configs:
   - targets:
-    - localhost:9090
+    - prometheus:9090
 - job_name: 'web'
   scrape_interval: 10s
   metrics_path: /metrics
diff --git a/docker-compose.yml b/docker-compose.yml
index 774a575..9921d38 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -13,6 +13,7 @@ version: '3.8'
 
 services:
   web:
+    container_name: web
     build:
       context: .
       dockerfile: Dockerfile
@@ -34,6 +35,7 @@ services:
     logging: *default-logging
 
   worker:
+    container_name: worker
     build:
       context: .
       dockerfile: Dockerfile
@@ -48,6 +50,7 @@ services:
     logging: *default-logging
 
   db:
+    container_name: db
     image: postgres:13
     env_file:
       - ./src/.env
@@ -60,6 +63,7 @@ services:
     #  - 5432:5432
 
   redis:
+    container_name: redis
     image: redis:alpine
     volumes:
       - redis-data:/data
@@ -74,8 +78,8 @@ services:
       - "3100:3100"
 
   prometheus:
-    image: prom/prometheus
     container_name: prometheus
+    image: prom/prometheus
     ports:
       - 9090:9090
     volumes:
@@ -88,8 +92,8 @@ services:
     logging: *default-logging
 
   grafana:
-    image: grafana/grafana
     container_name: grafana
+    image: grafana/grafana
     ports:
       - 3000:3000
     volumes:

From fab2607f8bc90a36b2049d157e98bf92896323a7 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:54:48 +0900
Subject: [PATCH 05/20] add prometheus middleware

---
 src/app/middleware/prometheus_middleware.py | 85 +++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 src/app/middleware/prometheus_middleware.py

diff --git a/src/app/middleware/prometheus_middleware.py b/src/app/middleware/prometheus_middleware.py
new file mode 100644
index 0000000..1444348
--- /dev/null
+++ b/src/app/middleware/prometheus_middleware.py
@@ -0,0 +1,85 @@
+import time
+
+from prometheus_client import Counter, Gauge, Histogram
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Match
+from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR
+from starlette.types import ASGIApp
+
+INFO = Gauge(
+    "fastapi_app_info",
+    "FastAPI application information.",
+    ["app_name"],
+    multiprocess_mode="livesum",
+)
+REQUESTS = Counter(
+    "fastapi_requests_total", "Total count of requests by method and path.", ["method", "path", "app_name"]
+)
+RESPONSES = Counter(
+    "fastapi_responses_total",
+    "Total count of responses by method, path and status codes.",
+    ["method", "path", "status_code", "app_name"],
+)
+REQUESTS_PROCESSING_TIME = Histogram(
+    "fastapi_requests_duration_seconds",
+    "Histogram of requests processing time by path (in seconds)",
+    ["method", "path", "app_name"],
+)
+EXCEPTIONS = Counter(
+    "fastapi_exceptions_total",
+    "Total count of exceptions raised by path and exception type",
+    ["method", "path", "exception_type", "app_name"],
+)
+REQUESTS_IN_PROGRESS = Gauge(
+    "fastapi_requests_in_progress",
+    "Gauge of requests by method and path currently being processed",
+    ["method", "path", "app_name"],
+    multiprocess_mode="livesum",
+)
+
+
+class PrometheusMiddleware(BaseHTTPMiddleware):
+    def __init__(self, app: ASGIApp, app_name: str = "fastapi-app") -> None:
+        super().__init__(app)
+        self.app_name = app_name
+        INFO.labels(app_name=self.app_name).inc()
+
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        method = request.method
+        path, is_handled_path = self.get_path(request)
+
+        if not is_handled_path:
+            return await call_next(request)
+
+        REQUESTS_IN_PROGRESS.labels(method=method, path=path, app_name=self.app_name).inc()
+        REQUESTS.labels(method=method, path=path, app_name=self.app_name).inc()
+        before_time = time.perf_counter()
+        try:
+            response = await call_next(request)
+        except BaseException as e:
+            status_code = HTTP_500_INTERNAL_SERVER_ERROR
+            EXCEPTIONS.labels(method=method, path=path, exception_type=type(e).__name__, app_name=self.app_name).inc()
+            raise e from None
+        else:
+            status_code = response.status_code
+            after_time = time.perf_counter()
+
+            REQUESTS_PROCESSING_TIME.labels(method=method, path=path, app_name=self.app_name).observe(
+                after_time - before_time
+            )
+        finally:
+            RESPONSES.labels(method=method, path=path, status_code=status_code, app_name=self.app_name).inc()
+            REQUESTS_IN_PROGRESS.labels(method=method, path=path, app_name=self.app_name).dec()
+
+        return response
+
+    @staticmethod
+    def get_path(request: Request) -> tuple[str, bool]:
+        for route in request.app.routes:
+            match, child_scope = route.matches(request.scope)
+            if match == Match.FULL:
+                return route.path, True
+
+        return request.url.path, False

From 280e1e6ed9586753b4f8f5043b3083a3f8bbe1ac Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:55:24 +0900
Subject: [PATCH 06/20] update setup optl, prometheus

---
 src/app/core/config.py   |  5 +++++
 src/app/core/setup.py    | 11 +++++++++++
 src/app/core/tracking.py | 36 ++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+)
 create mode 100644 src/app/core/tracking.py

diff --git a/src/app/core/config.py b/src/app/core/config.py
index e8b85bf..be80ebf 100644
--- a/src/app/core/config.py
+++ b/src/app/core/config.py
@@ -99,6 +99,10 @@ class DefaultRateLimitSettings(BaseSettings):
     DEFAULT_RATE_LIMIT_PERIOD: int = config("DEFAULT_RATE_LIMIT_PERIOD", default=3600)
 
 
+class OpenTelemeterySettings(BaseSettings):
+    OTLP_GRPC_ENDPOINT: str = config("OTLP_GRPC_ENDPOINT", default="http://tempo:4317")
+
+
 class EnvironmentOption(Enum):
     LOCAL = "local"
     STAGING = "staging"
@@ -121,6 +125,7 @@ class Settings(
     RedisRateLimiterSettings,
     DefaultRateLimitSettings,
     EnvironmentSettings,
+    OpenTelemeterySettings,
 ):
     pass
 
diff --git a/src/app/core/setup.py b/src/app/core/setup.py
index c126cd3..ecc3e47 100644
--- a/src/app/core/setup.py
+++ b/src/app/core/setup.py
@@ -13,12 +13,14 @@
 
 from ..api.dependencies import get_current_superuser
 from ..middleware.client_cache_middleware import ClientCacheMiddleware
+from ..middleware.prometheus_middleware import PrometheusMiddleware
 from .config import (
     AppSettings,
     ClientSideCacheSettings,
     DatabaseSettings,
     EnvironmentOption,
     EnvironmentSettings,
+    OpenTelemeterySettings,
     RedisCacheSettings,
     RedisQueueSettings,
     RedisRateLimiterSettings,
@@ -26,6 +28,7 @@
 )
 from .db.database import Base
 from .db.database import async_engine as engine
+from .tracking import metrics, setting_otlp
 from .utils import cache, queue, rate_limit
 
 
@@ -125,6 +128,7 @@ def create_application(
         | RedisQueueSettings
         | RedisRateLimiterSettings
         | EnvironmentSettings
+        | OpenTelemeterySettings
     ),
     create_tables_on_start: bool = True,
     **kwargs: Any,
@@ -186,6 +190,13 @@ def create_application(
 
     application = FastAPI(lifespan=lifespan, **kwargs)
 
+    if isinstance(settings, AppSettings):
+        # Setting metrics middleware
+        application.add_middleware(PrometheusMiddleware, app_name=settings.APP_NAME)
+        application.add_route("/metrics", metrics)
+        if isinstance(settings, OpenTelemeterySettings):
+            setting_otlp(application, settings.APP_NAME, settings.OTLP_GRPC_ENDPOINT)
+
     if isinstance(settings, ClientSideCacheSettings):
         application.add_middleware(ClientCacheMiddleware, max_age=settings.CLIENT_CACHE_MAX_AGE)
 
diff --git a/src/app/core/tracking.py b/src/app/core/tracking.py
new file mode 100644
index 0000000..c3cc0e2
--- /dev/null
+++ b/src/app/core/tracking.py
@@ -0,0 +1,36 @@
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, generate_latest
+from prometheus_client.multiprocess import MultiProcessCollector
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.types import ASGIApp
+
+
+def metrics(request: Request) -> Response:
+    registry = CollectorRegistry()
+    MultiProcessCollector(registry)
+    data = generate_latest(registry)
+    return Response(data, headers={"Content-Type": CONTENT_TYPE_LATEST})
+
+
+def setting_otlp(app: ASGIApp, app_name: str, endpoint: str, log_correlation: bool = True) -> None:
+    # Setting OpenTelemetry
+    # set the service name to show in traces
+    resource = Resource.create(attributes={"service.name": app_name, "compose_service": app_name})
+
+    # set the tracer provider
+    tracer = TracerProvider(resource=resource)
+    trace.set_tracer_provider(tracer)
+
+    tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))
+
+    if log_correlation:
+        LoggingInstrumentor().instrument(set_logging_format=True)
+
+    FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer)

From f6dd40dda72c19e6a6586d672bb0f0895cee4a3b Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:55:40 +0900
Subject: [PATCH 07/20] consistency

---
 dashboard/prometheus/prometheus.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dashboard/prometheus/prometheus.yml b/dashboard/prometheus/prometheus.yml
index 7037f23..9d7f7bd 100644
--- a/dashboard/prometheus/prometheus.yml
+++ b/dashboard/prometheus/prometheus.yml
@@ -23,8 +23,9 @@ scrape_configs:
   static_configs:
   - targets:
     - prometheus:9090
-- job_name: 'web'
+- job_name: web
   scrape_interval: 10s
   metrics_path: /metrics
   static_configs:
-    - targets: ['web:5000']
+    - targets:
+      - web:8000

From 3668dd975012dbb2ce88b1d7b7867ce86a128072 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:56:05 +0900
Subject: [PATCH 08/20] add prometheus for gunicorn (multiprocess)

---
 src/app/gunicorn_conf.py | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 src/app/gunicorn_conf.py

diff --git a/src/app/gunicorn_conf.py b/src/app/gunicorn_conf.py
new file mode 100644
index 0000000..cbfb88b
--- /dev/null
+++ b/src/app/gunicorn_conf.py
@@ -0,0 +1,5 @@
+from prometheus_client import multiprocess
+
+
+def worker_exit(server, worker):
+    multiprocess.mark_process_dead(worker.pid)

From 0ca00c4a773e553ae0847075544f3e7d12e132d6 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:56:26 +0900
Subject: [PATCH 09/20] update logger for tracing

---
 src/app/core/logger.py | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/app/core/logger.py b/src/app/core/logger.py
index 91b35a1..c1e44d9 100644
--- a/src/app/core/logger.py
+++ b/src/app/core/logger.py
@@ -8,13 +8,42 @@
 
 LOG_FILE_PATH = os.path.join(LOG_DIR, "app.log")
 
+
+class SafeOtelFormatter(logging.Formatter):
+    def format(self, record):
+        # Default values if OpenTelemetry fields are not present
+        record.otelTraceID = getattr(record, "otelTraceID", "none")
+        record.otelSpanID = getattr(record, "otelSpanID", "none")
+        record.otelServiceName = getattr(record, "otelServiceName", "none")
+        return super().format(record)
+
+
 LOGGING_LEVEL = logging.INFO
-LOGGING_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+LOGGING_FORMAT = "%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [trace_id=%(otelTraceID)s \
+                span_id=%(otelSpanID)s resource.service.name=%(otelServiceName)s] - %(message)s"
+
+
+logging.basicConfig(level=logging.INFO, format=LOGGING_FORMAT)
 
-logging.basicConfig(level=LOGGING_LEVEL, format=LOGGING_FORMAT)
+# Create the formatter
+formatter = SafeOtelFormatter(LOGGING_FORMAT)
+
+# Apply the formatter to all handlers
+for handler in logging.getLogger().handlers:
+    handler.setFormatter(formatter)
 
 file_handler = RotatingFileHandler(LOG_FILE_PATH, maxBytes=10485760, backupCount=5)
 file_handler.setLevel(LOGGING_LEVEL)
-file_handler.setFormatter(logging.Formatter(LOGGING_FORMAT))
+file_handler.setFormatter(SafeOtelFormatter(LOGGING_FORMAT))
 
 logging.getLogger("").addHandler(file_handler)
+
+
+class EndpointFilter(logging.Filter):
+    # Uvicorn endpoint access log filter
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.getMessage().find("GET /metrics") == -1
+
+
+# Filter out /endpoint
+logging.getLogger("uvicorn.access").addFilter(EndpointFilter())

From 845bf1082d472fed75ba22f5154aaf81c357d26b Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:56:47 +0900
Subject: [PATCH 10/20] add prometheus multiprocessing folder

---
 Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 49538b8..f8c2323 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,6 +22,8 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 COPY ./src/app /code/app
 
+RUN mkdir -p /tmp/prometheus_multiproc
+
 # -------- replace with comment to run with gunicorn --------
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
 # CMD ["gunicorn", "app.main:app", "-w", "4", "-k", "uvicorn.workers.UvicornWorker". "-b", "0.0.0.0:8000"]

From d4d2a77eb84622103348238c44c28a458666f7d8 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:57:58 +0900
Subject: [PATCH 11/20] fix docker-compose for multiproc gunicorn

---
 docker-compose.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 2ff1599..8222375 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -19,9 +19,11 @@ services:
       dockerfile: Dockerfile
     # -------- replace with comment to run with gunicorn --------
     command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
-    # command: gunicorn app.main:app -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0:8000
+    # command: gunicorn app.main:app -c app/gunicorn_conf.py -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0:8000
     env_file:
       - ./src/.env
+    environment:
+      - PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_multiproc
     # -------- replace with comment if you are using nginx --------
     ports:
       - "8000:8000"

From da6d2c51d86bb77ba2b8550426973d7221ca65ab Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 11:58:11 +0900
Subject: [PATCH 12/20] add dependencies

---
 pyproject.toml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 807c854..bed15dc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,6 +31,11 @@ arq = "^0.25.0"
 gunicorn = "^21.2.0"
 bcrypt = "^4.1.1"
 fastcrud = "^0.6.0"
+prometheus-client = "^0.17.1"
+opentelemetry-distro = "^0.42b0"
+opentelemetry-instrumentation-fastapi = "^0.42b0"
+opentelemetry-instrumentation-logging = "^0.42b0"
+opentelemetry-exporter-otlp = "^1.21.0"
 
 
 [build-system]

From 9a059963a18e281d24717cb3097390cbd512d018 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 15:39:13 +0900
Subject: [PATCH 13/20] Squashed commit of the following:

commit b1a8370442cbf94bd2bbac4dad5ab2848367995f
Merge: ddec40e 169e93f
Author: Igor Benav <43156212+igorbenav@users.noreply.github.com>
Date:   Thu Feb 15 00:26:05 2024 -0300

    Merge pull request #118 from luca-medeiros/fix/router

    bug: fix missing router when creating application

commit 169e93f83d1843728d2e1b3d3a8a63af41729ee6
Author: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date:   Thu Feb 15 12:24:15 2024 +0900

    bug: fix missing router when creating application
---
 src/app/core/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/app/core/setup.py b/src/app/core/setup.py
index ecc3e47..29badf2 100644
--- a/src/app/core/setup.py
+++ b/src/app/core/setup.py
@@ -189,6 +189,7 @@ def create_application(
     lifespan = lifespan_factory(settings, create_tables_on_start=create_tables_on_start)
 
     application = FastAPI(lifespan=lifespan, **kwargs)
+    application.include_router(router)
 
     if isinstance(settings, AppSettings):
         # Setting metrics middleware

From 1cc2e0e267ed0517fd545e8628f2abb38490fc29 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 17:31:41 +0900
Subject: [PATCH 14/20] add excluded handlers to prom

---
 .../dashboards/fastapi-observability.json     | 22 +++++++++----------
 src/app/core/setup.py                         |  6 ++++-
 src/app/middleware/prometheus_middleware.py   | 16 ++++++++------
 3 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/dashboard/grafana/provisioning/dashboards/fastapi-observability.json b/dashboard/grafana/provisioning/dashboards/fastapi-observability.json
index a19e9bb..92257a3 100644
--- a/dashboard/grafana/provisioning/dashboards/fastapi-observability.json
+++ b/dashboard/grafana/provisioning/dashboards/fastapi-observability.json
@@ -79,7 +79,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "sum(increase(fastapi_requests_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[$__range]))",
+                    "expr": "sum(increase(fastapi_requests_total{app_name=\"$app_name\"}[$__range]))",
                     "instant": true,
                     "interval": "",
                     "legendFormat": "",
@@ -163,7 +163,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "increase(fastapi_requests_duration_seconds_count{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[$__range])",
+                    "expr": "increase(fastapi_requests_duration_seconds_count{app_name=\"$app_name\"}[$__range])",
                     "instant": true,
                     "interval": "",
                     "legendFormat": "{{method}} {{path}}",
@@ -257,7 +257,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "fastapi_requests_duration_seconds_sum{app_name=\"$app_name\", path!~\"/metrics|/health|/\"} / fastapi_requests_duration_seconds_count{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}",
+                    "expr": "fastapi_requests_duration_seconds_sum{app_name=\"$app_name\"} / fastapi_requests_duration_seconds_count{app_name=\"$app_name\"}",
                     "interval": "",
                     "legendFormat": "{{method}} {{path}}",
                     "range": true,
@@ -433,7 +433,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"2.*\", path!~\"/metrics|/health|/\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"})",
+                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"2.*\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\"})",
                     "interval": "",
                     "legendFormat": "{{path}}",
                     "range": true,
@@ -529,7 +529,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"4.*\", path!~\"/metrics|/health|/\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"})",
+                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"4.*\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\"})",
                     "interval": "",
                     "legendFormat": "{{path}}",
                     "range": true,
@@ -625,7 +625,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"5.*\", path!~\"/metrics|/health|/\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"})",
+                    "expr": "sum by(path) (fastapi_responses_total{app_name=\"$app_name\", status_code=~\"5.*\"}) / sum by(path) (fastapi_responses_total{app_name=\"$app_name\"})",
                     "interval": "",
                     "legendFormat": "{{path}}",
                     "range": true,
@@ -720,7 +720,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "histogram_quantile(.99,sum(rate(fastapi_requests_duration_seconds_bucket{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[1m])) by(path, le))",
+                    "expr": "histogram_quantile(.99,sum(rate(fastapi_requests_duration_seconds_bucket{app_name=\"$app_name\"}[1m])) by(path, le))",
                     "interval": "",
                     "legendFormat": "{{path}}",
                     "range": true,
@@ -814,7 +814,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "fastapi_requests_in_progress{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}",
+                    "expr": "fastapi_requests_in_progress{app_name=\"$app_name\"}",
                     "interval": "",
                     "legendFormat": "{{path}}",
                     "range": true,
@@ -909,7 +909,7 @@
                     },
                     "editorMode": "code",
                     "exemplar": true,
-                    "expr": "rate(fastapi_requests_total{app_name=\"$app_name\", path!~\"/metrics|/health|/\"}[1m])",
+                    "expr": "rate(fastapi_requests_total{app_name=\"$app_name\"}[1m])",
                     "interval": "",
                     "legendFormat": "{{path}}",
                     "range": true,
@@ -1050,7 +1050,7 @@
                         "type": "loki",
                         "uid": "loki"
                     },
-                    "expr": "sum by(type) (rate({compose_service=~\"web-.*\"} | pattern `<date> <time> <type> <logger> <code> [<_>=<trace_id> <_>=<span_id> <_>=<service_name>] - <msg>` | type != \"\" |= \"$log_keyword\" [1m]))",
+                    "expr": "sum by(type) (rate({compose_service=~\"web.*\"} | pattern `<date> <time> <type> <logger> <code> [<_>=<trace_id> <_>=<span_id> <_>=<service_name>] - <msg>` | type != \"\" |= \"$log_keyword\" [1m]))",
                     "legendFormat": "{{type}}",
                     "refId": "A"
                 }
@@ -1088,7 +1088,7 @@
                         "uid": "loki"
                     },
                     "editorMode": "code",
-                    "expr": "{compose_service=~\"web-.*\", source=\"stderr\"} | pattern `<date> <time> <type> <logger> <code> [<_>=<trace_id> <_>=<span_id> <_>=<service_name>] - <msg>` | line_format \"{{.compose_service}}\\t{{.type}}\\t trace_id={{.trace_id}}\\t {{.msg}}\" |= \"$log_keyword\"",
+                    "expr": "{compose_service=~\"web.*\", source=\"stderr\"} | pattern `<date> <time> <type> <logger> <code> [<_>=<trace_id> <_>=<span_id> <_>=<service_name>] - <msg>` | line_format \"{{.compose_service}}\\t{{.type}}\\t trace_id={{.trace_id}}\\t {{.msg}}\" |= \"$log_keyword\"",
                     "hide": false,
                     "queryType": "range",
                     "refId": "A"
diff --git a/src/app/core/setup.py b/src/app/core/setup.py
index 29badf2..f3017fc 100644
--- a/src/app/core/setup.py
+++ b/src/app/core/setup.py
@@ -193,7 +193,11 @@ def create_application(
 
     if isinstance(settings, AppSettings):
         # Setting metrics middleware
-        application.add_middleware(PrometheusMiddleware, app_name=settings.APP_NAME)
+        application.add_middleware(
+            PrometheusMiddleware,
+            app_name=settings.APP_NAME,
+            excluded_handlers=["/docs", "/redoc", "/openapi.json", "/metrics"],
+        )
         application.add_route("/metrics", metrics)
         if isinstance(settings, OpenTelemeterySettings):
             setting_otlp(application, settings.APP_NAME, settings.OTLP_GRPC_ENDPOINT)
diff --git a/src/app/middleware/prometheus_middleware.py b/src/app/middleware/prometheus_middleware.py
index 1444348..11b380e 100644
--- a/src/app/middleware/prometheus_middleware.py
+++ b/src/app/middleware/prometheus_middleware.py
@@ -1,3 +1,4 @@
+import re
 import time
 
 from prometheus_client import Counter, Gauge, Histogram
@@ -41,9 +42,10 @@
 
 
 class PrometheusMiddleware(BaseHTTPMiddleware):
-    def __init__(self, app: ASGIApp, app_name: str = "fastapi-app") -> None:
+    def __init__(self, app: ASGIApp, app_name: str, excluded_handlers: list[str]) -> None:
         super().__init__(app)
         self.app_name = app_name
+        self.excluded_handlers = [re.compile(path) for path in excluded_handlers]
         INFO.labels(app_name=self.app_name).inc()
 
     async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
@@ -75,11 +77,11 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
 
         return response
 
-    @staticmethod
-    def get_path(request: Request) -> tuple[str, bool]:
-        for route in request.app.routes:
-            match, child_scope = route.matches(request.scope)
-            if match == Match.FULL:
-                return route.path, True
+    def get_path(self, request: Request) -> tuple[str, bool]:
+        if not any(pattern.search(request.url.path) for pattern in self.excluded_handlers):
+            for route in request.app.routes:
+                match, child_scope = route.matches(request.scope)
+                if match == Match.FULL:
+                    return route.path, True
 
         return request.url.path, False

From 294a871e40880ac7d7dbbae6f06c085912f57064 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Thu, 15 Feb 2024 17:38:17 +0900
Subject: [PATCH 15/20] compose

---
 docker-compose.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 8222375..3f9ea05 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -192,5 +192,6 @@ services:
   #     - ./src:/code/src
 
 volumes:
+  #pgadmin-data:
   postgres-data:
-  redis-data: #pgadmin-data:
+  redis-data:

From 6d277f41d3e897aa9dce3816937c68a8d849700a Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Mon, 13 May 2024 10:57:36 +0900
Subject: [PATCH 16/20] add opentel settings to init

---
 src/app/core/setup.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/app/core/setup.py b/src/app/core/setup.py
index f3017fc..676ca94 100644
--- a/src/app/core/setup.py
+++ b/src/app/core/setup.py
@@ -82,6 +82,7 @@ def lifespan_factory(
         | RedisQueueSettings
         | RedisRateLimiterSettings
         | EnvironmentSettings
+        | OpenTelemeterySettings
     ),
     create_tables_on_start: bool = True,
 ) -> Callable[[FastAPI], _AsyncGeneratorContextManager[Any]]:
@@ -155,6 +156,7 @@ def create_application(
         - RedisRateLimiterSettings: Sets up event handlers for creating and closing a Redis rate limiter pool.
         - EnvironmentSettings: Conditionally sets documentation URLs and integrates custom routes for API documentation
           based on the environment type.
+        - OpenTelemeterySettings: Configures OpenTelemetry tracing for the FastAPI application.
 
     create_tables_on_start : bool
         A flag to indicate whether to create database tables on application startup.

From bc8e6c1aeace4ce77d0f791d1b055b2868491b10 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Mon, 13 May 2024 12:46:41 +0900
Subject: [PATCH 17/20] comment out docker compose

---
 docker-compose.yml | 122 +++++++++++++++++++++++----------------------
 1 file changed, 63 insertions(+), 59 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 3f9ea05..2864064 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,13 +1,14 @@
-x-logging: &default-logging
-  driver: loki
-  options:
-    loki-url: 'http://localhost:3100/api/prom/push'
-    loki-pipeline-stages: |
-      - multiline:
-          firstline: '^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}'
-          max_wait_time: 3s
-      - regex:
-          expression: '^(?P<time>\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2},d{3}) (?P<message>(?s:.*))$$'
+#-------- uncomment to run with monitoring --------
+# x-logging: &default-logging
+#   driver: loki
+#   options:
+#     loki-url: 'http://localhost:3100/api/prom/push'
+#     loki-pipeline-stages: |
+#       - multiline:
+#           firstline: '^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}'
+#           max_wait_time: 3s
+#       - regex:
+#           expression: '^(?P<time>\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2},d{3}) (?P<message>(?s:.*))$$'
 
 version: '3.8'
 
@@ -35,7 +36,8 @@ services:
     volumes:
       - ./src/app:/code/app
       - ./src/.env:/code/.env
-    logging: *default-logging
+    #-------- uncomment to run with monitoring --------
+    # logging: *default-logging
 
   worker:
     container_name: worker
@@ -51,7 +53,8 @@ services:
     volumes:
       - ./src/app:/code/app
       - ./src/.env:/code/.env
-    logging: *default-logging
+    #-------- uncomment to run with monitoring --------
+    # logging: *default-logging
 
   db:
     container_name: db
@@ -73,57 +76,58 @@ services:
       - redis-data:/data
     expose:
       - "6379"
+  #-------- uncomment to run with monitoring --------
+  # loki:
+  #   container_name: loki
+  #   image: grafana/loki:2.8.3
+  #   command: -config.file=/etc/loki/local-config.yaml
+  #   ports:
+  #     - "3100:3100"
 
-  loki:
-    container_name: loki
-    image: grafana/loki:2.8.3
-    command: -config.file=/etc/loki/local-config.yaml
-    ports:
-      - "3100:3100"
+  # prometheus:
+  #   container_name: prometheus
+  #   image: prom/prometheus
+  #   ports:
+  #     - 9090:9090
+  #   volumes:
+  #     - ./dashboard/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+  #   command:
+  #     - '--config.file=/etc/prometheus/prometheus.yml'
+  #   depends_on:
+  #     - loki
+  #     - web
+  #   logging: *default-logging
 
-  prometheus:
-    container_name: prometheus
-    image: prom/prometheus
-    ports:
-      - 9090:9090
-    volumes:
-      - ./dashboard/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-    depends_on:
-      - loki
-      - web
-    logging: *default-logging
+  # grafana:
+  #   container_name: grafana
+  #   image: grafana/grafana
+  #   ports:
+  #     - 3000:3000
+  #   volumes:
+  #     - ./dashboard/grafana/datasources:/etc/grafana/provisioning/datasources
+  #     - ./dashboard/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+  #   depends_on:
+  #     - loki
+  #     - prometheus
+  #     - tempo
+  #   logging: *default-logging
 
-  grafana:
-    container_name: grafana
-    image: grafana/grafana
-    ports:
-      - 3000:3000
-    volumes:
-      - ./dashboard/grafana/datasources:/etc/grafana/provisioning/datasources
-      - ./dashboard/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
-    depends_on:
-      - loki
-      - prometheus
-      - tempo
-    logging: *default-logging
+  # tempo:
+  #   container_name: tempo
+  #   image: grafana/tempo:2.1.1
+  #   command:
+  #     [
+  #       "--target=all",
+  #       "--storage.trace.backend=local",
+  #       "--storage.trace.local.path=/var/tempo",
+  #       "--auth.enabled=false"
+  #     ]
+  #   ports:
+  #     - "14250:14250"
+  #   depends_on:
+  #     - loki
+  #   logging: *default-logging
 
-  tempo:
-    container_name: tempo
-    image: grafana/tempo:2.1.1
-    command:
-      [
-        "--target=all",
-        "--storage.trace.backend=local",
-        "--storage.trace.local.path=/var/tempo",
-        "--auth.enabled=false"
-      ]
-    ports:
-      - "14250:14250"
-    depends_on:
-      - loki
-    logging: *default-logging
   #-------- uncomment to run with pgadmin --------
   # pgadmin:
   #   container_name: pgadmin4

From f451eef42e0b439eca828da1c4377faa0f4fdb3a Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Mon, 13 May 2024 12:47:10 +0900
Subject: [PATCH 18/20] update readme

---
 README.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 55 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 3115bb7..b502019 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,7 @@
 - [`Docker Compose`](https://docs.docker.com/compose/) With a single command, create and start all the services from your configuration.
 - [`NGINX`](https://nginx.org/en/) High-performance low resource consumption web server used for Reverse Proxy and Load Balancing.
 
-> \[!TIP\] 
+> \[!TIP\]
 > If you want the `SQLModel` version instead, head to [SQLModel-boilerplate](https://github.com/igorbenav/SQLModel-boilerplate).
 
 ## 1. Features
@@ -66,6 +66,8 @@
 - 🤸‍♂️ Flexible
 - 🚚 Easy running with docker compose
 - ⚖️ NGINX Reverse Proxy and Load Balancing
+- 📊 Prometheus metrics with Grafana dashboards monitoring
+- 📖 Loki and open telemetrics for logging
 
 ## 2. Contents
 
@@ -100,9 +102,10 @@
    1. [ARQ Job Queues](#510-arq-job-queues)
    1. [Rate Limiting](#511-rate-limiting)
    1. [JWT Authentication](#512-jwt-authentication)
-   1. [Running](#513-running)
-   1. [Create Application](#514-create-application)
-   2. [Opting Out of Services](#515-opting-out-of-services)
+   1. [Grafana Dashboards](#513-grafana-dashboards)
+   1. [Running](#514-running)
+   1. [Create Application](#515-create-application)
+   1. [Opting Out of Services](#516-opting-out-of-services)
 1. [Running in Production](#6-running-in-production)
    1. [Uvicorn Workers with Gunicorn](#61-uvicorn-workers-with-gunicorn)
    1. [Running With NGINX](#62-running-with-nginx)
@@ -587,6 +590,7 @@ First, you may want to take a look at the project structure and understand what
     │   │   ├── schemas.py            # Pydantic schemas for data validation.
     │   │   ├── security.py           # Security utilities, such as password hashing.
     │   │   ├── setup.py              # Setup file for the FastAPI app instance.
+    │   │   ├── tracking.py           # Metrics endpoint and open telemetrics setup
     │   │   │
     │   │   ├── db                    # Core Database related modules.
     │   │   │   ├── __init__.py
@@ -625,6 +629,7 @@ First, you may want to take a look at the project structure and understand what
     │   │
     │   ├── middleware                # Middleware components for the application.
     │   │   └── client_cache_middleware.py  # Middleware for client-side caching.
+    │   │   └── prometheus_middleware.py    # Middleware for prometheus metrics.
     │   │
     │   ├── models                    # ORM models for the application.
     │   │   ├── __init__.py
@@ -1459,7 +1464,48 @@ What you should do with the client is:
 
 This authentication setup in the provides a robust, secure, and user-friendly way to handle user sessions in your API applications.
 
-### 5.13 Running
+### 5.13 Grafana Dashboards
+
+For real-time monitoring, make sure to uncomment the containers related to monitoring:
+
+- loki
+- prometheus
+- grafana
+- tempo
+
+Also, for propper uniform logging, uncomment the x-logging and add the default logging to the worker and web.
+
+```Dockerfile
+x-logging: &default-logging
+  driver: loki
+  options:
+    loki-url: 'http://localhost:3100/api/prom/push'
+    loki-pipeline-stages: |
+      - multiline:
+          firstline: '^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}'
+          max_wait_time: 3s
+      - regex:
+          expression: '^(?P<time>\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2},d{3}) (?P<message>(?s:.*))$$'
+...
+web:
+  ...
+  logging: *default-logging
+
+worker:
+  ...
+  logging: *default-logging
+
+```
+
+Lastly, Install the loki plugin:
+
+```sh
+docker plugin install grafana/loki-docker-driver:2.9.2 --alias loki --grant-all-permissions
+```
+
+The dashboard graphs can be edited in `dashboard/grafana/provisioning/dashboards/fastapi-observability.json`
+
+### 5.14 Running
 
 If you are using docker compose, just running the following command should ensure everything is working:
 
@@ -1479,7 +1525,8 @@ And for the worker:
 ```sh
 poetry run arq src.app.core.worker.settings.WorkerSettings
 ```
-### 5.14 Create Application
+
+### 5.15 Create Application
 
 If you want to stop tables from being created every time you run the api, you should disable this here:
 
@@ -1502,7 +1549,7 @@ A few examples:
 - Add client-side cache middleware
 - Add Startup and Shutdown event handlers for cache, queue and rate limit
 
-### 5.15 Opting Out of Services
+### 5.16 Opting Out of Services
 
 To opt out of services (like `Redis`, `Queue`, `Rate Limiter`), head to the `Settings` class in `src/app/core/config`:
 
@@ -1519,6 +1566,7 @@ env_path = os.path.join(current_file_dir, "..", "..", ".env")
 config = Config(env_path)
 ...
 
+
 class Settings(
     AppSettings,
     PostgresSettings,

From 3f5e75108778a3d0b7026ae34028d595a1e15b29 Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Tue, 14 May 2024 14:31:35 +0900
Subject: [PATCH 19/20] add option to turn off monitoring

---
 README.md              | 13 ++++++++++---
 src/app/core/config.py |  5 +++--
 src/app/core/setup.py  | 29 +++++++++++++++--------------
 3 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index b502019..14d69e8 100644
--- a/README.md
+++ b/README.md
@@ -66,7 +66,7 @@
 - 🤸‍♂️ Flexible
 - 🚚 Easy running with docker compose
 - ⚖️ NGINX Reverse Proxy and Load Balancing
-- 📊 Prometheus metrics with Grafana dashboards monitoring
+- 📊 Prometheus metrics with Monitoring monitoring
 - 📖 Loki and open telemetrics for logging
 
 ## 2. Contents
@@ -102,7 +102,7 @@
    1. [ARQ Job Queues](#510-arq-job-queues)
    1. [Rate Limiting](#511-rate-limiting)
    1. [JWT Authentication](#512-jwt-authentication)
-   1. [Grafana Dashboards](#513-grafana-dashboards)
+   1. [Monitoring](#513-monitoring)
    1. [Running](#514-running)
    1. [Create Application](#515-create-application)
    1. [Opting Out of Services](#516-opting-out-of-services)
@@ -1464,7 +1464,7 @@ What you should do with the client is:
 
 This authentication setup in the provides a robust, secure, and user-friendly way to handle user sessions in your API applications.
 
-### 5.13 Grafana Dashboards
+### 5.13 Monitoring
 
 For real-time monitoring, make sure to uncomment the containers related to monitoring:
 
@@ -1473,6 +1473,13 @@ For real-time monitoring, make sure to uncomment the containers related to monit
 - grafana
 - tempo
 
+Inside of `.env`, create the following setting:
+
+```
+# ------------- monitoring settings -------------
+MONITORING=true
+```
+
 Also, for propper uniform logging, uncomment the x-logging and add the default logging to the worker and web.
 
 ```Dockerfile
diff --git a/src/app/core/config.py b/src/app/core/config.py
index be80ebf..142d550 100644
--- a/src/app/core/config.py
+++ b/src/app/core/config.py
@@ -99,7 +99,8 @@ class DefaultRateLimitSettings(BaseSettings):
     DEFAULT_RATE_LIMIT_PERIOD: int = config("DEFAULT_RATE_LIMIT_PERIOD", default=3600)
 
 
-class OpenTelemeterySettings(BaseSettings):
+class MonitoringSettings(BaseSettings):
+    MONITORING: bool = config("MONITORING", default=False)
     OTLP_GRPC_ENDPOINT: str = config("OTLP_GRPC_ENDPOINT", default="http://tempo:4317")
 
 
@@ -125,7 +126,7 @@ class Settings(
     RedisRateLimiterSettings,
     DefaultRateLimitSettings,
     EnvironmentSettings,
-    OpenTelemeterySettings,
+    MonitoringSettings,
 ):
     pass
 
diff --git a/src/app/core/setup.py b/src/app/core/setup.py
index 676ca94..36155da 100644
--- a/src/app/core/setup.py
+++ b/src/app/core/setup.py
@@ -20,7 +20,7 @@
     DatabaseSettings,
     EnvironmentOption,
     EnvironmentSettings,
-    OpenTelemeterySettings,
+    MonitoringSettings,
     RedisCacheSettings,
     RedisQueueSettings,
     RedisRateLimiterSettings,
@@ -82,7 +82,7 @@ def lifespan_factory(
         | RedisQueueSettings
         | RedisRateLimiterSettings
         | EnvironmentSettings
-        | OpenTelemeterySettings
+        | MonitoringSettings
     ),
     create_tables_on_start: bool = True,
 ) -> Callable[[FastAPI], _AsyncGeneratorContextManager[Any]]:
@@ -129,7 +129,7 @@ def create_application(
         | RedisQueueSettings
         | RedisRateLimiterSettings
         | EnvironmentSettings
-        | OpenTelemeterySettings
+        | MonitoringSettings
     ),
     create_tables_on_start: bool = True,
     **kwargs: Any,
@@ -156,7 +156,7 @@ def create_application(
         - RedisRateLimiterSettings: Sets up event handlers for creating and closing a Redis rate limiter pool.
         - EnvironmentSettings: Conditionally sets documentation URLs and integrates custom routes for API documentation
           based on the environment type.
-        - OpenTelemeterySettings: Configures OpenTelemetry tracing for the FastAPI application.
+        - MonitoringSettings: Configures OpenTelemetry tracing for the FastAPI application.
 
     create_tables_on_start : bool
         A flag to indicate whether to create database tables on application startup.
@@ -193,16 +193,17 @@ def create_application(
     application = FastAPI(lifespan=lifespan, **kwargs)
     application.include_router(router)
 
-    if isinstance(settings, AppSettings):
-        # Setting metrics middleware
-        application.add_middleware(
-            PrometheusMiddleware,
-            app_name=settings.APP_NAME,
-            excluded_handlers=["/docs", "/redoc", "/openapi.json", "/metrics"],
-        )
-        application.add_route("/metrics", metrics)
-        if isinstance(settings, OpenTelemeterySettings):
-            setting_otlp(application, settings.APP_NAME, settings.OTLP_GRPC_ENDPOINT)
+    if isinstance(settings, MonitoringSettings):
+        if settings.MONITORING:
+            # Setting metrics middleware
+            application.add_middleware(
+                PrometheusMiddleware,
+                app_name=application.title,
+                excluded_handlers=["/docs", "/redoc", "/openapi.json", "/metrics"],
+            )
+            application.add_route("/metrics", metrics)
+
+            setting_otlp(application, application.title, settings.OTLP_GRPC_ENDPOINT)
 
     if isinstance(settings, ClientSideCacheSettings):
         application.add_middleware(ClientCacheMiddleware, max_age=settings.CLIENT_CACHE_MAX_AGE)

From b6b1e49aae58b733ce8af2d18904f72172eb248a Mon Sep 17 00:00:00 2001
From: Luca Medeiros <luca.medeiros@nuvi-labs.com>
Date: Tue, 14 May 2024 14:34:11 +0900
Subject: [PATCH 20/20] fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 14d69e8..997401b 100644
--- a/README.md
+++ b/README.md
@@ -66,7 +66,7 @@
 - 🤸‍♂️ Flexible
 - 🚚 Easy running with docker compose
 - ⚖️ NGINX Reverse Proxy and Load Balancing
-- 📊 Prometheus metrics with Monitoring monitoring
+- 📊 Prometheus metrics with Grafana dashboards for monitoring
 - 📖 Loki and open telemetrics for logging
 
 ## 2. Contents