stats: add new fields for migrated stats

invenio.cfg

@@ -19,8 +19,8 @@ from cds_rdm.permissions import (
     CDSRDMPreservationSyncPermissionPolicy,
 )
 from cds_rdm.files import storage_factory
-from invenio_app_rdm.config import CELERY_BEAT_SCHEDULE as APP_RDM_CELERY_BEAT_SCHEDULE
 from celery.schedules import crontab
+from invenio_app_rdm.config import STATS_EVENTS as _APP_RDM_STATS_EVENTS
 from invenio_vocabularies.services.custom_fields import VocabularyCF
 from invenio_records_resources.services.custom_fields import KeywordCF
 from invenio_rdm_records.config import (
@@ -550,3 +550,14 @@ VOCABULARIES_NAMES_SCHEMES = {
     "lcds": {"label": _("CDS"), "validator": schemes.is_legacy_cds, "datacite": "CDS"},
 }
 """Names allowed identifier schemes."""
+
+
+# Invenio Stats
+# =============
+
+# We need to add explicitely the `filter_robots` as the migrated legacy statistics are
+# were not being processed through the event preprocessors
+
+# We override the templates to add new fields needed for the migrated statistic events
+_APP_RDM_STATS_EVENTS["file-download"]["templates"] = "cds_rdm.stats.templates.events.file_download"
+_APP_RDM_STATS_EVENTS["record-view"]["templates"] = "cds_rdm.stats.templates.events.record_view"

site/cds_rdm/stats/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2018 CERN.
+# Copyright (C) 2023 TU Wien.
+#
+# Invenio RDM Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""CDS-RDM Statistics search index templates."""

site/cds_rdm/stats/templates/events/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2018 CERN.
+# Copyright (C) 2023 TU Wien.
+#
+# Invenio RDM Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Statistics events search index templates."""

site/cds_rdm/stats/templates/events/file_download/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2019 CERN.
+# Copyright (C) 2023 TU Wien.
+#
+# Invenio RDM Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""File download event search index templates."""

site/cds_rdm/stats/templates/events/file_download/os-v2/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2019 CERN.
+# Copyright (C) 2023 TU Wien.
+#
+# Invenio RDM Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""File download event OpenSearch index templates."""

site/cds_rdm/stats/templates/events/file_download/os-v2/file-download-v1.0.0.json

@@ -0,0 +1,102 @@
+{
+  "index_patterns": ["__SEARCH_INDEX_PREFIX__events-stats-file-download-*"],
+  "settings": {
+    "index": {
+      "refresh_interval": "5s"
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "date_fields": {
+          "match_mapping_type": "date",
+          "mapping": {
+            "type": "date",
+            "format": "strict_date_hour_minute_second"
+          }
+        }
+      }
+    ],
+    "date_detection": false,
+    "dynamic": "strict",
+    "numeric_detection": false,
+    "properties": {
+      "timestamp": {
+        "type": "date",
+        "format": "strict_date_hour_minute_second"
+      },
+      "bucket_id": {
+        "type": "keyword"
+      },
+      "file_id": {
+        "type": "keyword"
+      },
+      "file_key": {
+        "type": "keyword"
+      },
+      "unique_id": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "keyword"
+      },
+      "visitor_id": {
+        "type": "keyword"
+      },
+      "is_machine": {
+        "type": "boolean"
+      },
+      "is_robot": {
+        "type": "boolean"
+      },
+      "unique_session_id": {
+        "type": "keyword"
+      },
+      "size": {
+        "type": "double"
+      },
+      "referrer": {
+        "type": "keyword"
+      },
+      "ip_address": {
+        "type": "keyword"
+      },
+      "user_agent": {
+        "type": "keyword"
+      },
+      "user_id": {
+        "type": "keyword"
+      },
+      "session_id": {
+        "type": "keyword"
+      },
+      "record_id": {
+        "type": "keyword"
+      },
+      "recid": {
+        "type": "keyword"
+      },
+      "parent_id": {
+        "type": "keyword"
+      },
+      "parent_recid": {
+        "type": "keyword"
+      },
+      "via_api": {
+        "type": "boolean"
+      },
+      "is_lcds": {
+        "type": "boolean"
+      },
+      "before_COUNTER": {
+        "type": "boolean"
+      },
+      "updated_timestamp": {
+        "type": "date"
+      }
+    }
+  },
+  "aliases": {
+    "__SEARCH_INDEX_PREFIX__events-stats-file-download": {}
+  }
+}

site/cds_rdm/stats/templates/events/record_view/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2018 CERN.
+# Copyright (C) 2023 TU Wien.
+#
+# Invenio RDM Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record views search index templates."""

site/cds_rdm/stats/templates/events/record_view/os-v2/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2019 CERN.
+# Copyright (C) 2023 TU Wien.
+#
+# Invenio RDM Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record view event OpenSearch index templates."""

site/cds_rdm/stats/templates/events/record_view/os-v2/record-view-v1.0.0.json

@@ -0,0 +1,82 @@
+{
+  "index_patterns": ["__SEARCH_INDEX_PREFIX__events-stats-record-view-*"],
+  "settings": {
+    "index": {
+      "refresh_interval": "5s"
+    }
+  },
+  "mappings": {
+    "date_detection": false,
+    "dynamic": "strict",
+    "numeric_detection": false,
+    "properties": {
+      "timestamp": {
+        "type": "date",
+        "format": "strict_date_hour_minute_second"
+      },
+      "labels": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "keyword"
+      },
+      "visitor_id": {
+        "type": "keyword"
+      },
+      "is_machine": {
+        "type": "boolean"
+      },
+      "is_robot": {
+        "type": "boolean"
+      },
+      "unique_id": {
+        "type": "keyword"
+      },
+      "unique_session_id": {
+        "type": "keyword"
+      },
+      "referrer": {
+        "type": "keyword"
+      },
+      "ip_address": {
+        "type": "keyword"
+      },
+      "user_agent": {
+        "type": "keyword"
+      },
+      "user_id": {
+        "type": "keyword"
+      },
+      "session_id":{
+        "type": "keyword"
+      },
+      "record_id": {
+        "type": "keyword"
+      },
+      "recid": {
+        "type": "keyword"
+      },
+      "parent_id": {
+        "type": "keyword"
+      },
+      "parent_recid": {
+        "type": "keyword"
+      },
+      "via_api": {
+        "type": "boolean"
+      },
+      "is_lcds": {
+        "type": "boolean"
+      },
+      "before_COUNTER": {
+        "type": "boolean"
+      },
+      "updated_timestamp": {
+        "type": "date"
+      }
+    }
+  },
+  "aliases": {
+    "__SEARCH_INDEX_PREFIX__events-stats-record-view": {}
+  }
+}