⚡️(dashboards) optimize ES index mapping

Use keywords when we can instead of derived keywords from text fields.

CHANGELOG.md

@@ -8,6 +8,11 @@ Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to
 
 ## [Unreleased]
 
+### Changed
+
+- Moved `object.id` and `context.contextActivities.parent.id` to `keyword`
+  mapping type.
+
 ### Removed
 
 - Support for legacy course key building from object definition extensions [BC]

Makefile

@@ -5,6 +5,7 @@ DOCKER_GID           = $(shell id -g)
 DOCKER_USER          = $(DOCKER_UID):$(DOCKER_GID)
 COMPOSE              = DOCKER_USER=$(DOCKER_USER) docker compose
 COMPOSE_RUN          = $(COMPOSE) run --rm
+ES_INDEX             = statements-fixtures
 
 # -- Node
 COMPOSE_RUN_NODE     = $(COMPOSE_RUN) node
@@ -83,10 +84,11 @@ fixtures: \
 fixtures: ## Load test data (for development)
 	@echo "Wait for databases to be up..."
 	@$(WAIT_ES)
+	$(COMPOSE_RUN) elasticsearch /usr/local/bin/create_es_index.sh $(ES_INDEX)
 	@$(WAIT_MYSQL)
 	zcat ./fixtures/elasticsearch/lrs.json.gz | \
 	  $(COMPOSE_RUN) patch_statements_date | \
-	  $(COMPOSE_RUN) -T ralph push -b es --es-index statements-fixtures  && \
+	  $(COMPOSE_RUN) -T ralph push -b es --es-index $(ES_INDEX) && \
 	  $(COMPOSE_RUN) users-permissions sh /scripts/users-permissions.sh
 .PHONY: fixtures
 

docker-compose.yml

@@ -88,6 +88,8 @@ services:
       xpack.security.enabled: "false"
     ports:
       - "9200:9200"
+    volumes:
+      - ./scripts/create_es_index.sh:/usr/local/bin/create_es_index.sh
 
   edx_mysql:
     image: mariadb:10.3.32

scripts/create_es_index.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# Usage: scripts/create_es_index.sh [INDEX]
+
+# Defaults
+declare -r INDEX="${1:-statements-fixtures}"
+
+# Create the statements index with appropriate mapping
+echo "Will create '${INDEX}' elasticsearch index..."
+
+curl -X PUT "elasticsearch:9200/${INDEX}?pretty"
+curl -X PUT "elasticsearch:9200/${INDEX}/_mapping?pretty" -H 'Content-Type: application/json' -d'
+{
+  "properties": {
+    "verb.id": {
+      "type": "keyword"
+    },
+    "object.id": {
+      "type": "keyword"
+    },
+    "object.definition.type": {
+      "type": "keyword"
+    },
+    "context.contextActivities.category.id": {
+      "type": "keyword"
+    },
+    "context.contextActivities.parent.definition.type": {
+      "type": "keyword"
+    },
+    "context.contextActivities.parent.id": {
+      "type": "keyword"
+    }
+  }
+}
+'
+

src/dashboards/common.libsonnet

@@ -8,7 +8,7 @@
   },
   fields: {
     actor_account_name: 'actor.account.name.keyword',
-    video_id: 'object.id.keyword',
+    video_id: 'object.id',
     subtitle_enabled: 'context.extensions.https://w3id.org/xapi/video/extensions/cc-subtitle-enabled',
     full_screen: 'context.extensions.https://w3id.org/xapi/video/extensions/full-screen',
     speed: 'context.extensions.https://w3id.org/xapi/video/extensions/speed.keyword',
@@ -61,5 +61,6 @@
     initialized: 'http://adlnet.gov/expapi/verbs/initialized',
     played: 'https://w3id.org/xapi/video/verbs/played',
     downloaded: 'http://id.tincanapi.com/verb/downloaded',
+    interacted: 'http://adlnet.gov/expapi/verbs/interacted',
   },
 }

src/dashboards/teachers/common.libsonnet

@@ -32,21 +32,25 @@ local common = import '../common.libsonnet';
       video_query: $.queries.video_id,
       verb_completed: common.verb_ids.completed,
     },
-    course_key: 'context.contextActivities.parent.id.keyword:${EDX_COURSE_KEY:doublequote}',
+    course_key: 'context.contextActivities.parent.id:${EDX_COURSE_KEY:doublequote}',
     course_enrollments: 'SELECT DISTINCT COUNT(`user_id`) FROM `student_courseenrollment` WHERE (`is_active`=1 AND `course_id`="${EDX_COURSE_KEY}")',
     course_title: 'SELECT `title` FROM courses_course WHERE `key`="${EDX_COURSE_KEY}"',
     course_start_date: 'SELECT DATE_FORMAT(start_date, "%d/%m/%Y") FROM courses_course WHERE `key`="${EDX_COURSE_KEY}"',
     course_end_date: 'SELECT DATE_FORMAT(end_date, "%d/%m/%Y") FROM courses_course WHERE `key`="${EDX_COURSE_KEY}"',
     course_query: '%(course_key)s' % {
       course_key: $.queries.course_key,
     },
-    course_videos: 'object.id.keyword:${COURSE_VIDEOS_IDS_WITH_UUID:lucene}',
+    course_videos: 'object.id:${COURSE_VIDEOS_IDS_WITH_UUID:lucene}',
     downloads: '%(video_query)s AND verb.id:"%(verb_downloaded)s"' % {
       video_query: $.queries.video_id,
       verb_downloaded: common.verb_ids.downloaded,
     },
     edx_course_key: 'SELECT `key` FROM courses_course WHERE `key`="${EDX_COURSE_KEY}"',
-    video_id: 'object.id.keyword:${VIDEO:doublequote}',
+    video_interacted: '%(video_query)s AND verb.id:"%(verb_interacted)s"' % {
+      video_query: $.queries.video_id,
+      verb_interacted: common.verb_ids.interacted,
+    },
+    video_id: 'object.id:${VIDEO:doublequote}',
     views: '%(video_query)s AND verb.id:"%(verb_played)s" AND %(time)s:[0 TO %(view_count_threshold)s]' % {
       video_query: $.queries.video_id,
       verb_played: common.verb_ids.played,

src/dashboards/teachers/course.jsonnet

@@ -387,7 +387,7 @@ dashboard.new(
       {
         id: 'seriesToColumns',
         options: {
-          byField: 'object.id.keyword',
+          byField: 'object.id',
         },
       },
     ],
@@ -407,7 +407,7 @@ dashboard.new(
         {
           matcher: {
             id: 'byName',
-            options: 'object.id.keyword',
+            options: 'object.id',
           },
           properties: [
             {
@@ -547,7 +547,7 @@ dashboard.new(
         ],
         query: '',
         rawQuery: true,
-        rawSql: "SELECT 'uuid://' || id AS \"object.id.keyword\",title FROM video where id IN (${COURSE_VIDEOS_IDS:sqlstring})",
+        rawSql: "SELECT 'uuid://' || id AS \"object.id\",title FROM video where id IN (${COURSE_VIDEOS_IDS:sqlstring})",
         refId: 'B',
         select: [
           [

src/dashboards/teachers/details.jsonnet

@@ -458,8 +458,8 @@ dashboard.new(
   ).addTarget(
     elasticsearch.target(
       datasource=common.datasources.lrs,
-      query='%(video_query)s AND verb.id:"interacted" AND %(subtitle_enabled)s:true' % {
-        video_query: teachers_common.queries.video_id,
+      query='%(interacted_query)s AND %(subtitle_enabled)s:true' % {
+        interacted_query: teachers_common.queries.video_interacted,
         subtitle_enabled: common.utils.single_escape_string(common.fields.subtitle_enabled),
       },
       metrics=[common.metrics.count],
@@ -511,8 +511,8 @@ dashboard.new(
     targets: [
       {
         datasource: common.datasources.lrs,
-        query: '%(video_query)s AND verb.id:"interacted"' % {
-          video_query: teachers_common.queries.video_id,
+        query: '%(interacted_query)s' % {
+          interacted_query: teachers_common.queries.video_interacted,
         },
         metrics: [common.metrics.cardinality(common.fields.actor_account_name)],
         bucketAggs: [
@@ -549,8 +549,8 @@ dashboard.new(
   ).addTarget(
     elasticsearch.target(
       datasource=common.datasources.lrs,
-      query='%(video_query)s AND verb.id:"interacted" AND %(full_screen)s:true' % {
-        video_query: teachers_common.queries.video_id,
+      query='%(interacted_query)s AND %(full_screen)s:true' % {
+        interacted_query: teachers_common.queries.video_interacted,
         full_screen: common.utils.single_escape_string(common.fields.full_screen),
       },
       metrics=[common.metrics.count],
@@ -603,8 +603,8 @@ dashboard.new(
     targets: [
       {
         datasource: common.datasources.lrs,
-        query: '%(video_query)s AND verb.id:"interacted"' % {
-          video_query: teachers_common.queries.video_id,
+        query: '%(video_interacted)s' % {
+          video_interacted: teachers_common.queries.video_interacted,
         },
         metrics: [common.metrics.cardinality(common.fields.actor_account_name)],
         bucketAggs: [
@@ -678,8 +678,8 @@ dashboard.new(
     targets: [
       {
         datasource: common.datasources.lrs,
-        query: '%(video_query)s AND verb.id:"interacted"' % {
-          video_query: teachers_common.queries.video_id,
+        query: '%(interacted_query)s' % {
+          interacted_query: teachers_common.queries.video_interacted,
         },
         metrics: [common.metrics.cardinality(common.fields.actor_account_name)],
         bucketAggs: [
@@ -799,8 +799,8 @@ dashboard.new(
     targets: [
       {
         datasource: common.datasources.lrs,
-        query: '%(video_query)s AND verb.id:"interacted"' % {
-          video_query: teachers_common.queries.video_id,
+        query: '%(interacted_query)s' % {
+          interacted_query: teachers_common.queries.video_interacted,
         },
         metrics: [common.metrics.cardinality(common.fields.actor_account_name)],
         bucketAggs: [