Merge pull request #569 from swrlab/dev/dts-api-migration

refactor: migrate DTS APIs
swrlab · Sep 25, 2023 · 8bf66b2 · 8bf66b2
2 parents 45344b8 + 2700dcd
commit 8bf66b2
Show file tree

Hide file tree

Showing 11 changed files with 1,306 additions and 810 deletions.
diff --git a/.github/workflows/ingest-pull.yml b/.github/workflows/ingest-pull.yml
@@ -24,14 +24,14 @@ jobs:
         STAGES: [dev]
     steps:
       - name: 👀 Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 🏗 Install Node.js
         uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}
-          cache: "yarn"
-          cache-dependency-path: "**/yarn.lock"
+          cache: 'yarn'
+          cache-dependency-path: '**/yarn.lock'
 
       - name: 📦 Install Dependencies
         run: yarn
@@ -63,14 +63,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 👀 Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: ⛺️ Install Node.js
         uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}
-          cache: "yarn"
-          cache-dependency-path: "**/yarn.lock"
+          cache: 'yarn'
+          cache-dependency-path: '**/yarn.lock'
 
       - name: 📦 Install Dependencies
         run: yarn

diff --git a/.github/workflows/ingest-push.yml b/.github/workflows/ingest-push.yml
@@ -17,7 +17,7 @@ jobs:
   security:
     name: Check permissions for action runs
     runs-on: ubuntu-latest
-    if: "github.event.pull_request.merged == true || github.event_name == 'push'"
+    if: github.event.pull_request.merged == true || github.event_name == 'push'
     steps:
       - run: |
           echo "Security checks passed!"
@@ -30,14 +30,14 @@ jobs:
         STAGES: [dev]
     steps:
       - name: 👀 Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 🏗 Install Node.js
         uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}
-          cache: "yarn"
-          cache-dependency-path: "**/yarn.lock"
+          cache: 'yarn'
+          cache-dependency-path: '**/yarn.lock'
 
       - name: 📦 Install Dependencies
         run: yarn
@@ -71,18 +71,18 @@ jobs:
       VERSION: ${{ steps.push.outputs.VERSION }}
     steps:
       - name: 👀 Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 🔑 Setup Google Cloud Auth
         uses: google-github-actions/auth@v1
         with:
           credentials_json: ${{ secrets.GCP_GITHUB_SERVICE_ACCOUNT_KEY }}
 
       - name: 🔑 Login to Registry
-        run: "gcloud auth configure-docker $REGISTRY_DOMAIN"
+        run: 'gcloud auth configure-docker $REGISTRY_DOMAIN'
 
       - name: 🚧 Building docker image
-        run: "docker build ./ --file ./Dockerfile -t image"
+        run: 'docker build ./ --file ./Dockerfile -t image'
 
       - name: 🔖 Get Package Version
         id: package-version
@@ -130,7 +130,7 @@ jobs:
             ```
 
       - name: 👋 Logout
-        run: "docker logout"
+        run: 'docker logout'
 
   deploy-A-dev-kubernetes:
     if: github.event.pull_request.merged == true

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.0] - 2023-09-22
+
+- feat: prepare feature flat to toggle `dts` plugin to be opt-out
+- feat: add common pubsub topic
+- refactor: migrate DTS APIs
+
 ## [1.5.2] - 2023-08-08
 
 - fix: replace docker-scan with docker-scout

diff --git a/openapi.json b/openapi.json
@@ -11,7 +11,7 @@
 			"name": "European Union Public License 1.2",
 			"url": "https://spdx.org/licenses/EUPL-1.2.html"
 		},
-		"version": "1.5.2"
+		"version": "1.6.0"
 	},
 	"externalDocs": {
 		"description": "ARD-Eventhub Documentation",

diff --git a/openapi.yaml b/openapi.yaml
@@ -10,7 +10,7 @@ info:
   license:
     name: European Union Public License 1.2
     url: "https://spdx.org/licenses/EUPL-1.2.html"
-  version: 1.5.2
+  version: 1.6.0
 externalDocs:
   description: ARD-Eventhub Documentation
   url: "https://swrlab.github.io/ard-eventhub/"

diff --git a/package.json b/package.json
@@ -1,10 +1,10 @@
 {
 	"name": "ard-eventhub",
-	"version": "1.5.2",
+	"version": "1.6.0",
 	"description": "ARD system to distribute real-time (live) metadata for primarily radio broadcasts.",
 	"main": "./src/ingest/index.js",
 	"engines": {
-		"node": ">=16.0.0"
+		"node": ">=18.0.0"
 	},
 	"scripts": {
 		"docs:serve": "docsify serve . --port 3000",
@@ -29,37 +29,37 @@
 	"author": "SWR Audio Lab <[email protected]>",
 	"license": "EUPL-1.2",
 	"dependencies": {
-		"@google-cloud/datastore": "7.5.1",
-		"@google-cloud/pubsub": "3.7.1",
-		"@google-cloud/secret-manager": "4.2.2",
+		"@google-cloud/datastore": "8.1.0",
+		"@google-cloud/pubsub": "4.0.6",
+		"@google-cloud/secret-manager": "5.0.1",
 		"@swrlab/utils": "1.1.3",
 		"compression": "1.7.4",
-		"dd-trace": "4.11.1",
+		"dd-trace": "4.15.0",
 		"dotenv": "16.3.1",
 		"express": "4.18.2",
-		"express-openapi-validator": "5.0.4",
-		"firebase-admin": "11.9.0",
-		"google-auth-library": "8.8.0",
-		"jsonwebtoken": "9.0.0",
-		"luxon": "3.3.0",
-		"slug": "8.2.2",
-		"swagger-ui-express": "4.6.3",
-		"uuid": "9.0.0",
-		"winston": "3.9.0"
+		"express-openapi-validator": "5.0.6",
+		"firebase-admin": "11.10.1",
+		"google-auth-library": "9.0.0",
+		"jsonwebtoken": "9.0.2",
+		"luxon": "3.4.3",
+		"slug": "8.2.3",
+		"swagger-ui-express": "5.0.0",
+		"uuid": "9.0.1",
+		"winston": "3.10.0"
 	},
 	"devDependencies": {
-		"@swrlab/eslint-plugin-swr": "^0.2.0",
-		"@swrlab/swr-prettier-config": "^0.2.0",
-		"chai": "^4.3.7",
+		"@swrlab/eslint-plugin-swr": "^0.3.0",
+		"@swrlab/swr-prettier-config": "^0.3.0",
+		"chai": "^4.3.8",
 		"chai-http": "^4.4.0",
 		"docsify-cli": "^4.4.4",
-		"eslint": "^8.43.0",
+		"eslint": "^8.50.0",
 		"eslint-plugin-chai-friendly": "^0.7.2",
 		"license-compliance": "^2.0.0",
 		"mocha": "^10.2.0",
-		"nodemon": "^2.0.22",
-		"prettier": "^2.8.8",
-		"typescript": "^5.1.3"
+		"nodemon": "^3.0.1",
+		"prettier": "^3.0.3",
+		"typescript": "^5.2.2"
 	},
 	"resolutions": {
 		"ansi-regex": "^5.0.1",

diff --git a/src/ingest/events/post.js b/src/ingest/events/post.js
@@ -21,11 +21,16 @@ const config = require('../../../config')
 const source = 'ingest/events/post'
 const DEFAULT_ZONE = 'Europe/Berlin'
 
+// feature flags
+const IS_DTS_OPT_OUT_ENABLED = false
+const IS_COMMON_TOPIC_ENABLED = true
+
 module.exports = async (req, res) => {
 	try {
 		// fetch inputs
 		const { eventName } = req.params
 		const start = DateTime.fromISO(req.body.start, { zone: DEFAULT_ZONE })
+		const pluginMessages = []
 
 		// check eventName consistency
 		if (req.body?.event && req.body.event !== eventName) {
@@ -42,6 +47,7 @@ module.exports = async (req, res) => {
 			name: eventName,
 			creator: req.user.email,
 			created: DateTime.now().toLocal().toISO(),
+			plugins: [],
 
 			// use entire POST body to include potentially new fields
 			...structuredClone(req.body),
@@ -92,29 +98,69 @@ module.exports = async (req, res) => {
 		// replace services
 		message.services = newServices
 
+		// send event to common topic
+		if (IS_COMMON_TOPIC_ENABLED) {
+			// prepare common post
+			const topicName = pubsub.buildId(eventName.replace('de.ard.eventhub.', ''))
+			const commonEvent = {
+				type: 'common',
+				topic: {
+					id: eventName,
+					name: topicName,
+				},
+			}
+
+			// try sending message
+			commonEvent.messageId = await pubsub.publishMessage(topicName, message, attributes)
+
+			// handle errors
+			if (commonEvent.messageId === 'TOPIC_ERROR' || commonEvent.messageId === 'TOPIC_NOT_FOUND') {
+				logger.log({
+					level: 'warning',
+					message: `failed common plugin > ${eventName} > ${message.services[0]?.publisherId}`,
+					source,
+					data: { message, body: req.body, commonEvent },
+				})
+			}
+
+			// add to output
+			pluginMessages.push(commonEvent)
+		}
+
+		// add opt-out plugins
+		const isDtsPluginSet = message.plugins?.find((plugin) => plugin.type === 'dts')
+		if (!isDtsPluginSet && IS_DTS_OPT_OUT_ENABLED) {
+			message.plugins.push({
+				type: 'dts',
+				isDeactivated: false,
+				note: 'automatically enabled by opt-out',
+			})
+		}
+
 		// handle plugin integrations
-		const pluginMessages = []
-		if (message?.plugins?.length > 0) {
+		if (message.plugins?.length > 0) {
 			for await (const plugin of message.plugins) {
-				const pluginMessage = {
-					action: `plugins.${plugin.type}.event`,
-					event: message,
-					plugin,
-					institutionId: req.user.institutionId,
+				if (!plugin.isDeactivated) {
+					const pluginMessage = {
+						action: `plugins.${plugin.type}.event`,
+						event: message,
+						plugin,
+						institutionId: req.user.institutionId,
+					}
+
+					// try sending message
+					const messageId = await pubsub.publishMessage(
+						config.pubSubTopicSelf,
+						pluginMessage,
+						attributes
+					)
+
+					// add to output
+					pluginMessages.push({
+						type: plugin.type,
+						messageId,
+					})
 				}
-
-				// try sending message
-				const messageId = await pubsub.publishMessage(
-					config.pubSubTopicSelf,
-					pluginMessage,
-					attributes
-				)
-
-				// add to output
-				pluginMessages.push({
-					type: plugin.type,
-					messageId,
-				})
 			}
 		}
 
@@ -136,7 +182,7 @@ module.exports = async (req, res) => {
 			level: 'info',
 			message: `event processed > ${eventName} > ${message.services.length}x services (${message.services[0]?.publisherId})`,
 			source,
-			data: { ...data, body: req.body },
+			data: { ...data, body: req.body, isDtsPluginSet },
 		})
 
 		// return ok

diff --git a/src/utils/events/processServices.js b/src/utils/events/processServices.js
@@ -17,8 +17,8 @@ const pubsub = require('../pubsub')
 const { coreIdPrefixes } = require('../../../config')
 
 const source = 'utils.events.processServices'
-const urnPublisherRegex = /(?=urn:ard:publisher:[a-z0-9]{16})/g
-const urnPublisherPrefix = coreIdPrefixes.Publisher
+const URN_PUBLISHER_PREFIX = coreIdPrefixes.Publisher
+const URN_PUBLISHER_REGEX = /(?=urn:ard:publisher:[a-z0-9]{16})/g
 
 module.exports = async (service, req) => {
 	// fetch prefix from configured list
@@ -35,12 +35,12 @@ module.exports = async (service, req) => {
 	}
 
 	// convert publisher if not in new ARD urn format
-	if (!service.publisherId.match(urnPublisherRegex)) {
+	if (!service.publisherId.match(URN_PUBLISHER_REGEX)) {
 		// add trailing 0 if number is only 5 digits
 		if (service.publisherId.length === 5) service.publisherId = `${service.publisherId}0`
 
 		// create hash using given publisherId
-		service.publisherId = `${urnPublisherPrefix}${createHashedId(service.publisherId)}`
+		service.publisherId = `${URN_PUBLISHER_PREFIX}${createHashedId(service.publisherId)}`
 	}
 
 	// fetch publisher