Merge pull request #95 from swrlab/feature/dts-plugin

Feature/dts plugin
swrlab · Jun 1, 2021 · 034f09d · 034f09d
2 parents fc666f2 + 19378f7
commit 034f09d
Show file tree

Hide file tree

Showing 24 changed files with 824 additions and 162 deletions.
diff --git a/.env.example b/.env.example
@@ -2,6 +2,7 @@ STAGE=dev
 GCP_PROJECT_ID=ard-eventhub
 FIREBASE_API_KEY=<firebase-api-key>
 GOOGLE_APPLICATION_CREDENTIALS=./keys/ingest.json
+PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL=<service-account-email>
 #PORT=<custom-port>
 #DEBUG=true
 

diff --git a/.github/workflows/ingest-pull.yml b/.github/workflows/ingest-pull.yml
@@ -7,7 +7,7 @@ on:
 
 
 env:
-  NODE_VERSION: 14.5
+  NODE_VERSION: 14.17
 
 
 jobs:
@@ -17,18 +17,18 @@ jobs:
       matrix:
         STAGES: [dev]
     steps:
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v2.1.5
       with:
         node-version: ${{ env.NODE_VERSION }}
 
     - name: Checkout repo 👀
-      uses: actions/checkout@v2
+      uses: actions/checkout@v2.3.4
 
     - name: Get yarn cache directory path
       id: yarn-cache-dir-path
       run: echo "::set-output name=dir::$(yarn cache dir)"
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v2.1.5
       with:
         path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
         key: ${{ runner.os }}-node-${{ hashFiles('**/yarn.lock') }}
@@ -51,23 +51,24 @@ jobs:
         FIREBASE_API_KEY: ${{ secrets.TEST_FIREBASE_API_KEY }}
         TEST_USER: ${{ secrets.TEST_USER }}
         TEST_USER_PW: ${{ secrets.TEST_USER_PW }}
+        PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL: ${{ secrets.PUBSUB_SERVICE_ACCOUNT_EMAIL }}
       run: yarn ingest:test
 
   license:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v2.1.5
       with:
         node-version: ${{ env.NODE_VERSION }}
 
     - name: Checkout repo 👀
-      uses: actions/checkout@v2
+      uses: actions/checkout@v2.3.4
 
     - name: Get yarn cache directory path
       id: yarn-cache-dir-path
       run: echo "::set-output name=dir::$(yarn cache dir)"
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v2.1.5
       with:
         path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
         key: ${{ runner.os }}-node-${{ hashFiles('**/yarn.lock') }}

diff --git a/.github/workflows/ingest-push.yml b/.github/workflows/ingest-push.yml
@@ -11,7 +11,7 @@ on:
 
 
 env:
-  NODE_VERSION: 14.5
+  NODE_VERSION: 14.17
   IMAGE_NAME: de.swr.ard.eventhub/ingest
   REGISTRY_DOMAIN: eu.gcr.io
   GCP_RUN_REGION: europe-west1
@@ -26,18 +26,18 @@ jobs:
       matrix:
         STAGES: [dev]
     steps:
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v2.1.5
       with:
         node-version: ${{ env.NODE_VERSION }}
 
     - name: Checkout repo 👀
-      uses: actions/checkout@v2
+      uses: actions/checkout@v2.3.4
 
     - name: Get yarn cache directory path
       id: yarn-cache-dir-path
       run: echo "::set-output name=dir::$(yarn cache dir)"
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v2.1.5
       with:
         path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
         key: ${{ runner.os }}-node-${{ hashFiles('**/yarn.lock') }}
@@ -60,6 +60,7 @@ jobs:
         FIREBASE_API_KEY: ${{ secrets.TEST_FIREBASE_API_KEY }}
         TEST_USER: ${{ secrets.TEST_USER }}
         TEST_USER_PW: ${{ secrets.TEST_USER_PW }}
+        PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL: ${{ secrets.PUBSUB_SERVICE_ACCOUNT_EMAIL }}
       run: yarn ingest:test
 
 
@@ -71,7 +72,7 @@ jobs:
       VERSION: ${{ steps.push.outputs.VERSION }}
     steps:
     - name: 👀 Checkout repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v2.3.4
 
     - name: 🔑 Setup Google Cloud Auth
       uses: google-github-actions/setup-gcloud@master
@@ -145,7 +146,7 @@ jobs:
           --allow-unauthenticated \
           --timeout $GCP_RUN_TIMEOUT \
           --memory $GCP_RUN_MEMORY \
-          --update-env-vars STAGE=dev,DD_TRACER_ENABLED=false \
+          --update-env-vars STAGE=dev,DD_TRACER_ENABLED=false,PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL=${{ secrets.PUBSUB_SERVICE_ACCOUNT_EMAIL }} \
           --service-account ${{ secrets.GCP_SERVICE_ACCOUNT_INGEST }}
 
   deploy-B-test-kubernetes:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.0] - 2021-05-19
+
+### Added
+
+- Add first external plugin with dts-integration
+
 ## [1.0.2] - 2021-05-11
 
 ### Changed

diff --git a/Dockerfile.ingest b/Dockerfile.ingest
@@ -1,5 +1,5 @@
 # Load desired node pckg
-FROM node:14.16-alpine
+FROM node:14.17-alpine
 
 # Add python
 RUN apk add g++ make python

diff --git a/README.md b/README.md
@@ -48,38 +48,40 @@ This source code is provided under EUPL v1.2, except for the [`spdx-exceptions`]
 
 ## Third-Party Components
 
-| Type    | Name                          | License                                                                                   |
-| ------- | ----------------------------- | ----------------------------------------------------------------------------------------- |
-| Docker  | `node:14.15-alpine`           | [MIT](https://github.com/nodejs/node/blob/master/LICENSE)                                 |
-| NPM     | `@google-cloud/datastore`     | [Apache License 2.0](https://github.com/googleapis/nodejs-datastore/blob/master/LICENSE)  |
-| NPM     | `@google-cloud/pubsub`        | [Apache License 2.0](https://github.com/googleapis/nodejs-pubsub/blob/master/LICENSE)     |
-| NPM     | `body-parser`                 | [MIT](https://github.com/expressjs/body-parser/blob/master/LICENSE)                       |
-| NPM     | `compression`                 | [MIT](https://github.com/expressjs/compression/blob/master/LICENSE)                       |
-| NPM     | `dd-trace`                    | [Apache-2.0 OR BSD-3-Clause](https://github.com/DataDog/dd-trace-js/blob/master/LICENSE)  |
-| NPM     | `express`                     | [MIT](https://github.com/expressjs/express/blob/master/LICENSE)                           |
-| NPM     | `express-openapi-validator`   | [MIT](https://github.com/cdimascio/express-openapi-validator/blob/master/LICENSE)         |
-| NPM     | `firebase-admin`              | [Apache License 2.0](https://github.com/firebase/firebase-admin-node/blob/master/LICENSE) |
-| NPM     | `jsonwebtoken`                | [MIT](https://github.com/auth0/node-jsonwebtoken/blob/master/LICENSE)                     |
-| NPM     | `moment`                      | [MIT](https://github.com/moment/moment/blob/develop/LICENSE)                              |
-| NPM     | `node-crc`                    | [MIT](https://github.com/magiclen/node-crc/blob/master/LICENSE)                           |
-| NPM     | `node-fetch`                  | [MIT](https://github.com/node-fetch/node-fetch/blob/master/LICENSE.md)                    |
-| NPM     | `slug`                        | [MIT](https://github.com/Trott/slug/blob/master/LICENSE)                                  |
-| NPM     | `swagger-ui-express`          | [MIT](https://github.com/scottie1984/swagger-ui-express/blob/master/LICENSE)              |
-| NPM     | `uuid`                        | [MIT](https://github.com/uuidjs/uuid/blob/master/LICENSE.md)                              |
-| NPM     | `winston`                     | [MIT](https://github.com/winstonjs/winston/blob/master/LICENSE)                           |
-| NPM DEV | `@swrlab/eslint-plugin-swr`   | [ISC](https://github.com/swrlab/eslint-plugin-swr/)                                       |
-| NPM DEV | `@swrlab/swr-prettier-config` | [ISC](https://github.com/swrlab/prettier-config/blob/main/license.md)                     |
-| NPM DEV | `chai`                        | [MIT](https://github.com/chaijs/chai/blob/master/LICENSE)                                 |
-| NPM DEV | `chai-http`                   | [MIT](https://github.com/chaijs/chai-http/blob/master/package.json)                       |
-| NPM DEV | `docsify-cli`                 | [MIT](https://github.com/docsifyjs/docsify-cli/blob/master/LICENSE)                       |
-| NPM DEV | `dotenv`                      | [BSD-2-Clause](https://github.com/motdotla/dotenv/blob/master/LICENSE)                    |
-| NPM DEV | `eslint`                      | [MIT](https://github.com/eslint/eslint/blob/master/LICENSE)                               |
-| NPM DEV | `eslint-plugin-chai-friendly` | [MIT](https://github.com/ihordiachenko/eslint-plugin-chai-friendly/blob/master/LICENSE)   |
-| NPM DEV | `eslint-plugin-swr`           | [ISC](https://github.com/swrlab/eslint-plugin-swr/blob/main/package.json)                 |
-| NPM DEV | `license-compliance`          | [MIT](https://github.com/tmorell/license-compliance/blob/master/LICENSE)                  |
-| NPM DEV | `mocha`                       | [MIT](https://github.com/mochajs/mocha/blob/master/LICENSE)                               |
-| NPM DEV | `nodemon`                     | [MIT](https://github.com/remy/nodemon/blob/master/LICENSE)                                |
-| NPM DEV | `prettier`                    | [MIT](https://github.com/prettier/prettier/blob/main/LICENSE)                             |
+| Type    | Name                           | License                                                                                            |
+| ------- | ------------------------------ | -------------------------------------------------------------------------------------------------- |
+| Docker  | `node:14.17-alpine`            | [MIT](https://github.com/nodejs/node/blob/master/LICENSE)                                          |
+| NPM     | `@google-cloud/datastore`      | [Apache License 2.0](https://github.com/googleapis/nodejs-datastore/blob/master/LICENSE)           |
+| NPM     | `@google-cloud/pubsub`         | [Apache License 2.0](https://github.com/googleapis/nodejs-pubsub/blob/master/LICENSE)              |
+| NPM     | `@google-cloud/secret-manager` | [Apache License 2.0](https://github.com/googleapis/nodejs-secret-manager/blob/master/LICENSE)      |
+| NPM     | `body-parser`                  | [MIT](https://github.com/expressjs/body-parser/blob/master/LICENSE)                                |
+| NPM     | `compression`                  | [MIT](https://github.com/expressjs/compression/blob/master/LICENSE)                                |
+| NPM     | `dd-trace`                     | [Apache-2.0 OR BSD-3-Clause](https://github.com/DataDog/dd-trace-js/blob/master/LICENSE)           |
+| NPM     | `express`                      | [MIT](https://github.com/expressjs/express/blob/master/LICENSE)                                    |
+| NPM     | `express-openapi-validator`    | [MIT](https://github.com/cdimascio/express-openapi-validator/blob/master/LICENSE)                  |
+| NPM     | `firebase-admin`               | [Apache License 2.0](https://github.com/firebase/firebase-admin-node/blob/master/LICENSE)          |
+| NPM     | `google-auth-library`          | [Apache License 2.0](https://github.com/googleapis/google-auth-library-nodejs/blob/master/LICENSE) |
+| NPM     | `jsonwebtoken`                 | [MIT](https://github.com/auth0/node-jsonwebtoken/blob/master/LICENSE)                              |
+| NPM     | `moment`                       | [MIT](https://github.com/moment/moment/blob/develop/LICENSE)                                       |
+| NPM     | `node-crc`                     | [MIT](https://github.com/magiclen/node-crc/blob/master/LICENSE)                                    |
+| NPM     | `node-fetch`                   | [MIT](https://github.com/node-fetch/node-fetch/blob/master/LICENSE.md)                             |
+| NPM     | `slug`                         | [MIT](https://github.com/Trott/slug/blob/master/LICENSE)                                           |
+| NPM     | `swagger-ui-express`           | [MIT](https://github.com/scottie1984/swagger-ui-express/blob/master/LICENSE)                       |
+| NPM     | `uuid`                         | [MIT](https://github.com/uuidjs/uuid/blob/master/LICENSE.md)                                       |
+| NPM     | `winston`                      | [MIT](hhttps://github.com/winstonjs/winston/blob/master/LICENSE)                                   |
+| NPM DEV | `@swrlab/eslint-plugin-swr`    | [ISC](https://github.com/swrlab/eslint-plugin-swr/)                                                |
+| NPM DEV | `@swrlab/swr-prettier-config`  | [ISC](https://github.com/swrlab/prettier-config/blob/main/license.md)                              |
+| NPM DEV | `chai`                         | [MIT](https://github.com/chaijs/chai/blob/master/LICENSE)                                          |
+| NPM DEV | `chai-http`                    | [MIT](https://github.com/chaijs/chai-http/blob/master/package.json)                                |
+| NPM DEV | `docsify-cli`                  | [MIT](https://github.com/docsifyjs/docsify-cli/blob/master/LICENSE)                                |
+| NPM DEV | `dotenv`                       | [BSD-2-Clause](https://github.com/motdotla/dotenv/blob/master/LICENSE)                             |
+| NPM DEV | `eslint`                       | [MIT](https://github.com/eslint/eslint/blob/master/LICENSE)                                        |
+| NPM DEV | `eslint-plugin-chai-friendly`  | [MIT](https://github.com/ihordiachenko/eslint-plugin-chai-friendly/blob/master/LICENSE)            |
+| NPM DEV | `eslint-plugin-swr`            | [ISC](https://github.com/swrlab/eslint-plugin-swr/blob/main/package.json)                          |
+| NPM DEV | `license-compliance`           | [MIT](https://github.com/tmorell/license-compliance/blob/master/LICENSE)                           |
+| NPM DEV | `mocha`                        | [MIT](https://github.com/mochajs/mocha/blob/master/LICENSE)                                        |
+| NPM DEV | `nodemon`                      | [MIT](https://github.com/remy/nodemon/blob/master/LICENSE)                                         |
+| NPM DEV | `prettier`                     | [MIT](https://github.com/prettier/prettier/blob/main/LICENSE)                                      |
 
 ## Authors
 

diff --git a/config/index.js b/config/index.js
@@ -16,6 +16,9 @@ if (!process.env.GCP_PROJECT_ID) {
 } else if (!process.env.FIREBASE_API_KEY) {
 	console.error('process.env.FIREBASE_API_KEY not found')
 	process.exit(1)
+} else if (!process.env.PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL) {
+	console.error('process.env.PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL not found')
+	process.exit(1)
 }
 
 // read env vars
@@ -26,6 +29,7 @@ const serviceName = 'ard-eventhub'
 const baseConfig = {
 	coreIdPrefixes,
 	pubSubPrefix: `de.ard.eventhub.${stage}.`,
+	pubSubTopicSelf: `de.ard.eventhub.${stage}.internal`,
 	stage,
 	userAgent: `${serviceName}/${version}`,
 	version,

diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md
@@ -20,7 +20,7 @@ If you are a radio station that wants to start publishing events to ARD-Eventhub
 
 - Set up your account and understand the authentication process
 - Use the POST `/events/{eventName}` endpoint to add your events
-- Note: Even if GET `/topics` does not list your radio station(s) beforehand, the topic(s) will be created during your first published event (response will contain:
+- Note: Even if GET `/topics` does not list your radio station(s) beforehand, the topic(s) will be created during your first published event (response will contain):
 
 ```js
 {
@@ -84,6 +84,7 @@ In your system for every new event, you might follow a workflow like this:
 
 ```js
 {
+   "type": "music",
    "start": "2021-03-17T10:04:35+01:00",
    "length": 215.2,
    "title": "Save your tears",
@@ -119,7 +120,6 @@ In your system for every new event, you might follow a workflow like this:
          "attribution": ""
       }
    ],
-   "type": "music",
    "hfdbIds": [
       "swrhfdb1.KONF.12345"
    ]

diff --git a/openapi.json b/openapi.json
@@ -11,7 +11,7 @@
       "name": "European Union Public License 1.2",
       "url": "https://spdx.org/licenses/EUPL-1.2.html"
     },
-    "version": "1.0.2"
+    "version": "1.1.0"
   },
   "externalDocs": {
     "description": "ARD-Eventhub Documentation",

diff --git a/openapi.yaml b/openapi.yaml
@@ -10,7 +10,7 @@ info:
   license:
     name: European Union Public License 1.2
     url: 'https://spdx.org/licenses/EUPL-1.2.html'
-  version: 1.0.2
+  version: 1.1.0
 externalDocs:
   description: ARD-Eventhub Documentation
   url: 'https://swrlab.github.io/ard-eventhub/'

diff --git a/package.json b/package.json
@@ -1,10 +1,11 @@
 {
 	"name": "ard-eventhub",
-	"version": "1.0.2",
+	"version": "1.1.0",
 	"description": "ARD system to distribute real-time (live) metadata for primarily radio broadcasts.",
 	"main": "./src/ingest/index.js",
 	"scripts": {
 		"docs:serve": "docsify serve . --port 3000",
+		"coreId": "node ./src/coreId/index.js",
 		"ingest:test": "mocha test/ingest.test.js --timeout 15000 --exit -r dotenv/config",
 		"ingest:local": "nodemon -r dotenv/config ./src/ingest/index.js",
 		"ingest:cloud": "node -r dotenv/config ./src/ingest/index.js",
@@ -16,20 +17,22 @@
 		"metadata",
 		"pubsub"
 	],
-	"author": "SWR audio lab <[email protected]>",
+	"author": "SWR Audio Lab <[email protected]>",
 	"license": "EUPL-1.2",
 	"dependencies": {
 		"@google-cloud/datastore": "^6.4.0",
-		"@google-cloud/pubsub": "^2.11.0",
+		"@google-cloud/pubsub": "^2.12.0",
+		"@google-cloud/secret-manager": "^3.7.1",
 		"body-parser": "^1.19.0",
 		"compression": "^1.7.4",
-		"dd-trace": "^0.33.1",
+		"dd-trace": "^0.34.0",
 		"express": "4.17.1",
-		"express-openapi-validator": "^4.12.9",
-		"firebase-admin": "^9.7.0",
+		"express-openapi-validator": "^4.12.11",
+		"firebase-admin": "^9.9.0",
+		"google-auth-library": "^7.1.0",
 		"jsonwebtoken": "^8.5.1",
 		"moment": "^2.29.1",
-		"node-crc": "^1.3.0",
+		"node-crc": "^1.3.2",
 		"node-fetch": "2.6.1",
 		"slug": "^5.0.0",
 		"swagger-ui-express": "^4.1.6",
@@ -42,9 +45,9 @@
 		"chai": "^4.3.4",
 		"chai-http": "^4.3.0",
 		"docsify-cli": "^4.4.3",
-		"dotenv": "^9.0.1",
-		"eslint": "^7.25.0",
-		"eslint-plugin-chai-friendly": "^0.6.0",
+		"dotenv": "^10.0.0",
+		"eslint": "^7.27.0",
+		"eslint-plugin-chai-friendly": "^0.7.1",
 		"eslint-plugin-swr": "0.0.5",
 		"license-compliance": "^1.2.1",
 		"mocha": "^8.3.2",

diff --git a/src/coreId/index.js b/src/coreId/index.js
@@ -0,0 +1,21 @@
+/*
+
+	ard-eventhub
+	by SWR audio lab
+
+	this file creates a CLI for hashing ids
+
+*/
+
+// load utils
+const createHashedId = require('../utils/core/createHashedId')
+
+// parse input
+const input = process.argv[2]
+
+// log output
+console.log('INPUT:')
+console.log(input)
+console.log(' ')
+console.log('OUTPUT (CRC64-ECMA182):')
+console.log(createHashedId(input))
diff --git a/src/ingest/README.md b/src/ingest/README.md
@@ -13,6 +13,7 @@ Several environment variables need to be set in `.env` config in order to run th
 - REQUIRED `GCP_PROJECT_ID` - which GCP project ID to use for Pub/Sub and Datastore requests
 - REQUIRED `FIREBASE_API_KEY` - corresponding `API_KEY` which matches the `GCP_PROJECT_ID`
 - REQUIRED `GOOGLE_APPLICATION_CREDENTIALS` - where the Google Cloud Service Account Key can be found (usually a path to a .json file)
+- REQUIRED `PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL` - for verification of internal publisher service account
 - REQUIRED `STAGE` - can be one of the Stages below to switch several settings
 - OPTIONAL `PORT` - override server port setting, default is 8080
 - OPTIONAL `DEBUG` - set true to enable more detailed logging

diff --git a/src/ingest/events/post.js b/src/ingest/events/post.js
@@ -254,6 +254,27 @@ module.exports = async (req, res) => {
 		// replace services
 		message.services = newServices
 
+		// handle plugin integrations
+		const pluginMessages = []
+		if (message?.plugins?.length > 0) {
+			for await (const plugin of message.plugins) {
+				const pluginMessage = {
+					action: `plugins.${plugin.type}.event`,
+					event: message,
+					plugin,
+				}
+
+				// try sending message
+				const messageId = await pubsub.publishMessage(config.pubSubTopicSelf, pluginMessage)
+
+				// add to output
+				pluginMessages.push({
+					type: plugin.type,
+					messageId,
+				})
+			}
+		}
+
 		// prepare output data
 		const data = {
 			statuses: {
@@ -263,6 +284,7 @@ module.exports = async (req, res) => {
 					(service) => !service.topic?.messageId && !service.blocked
 				).length,
 			},
+			plugins: pluginMessages,
 			event: message,
 		}