diff --git a/.eslintrc.json b/.eslintrc.json index 23deaa49..0199bfc5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,21 +1,9 @@ { - "parserOptions": { - "ecmaVersion": 2020 - }, "root": true, - "plugins": ["@swrlab/eslint-plugin-swr", "chai-friendly"], - "extends": ["plugin:@swrlab/eslint-plugin-swr/recommended", "plugin:chai-friendly/recommended"], - "ignorePatterns": ["docs/_SIDEBAR.md"], + "env": {}, + "extends": [], "rules": { - "radix": 0, - "no-unused-vars": [ - "error", - { - "argsIgnorePattern": "next", - "varsIgnorePattern": "should" - } - ], - "no-param-reassign": [2, { "props": false }], - "no-restricted-syntax": 0 + "no-undef": "off", + "no-unused-vars": "off" } } diff --git a/.github/workflows/ingest-pull.yml b/.github/workflows/ingest-pull.yml index bf60458f..7930f39d 100644 --- a/.github/workflows/ingest-pull.yml +++ b/.github/workflows/ingest-pull.yml @@ -5,7 +5,7 @@ on: types: [labeled] env: - NODE_VERSION: 18 + NODE_VERSION: 22 jobs: security: @@ -30,11 +30,14 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} - cache: 'yarn' - cache-dependency-path: '**/yarn.lock' + + - name: πŸ›  Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest - name: πŸ“¦ Install Dependencies - run: yarn + run: bun install - name: πŸ”‘ Setup Google Cloud Auth uses: google-github-actions/auth@v2 @@ -43,7 +46,7 @@ jobs: create_credentials_file: true - name: πŸ“š Lint Code - run: yarn ingest:lint + run: bun ingest:lint - name: πŸ§ͺ Run Ingest Tests env: @@ -56,7 +59,7 @@ jobs: STAGE: ${{ matrix.STAGES }} TEST_USER: ${{ secrets.TEST_USER }} TEST_USER_PW: ${{ secrets.TEST_USER_PW }} - run: yarn ingest:test + run: bun ingest:test license: needs: security @@ -69,11 +72,14 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} - cache: 'yarn' - cache-dependency-path: '**/yarn.lock' + + - name: πŸ›  Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest - name: πŸ“¦ Install Dependencies - run: yarn + run: bun install - name: πŸ“š Run License Check - run: yarn license + run: bun license diff --git a/.github/workflows/ingest-push.yml b/.github/workflows/ingest-push.yml index 6e076e33..d8c185b2 100644 --- a/.github/workflows/ingest-push.yml +++ b/.github/workflows/ingest-push.yml @@ -9,7 +9,7 @@ on: types: [closed] env: - NODE_VERSION: 18 + NODE_VERSION: 22 IMAGE_NAME: ard-eventhub-europe-west3/ingest REGISTRY_DOMAIN: europe-west3-docker.pkg.dev @@ -25,9 +25,6 @@ jobs: test: needs: security runs-on: ubuntu-latest - strategy: - matrix: - STAGES: [dev] steps: - name: πŸ‘€ Checkout Code uses: actions/checkout@v4 @@ -36,11 +33,14 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} - cache: 'yarn' - cache-dependency-path: '**/yarn.lock' + + - name: πŸ›  Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest - name: πŸ“¦ Install Dependencies - run: yarn + run: bun install - name: πŸ”‘ Setup Google Cloud Auth uses: google-github-actions/auth@v2 @@ -49,7 +49,7 @@ jobs: create_credentials_file: true - name: πŸ“š Lint Code - run: yarn ingest:lint + run: bun ingest:lint - name: πŸ§ͺ Run Ingest Tests env: @@ -59,10 +59,10 @@ jobs: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} PUBSUB_SERVICE_ACCOUNT_EMAIL_INTERNAL: ${{ secrets.PUBSUB_SERVICE_ACCOUNT_EMAIL }} SERVICE_NAME: ard-eventhub-ingest - STAGE: ${{ matrix.STAGES }} + STAGE: dev TEST_USER: ${{ secrets.TEST_USER }} TEST_USER_PW: ${{ secrets.TEST_USER_PW }} - run: yarn ingest:test + run: bun ingest:test docker: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 7e941486..1250342f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ logs *.log npm-debug.log* -yarn-debug.log* -yarn-error.log* .DS_STORE # keys @@ -23,8 +21,5 @@ lib-cov # Optional eslint cache .eslintcache -# Yarn Integrity file -.yarn-integrity - # IntelliJ Idea .idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b7bde713..ab3f4b29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,20 @@ -# ARD-Eventhub / Changelog +# ARD Eventhub / Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.8.0] - 2024-06-26 + +- chore: upgrade to Node.js v22 +- feat: switch subscription names from UUIDv4 to ULID +- chore: swap yarn for bun (package manager only) +- chore: swap eslint for biome (experimental) +- chore: remove table of contents from docs +- chore: add docs about plugins +- chore: add docs about pubsub defaults + ## [1.7.3] - 2024-04-23 - chore: updated lots of dependencies diff --git a/Dockerfile b/Dockerfile index 8b90b1fe..b23a9940 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Load desired node image -FROM node:18-alpine +# select bun +FROM oven/bun:1 as bun # Create app directory WORKDIR /web/app @@ -7,13 +7,20 @@ WORKDIR /web/app # Copy app source COPY . . -# Install node dependencies with clean slate -# Also download tokens -RUN rm -rf node_modules && \ - yarn +# Install dependencies +RUN bun install + +# Load desired node image +FROM node:22-alpine + +# Create app directory +WORKDIR /web/app + +# Copy compiled app source +COPY --from=bun /web/app /web/app # Expose port EXPOSE 80 # Run app -CMD [ "yarn", "ingest:cloud" ] +CMD [ "npm", "run", "ingest:cloud" ] diff --git a/README.md b/README.md index dc193ee1..8b559a48 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,7 @@ -# ARD-Eventhub +# ARD Eventhub ARD system to distribute real-time (live) metadata for primarily radio broadcasts. -- [ARD-Eventhub](#ard-eventhub) - - [Active Data Suppliers](#active-data-suppliers) - - [Get Started and Documentation](#get-started-and-documentation) - - [Get involved](#get-involved) - - [Modules](#modules) - - [Ingest](#ingest) - - [Changelog](#changelog) - - [License](#license) - - [Third-Party Components](#third-party-components) - - [Authors](#authors) - ## Active Data Suppliers These ARD broadcasters are currently sending live metadata via ARD Eventhub: @@ -54,13 +43,29 @@ This project will include two modules: Ingest and API. The first development ste The Ingest service is responsible for receiving and publishing events and managing subscriptions. You'll find the core code in [`./src/ingest/`](./src/ingest/). +## Local Setup + +This project uses `bun` as package manager and runtime environment. Please note that this only applies to the package manager and CLI commands. JavaScript still uses the 'traditional' `node` runtime. Check the [Bun documentation](https://bun.sh/docs/installation) for installation instructions. + +To install dependencies run: + +```sh +bun install +``` + +To check for minor updates: + +```sh +bun update +``` + ## Changelog See [CHANGELOG](CHANGELOG.md) for latest changes. ## License -This project is licensed under **European Union Public License 1.2** ([`EUPL-1.2`](https://spdx.org/licenses/EUPL-1.2.html)), which can be found in [LICENSE](LICENSE.txt). Detailed information and translations to all 23 official languages of the European Union are available at [joinup.ec.europa.eu](https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12). The usage of this license does not imply any relationship to or endorsement by the European Union, the joinup initiative or other participating parties. +This project is licensed under **European Union Public License 1.2** ([`EUPL-1.2`](https://spdx.org/licenses/EUPL-1.2.html)), which can be found in [LICENSE](LICENSE.txt). Detailed information and translations to all 23 official languages of the European Union are available at [joinup.ec.europa.eu](https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12). The usage of this license does not imply any relationship to or endorsement by the European Union, the joinup initiative or other participating parties. A [compatibility matrix](https://joinup.ec.europa.eu/collection/eupl/matrix-eupl-compatible-open-source-licences) is also available and describes the relationships between EUPL-1.2 in upstream and downstream projects. This source code is provided under EUPL v1.2, except for the [`spdx-exceptions`](https://www.npmjs.com/package/spdx-exceptions) package, which uses the `CC-BY-3.0` license, without adding any legal or technical restrictions guaranteed by their license. @@ -69,7 +74,7 @@ This source code is provided under EUPL v1.2, except for the [`spdx-exceptions`] | Type | Name | License | | ------- | ------------------------------ | -------------------------------------------------------------------------------------------------- | -| Docker | `node:18-alpine` | [MIT](https://github.com/nodejs/node/blob/master/LICENSE) | +| Docker | `node:22-alpine` | [MIT](https://github.com/nodejs/node/tree/main?tab=readme-ov-file#license) | | NPM | `@google-cloud/datastore` | [Apache License 2.0](https://github.com/googleapis/nodejs-datastore/blob/master/LICENSE) | | NPM | `@google-cloud/pubsub` | [Apache License 2.0](https://github.com/googleapis/nodejs-pubsub/blob/master/LICENSE) | | NPM | `@google-cloud/secret-manager` | [Apache License 2.0](https://github.com/googleapis/nodejs-secret-manager/blob/master/LICENSE) | @@ -84,20 +89,17 @@ This source code is provided under EUPL v1.2, except for the [`spdx-exceptions`] | NPM | `luxon` | [MIT](https://github.com/moment/luxon/blob/master/LICENSE.md) | | NPM | `slug` | [MIT](https://github.com/Trott/slug/blob/master/LICENSE) | | NPM | `swagger-ui-express` | [MIT](https://github.com/scottie1984/swagger-ui-express/blob/master/LICENSE) | +| NPM | `ulid` | [MIT](https://github.com/ulid/javascript/blob/master/LICENSE) | | NPM | `winston` | [MIT](https://github.com/winstonjs/winston/blob/master/LICENSE) | -| NPM DEV | `@swrlab/eslint-plugin-swr` | [ISC](https://github.com/swrlab/eslint-plugin-swr/blob/main/package.json) | +| NPM DEV | `@biomejs/biome` | [MIT](https://github.com/biomejs/biome/blob/main/LICENSE-MIT) | | NPM DEV | `@swrlab/swr-prettier-config` | [MIT](https://github.com/swrlab/swr-prettier-config/blob/main/LICENSE.txt) | | NPM DEV | `chai` | [MIT](https://github.com/chaijs/chai/blob/master/LICENSE) | | NPM DEV | `chai-http` | [MIT](https://github.com/chaijs/chai-http/blob/master/package.json) | | NPM DEV | `docsify-cli` | [MIT](https://github.com/docsifyjs/docsify-cli/blob/master/LICENSE) | -| NPM DEV | `dotenv` | [BSD-2-Clause](https://github.com/motdotla/dotenv/blob/master/LICENSE) | -| NPM DEV | `eslint` | [MIT](https://github.com/eslint/eslint/blob/master/LICENSE) | -| NPM DEV | `eslint-plugin-chai-friendly` | [MIT](https://github.com/ihordiachenko/eslint-plugin-chai-friendly/blob/master/LICENSE) | | NPM DEV | `license-compliance` | [MIT](https://github.com/tmorell/license-compliance/blob/master/LICENSE) | | NPM DEV | `mocha` | [MIT](https://github.com/mochajs/mocha/blob/master/LICENSE) | | NPM DEV | `nodemon` | [MIT](https://github.com/remy/nodemon/blob/master/LICENSE) | | NPM DEV | `prettier` | [MIT](https://github.com/prettier/prettier/blob/main/LICENSE) | -| NPM DEV | `typescript` | [Apache License 2.0](https://github.com/microsoft/TypeScript/blob/main/LICENSE.txt) | ## Authors @@ -106,3 +108,5 @@ This project was realized by - [Daniel Freytag](https://github.com/frytg) - [Rafael MΓ€uer](https://github.com/rafaelmaeuer) - [Christian Hufnagel](https://github.com/chhufnagel) + +at [SWR Audio Lab](https://www.swr.de/audiolab) diff --git a/test/README.md b/TESTING.md similarity index 85% rename from test/README.md rename to TESTING.md index 644f3467..87ac014b 100644 --- a/test/README.md +++ b/TESTING.md @@ -1,8 +1,8 @@ -# ARD-Eventhub - Unit Tests +# ARD Eventhub - Unit Tests ## API-Tests -API unit tests are designed to check and verify existing and new implementations with the ARD-Eventhub. +API unit tests are designed to check and verify existing and new implementations with the ARD Eventhub. As test-environment [Mocha](https://mochajs.org/) is used in combination with [Chai](https://www.chaijs.com/) for a readable chaining of test-arguments. ### Environments @@ -17,6 +17,6 @@ In addition to the [ingest-env](../src/ingest/README.md#Environments), following Follow the [ingest-setup](../src/ingest/README.md) first, then run tests with - ```sh - yarn ingest:test - ``` +```sh +bun ingest:test +``` diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 00000000..9a31b1ff Binary files /dev/null and b/bun.lockb differ diff --git a/bunfig.toml b/bunfig.toml new file mode 100644 index 00000000..5946c175 --- /dev/null +++ b/bunfig.toml @@ -0,0 +1,8 @@ +# disable analytics +# https://bun.sh/docs/runtime/bunfig#telemetry +telemetry = false + +# exclude commands from logs +# https://bun.sh/docs/runtime/bunfig#run-silent-suppress-reporting-the-command-being-run +[run] +silent = true diff --git a/config/swaggerUI.js b/config/swaggerUI.js index dbcccf12..21fe20a1 100644 --- a/config/swaggerUI.js +++ b/config/swaggerUI.js @@ -8,7 +8,7 @@ module.exports = { explorer: true, customCss: '.topbar {display: none;}', - customSiteTitle: 'ARD-Eventhub API Documentation', + customSiteTitle: 'ARD Eventhub API Documentation', swaggerOptions: { url: '/openapi/openapi.json', }, diff --git a/docs/AUTHENTICATION.md b/docs/AUTHENTICATION.md index c1d02664..fa5fe82e 100644 --- a/docs/AUTHENTICATION.md +++ b/docs/AUTHENTICATION.md @@ -1,19 +1,12 @@ -# ARD-Eventhub / Authentication +# ARD Eventhub / Authentication -To authenticate and work with Eventhub API you will need a valid user. For now these user logins are kept separate from the ARD Core API, but are using a similar login method. This page covers the authentication process. - -- [ARD-Eventhub / Authentication](#ard-eventhub--authentication) - - [Authentication Overview](#authentication-overview) - - [Exchange Credentials for Tokens](#exchange-credentials-for-tokens) - - [Refresh Token](#refresh-token) - - [Expiration](#expiration) - - [Reset password](#reset-password) +To authenticate and work with Eventhub API you will need a valid user. For now these user logins are kept separate from the ARD Core API, but are using a similar login method. This page covers the authentication process. ## Authentication Overview -The app uses an authentication workflow, that is kept very similar to the new ARD Core API, so once it goes live there, it can be easily migrated (variations may apply). You can find their documentation for ARD in the [developer portal](https://developer.ard.de/core-api-v2-roles-and-access-control). +The app uses an authentication workflow, that is kept very similar to the new ARD Core API, so once it goes live there, it can be easily migrated (variations may apply). You can find their documentation for ARD in the [developer portal](https://developer.ard.de/core-api-v2-roles-and-access-control). -This page explicitly covers the login methods for the ARD-Eventhub API. Compared to the ARD Core API the token exchange for ARD-Eventhub is handled in this service directly, not externally so it does not need to expose the `API_KEY` to clients. +This page explicitly covers the login methods for the ARD Eventhub API. Compared to the ARD Core API the token exchange for ARD Eventhub is handled in this service directly, not externally so it does not need to expose the `API_KEY` to clients. ## Exchange Credentials for Tokens @@ -45,7 +38,7 @@ This endpoint will return a `token` and `refreshToken` alongside an expiry durat ## Refresh Token -While the normal `token` expires, the `refreshToken` can be used for a longer period of time. Therefore it can be used to be exchanged for a new `token`. +While the normal `token` expires, the `refreshToken` can be used for a longer period of time. Therefore it can be used to be exchanged for a new `token`. **POST `{HOST}/auth/refresh`** @@ -76,13 +69,13 @@ Returns `200 OK` > > - The user is deleted > - The user is disabled -> - A major account change is detected for the user. This includes events like password or email address updates. +> - A major account change is detected for the user. This includes events like password or email address updates. [Source: Firebase docs](https://firebase.google.com/docs/auth/admin/manage-sessions) ## Reset password -Sometimes you might loose your old password and need to reset it. If this happens, use this endpoint to request a password reset email. This endpoint might be subject to throttling/ rate-limits in the future. +Sometimes you might loose your old password and need to reset it. If this happens, use this endpoint to request a password reset email. This endpoint might be subject to throttling/ rate-limits in the future. **POST `{HOST}/auth/reset`** diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 9600a355..491fb916 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -1,12 +1,12 @@ -# ARD-Eventhub / Events +# ARD Eventhub / Events -In its first version ARD-Eventhub expects the `event` value to be one of the following: +In its first version ARD Eventhub expects the `event` value to be one of the following: ## `de.ard.eventhub.v1.radio.track.playing` -This event marks the beginning of a new playing element/ track for the providing radio station. It should have the `start` time information of the source information, to provide an as detailed as possible start time and avoid misalignments during network latencies. +This event marks the beginning of a new playing element/ track for the providing radio station. It should have the `start` time information of the source information, to provide an as detailed as possible start time and avoid misalignments during network latencies. ## `de.ard.eventhub.v1.radio.track.next` -The `next` event is similar to `playing` but only signalizes the next scheduled upcoming track. The next element can be replaced by a new next element before a playing element to signalize a new scheduled track. +The `next` event is similar to `playing` but only signalizes the next scheduled upcoming track. The next element can be replaced by a new next element before a playing element to signalize a new scheduled track. A pair of next and playing events should have a reference between each other (`playlistId`), so subscribers can link these two incoming events. diff --git a/docs/EXTERNAL_IDS.md b/docs/EXTERNAL_IDS.md new file mode 100644 index 00000000..951eb2e6 --- /dev/null +++ b/docs/EXTERNAL_IDS.md @@ -0,0 +1,17 @@ +# Importance of External IDs + +For the Eventhub to work it needs to be able to uniquely identify a service. This is defined as the so-called `externalId` in ARD's new Core API. You might currently know this as _CRID_, which you are using in the TVA documents. + +⚠️ Please make sure to use the **exact** `externalId` that you will be using to deliver the metadata of your livestreams to ARD Core (_PermanentLivestream_). When in doubt please reach out to your metadata contacts or to SWR Audio Lab. + +> **External ID Requirements and Recommendations** +> The external ID may be provided through the field `externalId` during an entity creation request. +> +> If you do not already deliver content via TVA you are free in your choice of external ID. However, your choice **must** meet the following criteria: +> +> (a) The external ID of a single entity does not change over time +> (b) The external ID is referring to the local entity you want to import +> (c) The external ID is unique in your own local context +> (d) The external ID is unique in the whole ARD context + +[Source: developer.ard.de](https://developer.ard.de/core-api-v2-delivering-content#ExternalIDRequirementsRecommendations) diff --git a/docs/NAMING.md b/docs/NAMING.md index a8196b4c..59fa4b70 100644 --- a/docs/NAMING.md +++ b/docs/NAMING.md @@ -1,11 +1,6 @@ -# ARD-Eventhub / Naming Conventions +# ARD Eventhub / Naming Conventions -Several parts of this service require naming conventions that are followed throughout all parts. - -- [ARD-Eventhub / Naming Conventions](#ard-eventhub--naming-conventions) - - [Google Cloud Pub/Sub](#google-cloud-pubsub) - - [Pub/Sub Topics](#pubsub-topics) - - [Pub/Sub Subscriptions](#pubsub-subscriptions) +Several parts of this service require naming conventions that are followed throughout all parts. ## Google Cloud Pub/Sub diff --git a/docs/OPENAPI.md b/docs/OPENAPI.md index 2ad965c6..6666070d 100644 --- a/docs/OPENAPI.md +++ b/docs/OPENAPI.md @@ -1,6 +1,6 @@ -# ARD-Eventhub / OpenAPI +# ARD Eventhub / OpenAPI -The ARD-Eventhub APIs are documented with [OpenAPI](https://swagger.io/specification/) specification (a standard, language-agnostic interface to RESTful APIs). With [Swagger-UI](https://swagger.io/tools/swagger-ui/) a testable interface is provided at `{service-endpoint}/openapi/`. +The ARD Eventhub APIs are documented with [OpenAPI](https://swagger.io/specification/) specification (a standard, language-agnostic interface to RESTful APIs). With [Swagger-UI](https://swagger.io/tools/swagger-ui/) a testable interface is provided at `{service-endpoint}/openapi/`. All changes in the API should be documented in [openapi.yaml](https://eventhub-ingest.ard.de/openapi/openapi.yaml) which must be converted to a JSON format afterwards. diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md new file mode 100644 index 00000000..22313791 --- /dev/null +++ b/docs/PLUGINS.md @@ -0,0 +1,39 @@ +# ARD Eventhub / Plugins + +The Eventhub is a platform that allows for the integration of various plugins. These plugins are used to process and transform data, and to send it to the appropriate destination. The plugins are written in JavaScript. This page provides an overview of the plugins that are currently available. + +## DTS Xperi + +This plugin makes it possible to send data to their system. It is currently set as an opt-out feature, meaning that data is sent by default. The opt-out feature only applies to music titles at the moment (`type: music`). Sending data can be toggled for every individual event, if you want to set it yourself: + +```json5 +{ + "type": "music", + // ... + "plugins": [ + { + "type": "dts", + "isDeactivated": false + } + ] +} +``` + +There are also a couple of additional properties that can be set: + +- `delay` (int, default: `0`) - if there is an offset to wait until this data is shown +- `album` (string, default `null`) - album title, if available +- `composer` (string, default `null`) - composer, if available +- `program` (string, default `null`) - program title, if available +- `subject` (string, default `null`) - subject, if available +- `webUrl` (string, default `null`) - URL to the web page of the event +- `preferArtistMedia` (boolean, default `false`) - if `true`, the artist media will be preferred over the cover (both, if available) +- `excludeFields` (array, default `[]`) - fields to exclude from the data sent to the external system + +If `plugins[].type === 'dts'` is not set, it will be added automatically (opt-out principle). The `isDeactivated` property is optional and defaults to `false`. Setting it to `true` will prevent the data from being sent to the external system. + +ARD Core IDs are already mapped in the external system. Changing your IDs (not recommended) requires manual updates (please contact for more details). + +## Additional plugins + +While there are technically endless possibilities for plugins, there are guidelines to follow. Please reach out internally to the partner management for more details. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 90239d42..6184ed4d 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -1,22 +1,14 @@ -# ARD-Eventhub / Quickstart +# ARD Eventhub / Quickstart -This guide will help you get started with ARD-Eventhub. +This guide will help you get started with ARD Eventhub. No matter if you are a Publisher or Subscriber, you will need a user account to interact with the API. Request one through your contacts at SWR Audio Lab or ARD Online. Admins can reference the Users docs for account registrations. Once this has been set up, check the Authentication docs to learn more about the login and token exchange process. -- [ARD-Eventhub / Quickstart](#ard-eventhub--quickstart) - - [Publishers](#publishers) - - [Importance of External IDs](#importance-of-external-ids) - - [Workflow Example](#workflow-example) - - [Subscribers](#subscribers) - - [Security](#security) - - [Receiver Example](#receiver-example) - ## Publishers -If you are a radio station that wants to start publishing events to ARD-Eventhub, follow these easy steps: +If you are a radio station that wants to start publishing events to ARD Eventhub, follow these easy steps: - Set up your account and understand the authentication process - Use the POST `/events/{eventName}` endpoint to add your events @@ -54,23 +46,19 @@ It is recommended to use the Eventhub `test` system first, to make sure everythi Security Note: Every user account can only publish to `publisherId`s from their own institution. If you are receiving an error, the Id could be misspelled, or the user account was wrongly configured by an admin. -### Importance of External IDs +### Handling Events with External Data + +Let's say you are using Eventhub to receive events from a station that belongs to another broadcaster (i.e. nightly broadcasts) for your own station. For this case, it is expected to also re-publish these events to Eventhub. -For the Eventhub to work it needs to be able to uniquely identify a service. This is defined as the so-called `externalId` in ARD's new Core API. You might currently know this as _CRID_, which you are using in the TVA documents. +This is important because your subscribers expect to receive all events from your station, including the ones that are re-broadcasted from other stations. They might not know that you are re-broadcasting the signal from another station and are simply using a subscription to your station to receive all events. -⚠️ Please make sure to use the **exact** `externalId` that you will be using to deliver the metadata of your livestreams to ARD Core (_PermanentLivestream_). When in doubt please reach out to your metadata contacts or to SWR Audio Lab. +For ARD Audiothek, this is also important. Otherwise, your station might have incomplete live metadata when re-broadcasting other stations. -> **External ID Requirements and Recommendations** -> The external ID may be provided through the field `externalId` during an entity creation request. -> -> If you do not already deliver content via TVA you are free in your choice of external ID. However, your choice **must** meet the following criteria: -> -> (a) The external ID of a single entity does not change over time -> (b) The external ID is referring to the local entity you want to import -> (c) The external ID is unique in your own local context -> (d) The external ID is unique in the whole ARD context +For this case, it is important to make sure your internal filtering works correctly when receiving events from other stations and only publish them when the station is actually on air. Otherwise, it could create a loop. -[Source: developer.ard.de](https://developer.ard.de/core-api-v2-delivering-content#ExternalIDRequirementsRecommendations) +### Importance of External IDs + +This has moved to [EXTERNAL_IDS.md](EXTERNAL_IDS.md). ### Workflow Example @@ -135,6 +123,8 @@ Please be aware that the type of events published to this service may be extende In case of nightly re-broadcasts you should create a permanent subscription and keep this one running 24/7. The filter based on the program schedule should be done on your side. Pub/Sub should not be used to create and delete subscriptions once the re-broadcast starts and ends. +Make sure that your endpoint is reachable from the internet and that you have a valid SSL certificate installed. If the endpoint is periodically unreachable, the subscription will collect past events and will retry delivering them to you. Reference [`src/utils/pubsub/createSubscription.js`](../src/utils/pubsub/createSubscription.js) and [cloud.google.com/pubsub/docs/push](https://cloud.google.com/pubsub/docs/push#push_backoff) for more details about the default subscription config and retry behavior. + Start receiving events with these steps: - Set up your account and understand the authentication process @@ -144,12 +134,12 @@ Start receiving events with these steps: - Check the Google Cloud page ["Receiving messages using Push"](https://cloud.google.com/pubsub/docs/push#receiving_messages) to learn more about the format that you will be receiving those events in - Use GET `/subcriptions` to verify your new or existing subscriptions -Security Note: When a user is registered, it is linked to a specific institution (_Landesrundfunkanstalt_ or _GSEA_). Users can manage all subscriptions within this institution, so be careful not to delete your colleagues' (production) entries. +Security Note: When a user is registered, it is linked to a specific institution (_Landesrundfunkanstalt_ or _GSEA_). Users can manage all subscriptions within this institution, so be careful not to delete your colleagues' (production) entries. With this method you will still have access to all subscriptions, even if a person leaves your institution or their account is deactivated. ### Security -Generally it is recommended to keep your endpoints hidden from public indexes. To be absolutely sure that an event is actually being received from Eventhub, you can make use of the provided JWT token and service account. +Generally it is recommended to keep your endpoints hidden from public indexes. To be absolutely sure that an event is actually being received from Eventhub, you can make use of the provided JWT token and service account. For every subscription that you create, the response will (amongst other metadata) also include a field about the used service account: ```js @@ -164,7 +154,7 @@ Please note that for now the service account usually contains the same response. ### Receiver Example -In a simplified way, your receiver might look something like this (example for NodeJS with Express). The Google Cloud section ["Authentication and authorization by the push endpoint"](https://cloud.google.com/pubsub/docs/push#authentication_and_authorization_by_the_push_endpoint) also holds more information about this process. +In a simplified way, your receiver might look something like this (example for NodeJS with Express). The Google Cloud section ["Authentication and authorization by the push endpoint"](https://cloud.google.com/pubsub/docs/push#authentication_and_authorization_by_the_push_endpoint) also holds more information about this process. ```js // load node packages diff --git a/docs/SECRETS.md b/docs/SECRETS.md index ad9d3c28..206dadba 100644 --- a/docs/SECRETS.md +++ b/docs/SECRETS.md @@ -1,18 +1,12 @@ -# ARD-Eventhub / Secrets +# ARD Eventhub / Secrets -This repository obviously needs a number of secrets and configuration files that are kept in various places. This page documents what goes where. -Since this project is designed to be kept public to allow a collaborative development process, the full configuration around secrets and their deployment process is not described to the fullest extend here. There are other places, which include more about the internal maintenance structure. - -- [ARD-Eventhub / Secrets](#ard-eventhub--secrets) - - [Code](#code) - - [Github](#github) - - [Google Cloud](#google-cloud) - - [Docker Image](#docker-image) +This repository obviously needs a number of secrets and configuration files that are kept in various places. This page documents what goes where. +Since this project is designed to be kept public to allow a collaborative development process, the full configuration around secrets and their deployment process is not described to the fullest extend here. There are other places, which include more about the internal maintenance structure. ## Code -Different modules might need varying sets of variables. Check the README of each provided module to see more about those. -Usually it only requires few API keys to external services. Access to Google Cloud services is given by a [service account](https://cloud.google.com/iam/docs/service-accounts) (SA), with only the minimum set of permissions needed. This SA is added using an environment variable. +Different modules might need varying sets of variables. Check the README of each provided module to see more about those. +Usually it only requires few API keys to external services. Access to Google Cloud services is given by a [service account](https://cloud.google.com/iam/docs/service-accounts) (SA), with only the minimum set of permissions needed. This SA is added using an environment variable. ## Github @@ -37,9 +31,9 @@ Secrets in Github are write-only by default for users. Admins can change them in ## Google Cloud -When this project is deployed to Google Cloud for example, it also needs environment variables and keys. Those are usually provided by the runtime and its deployment configuration. The location of all Kubernetes deployments files is explicitly not mentioned here, but if have access to said environment, you should know where to start searching. +When this project is deployed to Google Cloud for example, it also needs environment variables and keys. Those are usually provided by the runtime and its deployment configuration. The location of all Kubernetes deployments files is explicitly not mentioned here, but if have access to said environment, you should know where to start searching. ### Docker Image -Kubernetes needs to pull the image from some specified registry. Usually there's one central place for these images. For eventhub we are not using this place to avoid adding their key to this repository. Instead we are storing containers in our eventhub project and are giving access to their SA. -To do this, open the console, navigate to the eventhub project, go to storage, select the artifact bucket. In the info panel, add their SA email address with the permission "_Storage Object Viewer_". +Kubernetes needs to pull the image from some specified registry. Usually there's one central place for these images. For eventhub we are not using this place to avoid adding their key to this repository. Instead we are storing containers in our eventhub project and are giving access to their SA. +To do this, open the console, navigate to the eventhub project, go to storage, select the artifact bucket. In the info panel, add their SA email address with the permission "_Storage Object Viewer_". diff --git a/docs/STAGES.md b/docs/STAGES.md index 51ddb0df..6d3015d2 100644 --- a/docs/STAGES.md +++ b/docs/STAGES.md @@ -1,12 +1,7 @@ -# ARD-Eventhub / Stages +# ARD Eventhub / Stages The Eventhub differentiates between stages given to the service via env `STAGE` and different runtime environments or deployments, such as beta, test, or similar. -- [ARD-Eventhub / Stages](#ard-eventhub--stages) - - [Ingest](#ingest) - - [Ingest Service Stages](#ingest-service-stages) - - [Ingest Deployment Stages](#ingest-deployment-stages) - ## Ingest ### Ingest Service Stages diff --git a/docs/TYPES.md b/docs/TYPES.md index 4b6f4d2c..d54035cb 100644 --- a/docs/TYPES.md +++ b/docs/TYPES.md @@ -1,17 +1,7 @@ -# ARD-Eventhub / Types +# ARD Eventhub / Types Each triggered `track` can and must be of a certain type, to be properly displayed by receiving subscribers. -- [ARD-Eventhub / Types](#ard-eventhub--types) - - [`music`](#music) - - [`live`](#live) - - [`news`](#news) - - [`weather`](#weather) - - [`traffic`](#traffic) - - [`audio`](#audio) - - [`commercial`](#commercial) - - [`jingle`](#jingle) - ## `music` A song or commercially produced piece of music. It is highly recommended and expected to set both `title` and `artist`. Full details about participating artists inside `contributors` is a bonus. This type should ideally also include references to its source element, like `confId` from ARD's HFDB, `isrc` and `upc`. @@ -48,7 +38,7 @@ A song or commercially produced piece of music. It is highly recommended and exp "attribution": "Photographer XYZ" } ], - … + … } ``` @@ -63,7 +53,7 @@ If a live element is starting. This can be an moderation by an anchor, interview "start": "2020-01-19T06:20:00+01:00", "length": 240, "title": "Moderation", - … + … } ``` @@ -100,7 +90,7 @@ This indicates the beginning of the news in general or a new news item. Get as d "url": "https://www.deutschlandfunkkultur.de/kommerzielle-us-raumfahrt-die-neue-weltraumoekonomie-100.html" } ] - … + … } ``` @@ -115,7 +105,7 @@ Similar to `news` it marks the beginning of a weather segment. "start": "2020-01-19T06:03:00+01:00", "length": 30, "title": "Wetter", - … + … } ``` @@ -130,7 +120,7 @@ Similar to `news` and `weather` it can mark the beginning of a traffic announcem "start": "2020-01-19T06:03:30+01:00", "length": 10, "title": "Verkehr", - … + … } ``` diff --git a/docs/USERS.md b/docs/USERS.md index 4673c58d..4de7425b 100644 --- a/docs/USERS.md +++ b/docs/USERS.md @@ -1,11 +1,6 @@ -# ARD-Eventhub / Users +# ARD Eventhub / Users -To authenticate and work with Eventhub API you will need a valid user. For now these user logins are kept separate from the ARD Core API, but are using a similar login method. This page covers the process of administrating users. - -- [ARD-Eventhub / Users](#ard-eventhub--users) - - [Authentication](#authentication) - - [Registering new users](#registering-new-users) - - [Removing users](#removing-users) +To authenticate and work with Eventhub API you will need a valid user. For now these user logins are kept separate from the ARD Core API, but are using a similar login method. This page covers the process of administrating users. ## Authentication @@ -13,17 +8,17 @@ Please see the [Authentication document](AUTHENTICATION.md) to learn more about ## Registering new users -New users cannot sign up themselves, but need to go through the ARD Online team to receive access to this service. The following step-by-step guide is targeted at admins, that want to add a new account for a user. +New users cannot sign up themselves, but need to go through the ARD Online team to receive access to this service. The following step-by-step guide is targeted at admins, that want to add a new account for a user. - Open Datastore in the GCP Eventhub project, go to the appropriate namespace (usually `prod`) and kind `users` - Check that the user really hasn't been registered, then add a new entity - The entity key needs to be '_Custom Name_', with the user's email address (**in lowercase**) - Set `active` to `true` - - In the field `institutionId`, you will need to add a string with the same ID that is being used in ARD Core in this format: `urn:ard:institution:hex` -- Now that the user is entered in Datastore with its profile, you can register it in Firebase. Therefore go to the [Firebase Console](https://console.firebase.google.com/) in the section _Build_ -> _Authentication_. + - In the field `institutionId`, you will need to add a string with the same ID that is being used in ARD Core in this format: `urn:ard:institution:hex` +- Now that the user is entered in Datastore with its profile, you can register it in Firebase. Therefore go to the [Firebase Console](https://console.firebase.google.com/) in the section _Build_ -> _Authentication_. - On this page, click on _Add user_ and enter the same email address (again in lowercase). The password can be something random, that will never be seen. - Once the user has been added, click on the dropdown menu and select _Reset password_ and confirm the pop-up. -- The user will now receive an email to set their password, that they can then use to interact with the API. +- The user will now receive an email to set their password, that they can then use to interact with the API. ## Removing users diff --git a/docs/_SIDEBAR.md b/docs/_SIDEBAR.md index b0f38ee7..ef5f5600 100644 --- a/docs/_SIDEBAR.md +++ b/docs/_SIDEBAR.md @@ -1,7 +1,9 @@ -- [ARD-Eventhub Overview](/ 'ARD-Eventhub') +- [ARD Eventhub Overview](/ 'ARD Eventhub') - [Quickstart](/docs/QUICKSTART.md) - [Authentication](/docs/AUTHENTICATION.md) + - [External IDs](/docs/EXTERNAL_IDS.md) - [Events](/docs/EVENTS.md) + - [Plugins](/docs/PLUGINS.md) - [Types](/docs/TYPES.md) - [OpenAPI](/docs/OPENAPI.md) - [OpenAPI Swagger UI](https://eventhub-ingest.ard.de/openapi) diff --git a/index.html b/index.html index f3c977a2..1f42b8c0 100644 --- a/index.html +++ b/index.html @@ -1,4 +1,4 @@ - + @@ -10,7 +10,7 @@