diff --git a/.gitignore b/.gitignore index 1c552a5b..098e6ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,13 @@ yarn-error.log* # keys keys/*.json +.env # Dependency directories node_modules/ +coverage +lib-cov +.nyc_output # Optional npm cache directory .npm diff --git a/CHANGELOG.md b/CHANGELOG.md index e1e8b580..d7e79ba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,28 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.0-beta1] - 2021-03-24 + +🚧 BREAKING CHANGES for `serviceIds` 🚧 + +### Added + +- New detailed return output when posting events for each service (`published`, `blocked`, `failed`) +- Improved log output in JSON format for better monitoring + +### Changed + +- Now using `services` with required fields `type`, `externalId` and `publisherId` to identify a publishers' channel + +### Removed + +- No longer using `serviceIds` as required identification keys +- `event` in the POST body for new events is now called `name` and is no longer required + - variable is inserted using the event name provided by the URL + ## [0.1.7] - 2021-03-18 -### Changes +### Changed - Add auth verification for unit tests - Check `content-type` in unit tests @@ -15,14 +34,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.6] - 2021-03-17 -### Changes +### Changed - Allow optional fields in POST /events to be `null` - Remove field `isInternal` from POST /events ## [0.1.5] - 2021-03-17 -### Changes +### Changed - Preventing errors when `institution.name` isn't properly set in the user account - Enforcing separation between dev/prod topics and subscriptions @@ -30,7 +49,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.4] - 2021-03-16 -### Changes +### Changed - Removed `attribution` from required media fields of new events - Added OpenAPI documentation to the docs @@ -39,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.3] - 2021-03-16 -### Changes +### Changed - Hotfix `content-type` bug for error responses - Updated endpoint structure for `/events` diff --git a/Dockerfile.ingest b/Dockerfile.ingest index 4535aa51..c0e05641 100644 --- a/Dockerfile.ingest +++ b/Dockerfile.ingest @@ -1,6 +1,9 @@ # Load desired node pckg FROM node:14.16-alpine +# Add python +RUN apk add g++ make python + # Create app directory WORKDIR /web/app diff --git a/README.md b/README.md index bcd7ec17..19b21f33 100644 --- a/README.md +++ b/README.md @@ -55,15 +55,18 @@ This source code is provided under EUPL v1.2, except for the [`spdx-exceptions`] | NPM | `firebase-admin` | [Apache License 2.0](https://github.com/firebase/firebase-admin-node/blob/master/LICENSE) | | NPM | `jsonwebtoken` | [MIT](https://github.com/auth0/node-jsonwebtoken/blob/master/LICENSE) | | NPM | `moment` | [MIT](https://github.com/moment/moment/blob/develop/LICENSE) | +| NPM | `node-crc` | [MIT](https://github.com/magiclen/node-crc/blob/master/LICENSE) | | NPM | `node-fetch` | [MIT](https://github.com/node-fetch/node-fetch/blob/master/LICENSE.md) | | NPM | `slug` | [MIT](https://github.com/Trott/slug/blob/master/LICENSE) | | NPM | `swagger-ui-express` | [MIT](https://github.com/scottie1984/swagger-ui-express/blob/master/LICENSE) | | NPM | `uuid` | [MIT](https://github.com/uuidjs/uuid/blob/master/LICENSE.md) | +| NPM | `winston` | [MIT](hhttps://github.com/winstonjs/winston/blob/master/LICENSE) | | NPM DEV | `@swrlab/eslint-plugin-swr` | [ISC](https://github.com/swrlab/eslint-plugin-swr/) | | NPM DEV | `@swrlab/swr-prettier-config` | [ISC](https://github.com/swrlab/prettier-config/blob/main/license.md) | | NPM DEV | `chai` | [MIT](https://github.com/chaijs/chai/blob/master/LICENSE) | | NPM DEV | `chai-http` | [MIT](https://github.com/chaijs/chai-http/blob/master/package.json) | | NPM DEV | `docsify-cli` | [MIT](https://github.com/docsifyjs/docsify-cli/blob/master/LICENSE) | +| NPM DEV | `dotenv` | [BSD-2-Clause](<[BSD-2-Clause](https://github.com/motdotla/dotenv/blob/master/LICENSE)>) | | NPM DEV | `eslint` | [MIT](https://github.com/eslint/eslint/blob/master/LICENSE) | | NPM DEV | `eslint-plugin-swr` | [ISC](https://github.com/swrlab/eslint-plugin-swr/blob/main/package.json) | | NPM DEV | `eslint-plugin-chai-friendly` | [MIT](https://github.com/ihordiachenko/eslint-plugin-chai-friendly/blob/master/LICENSE) | diff --git a/config/coreIdPrefixes.json b/config/coreIdPrefixes.json new file mode 100644 index 00000000..4fabff9c --- /dev/null +++ b/config/coreIdPrefixes.json @@ -0,0 +1,5 @@ +{ + "EventLivestream": "urn:ard:event-livestream:", + "Publisher": "urn:ard:publisher:", + "PermanentLivestream": "urn:ard:permanent-livestream:" +} diff --git a/config/index.js b/config/index.js index ee9abdb5..67d0cb78 100644 --- a/config/index.js +++ b/config/index.js @@ -7,6 +7,7 @@ // import version from package.json const { version } = require('../package.json') +const coreIdPrefixes = require('./coreIdPrefixes.json') // check existence of several process vars if (!process.env.GCP_PROJECT_ID) { @@ -23,10 +24,12 @@ const stage = process.env.STAGE // set config const serviceName = 'ard-eventhub' const baseConfig = { - userAgent: `${serviceName}/${version}`, - pubsubPrefix: 'de.ard.eventhub', + coreIdPrefixes, + pubSubPrefix: `de.ard.eventhub.${stage}.`, stage, + userAgent: `${serviceName}/${version}`, version, + isDebug: process.env.DEBUG === 'true', } // set config based on stages @@ -35,6 +38,10 @@ const config = { ...baseConfig, serviceName: `${serviceName}-dev`, }, + test: { + ...baseConfig, + serviceName: `${serviceName}-test`, + }, prod: { ...baseConfig, serviceName, diff --git a/docs/NAMING.md b/docs/NAMING.md index 1a5c50a7..a8196b4c 100644 --- a/docs/NAMING.md +++ b/docs/NAMING.md @@ -20,17 +20,17 @@ Pub/Sub includes a number of restrictions around names, keys and values. For all ## Pub/Sub Topics ```txt -<domain-prefix> . <service> . <module> . <stage> . <service-id> - de.ard . eventhub . publisher . dev . 284680 +<domain-prefix> . <service> . <stage> . <encoded-core-id> + de.ard . eventhub . dev . urn%3Aard%3Aper... -=> de.ard.eventhub.publisher.dev.284680 +=> de.ard.eventhub.dev.urn%3Aard%3Apermanent-livestream%3Aa315d3e482f09e1b ``` ## Pub/Sub Subscriptions ```txt -<domain-prefix> . <service> . <module> . <stage> . <institution> . <uid> - de.ard . eventhub . subscription . dev . swr . 9bdb9316-c78a-4ebe-a131-30b2738435a3 +<domain-prefix> . <service> . <module> . <stage> . <uid> + de.ard . eventhub . subscription . dev . 9bdb9316-c78a-4ebe-a131-30b2738435a3 -=> de.ard.eventhub.subscription.dev.swr.9bdb9316-c78a-4ebe-a131-30b2738435a3 +=> de.ard.eventhub.subscription.dev.9bdb9316-c78a-4ebe-a131-30b2738435a3 ``` diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index f508ae08..f1e09392 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -1,39 +1,182 @@ # ARD-Eventhub / Quickstart -This guide will help you get started with ARD-Eventhub. +This guide will help you get started with ARD-Eventhub. -No matter if you are a Publisher or Subscriber, you will need a user account to interact with the API. Request one through your contacts at SWR Audio Lab or ARD Online. Admins can reference the Users docs for account registrations. +No matter if you are a Publisher or Subscriber, you will need a user account to interact with the API. Request one through your contacts at SWR Audio Lab or ARD Online. Admins can reference the Users docs for account registrations. Once this has been set up, check the Authentication docs to learn more about the login and token exchange process. +- [ARD-Eventhub / Quickstart](#ard-eventhub--quickstart) + - [Publishers](#publishers) + - [Importance of External IDs](#importance-of-external-ids) + - [Workflow Example](#workflow-example) + - [Subscribers](#subscribers) + - [Security](#security) + - [Receiver Example](#receiver-example) + ## Publishers -If you are a radio station that wants to start publishing events to ARD-Eventhub, follow these easy steps: +If you are a radio station that wants to start publishing events to ARD-Eventhub, follow these easy steps: - Set up your account and understand the authentication process - Use the POST `/events/{eventName}` endpoint to add your events - - Note: Even if GET `/topics` does not list your radio station(s) beforehand, the topic(s) will be created during your first published event (response will contain: `"topics": {"de.ard.eventhub.publisher.dev.{serviceId}": "TOPIC_CREATED"}`) +- Note: Even if GET `/topics` does not list your radio station(s) beforehand, the topic(s) will be created during your first published event (response will contain: `"topic": { ..., "status": "TOPIC_CREATED" }`) It is recommended to use the Eventhub `test` system first, to make sure everything works. Then bring it to production on `prod`. The host names are listed in the Stages document. -Security Note: Every user account has a list of permitted `serviceIds` that they can publish to. If you are receiving an error, the Id could be misspelled, or the user account was wrongly configured by an admin. +Security Note: Every user account can only publish to `publisherId`s from their own institution. If you are receiving an error, the Id could be misspelled, or the user account was wrongly configured by an admin. + +### Importance of External IDs + +For the Eventhub to work it needs to be able to uniquely identify a service. This is defined as the so-called `externalId` in ARD's new Core API. You might currently know this as _CRID_, which you are using in the TVA documents. + +⚠️ Please make sure to use the **exact** `externalId` that you will be using to deliver the metadata of your livestreams to ARD Core (_PermanentLivestream_). When in doubt please reach out to your metadata contacts or to SWR Audio Lab. + +> **External ID Requirements and Recommendations** +> The external ID may be provided through the field `externalId` during an entity creation request. +> +> If you do not already deliver content via TVA you are free in your choice of external ID. However, your choice **must** meet the following criteria: +> +> (a) The external ID of a single entity does not change over time +> (b) The external ID is referring to the local entity you want to import +> (c) The external ID is unique in your own local context +> (d) The external ID is unique in the whole ARD context + +[Source: developer.ard.de](https://developer.ard.de/core-api-v2-delivering-content#ExternalIDRequirementsRecommendations) + +### Workflow Example + +In your system for every new event, you might follow a workflow like this: + +1. Check if you have `token` from a previous call that has not expired +2. If not found, check if you have a `refreshToken` from a previous call + 1. If found exchange it for a new `token` + 2. If not found, create a new login +3. POST the event using the pre-defined format. The example below might help you understand the different fields: + +```js +{ + "start": "2021-03-17T10:04:35+01:00", + "length": 215.2, + "title": "Save your tears", + "artist": "The Weeknd", + "contributors": [ + { + "name": "The Weeknd", + "role": "artist", + "normDb": { + "type": "Person", + "id": "12345" + } + } + ], + "services": [ + { + "type": "PermanentLivestream", + "externalId": "crid://swr.de/282310", + "publisherId": "282310" + } + ], + "playlistItemId": "radiomax:SWR3-BAD-MAX:12569153", + "externalId": "M0589810001", + "isrc": null, + "upc": null, + "mpn": null, + "media": [ + { + "type": "cover", + "url": "http://rdz-dev:4001/covers/M0589810.001", + "templateUrl": null, + "description": "SWR Cover zu Save your tears von The Weeknd", + "attribution": "" + } + ], + "type": "music", + "hfdbIds": [ + "swrhfdb1.KONF.12345" + ] +} + +``` ## Subscribers -If you plan to receive events published by other stations, add yourself as one of their subscribers and receive real-time POST webhooks for all published events. Those can then be used to improve your products such as websites and apps during re-broadcasts in the nightly tracks. +If you plan to receive events published by other stations, add yourself as one of their subscribers and receive real-time POST webhooks for all published events. Those can then be used to improve your products such as websites and apps during re-broadcasts in the nightly tracks. -Please be aware that the type of events published to this service may be extended in the future. Make sure to filter them appropriately. The data format should and will always be backwards-compatible, but new fields may be added to this service as needed. +Please be aware that the type of events published to this service may be extended in the future. Make sure to filter them appropriately. The data format should and will always be backwards-compatible, but new fields may be added to this service as needed. -In case of nightly re-broadcasts you should create a permanent subscription and keep this one running 24/7. The filter based on the program schedule should be done on your side. Pub/Sub should not be used to create and delete subscriptions once the re-broadcast starts and ends. +In case of nightly re-broadcasts you should create a permanent subscription and keep this one running 24/7. The filter based on the program schedule should be done on your side. Pub/Sub should not be used to create and delete subscriptions once the re-broadcast starts and ends. -Start receiving events with these steps: +Start receiving events with these steps: - Set up your account and understand the authentication process - Use the GET `/topics` endpoint to see a list of available channels (topics) that you can subscribe to - - If a channel is not yet visible, no one has attempted to publish an event to it before. Topics are not created until someone starts publishing +- If a channel is not yet visible, no one has attempted to publish an event to it before. Topics are not created until someone starts publishing - Use the POST `/subscriptions` endpoint to create your own subscription. - - Check the Google Cloud page ["Receiving messages using Push"](https://cloud.google.com/pubsub/docs/push#receiving_messages) to learn more about the format that you will be receiving those events in +- Check the Google Cloud page ["Receiving messages using Push"](https://cloud.google.com/pubsub/docs/push#receiving_messages) to learn more about the format that you will be receiving those events in - Use GET `/subcriptions` to verify your new or existing subscriptions Security Note: When a user is registered, it is linked to a specific institution (_Landesrundfunkanstalt_ or _GSEA_). Users can manage all subscriptions within this institution, so be careful not to delete your colleagues' (production) entries. -With this method you will still have access to all subscriptions, even if a person leaves your institution or their account is deactivated. +With this method you will still have access to all subscriptions, even if a person leaves your institution or their account is deactivated. + +### Security + +Generally it is recommended to keep your endpoints hidden from public indexes. To be absolutely sure that an event is actually being received from Eventhub, you can make use of the provided JWT token and service account. +For every subscription that you create, the response will (amongst other metadata) also include a field about the used service account: + +```js +{ + ... + "serviceAccount": "somethin@something-else.iam.gserviceaccount.com", + ... +} +``` + +Please note that for now the service account usually contains the same response. However, for future subscriptions, it might contain a different account. Configure your service to validate the appropriate account for each subscription. + +### Receiver Example + +In a simplified way, your receiver might look something like this (example for NodeJS with Express). The Google Cloud section ["Authentication and authorization by the push endpoint"](https://cloud.google.com/pubsub/docs/push#authentication_and_authorization_by_the_push_endpoint) also holds more information about this process. + +```js +// load node packages +const { OAuth2Client } = require('google-auth-library') +const authClient = new OAuth2Client() + +// set received serviceAccount +const serviceAccountEmail = 'somethin@something-else.iam.gserviceaccount.com' + +module.exports = async (req, res) => { + try { + // read token from header + const bearer = req.header('Authorization') + const [, idToken] = bearer.match(/Bearer (.*)/) + + // verify token, throws error if invalid + const verification = await authClient.verifyIdToken({ + idToken, + }) + + // check token email vs. subscription email + if(verification?.payload?.email === serviceAccountEmail) { + // get message and metadata from pubsub body + const { attributes, messageId } = req.body.message + const { subscription } = req.body + let data = Buffer.from(req.body.message.data, 'base64').toString() + data = JSON.parse(data) + + // request successful, you can now use the received data + console.log({ attributes, messageId, subscription, data }) + + // close connection + return res.sendStatus(201) + } else { + // user provided valid token but failed email verification + return res.sendStatus(204) + } + } catch (err) { + // request failed or invalid token + return res.sendStatus(204) + } +} +``` diff --git a/docs/STAGES.md b/docs/STAGES.md index 872411b8..51ddb0df 100644 --- a/docs/STAGES.md +++ b/docs/STAGES.md @@ -11,20 +11,20 @@ The Eventhub differentiates between stages given to the service via env `STAGE` ### Ingest Service Stages -| Module / Stage | `dev` | `prod` | -| -------------- | --------------------- | ---------------------- | -| Database | Namespace `dev` | Namespace `prod` | -| Pub/Sub | Prefix includes `dev` | Prefix includes `prod` | -| Dev Logging | true | false | +| Module / Stage | `dev` | `test` | `prod` | +| -------------- | --------------------- | ---------------------- | ---------------------- | +| Database | Namespace `dev` | Namespace `test` | Namespace `prod` | +| Pub/Sub | Prefix includes `dev` | Prefix includes `test` | Prefix includes `prod` | +| Dev Logging | true | true | false | ### Ingest Deployment Stages -| Module / Stage | `dev` | `test` | `beta` | `prod` | -| ---------------------------- | ------------------------------- | ----------------------------------- | ----------------------------- | ------------------------ | -| Used Ingest Stage | `dev` | `dev` | `prod` | `prod` | -| Stable | No, used for internal tests | Yes, can be used for external tests | Yes, usually | Yes | -| Runtime | Cloud Run | Kubernetes | Kubernetes | Kubernetes | -| Container Registry | Eventhub project | Eventhub project | Eventhub project | Eventhub project | -| Host | For internal use only | `eventhub-ingest-test.ard.de` | `eventhub-ingest-beta.ard.de` | `eventhub-ingest.ard.de` | -| Automatic Deployment | Yes, with Github Actions | Yes, through API with Review | Yes, through API with Review | No, manual trigger | -| Deployment Branch Protection | `main`, `dev/*`, `feature/*` | `main` | `main` | `main` | +| Module / Stage | `dev` | `test` | `beta` | `prod` | +| ---------------------------- | ---------------------------- | ----------------------------------- | ----------------------------- | ------------------------ | +| Used Ingest Stage | `dev` | `test` | `prod` | `prod` | +| Stable | No, used for internal tests | Yes, can be used for external tests | Yes, usually | Yes | +| Runtime | Cloud Run | Kubernetes | Kubernetes | Kubernetes | +| Container Registry | Eventhub project | Eventhub project | Eventhub project | Eventhub project | +| Host | For internal use only | `eventhub-ingest-test.ard.de` | `eventhub-ingest-beta.ard.de` | `eventhub-ingest.ard.de` | +| Automatic Deployment | Yes, with Github Actions | Yes, through API with Review | Yes, through API with Review | No, manual trigger | +| Deployment Branch Protection | `main`, `dev/*`, `feature/*` | `main` | `main` | `main` | diff --git a/docs/USERS.md b/docs/USERS.md index bc786d78..4673c58d 100644 --- a/docs/USERS.md +++ b/docs/USERS.md @@ -19,36 +19,7 @@ New users cannot sign up themselves, but need to go through the ARD Online team - Check that the user really hasn't been registered, then add a new entity - The entity key needs to be '_Custom Name_', with the user's email address (**in lowercase**) - Set `active` to `true` - - In the `institution` field, you will need to add an object with `name` and `id`. You can use this template and fill in the name and id strings. Make sure that `name` does not contain any special characters. - -```json -{ - "properties": { - "name": { - "stringValue": "name-here" - }, - "id": { - "stringValue": "urn:id-here" - } - } -} -``` - -- For `serviceIds` enter an array of all the services, that the user is allowed to publish to (as strings). Only enter the ids needed for publishing, there are no requirements for subscribing! - -```json -{ - "values": [ - { - "stringValue": "123456" - }, - { - "stringValue": "789012" - } - ] -} -``` - + - In the field `institutionId`, you will need to add a string with the same ID that is being used in ARD Core in this format: `urn:ard:institution:hex` - Now that the user is entered in Datastore with its profile, you can register it in Firebase. Therefore go to the [Firebase Console](https://console.firebase.google.com/) in the section _Build_ -> _Authentication_. - On this page, click on _Add user_ and enter the same email address (again in lowercase). The password can be something random, that will never be seen. - Once the user has been added, click on the dropdown menu and select _Reset password_ and confirm the pop-up. diff --git a/openapi.json b/openapi.json index ba34c7eb..4881757a 100644 --- a/openapi.json +++ b/openapi.json @@ -11,7 +11,7 @@ "name": "European Union Public License 1.2", "url": "https://spdx.org/licenses/EUPL-1.2.html" }, - "version": "0.1.7" + "version": "1.0.0-beta2" }, "externalDocs": { "description": "ARD-Eventhub Documentation", @@ -215,7 +215,7 @@ "tags": [ "events" ], - "summary": "Publish a new event", + "summary": "Distribute a next track", "operationId": "eventPostV1RadioTrackNext", "security": [ { @@ -248,7 +248,7 @@ "tags": [ "events" ], - "summary": "Publish a new event", + "summary": "Distribute a now-playing track", "operationId": "eventPostV1RadioTrackPlaying", "security": [ { @@ -622,7 +622,7 @@ "components": { "requestBodies": { "eventV1RadioTrack": { - "description": "New event to be distributed to subscribers. \n \nThe Eventhub format validation expects only a subset of these variables as minimum set. All other fields are technically optional, but **highly encouraged** to be included, so a best-possible metadata exchange is possible. \nThe subset is defined in the list of required fields of Schemas `eventV1PostBody`, resulting in this body: \n```json\n{\n \"event\": \"de.ard.eventhub.v1.radio.track.playing\",\n \"start\": \"2020-01-19T06:00:00+01:00\",\n \"title\": \"Song name\",\n \"serviceIds\": [ \"1234\" ],\n \"playlistItemId\": \"swr3-5678\"\n}\n```\nRequired fields not specified in the Schema, will cause your request to fail. \n \nThe `id` is inserted by Eventhub as string-formatted number, but might be a true string in the future, do not expect this string to remain numbers only!\n", + "description": "New event to be distributed to subscribers. \n \nThe Eventhub format validation expects only a subset of these variables as minimum set. All other fields are technically optional, but **highly encouraged** to be included, so a best-possible metadata exchange is possible. \nThe subset is defined in the list of required fields of Schemas `eventV1PostBody`, resulting in this body: \n```json\n{\n \"start\": \"2020-01-19T06:00:00+01:00\",\n \"title\": \"Song name\",\n \"services\": [ { ... } ],\n \"playlistItemId\": \"swr3-5678\"\n}\n```\nRequired fields not specified in the Schema, will cause your request to fail. \n \nThe `id` is inserted by Eventhub as string-formatted number, but might be a true string in the future, do not expect this string to remain numbers only!\n", "content": { "application/json": { "schema": { @@ -803,14 +803,45 @@ } } }, + "services": { + "type": "object", + "required": [ + "type", + "externalId", + "publisherId" + ], + "properties": { + "type": { + "type": "string", + "example": "PermanentLivestream", + "enum": [ + "EventLivestream", + "PermanentLivestream" + ] + }, + "externalId": { + "type": "string", + "example": "crid://swr.de/123450" + }, + "publisherId": { + "type": "string", + "description": "External ID or globally unique identifier (Core ID) for the associated publisher.\nWhen no Core ID is provided, the External ID will be converted by Eventhub.\n", + "example": "248000" + }, + "id": { + "type": "string", + "description": "Globally unique identifier, created by Eventhub", + "example": "urn:ard:permanent-livestream:49267f7d67be180d" + } + } + }, "eventV1PostBody": { "additionalProperties": false, "required": [ - "event", "type", "start", "title", - "serviceIds", + "services", "playlistItemId" ], "type": "object", @@ -818,6 +849,7 @@ "properties": { "event": { "type": "string", + "description": "If set, it needs to match the URL event parameter", "example": "de.ard.eventhub.v1.radio.track.playing", "enum": [ "de.ard.eventhub.v1.radio.track.playing", @@ -920,16 +952,17 @@ } } }, - "serviceIds": { + "services": { "type": "array", "description": "The playing stations unique Service-IDs. Do not include the Service-Type suffix.", "items": { - "type": "string" - }, - "example": [ - "284680", - "284700" - ] + "minItems": 1, + "allOf": [ + { + "$ref": "#/components/schemas/services" + } + ] + } }, "playlistItemId": { "type": "string", diff --git a/openapi.yaml b/openapi.yaml index e6a13775..233fb39c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -10,7 +10,7 @@ info: license: name: European Union Public License 1.2 url: 'https://spdx.org/licenses/EUPL-1.2.html' - version: 0.1.7 + version: 1.0.0-beta2 externalDocs: description: ARD-Eventhub Documentation url: 'https://swrlab.github.io/ard-eventhub/' @@ -135,7 +135,7 @@ paths: post: tags: - events - summary: Publish a new event + summary: Distribute a next track operationId: eventPostV1RadioTrackNext security: - bearerAuth: [] @@ -155,7 +155,7 @@ paths: post: tags: - events - summary: Publish a new event + summary: Distribute a now-playing track operationId: eventPostV1RadioTrackPlaying security: - bearerAuth: [] @@ -398,10 +398,9 @@ components: ```json { - "event": "de.ard.eventhub.v1.radio.track.playing", "start": "2020-01-19T06:00:00+01:00", "title": "Song name", - "serviceIds": [ "1234" ], + "services": [ { ... } ], "playlistItemId": "swr3-5678" } @@ -540,15 +539,41 @@ components: trace: type: string example: null + + services: + type: object + required: + - type + - externalId + - publisherId + properties: + type: + type: string + example: 'PermanentLivestream' + enum: + - 'EventLivestream' + - 'PermanentLivestream' + externalId: + type: string + example: 'crid://swr.de/123450' + publisherId: + type: string + description: | + External ID or globally unique identifier (Core ID) for the associated publisher. + When no Core ID is provided, the External ID will be converted by Eventhub. + example: '248000' + id: + type: string + description: Globally unique identifier, created by Eventhub + example: 'urn:ard:permanent-livestream:49267f7d67be180d' eventV1PostBody: additionalProperties: false required: - - event - type - start - title - - serviceIds + - services - playlistItemId type: object description: > @@ -557,6 +582,7 @@ components: properties: event: type: string + description: If set, it needs to match the URL event parameter example: de.ard.eventhub.v1.radio.track.playing enum: - de.ard.eventhub.v1.radio.track.playing @@ -640,16 +666,15 @@ components: type: string description: Can link to external reference nullable: true - serviceIds: + services: type: array description: >- The playing stations unique Service-IDs. Do not include the Service-Type suffix. items: - type: string - example: - - '284680' - - '284700' + minItems: 1 + allOf: + - $ref: '#/components/schemas/services' playlistItemId: type: string description: >- diff --git a/package.json b/package.json index a8db56a0..50aff224 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { "name": "ard-eventhub", - "version": "0.1.7", + "version": "1.0.0-beta2", "description": "ARD system to distribute real-time (live) metadata for primarily radio broadcasts.", "main": "./src/ingest/index.js", "scripts": { "docs:serve": "docsify serve . --port 3000", - "ingest:test": "mocha test/ingest.test.js --timeout 10000 --exit", - "ingest:local": "nodemon ./src/ingest/index.js", + "ingest:test": "mocha test/ingest.test.js --timeout 15000 --exit", + "ingest:local": "nodemon -r dotenv/config ./src/ingest/index.js", "ingest:cloud": "node ./src/ingest/index.js", "license": "chmod +x license.sh && ./license.sh", "test": "mocha test/example.test.js" @@ -23,16 +23,18 @@ "@google-cloud/pubsub": "^2.10.0", "body-parser": "^1.19.0", "compression": "^1.7.4", - "dd-trace": "^0.31.1", + "dd-trace": "^0.31.2", "express": "4.17.1", - "express-openapi-validator": "^4.12.5", + "express-openapi-validator": "^4.12.6", "firebase-admin": "^9.5.0", "jsonwebtoken": "^8.5.1", "moment": "^2.29.1", + "node-crc": "^1.3.0", "node-fetch": "2.6.1", "slug": "^4.0.3", "swagger-ui-express": "^4.1.6", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "winston": "^3.3.3" }, "devDependencies": { "@swrlab/eslint-plugin-swr": "^0.1.0", @@ -40,10 +42,11 @@ "chai": "^4.3.4", "chai-http": "^4.3.0", "docsify-cli": "^4.4.3", + "dotenv": "^8.2.0", "eslint": "^7.22.0", "eslint-plugin-chai-friendly": "^0.6.0", "eslint-plugin-swr": "0.0.5", - "license-compliance": "^1.0.3", + "license-compliance": "^1.1.0", "mocha": "^8.3.2", "nodemon": "^2.0.7", "prettier": "^2.2.1" diff --git a/src/ingest/auth/middleware/verify.js b/src/ingest/auth/middleware/verify.js index 64b599f6..48c31b3d 100644 --- a/src/ingest/auth/middleware/verify.js +++ b/src/ingest/auth/middleware/verify.js @@ -5,12 +5,10 @@ */ -// load node utils -const slug = require('slug') - // load utils const datastore = require('../../../utils/datastore') const firebase = require('../../../utils/firebase') +const logger = require('../../../utils/logger') module.exports = async (req, res, next) => { try { @@ -32,13 +30,12 @@ module.exports = async (req, res, next) => { req.user = await firebase.verifyToken(authorization) res.set('x-ard-eventhub-uid', req.user.uid) } catch (err) { - console.warn( - 'ingest/auth/middleware/verify', - 'user token invalid', - JSON.stringify({ - error: err.stack || err, - }) - ) + logger.log({ + level: 'notice', + message: 'user token invalid', + source: 'ingest/auth/middleware/verify', + data: { ...req.headers, authorization: 'hidden' }, + }) return res.sendStatus(403) } @@ -51,20 +48,19 @@ module.exports = async (req, res, next) => { } // add user details to request profile - req.user.serviceIds = userDb.serviceIds - req.user.institution = userDb.institution - req.user.institution.name = slug(req.user.institution.name) + req.user.institutionId = userDb.institutionId // continue with normal workflow, user is authenticated 🎉 return next() - } catch (err) { - console.error( - 'ingest/auth/middleware/verify', - 'failed to verify user', - JSON.stringify({ - error: err.stack || err, - }) - ) + } catch (error) { + logger.log({ + level: 'error', + message: 'failed to verify user', + source: 'ingest/auth/middleware/verify', + error, + data: { ...req.headers, authorization: 'hidden' }, + }) + return res.sendStatus(500) } } diff --git a/src/ingest/events/post.js b/src/ingest/events/post.js index 7d65e4dd..1d7d783f 100644 --- a/src/ingest/events/post.js +++ b/src/ingest/events/post.js @@ -6,26 +6,19 @@ */ // load node utils -const slug = require('slug') +const moment = require('moment') // load eventhub utils +const core = require('../../utils/core') const datastore = require('../../utils/datastore') +const logger = require('../../utils/logger') const pubsub = require('../../utils/pubsub') const response = require('../../utils/response') -const config = require('../../../config') - -// TODO: check IDs in ARD Core-API instead of dump -const coreApi = require('../../data/coreApi.json') -// define functions -function getPubSubId(serviceId) { - const pubIdent = 'publisher' - return `${config.pubsubPrefix}.${pubIdent}.${config.stage}.${serviceId}` -} +// load config +const config = require('../../../config') -function getServiceId(pubSubId) { - return pubSubId.split('.').pop() -} +const source = 'ingest/events/post' module.exports = async (req, res) => { try { @@ -35,7 +28,20 @@ module.exports = async (req, res) => { const { user } = req // check eventName - if (eventName !== message.event) { + if (message.event && eventName !== message.event) { + // log access attempt + logger.log({ + level: 'warning', + message: 'User attempted event with missmatching names', + source, + data: { + email: req.user.email, + body: message.event, + params: eventName, + }, + }) + + // return 400 return response.badRequest(req, res, { message: 'request.body.event should match URL parameter', errors: [ @@ -48,110 +54,237 @@ module.exports = async (req, res) => { }) } - // save message to datastore - message = await datastore.save(message, 'events') - message.id = message.id.toString() + // check offset for start event + if (moment(message.start).add(2, 'm').isBefore()) { + // log access attempt + logger.log({ + level: 'notice', + message: `User attempted event with expired start time ${message.start}`, + source, + data: { + email: req.user.email, + message, + }, + }) - // get serviceIds from message - const { serviceIds } = message - const unauthorizedServiceIds = [] + // return 400 + return response.badRequest(req, res, { + message: 'request.body.start should be recent', + errors: [ + { + path: '.body.start', + message: 'should not be expired event', + errorCode: 'required.openapi.validation', + }, + ], + }) + } - // check allowed serviceIds for current user - serviceIds.forEach((serviceId) => { - if (user.serviceIds.indexOf(serviceId) === -1) { - // add forbidden ids to unauthorized array - unauthorizedServiceIds.push(serviceId) + // insert name, creator and timestamp into object + message = { + name: eventName, + creator: user.email, + created: moment().toISOString(), + ...message, + } - // remove forbidden ids from serviceId array - serviceIds.splice(serviceIds.indexOf(serviceId), 1) - } - }) + // use collector to check duplicates for externalId in services + const externalIdCollector = [] - // generate pubsub IDs with prefix - const pubSubIds = serviceIds.map((serviceId) => { - return getPubSubId(serviceId) - }) + // compile core hashes for every service + message.services = await Promise.all( + message.services.map(async (service) => { + // check for duplicates + if (externalIdCollector.includes(service.externalId)) service.blocked = true + externalIdCollector.push(service.externalId) - // try to publish message under given topics - const topics = await pubsub.publishMessage(pubSubIds, message) - const unknownTopics = [] + // fetch prefix from configured list + const urnPrefix = config.coreIdPrefixes[service.type] - // collect unknown topics from returning errors - Object.keys(topics).forEach((topic) => { - if (topics[topic] === 'TOPIC_ERROR' || topics[topic] === 'TOPIC_NOT_FOUND') { - const newTopic = { - id: getServiceId(topic), - pubsub: topic, - name: undefined, - label: undefined, - verified: false, - created: false, + // create hash based on prefix and id + service.topic = { id: `${urnPrefix}${core.createHashedId(service.externalId)}` } + + // convert publisher if needed + const urnRegex = /(?=urn:ard:publisher:[a-z0-9]{16})/g + if (!service.publisherId.match(urnRegex)) { + // fetch prefix + const urnPublisherPrefix = config.coreIdPrefixes.Publisher + + // add trailing 0 if number is only 5 digits + if (service.publisherId.length === 5) + service.publisherId = `${service.publisherId}0` + + // create hash using given publisherId + service.publisherId = `${urnPublisherPrefix}${core.createHashedId( + service.publisherId + )}` } - unknownTopics.push(newTopic) - } - }) - // check unknown topic IDs - if (unknownTopics.length > 0) { - // verify IDs of unknownTopics with coreApi - unknownTopics.forEach((topic) => { - coreApi.forEach((entry) => { - if (topic.id === entry.externalId) { - topic.name = entry.title - topic.label = slug(entry.title) - topic.verified = true - } - }) + // fetch publisher + const publisher = await core.getPublisher(service.publisherId) + + // block access if publisher not found + if (!service.blocked && !publisher) { + // set blocked flag to be filtered out + service.blocked = true + + // log access attempt + logger.log({ + level: 'warning', + message: `Publisher not found > ${service.publisherId}`, + source, + data: { + email: req.user.email, + service, + }, + }) + } + + // check allowed institutions for current user + if (!service.blocked && user.institutionId !== publisher?.institution?.id) { + // set blocked flag to be filtered out + service.blocked = true + + // log access attempt + logger.log({ + level: 'warning', + message: 'User unauthorized for service', + source, + data: { + email: req.user.email, + service, + user: user.institution, + publisher: publisher?.institution, + }, + }) + } + + // create pub/sub-compliant name + if (!service.blocked) service.topic.name = pubsub.buildId(service.topic.id) + + // final data + return service }) + ) + + // save message to datastore + message = await datastore.save(message, 'events') + message.id = message.id.toString() + delete message.creator + + // collect unknown topics from returning errors + const newServices = [] + for await (const service of message.services) { + // ignoring blocked services + if (!service.blocked && service.topic?.name) { + // try sending message + const messageId = await pubsub.publishMessage(service.topic.name, message) + + // handle errors + if (messageId === 'TOPIC_ERROR') { + // insert error message and empty id + service.topic.status = 'TOPIC_ERROR_1' + service.topic.messageId = null + } else if (messageId === 'TOPIC_NOT_FOUND') { + // fetch publisher + const publisher = await core.getPublisher(service.publisherId) + + // try creating new topic + const newTopic = { + created: moment().toISOString(), + creator: user.email, + + coreId: service.topic.id, + externalId: service.externalId, + name: service.topic.name, + + institution: { + id: user.institutionId, + title: publisher.institution.title, + }, + publisher: { + id: service.publisherId, + title: publisher.title, + }, + } - // create topics for verified IDs - for await (const topic of unknownTopics) { - if (topic.verified) { - const [result] = await pubsub.createTopic(topic) + // save topic to datastore + await datastore.save(newTopic, 'topics') + newTopic.id = newTopic.id.toString() - if (result?.name?.indexOf(topic.id) !== -1) { - topic.created = true + // create topic + const [result] = await pubsub.createTopic(newTopic) + + // handle feedback + if (result?.name?.indexOf(service.topic.name) !== -1) { // Update api result that topic was created - topics[topic.pubsub] = 'TOPIC_CREATED' + service.topic.status = 'TOPIC_CREATED' + + logger.log({ + level: 'notice', + message: `topic created > ${service.topic.name}`, + source, + data: { service, result }, + }) } else { // Update api result that topic was not created - topics[topic.pubsub] = 'TOPIC_NOT_CREATED' + service.topic.status = 'TOPIC_ERROR_2' + + logger.log({ + level: 'error', + message: `failed creating topic > ${service.topic.name}`, + source, + data: { service, result }, + }) } + + // insert empty id + service.topic.messageId = null + } else { + // insert messageId + service.topic.status = 'MESSAGE_SENT' + service.topic.messageId = messageId } } + + // send to new array + newServices.push(service) } - // check forbidden serviceIds - if (unauthorizedServiceIds.length > 0) { - console.error( - `User '${user.email}' is not allowed to publish events for serviceIds: [${unauthorizedServiceIds}]` - ) - unauthorizedServiceIds.forEach((unauthorizedServiceId) => { - const pubSubId = getPubSubId(unauthorizedServiceId) - topics[pubSubId] = 'TOPIC_NOT_ALLOWED' - }) + // replace services + message.services = newServices + + // prepare output data + const data = { + statuses: { + published: message.services.filter((service) => service.topic?.messageId).length, + blocked: message.services.filter((service) => service.blocked).length, + failed: message.services.filter( + (service) => !service.topic?.messageId && !service.blocked + ).length, + }, + event: message, } + // log success + logger.log({ + level: 'notice', + message: `event processed > ${eventName}`, + source, + data, + }) + // return ok - return response.ok( - req, - res, - { - topics, - message, - }, - 201 - ) - } catch (err) { - console.error( - 'ingest/events/post', - 'failed to publish event', - JSON.stringify({ - body: req.body, - headers: req.headers, - error: err.stack || err, - }) - ) - return response.internalServerError(req, res, err) + return response.ok(req, res, data, 201) + } catch (error) { + logger.log({ + level: 'error', + message: 'failed to publish event', + source, + error, + data: { body: req.body, headers: req.headers }, + }) + + return response.internalServerError(req, res, error) } } diff --git a/src/ingest/index.js b/src/ingest/index.js index 349e0b5a..18e2a1ce 100644 --- a/src/ingest/index.js +++ b/src/ingest/index.js @@ -8,9 +8,14 @@ // enable tracing require('../utils/tracer') -// load node utils and config +// load node utils const compression = require('compression') const express = require('express') + +// load utils +const logger = require('../utils/logger') + +// load config const config = require('../../config') // set up express server @@ -26,7 +31,12 @@ server.use((req, res, next) => { ...req.headers, authorization: 'hidden', } - console.log('DEV middleware', JSON.stringify(logHeaders)) + logger.log({ + level: 'debug', + message: `middleware logging`, + source: 'DEV', + data: logHeaders, + }) // continue with normal workflow next() diff --git a/src/ingest/subscriptions/delete.js b/src/ingest/subscriptions/delete.js index 65c4367b..69390295 100644 --- a/src/ingest/subscriptions/delete.js +++ b/src/ingest/subscriptions/delete.js @@ -7,6 +7,7 @@ // load eventhub utils const datastore = require('../../utils/datastore') +const logger = require('../../utils/logger') const pubsub = require('../../utils/pubsub') const response = require('../../utils/response') @@ -19,6 +20,7 @@ module.exports = async (req, res) => { // load single subscription to get owner try { subscription = await pubsub.getSubscription(subscriptionName) + subscription = subscription.full } catch (err) { console.error( 'ingest/subscription/delete', @@ -47,14 +49,14 @@ module.exports = async (req, res) => { } // check subscription permission by user institution - if (subscription.institution.id !== req.user.institution.id) { - const subsOrg = subscription.institution.name - const userOrg = req.user.institution.name + if (subscription.institutionId !== req.user.institutionId) { + const userInstitution = req.user.institutionId + // return 400 error return response.badRequest(req, res, { status: 400, message: `Mismatch of user and subscription institution`, - errors: `Subscription of institution '${subsOrg}' cannot be deleted by user of institution '${userOrg}'`, + errors: `Subscription of this institution cannot be deleted by user of institution '${userInstitution}'`, }) } @@ -66,25 +68,26 @@ module.exports = async (req, res) => { await datastore.delete('subscriptions', subscriptionId) // log progress - console.log( - 'ingest/subscriptions/delete', - 'removed subscription', - JSON.stringify({ subscriptionName, subscriptionId, email: req.user.email }) - ) + logger.log({ + level: 'info', + message: 'removed subscription', + source: 'ingest/subscriptions/delete', + data: { email: req.user.email, subscriptionName, subscriptionId, subscription }, + }) // return data return response.ok(req, res, { valid: true, }) - } catch (err) { - console.error( - 'ingest/subscriptions/delete', - 'failed to delete subscription', - JSON.stringify({ - body: req.body, - error: err.stack || err, - }) - ) - return response.internalServerError(req, res, err) + } catch (error) { + logger.log({ + level: 'error', + message: 'failed to delete subscription', + source: 'ingest/subscriptions/delete', + error, + data: { params: req.params }, + }) + + return response.internalServerError(req, res, error) } } diff --git a/src/ingest/subscriptions/get.js b/src/ingest/subscriptions/get.js index 63bb8cd0..a000a0be 100644 --- a/src/ingest/subscriptions/get.js +++ b/src/ingest/subscriptions/get.js @@ -6,6 +6,7 @@ */ // load eventhub utils +const logger = require('../../utils/logger') const pubsub = require('../../utils/pubsub') const response = require('../../utils/response') @@ -18,6 +19,7 @@ module.exports = async (req, res) => { // load single subscription try { subscription = await pubsub.getSubscription(subscriptionName) + subscription = subscription.limited } catch (err) { return response.notFound(req, res, { status: 404, @@ -26,29 +28,28 @@ module.exports = async (req, res) => { } // verify if user is allowed to get subscription (same institution) - if (subscription.institution.id !== req.user.institution.id) { - const subsInstitution = subscription.institution.name - const userInstitution = req.user.institution.name + if (subscription.institutionId !== req.user.institutionId) { + const userInstitution = req.user.institutionId // return 400 error return response.badRequest(req, res, { status: 400, message: `Mismatch of user and subscription institution`, - errors: `Subscription of institution '${subsInstitution}' is not visible for user of institution '${userInstitution}'`, + errors: `Subscription of this institution is not visible for user of institution '${userInstitution}'`, }) } // return data return res.status(200).json(subscription) - } catch (err) { - console.error( - 'ingest/subscriptions/get', - 'failed to get subscription', - JSON.stringify({ - body: req.body, - error: err.stack || err, - }) - ) - return response.internalServerError(req, res, err) + } catch (error) { + logger.log({ + level: 'error', + message: 'failed to get subscription', + source: 'ingest/subscriptions/get', + error, + data: { params: req.params }, + }) + + return response.internalServerError(req, res, error) } } diff --git a/src/ingest/subscriptions/list.js b/src/ingest/subscriptions/list.js index 97e22a19..2265f91e 100644 --- a/src/ingest/subscriptions/list.js +++ b/src/ingest/subscriptions/list.js @@ -6,6 +6,7 @@ */ // load eventhub utils +const logger = require('../../utils/logger') const pubsub = require('../../utils/pubsub') const response = require('../../utils/response') @@ -16,20 +17,20 @@ module.exports = async (req, res) => { // verify if user is allowed to list subscriptions (same institution) subscriptions = subscriptions.filter( - (subscription) => subscription?.institution?.id === req.user.institution?.id + (subscription) => subscription?.institutionId === req.user.institutionId ) // return data return res.status(200).json(subscriptions) - } catch (err) { - console.error( - 'ingest/subscriptions/list', - 'failed to list subscriptions', - JSON.stringify({ - body: req.body, - error: err.stack || err, - }) - ) - return response.internalServerError(req, res, err) + } catch (error) { + logger.log({ + level: 'error', + message: 'failed to list subscriptions', + source: 'ingest/subscriptions/list', + error, + data: {}, + }) + + return response.internalServerError(req, res, error) } } diff --git a/src/ingest/subscriptions/post.js b/src/ingest/subscriptions/post.js index 6c5d5a81..16066c20 100644 --- a/src/ingest/subscriptions/post.js +++ b/src/ingest/subscriptions/post.js @@ -11,6 +11,7 @@ const { v4: uuidv4 } = require('uuid') // load eventhub utils const datastore = require('../../utils/datastore') +const logger = require('../../utils/logger') const pubsub = require('../../utils/pubsub') const response = require('../../utils/response') const config = require('../../../config') @@ -18,18 +19,16 @@ const config = require('../../../config') // TODO: check IDs in ARD Core-API instead of dump const coreApi = require('../../data/coreApi.json') -const functionName = 'ingest/subscription/post' +const source = 'ingest/subscription/post' module.exports = async (req, res) => { try { // generate subscription name - const subIdent = 'subscription' - const prefix = `${config.pubsubPrefix}.${subIdent}.${config.stage}` - const topicName = req.body.topic + const prefix = `${config.pubSubPrefix}subscription.` // check existence of user institution const institutionExists = coreApi.some((entry) => { - return req.user.institution.id === entry.institution.id + return req.user.institutionId === entry.institution.id }) // check if user has institution set @@ -38,17 +37,18 @@ module.exports = async (req, res) => { const institutionName = req.user.institution.name // log action - console.warn( - functionName, - 'user attempted to create subscription without institution', - JSON.stringify({ - topicName, + logger.log({ + level: 'warning', + message: 'user attempted to create subscription without institution', + source, + data: { + topic: req.body.topic, stage: config.stage, email: req.user.email, institutionExists, userInstitution: req.user.institution, - }) - ) + }, + }) // return 401 error return response.badRequest(req, res, { @@ -58,38 +58,17 @@ module.exports = async (req, res) => { }) } - // check if topic is from this stage - if (topicName.indexOf(`.${config.stage}.`) === -1) { - // log action - console.warn( - functionName, - 'user attempted to create subscription from other stage', - JSON.stringify({ - topicName, - stage: config.stage, - email: req.user.email, - }) - ) - - // return 401 error - return response.badRequest(req, res, { - status: 400, - message: `Topic is not from this stage environment`, - errors: `The topic '${topicName}' does not belong to this stage (${config.stage})`, - }) - } - // map inputs let subscription = { - name: `${prefix}.${req.user.institution.name}.${uuidv4()}`, + name: `${prefix}${uuidv4()}`, type: req.body.type, method: req.body.method, url: req.body.url, contact: req.body.contact, - topic: topicName, + topic: pubsub.buildId(req.body.topic), - owner: req.user.email, - institution: req.user.institution, + creator: req.user.email, + institutionId: req.user.institutionId, created: moment().toISOString(), } @@ -99,16 +78,15 @@ module.exports = async (req, res) => { // check existence of topic try { await pubsub.getTopic(subscription.topic) - } catch (topicErr) { + } catch (error) { // log error - console.error( - functionName, - 'failed to find desired topic', - JSON.stringify({ - subscription, - error: topicErr.stack || topicErr, - }) - ) + logger.log({ + level: 'warning', + message: `failed to find desired topic > ${subscription.topic}`, + source, + error, + data: { subscription }, + }) // delete datastore object await datastore.delete('subscriptions', subscription.id) @@ -125,15 +103,15 @@ module.exports = async (req, res) => { // return data return res.status(201).json(createdSubscription) - } catch (err) { - console.error( - functionName, - 'failed to create subscription', - JSON.stringify({ - body: req.body, - error: err.stack || err, - }) - ) - return response.internalServerError(req, res, err) + } catch (error) { + logger.log({ + level: 'error', + message: 'failed to create subscription', + source, + error, + data: { body: req.body }, + }) + + return response.internalServerError(req, res, error) } } diff --git a/src/ingest/topics/list.js b/src/ingest/topics/list.js index 0d9e98de..b371580c 100644 --- a/src/ingest/topics/list.js +++ b/src/ingest/topics/list.js @@ -5,6 +5,7 @@ */ +const logger = require('../../utils/logger') const pubsub = require('../../utils/pubsub') const response = require('../../utils/response') @@ -15,15 +16,15 @@ module.exports = async (req, res) => { // return data return res.status(200).json(topics) - } catch (err) { - console.error( - 'ingest/topics/list', - 'failed to list topics', - JSON.stringify({ - body: req.body, - error: err.stack || err, - }) - ) - return response.internalServerError(req, res, err) + } catch (error) { + logger.log({ + level: 'error', + message: 'failed to list topics', + source: 'ingest/topics/list', + error, + data: {}, + }) + + return response.internalServerError(req, res, error) } } diff --git a/src/utils/core/createHashedId.js b/src/utils/core/createHashedId.js new file mode 100644 index 00000000..237b14f2 --- /dev/null +++ b/src/utils/core/createHashedId.js @@ -0,0 +1,16 @@ +/* + + ard-eventhub + by SWR audio lab + + this file creates a CRC64-ECMA182-compliant hash + based on an utf-8 encoded input string + +*/ + +// load node utils +const crc = require('node-crc') + +module.exports = (input) => { + return crc.crc64(Buffer.from(input, 'utf-8')).toString('hex') +} diff --git a/src/utils/core/getPublisher.js b/src/utils/core/getPublisher.js new file mode 100644 index 00000000..6575be76 --- /dev/null +++ b/src/utils/core/getPublisher.js @@ -0,0 +1,17 @@ +/* + + ard-eventhub + by SWR audio lab + +*/ + +// TODO: check IDs in ARD Core-API instead of dump +const coreApi = require('../../data/coreApi.json') + +module.exports = async (publisherId) => { + const publisher = coreApi.find((entry) => { + return publisherId === entry.id ? entry : null + }) + + return Promise.resolve(publisher) +} diff --git a/src/utils/core/index.js b/src/utils/core/index.js new file mode 100644 index 00000000..9a9ddc3d --- /dev/null +++ b/src/utils/core/index.js @@ -0,0 +1,14 @@ +/* + + ard-eventhub + by SWR audio lab + +*/ + +const createHashedId = require('./createHashedId') +const getPublisher = require('./getPublisher') + +module.exports = { + createHashedId, + getPublisher, +} diff --git a/src/utils/datastore/save.js b/src/utils/datastore/save.js index 3ba446d9..b979e15a 100644 --- a/src/utils/datastore/save.js +++ b/src/utils/datastore/save.js @@ -22,6 +22,7 @@ module.exports = async (data, kind, id) => { await datastoreClient.save({ key, data, + excludeFromIndexes: ['contributors'], }) // insert key diff --git a/src/utils/logger/index.js b/src/utils/logger/index.js new file mode 100644 index 00000000..848048b0 --- /dev/null +++ b/src/utils/logger/index.js @@ -0,0 +1,49 @@ +/* + + ard-eventhub + by SWR audio lab + +*/ + +// load node utils +const { createLogger, config, format, transports } = require('winston') + +// load config +const serviceConfig = require('../../../config') + +// add formatters +const convertError = format((event) => { + if (event?.error instanceof Error) { + event.error = { + code: event.error.code, + message: event.error.message, + stack: event.error.stack, + } + } + return event +}) + +const convertGlobals = format((event) => { + event.serviceName = 'eventhub-ingest' + event.stage = process.env.STAGE + event.version = serviceConfig.version + return event +}) + +const logger = createLogger({ + level: 'info', + levels: config.syslog.levels, + exitOnError: false, + format: serviceConfig.isDebug + ? format.combine( + convertError(), + convertGlobals(), + format.timestamp(), + format.json({ space: '\t' }), + format.colorize({ all: true, colors: { info: 'blue' } }) + ) + : format.combine(convertError(), convertGlobals(), format.json()), + transports: [new transports.Console()], +}) + +module.exports = logger diff --git a/src/utils/loggerDev.js b/src/utils/loggerDev.js deleted file mode 100644 index 6383fa82..00000000 --- a/src/utils/loggerDev.js +++ /dev/null @@ -1,29 +0,0 @@ -/* - - ard-eventhub - by SWR audio lab - -*/ - -// load config -const config = require('../../config') - -module.exports = (level, msg) => { - if (config.stage !== 'dev') { - return - } - - let thisMsg = msg - - if (msg instanceof Array) { - thisMsg = msg.join(' > ') - } - - if (level === 'log') { - console.log(thisMsg) - } else if (level === 'warn') { - console.warn(thisMsg) - } else if (level === 'error') { - console.error(thisMsg) - } -} diff --git a/src/utils/pubsub/buildId.js b/src/utils/pubsub/buildId.js new file mode 100644 index 00000000..fa51eb02 --- /dev/null +++ b/src/utils/pubsub/buildId.js @@ -0,0 +1,19 @@ +/* + + ard-eventhub + by SWR audio lab + + this file creates a PubSub-safe id, + which is just a URL-encoded version + +*/ + +// load util +const convertId = require('./convertId') + +// load config +const config = require('../../../config') + +module.exports = (input) => { + return `${config.pubSubPrefix}${convertId.encode(input)}` +} diff --git a/src/utils/pubsub/convertId.js b/src/utils/pubsub/convertId.js new file mode 100644 index 00000000..dea91a48 --- /dev/null +++ b/src/utils/pubsub/convertId.js @@ -0,0 +1,18 @@ +/* + + ard-eventhub + by SWR audio lab + + this file creates a PubSub-safe id, + which is just a URL-encoded version + +*/ + +module.exports = { + encode: (input) => { + return encodeURIComponent(input) + }, + decode: (input) => { + return decodeURIComponent(input) + }, +} diff --git a/src/utils/pubsub/createSubscription.js b/src/utils/pubsub/createSubscription.js index c69002a9..72be0b2d 100644 --- a/src/utils/pubsub/createSubscription.js +++ b/src/utils/pubsub/createSubscription.js @@ -7,13 +7,17 @@ // load node utils const moment = require('moment') +const slug = require('slug') // load utils +const logger = require('../logger') const pubSubSubscriberClient = require('./_subscriberClient') const mapSubscription = require('./mapSubscription') + +// load config const config = require('../../../config') -const functionName = 'utils/pubsub/createSubscription' +const source = 'utils/pubsub/createSubscription' module.exports = async (subscription) => { // map inputs for pubsub @@ -30,23 +34,38 @@ module.exports = async (subscription) => { }, labels: { id: subscription.id, - institution: subscription.institution.name, stage: config.stage, - created: moment().format('YYYY-MM-DD--x'), + 'creator-slug': slug(subscription.creator), + created: moment().format('YYYY-MM-DD'), }, ackDeadlineSeconds: 20, expirationPolicy: {}, } - console.log(functionName, 'built options', JSON.stringify({ subscription, options })) + logger.log({ + level: 'info', + message: 'built options', + source, + data: { subscription, options }, + }) // submit subscription - let [createdSubscription] = await pubSubSubscriberClient.createSubscription(options) - console.log(functionName, 'created subscription', JSON.stringify({ createdSubscription })) + const [createdSubscription] = await pubSubSubscriberClient.createSubscription(options) + logger.log({ + level: 'info', + message: 'created subscription', + source, + data: { createdSubscription }, + }) // map and filter values - createdSubscription = await mapSubscription(createdSubscription) - console.log(functionName, 'mapped subscription', JSON.stringify({ createdSubscription })) + const { limited: mappedSubscription } = await mapSubscription(createdSubscription) + logger.log({ + level: 'info', + message: 'mapped subscription', + source, + data: { mappedSubscription }, + }) // return data - return Promise.resolve(createdSubscription) + return Promise.resolve(mappedSubscription) } diff --git a/src/utils/pubsub/createTopic.js b/src/utils/pubsub/createTopic.js index 73e7ada2..1304de85 100644 --- a/src/utils/pubsub/createTopic.js +++ b/src/utils/pubsub/createTopic.js @@ -5,17 +5,27 @@ */ +// load node utils +const slug = require('slug') +const moment = require('moment') + // load pubsub for internal queues const publisherClient = require('./_publisherClient') const config = require('../../../config') module.exports = async (newTopic) => { // create new topic - const prefix = 'projects/ard-eventhub/topics/' const topic = { - name: prefix + newTopic.pubsub, + name: `projects/${process.env.GCP_PROJECT_ID}/topics/${newTopic.name}`, labels: { - name: newTopic.label, + created: moment().format('YYYY-MM-DD'), + 'creator-slug': slug(newTopic.creator), + + id: newTopic.id, + + 'institution-slug': slug(newTopic.institution.title), + 'publisher-slug': slug(newTopic.publisher.title), + stage: config.stage, }, } diff --git a/src/utils/pubsub/getSubscription.js b/src/utils/pubsub/getSubscription.js index d37c60c4..0bcdd0c1 100644 --- a/src/utils/pubsub/getSubscription.js +++ b/src/utils/pubsub/getSubscription.js @@ -9,15 +9,20 @@ const pubSubClient = require('./_client') const mapSubscription = require('./mapSubscription') +// load config +const config = require('../../../config') + module.exports = async (name) => { // fetch subscription list - let [subscription] = await pubSubClient.subscription(name).getMetadata() + const [subscription] = await pubSubClient.subscription(name).getMetadata() - // DEV filter subscriptions by prefix + // filter subscriptions by prefix + if (!subscription || subscription.name.indexOf(config.pubSubPrefix) === -1) + return Promise.reject(new Error(`subscription not found > ${name}`)) // map and filter values - subscription = await mapSubscription(subscription) + const mappedSubscription = await mapSubscription(subscription) // return data - return Promise.resolve(subscription) + return Promise.resolve(mappedSubscription) } diff --git a/src/utils/pubsub/getSubscriptions.js b/src/utils/pubsub/getSubscriptions.js index f76d8cf4..561a2a52 100644 --- a/src/utils/pubsub/getSubscriptions.js +++ b/src/utils/pubsub/getSubscriptions.js @@ -9,14 +9,23 @@ const pubSubClient = require('./_client') const mapSubscription = require('./mapSubscription') +// load config +const config = require('../../../config') + module.exports = async () => { // fetch subscriptions list let [subscriptions] = await pubSubClient.getSubscriptions() - // DEV filter subscriptions by prefix + // filter subscriptions by prefix (stage) + subscriptions = subscriptions.filter((subscription) => subscription.name.indexOf(config.pubSubPrefix) !== -1) // map and filter values - subscriptions = await Promise.all(subscriptions.map(async (subscription) => mapSubscription(subscription))) + subscriptions = await Promise.all( + subscriptions.map(async (subscription) => { + const { limited } = await mapSubscription(subscription) + return Promise.resolve(limited) + }) + ) // return data return Promise.resolve(subscriptions) diff --git a/src/utils/pubsub/getTopic.js b/src/utils/pubsub/getTopic.js index feb709ae..a2af50e8 100644 --- a/src/utils/pubsub/getTopic.js +++ b/src/utils/pubsub/getTopic.js @@ -8,19 +8,16 @@ // load pubsub for internal queues const pubSubClient = require('./_client') +// load config +const config = require('../../../config') + module.exports = async (topicName) => { // fetch topic list - let [topic] = await pubSubClient.topic(topicName).get() - - // DEV filter topics by prefix + const [topic] = await pubSubClient.topic(topicName).get() - // map values - topic = { - type: 'PUBSUB', - name: topic.name.split('/').pop(), - path: topic.name, - labels: topic.metadata.labels, - } + // filter topics by prefix (stage) + if (!topic || topic.name.indexOf(config.pubSubPrefix) === -1) + return Promise.reject(new Error(`topic not found > ${topicName}`)) // return data return Promise.resolve(topic) diff --git a/src/utils/pubsub/getTopics.js b/src/utils/pubsub/getTopics.js index 5f1af0a4..da9c3b5a 100644 --- a/src/utils/pubsub/getTopics.js +++ b/src/utils/pubsub/getTopics.js @@ -7,20 +7,26 @@ // load pubsub for internal queues const pubSubClient = require('./_client') +const convertId = require('./convertId') + +// load config const config = require('../../../config') module.exports = async () => { // fetch topic list let [topics] = await pubSubClient.getTopics() - // filter topics by stage - topics = topics.filter((topic) => topic.name.split('/').pop().indexOf(`.${config.stage}.`) !== -1) + // filter topics by prefix (stage) + topics = topics.filter((topic) => topic.name.indexOf(config.pubSubPrefix) !== -1) // map values topics = topics.map((topic) => { + const name = topic.name.split('/').pop() + return { type: 'PUBSUB', - name: topic.name.split('/').pop(), + id: convertId.decode(name).replace(config.pubSubPrefix, ''), + name, path: topic.name, labels: topic.metadata.labels, } diff --git a/src/utils/pubsub/index.js b/src/utils/pubsub/index.js index 6d6b658a..b47beb7c 100644 --- a/src/utils/pubsub/index.js +++ b/src/utils/pubsub/index.js @@ -5,13 +5,26 @@ */ +const buildId = require('./buildId') +const createSubscription = require('./createSubscription') +const createTopic = require('./createTopic') +const convertId = require('./convertId') +const deleteSubscription = require('./deleteSubscription') +const getSubscription = require('./getSubscription') +const getSubscriptions = require('./getSubscriptions') +const getTopic = require('./getTopic') +const getTopics = require('./getTopics') +const publishMessage = require('./publishMessage') + module.exports = { - createSubscription: require('./createSubscription'), - deleteSubscription: require('./deleteSubscription'), - getSubscription: require('./getSubscription'), - getSubscriptions: require('./getSubscriptions'), - getTopic: require('./getTopic'), - getTopics: require('./getTopics'), - createTopic: require('./createTopic'), - publishMessage: require('./publishMessage'), + buildId, + convertId, + createSubscription, + createTopic, + deleteSubscription, + getSubscription, + getSubscriptions, + getTopic, + getTopics, + publishMessage, } diff --git a/src/utils/pubsub/mapSubscription.js b/src/utils/pubsub/mapSubscription.js index 081935dd..c2f07c90 100644 --- a/src/utils/pubsub/mapSubscription.js +++ b/src/utils/pubsub/mapSubscription.js @@ -5,9 +5,13 @@ */ -// load pubsub for internal queues +// load utils +const convertId = require('./convertId') const datastore = require('../datastore') +// load config +const config = require('../../../config') + module.exports = async (subscription) => { // remap vars to metadata object // this is needed since pubsub feedback from new subscriptions is slightly different @@ -21,33 +25,35 @@ module.exports = async (subscription) => { : null // remap values - const subscriptionRemap = { + const topicName = subscription.metadata.topic.split('/').pop() + const limited = { type: 'PUBSUB', method: subscription.metadata.pushConfig?.pushEndpoint ? 'PUSH' : 'PULL', name: subscription.name.split('/').pop(), path: subscription.name, - url: subscription.metadata.pushConfig?.pushEndpoint ?? null, - topic: { - name: subscription.metadata.topic.split('/').pop(), + id: convertId.decode(topicName).replace(config.pubSubPrefix, ''), + name: topicName, path: subscription.metadata.topic, }, ackDeadlineSeconds: subscription.metadata.ackDeadlineSeconds, - retainAckedMessages: subscription.metadata.retainAckedMessages, retryPolicy: subscription.metadata.retryPolicy, serviceAccount: subscription.metadata.pushConfig?.oidcToken?.serviceAccountEmail ?? null, - labels: subscription.metadata.labels, - - created: lookup?.created ?? null, + url: subscription.metadata.pushConfig?.pushEndpoint ?? null, contact: lookup?.contact ?? null, - owner: lookup?.owner ?? null, - institution: lookup?.institution ?? null, + institutionId: lookup?.institutionId ?? null, + } + + const full = { + ...limited, + + labels: subscription.metadata.labels, } // return data - return Promise.resolve(subscriptionRemap) + return Promise.resolve({ limited, full }) } diff --git a/src/utils/pubsub/publishMessage.js b/src/utils/pubsub/publishMessage.js index eff02e28..e756dd18 100644 --- a/src/utils/pubsub/publishMessage.js +++ b/src/utils/pubsub/publishMessage.js @@ -6,18 +6,23 @@ */ // load pubsub for internal queues -const loggerDev = require('../loggerDev') +const logger = require('../logger') const pubSubClient = require('./_client') const config = require('../../../config') // set local config -const functionName = 'utils/pubsub/publishMessage' +const source = 'pubsub.publishMessage' -module.exports = async (topics, message) => { - loggerDev('log', [functionName, 'triggered', JSON.stringify({ topics, message })]) +module.exports = async (topic, message) => { + logger.log({ + level: 'info', + message: `sending message > ${topic}`, + source, + data: { topic }, + }) - // initialize output object - const messageIds = {} + // initialize output + let output // prepare buffer object const messageBuffer = Buffer.from(JSON.stringify(message)) @@ -29,31 +34,32 @@ module.exports = async (topics, message) => { } // send message for each topic - for await (const topicName of topics) { - try { - // attempt to send message - messageIds[topicName] = await pubSubClient - .topic(topicName) - .publish(messageBuffer, customAttributes) - - // log progress - loggerDev('log', [functionName, 'success', topicName, messageIds[topicName]]) - } catch (err) { - if (err?.code === 5) { - messageIds[topicName] = 'TOPIC_NOT_FOUND' - loggerDev('error', [functionName, 'topic missing', topicName, messageIds[topicName]]) - } else { - messageIds[topicName] = 'TOPIC_ERROR' - loggerDev('error', [ - functionName, - 'other error', - topicName, - messageIds[topicName], - JSON.stringify(err), - ]) - } + try { + // attempt to send message + output = await pubSubClient.topic(topic).publish(messageBuffer, customAttributes) + } catch (error) { + if (error?.code === 5) { + output = 'TOPIC_NOT_FOUND' + + logger.log({ + level: 'warning', + message: `topic not found > ${topic}`, + source, + error, + data: { topic }, + }) + } else { + output = 'TOPIC_ERROR' + + logger.log({ + level: 'error', + message: `failed sending message > ${topic}`, + source, + error, + data: { topic, message }, + }) } } - return Promise.resolve(messageIds) + return Promise.resolve(output) } diff --git a/src/utils/tracer/index.js b/src/utils/tracer/index.js index 3854c9a7..8895e49b 100644 --- a/src/utils/tracer/index.js +++ b/src/utils/tracer/index.js @@ -5,8 +5,17 @@ */ -require('dd-trace').init({ +const tracer = require('dd-trace').init({ enabled: process.env.DD_TRACER_ENABLED === 'true', logInjection: true, +}) + +tracer.use('express', { + headers: ['dnt', 'user-agent', 'x-forwarded-host'], +}) + +tracer.use('http', { blocklist: ['/', '/health'], }) + +module.exports = tracer diff --git a/test/ingest.test.js b/test/ingest.test.js index c1140d63..4b753723 100644 --- a/test/ingest.test.js +++ b/test/ingest.test.js @@ -13,6 +13,7 @@ // Require dependencies const chai = require('chai') +const moment = require('moment') const chaiHttp = require('chai-http') const server = require('../src/ingest/index') @@ -120,21 +121,26 @@ if (process.env.TEST_USER_RESET) { function testEventKeys(body) { body.should.be.a('object') - body.should.have.property('topics').to.be.a('object') - body.should.have.property('message').to.be.a('object') + body.should.have.property('statuses').to.be.a('object') + body.should.have.property('event').to.be.a('object') } const eventName = 'de.ard.eventhub.v1.radio.track.playing' const eventPath = `/events/${eventName}` const swrTV = '990030' -const ardDE = '990140' const event = { event: eventName, type: 'music', - start: '2020-01-01T06:00:00+01:00', + start: moment().toISOString(), title: 'Unit Test Song', - serviceIds: [swrTV, ardDE], + services: [ + { + type: 'PermanentLivestream', + externalId: 'crid://swr.de/282310/unit', + publisherId: '282310', + }, + ], playlistItemId: 'unit-test-playlist', } @@ -173,8 +179,8 @@ let topicName function testTopicKeys(body) { body.should.be.a('object') body.should.have.property('type').to.be.a('string') + body.should.have.property('id').to.be.a('string') body.should.have.property('name').to.be.a('string') - body.should.have.property('path').to.be.a('string') body.should.have.property('labels').to.be.a('object') } @@ -197,11 +203,7 @@ describe(`GET ${topicPath}`, () => { testResponse(res, 200) res.body.should.be.a('array') res.body.every((i) => testTopicKeys(i)) - res.body.forEach((topic) => { - if (topic.name.indexOf(swrTV) !== -1) { - topicName = topic.name - } - }) + topicName = res.body[0].id done() }) }) @@ -220,19 +222,15 @@ function testSubscriptionKeys(body) { body.should.have.property('method').to.be.a('string') body.should.have.property('name').to.be.a('string') body.should.have.property('path').to.be.a('string') - body.should.have.property('url').to.be.a('string') body.should.have.property('topic').to.be.a('object') + body.topic.should.have.property('id').to.be.a('string') body.topic.should.have.property('name').to.be.a('string') body.topic.should.have.property('path').to.be.a('string') body.should.have.property('ackDeadlineSeconds').to.be.a('number') - body.should.have.property('retainAckedMessages').to.be.a('boolean') body.should.have.property('serviceAccount').to.be.a('string') - body.should.have.property('labels').to.be.a('object') - body.labels.should.have.property('id').to.be.a('string') - body.labels.should.have.property('institution').to.be.a('string') - body.should.have.property('created').to.be.a('string') + body.should.have.property('url').to.be.a('string') body.should.have.property('contact').to.be.a('string') - body.should.have.property('owner').to.be.a('string') + body.should.have.property('institutionId').to.be.a('string') } describe(`POST ${subscriptPath}`, () => { diff --git a/yarn.lock b/yarn.lock index 97a94116..56f8560d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -306,6 +306,15 @@ lodash "^4.17.19" to-fast-properties "^2.0.0" +"@dabh/diagnostics@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.2.tgz#290d08f7b381b8f94607dc8f471a12c675f9db31" + integrity sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q== + dependencies: + colorspace "1.1.x" + enabled "2.0.x" + kuler "^2.0.0" + "@eslint/eslintrc@^0.4.0": version "0.4.0" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.0.tgz#99cc0a0584d72f1df38b900fb062ba995f395547" @@ -498,6 +507,18 @@ lodash.camelcase "^4.3.0" protobufjs "^6.8.6" +"@hapi/hoek@^9.0.0": + version "9.1.1" + resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.1.1.tgz#9daf5745156fd84b8e9889a2dc721f0c58e894aa" + integrity sha512-CAEbWH7OIur6jEOzaai83jq3FmKmv4PmX1JYfs9IrYcGEVI/lyL1EXJGCj7eFVJ0bg5QR8LMxBlEtA+xKiLpFw== + +"@hapi/topo@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.0.0.tgz#c19af8577fa393a06e9c77b60995af959be721e7" + integrity sha512-tFJlT47db0kMqVm3H4nQYgn6Pwg10GTZHb1pwmSiv1K4ks6drQOtfEF5ZnPjkvC+y4/bUPHK+bc87QvLcL+WMw== + dependencies: + "@hapi/hoek" "^9.0.0" + "@jsdevtools/ono@7.1.3", "@jsdevtools/ono@^7.1.3": version "7.1.3" resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796" @@ -601,6 +622,23 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= +"@sideway/address@^4.1.0": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.1.tgz#9e321e74310963fdf8eebfbee09c7bd69972de4d" + integrity sha512-+I5aaQr3m0OAmMr7RQ3fR9zx55sejEYR2BFJaxL+zT3VM2611X0SHvPWIbAUBZVTn/YzYKbV8gJ2oT/QELknfQ== + dependencies: + "@hapi/hoek" "^9.0.0" + +"@sideway/formula@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c" + integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg== + +"@sideway/pinpoint@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" + integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== + "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" @@ -744,6 +782,11 @@ resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== +"@types/parse-json@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" + integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== + "@types/qs@*": version "6.9.5" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.5.tgz#434711bdd49eb5ee69d90c1d67c354a9a8ecb18b" @@ -983,6 +1026,11 @@ async-retry@^1.3.1: dependencies: retry "0.12.0" +async@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" + integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -1032,6 +1080,13 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bindings@^1.3.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + body-parser@1.19.0, body-parser@^1.19.0: version "1.19.0" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a" @@ -1304,7 +1359,7 @@ clone-response@^1.0.2: dependencies: mimic-response "^1.0.0" -color-convert@^1.9.0: +color-convert@^1.9.0, color-convert@^1.9.1: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -1323,16 +1378,45 @@ color-name@1.1.3: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= -color-name@~1.1.4: +color-name@^1.0.0, color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-string@^1.5.2: + version "1.5.5" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.5.tgz#65474a8f0e7439625f3d27a6a19d89fc45223014" + integrity sha512-jgIoum0OfQfq9Whcfc2z/VhCNcmQjWbey6qBX0vqt7YICflUmBCh9E9CiQD5GSJ+Uehixm3NUwHVhqUAWRivZg== + dependencies: + color-name "^1.0.0" + simple-swizzle "^0.2.2" + +color@3.0.x: + version "3.0.0" + resolved "https://registry.yarnpkg.com/color/-/color-3.0.0.tgz#d920b4328d534a3ac8295d68f7bd4ba6c427be9a" + integrity sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w== + dependencies: + color-convert "^1.9.1" + color-string "^1.5.2" + colorette@^1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== +colors@^1.2.1: + version "1.4.0" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" + integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== + +colorspace@1.1.x: + version "1.1.2" + resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.2.tgz#e0128950d082b86a2168580796a0aa5d6c68d8c5" + integrity sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ== + dependencies: + color "3.0.x" + text-hex "1.0.x" + combined-stream@^1.0.6, combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -1340,10 +1424,10 @@ combined-stream@^1.0.6, combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" -commander@6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-6.1.0.tgz#f8d722b78103141006b66f4c7ba1e97315ba75bc" - integrity sha512-wl7PNrYWd2y5mp1OK/LhTlv8Ff4kQJQRXXAvF+uU/TPNiVJUxZLRYGj/B0y/lPGAVcSbJqH2Za/cvHmrPMC8mA== +commander@7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-7.1.0.tgz#f2eaecf131f10e36e07d894698226e36ae0eb5ff" + integrity sha512-pRxBna3MJe6HKnBGsDyMv8ETbptw3axEdYHoqNh7gu5oDcew8fs0xnivZGm06Ogk8zGAJ9VX+OPEr2GXEQK4dg== component-emitter@^1.2.0: version "1.3.0" @@ -1476,6 +1560,17 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +cosmiconfig@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3" + integrity sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA== + dependencies: + "@types/parse-json" "^4.0.0" + import-fresh "^3.2.1" + parse-json "^5.0.0" + path-type "^4.0.0" + yaml "^1.10.0" + cp-file@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/cp-file/-/cp-file-7.0.0.tgz#b9454cfd07fe3b974ab9ea0e5f29655791a9b8cd" @@ -1505,10 +1600,10 @@ date-and-time@^0.14.2: resolved "https://registry.yarnpkg.com/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e" integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA== -dd-trace@^0.31.1: - version "0.31.1" - resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-0.31.1.tgz#36339d9998780c44bf13fc5c99145b935bcf18a4" - integrity sha512-JExKxwYmXOl4+CiBviUuVjq/OSh1qiu/fCEtkPyEiLuUdTkvXP4Tji+ZQdTctLMiwfVMUy3XwADgyzu8s0x5PA== +dd-trace@^0.31.2: + version "0.31.2" + resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-0.31.2.tgz#0fe6a8720e57b884c8fb2765e9119cc62c895b9d" + integrity sha512-z7nZWJw8q074yM6l8sn+iB4MfF00P1ZheXcnuEc4r/Za4IsUdIXUTIOoXKC6w6wDHt8uQTrVWwAGfUyQJIIneg== dependencies: "@types/node" "^10.12.18" axios "^0.21.1" @@ -1554,13 +1649,6 @@ debug@4, debug@4.3.1, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: dependencies: ms "2.1.2" -debug@4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1" - integrity sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg== - dependencies: - ms "2.1.2" - debug@^3.1.0, debug@^3.2.6: version "3.2.7" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" @@ -1740,6 +1828,11 @@ dot-prop@^5.2.0: dependencies: is-obj "^2.0.0" +dotenv@^8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a" + integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw== + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" @@ -1782,6 +1875,11 @@ emoji-regex@^8.0.0: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +enabled@2.0.x: + version "2.0.0" + resolved "https://registry.yarnpkg.com/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2" + integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== + encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -2215,10 +2313,10 @@ event-target-shim@^5.0.0: resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== -express-openapi-validator@^4.12.5: - version "4.12.5" - resolved "https://registry.yarnpkg.com/express-openapi-validator/-/express-openapi-validator-4.12.5.tgz#9b89b90937296edd913459a76e6856cebc907334" - integrity sha512-d374zsM68x+SzGjFRjrH7dgPL+DvXfPLGlP9hP0nIvUkIATuXrILiX4PCkIOpabkKxxqDL5yRP567sp2nZ5SaA== +express-openapi-validator@^4.12.6: + version "4.12.6" + resolved "https://registry.yarnpkg.com/express-openapi-validator/-/express-openapi-validator-4.12.6.tgz#0738a0f7b8fb643839dd8c4a5f95809cb2d51f08" + integrity sha512-F8m1Kp2zNhwGq2a/cu5PJgZkIQs5WyH68nSxH0uIOoHmDF8PPixp2xEtuQVtj/XQeAtBT7Em+7q7GT7ASGjXzg== dependencies: "@types/multer" "^1.4.5" ajv "^6.12.6" @@ -2294,6 +2392,11 @@ fast-levenshtein@^2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= +fast-safe-stringify@^2.0.4: + version "2.0.7" + resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743" + integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA== + fast-text-encoding@^1.0.0, fast-text-encoding@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" @@ -2306,6 +2409,11 @@ faye-websocket@0.11.3: dependencies: websocket-driver ">=0.5.1" +fecha@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.0.tgz#3ffb6395453e3f3efff850404f0a59b6747f5f41" + integrity sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg== + figlet@^1.1.1: version "1.5.0" resolved "https://registry.yarnpkg.com/figlet/-/figlet-1.5.0.tgz#2db4d00a584e5155a96080632db919213c3e003c" @@ -2318,6 +2426,11 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + fill-range@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" @@ -2401,6 +2514,11 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.0.tgz#a5d06b4a8b01e3a63771daa5cb7a1903e2e57067" integrity sha512-tW+UkmtNg/jv9CSofAKvgVcO7c2URjhTdW1ZTkcAritblu8tajiYy7YisnIflEwtKssCtOxpnBRoCB7iap0/TA== +fn.name@1.x.x: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" + integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== + follow-redirects@^1.10.0: version "1.13.2" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.2.tgz#dd73c8effc12728ba5cf4259d760ea5fb83e3147" @@ -2890,6 +3008,11 @@ is-arrayish@^0.2.1: resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= +is-arrayish@^0.3.1: + version "0.3.2" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" + integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ== + is-binary-path@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" @@ -3052,6 +3175,17 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= +joi@17.4.0: + version "17.4.0" + resolved "https://registry.yarnpkg.com/joi/-/joi-17.4.0.tgz#b5c2277c8519e016316e49ababd41a1908d9ef20" + integrity sha512-F4WiW2xaV6wc1jxete70Rw4V/VuMd6IN+a5ilZsxG4uYtUXWu2kq9W5P2dz30e7Gmw8RCbY/u/uk+dMPma9tAg== + dependencies: + "@hapi/hoek" "^9.0.0" + "@hapi/topo" "^5.0.0" + "@sideway/address" "^4.1.0" + "@sideway/formula" "^3.0.0" + "@sideway/pinpoint" "^2.0.0" + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -3204,6 +3338,11 @@ koalas@^1.0.2: resolved "https://registry.yarnpkg.com/koalas/-/koalas-1.0.2.tgz#318433f074235db78fae5661a02a8ca53ee295cd" integrity sha1-MYQz8HQjXbePrlZhoCqMpT7ilc0= +kuler@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" + integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== + latest-version@^5.0.0: version "5.1.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face" @@ -3219,14 +3358,16 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" -license-compliance@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/license-compliance/-/license-compliance-1.0.3.tgz#cb9f9a6103ef899e32250b84c4b414b3a1c66ea6" - integrity sha512-9rLI24ru/c6Np/Lz1/I4N6GYfTlLbSl2xPiU8Ot7Ex32UCXXObFzx/JGOZDne/xbEKG8p8gNCquBS4RW2dOHow== +license-compliance@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/license-compliance/-/license-compliance-1.1.0.tgz#9f906999320afbdb484e5268f4d976a506c306ae" + integrity sha512-krgjD8nt/F80jeyoBzx+wcYmuhrKkI9mi9OiBLSI8cI8UYB1lwYhK7rcTJEJFkgxuyRIcw/yDwb5DYnF8RfGEw== dependencies: chalk "4.1.0" - commander "6.1.0" - debug "4.2.0" + commander "7.1.0" + cosmiconfig "7.0.0" + debug "4.3.1" + joi "17.4.0" spdx-expression-parse "3.0.1" spdx-satisfies "5.0.0" @@ -3407,6 +3548,17 @@ log-symbols@4.0.0: dependencies: chalk "^4.0.0" +logform@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/logform/-/logform-2.2.0.tgz#40f036d19161fc76b68ab50fdc7fe495544492f2" + integrity sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg== + dependencies: + colors "^1.2.1" + fast-safe-stringify "^2.0.4" + fecha "^4.2.0" + ms "^2.1.1" + triple-beam "^1.3.0" + long@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" @@ -3634,6 +3786,13 @@ nested-error-stacks@^2.0.0: resolved "https://registry.yarnpkg.com/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz#0fbdcf3e13fe4994781280524f8b96b0cdff9c61" integrity sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug== +node-crc@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/node-crc/-/node-crc-1.3.0.tgz#2ec7b75de98a92be6c12674e77a4a4300e09f0dc" + integrity sha512-XSs9gZhZKdiwpJDLSoRQsnn8N9q/KH4bc0ayO6+qnHbtb1YfIrdKfjbOQFCwup4q/D2sNxhbupvrZ3rWmzAk4A== + dependencies: + bindings "^1.3.0" + node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" @@ -3761,6 +3920,13 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" +one-time@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/one-time/-/one-time-1.0.0.tgz#e06bc174aed214ed58edede573b433bbf827cb45" + integrity sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g== + dependencies: + fn.name "1.x.x" + onetime@^5.1.0: version "5.1.2" resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" @@ -4010,6 +4176,11 @@ path-type@^3.0.0: dependencies: pify "^3.0.0" +path-type@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" + integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== + pathval@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.1.tgz#8534e77a77ce7ac5a2512ea21e0fdb8fcf6c3d8d" @@ -4257,7 +4428,7 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.2.2, readable-stream@^2.3.5: +readable-stream@^2.2.2, readable-stream@^2.3.5, readable-stream@^2.3.7: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -4270,7 +4441,7 @@ readable-stream@^2.2.2, readable-stream@^2.3.5: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.0.2, readable-stream@^3.1.1: +readable-stream@^3.0.2, readable-stream@^3.1.1, readable-stream@^3.4.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -4523,6 +4694,13 @@ signal-exit@^3.0.2: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== +simple-swizzle@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" + integrity sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo= + dependencies: + is-arrayish "^0.3.1" + slice-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" @@ -4621,6 +4799,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= +stack-trace@0.0.x: + version "0.0.10" + resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" + integrity sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA= + "statuses@>= 1.5.0 < 2", statuses@~1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" @@ -4839,6 +5022,11 @@ term-size@^2.1.0: resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.1.tgz#2a6a54840432c2fb6320fea0f415531e90189f54" integrity sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg== +text-hex@1.0.x: + version "1.0.0" + resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5" + integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg== + text-table@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" @@ -4883,6 +5071,11 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" +triple-beam@^1.2.0, triple-beam@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" + integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== + tsconfig-paths@^3.9.0: version "3.9.0" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz#098547a6c4448807e8fcb8eae081064ee9a3c90b" @@ -5100,6 +5293,29 @@ widest-line@^3.1.0: dependencies: string-width "^4.0.0" +winston-transport@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.4.0.tgz#17af518daa690d5b2ecccaa7acf7b20ca7925e59" + integrity sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw== + dependencies: + readable-stream "^2.3.7" + triple-beam "^1.2.0" + +winston@^3.3.3: + version "3.3.3" + resolved "https://registry.yarnpkg.com/winston/-/winston-3.3.3.tgz#ae6172042cafb29786afa3d09c8ff833ab7c9170" + integrity sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw== + dependencies: + "@dabh/diagnostics" "^2.0.2" + async "^3.1.0" + is-stream "^2.0.0" + logform "^2.2.0" + one-time "^1.0.0" + readable-stream "^3.4.0" + stack-trace "0.0.x" + triple-beam "^1.3.0" + winston-transport "^4.4.0" + word-wrap@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" @@ -5178,6 +5394,11 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== +yaml@^1.10.0: + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + yargonaut@^1.1.2: version "1.1.4" resolved "https://registry.yarnpkg.com/yargonaut/-/yargonaut-1.1.4.tgz#c64f56432c7465271221f53f5cc517890c3d6e0c"