diff --git a/package-lock.json b/package-lock.json index 87cbfa3e2..6e04d8ae0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "adm-zip": "^0.4.16", "apollo-server-core": "^3.12.0", "async": "^3.0.1", + "axios": "^1.6.7", "bcrypt-nodejs": "^0.0.3", "bluebird": "^3.5.5", "body-parser": "^1.19.0", @@ -36,6 +37,7 @@ "kafkajs": "^1.12.0", "kind-of": "^6.0.3", "lodash": "^4.17.21", + "memoizee": "^0.4.17", "migrate-mongo": "^8.2.3", "mock-http-server": "^1.4.1", "mongoose": "^5.13.14", @@ -43,6 +45,7 @@ "mongoose-sequence": "^5.2.2", "morgan": "^1.10.0", "mquery": ">=3.2.3", + "ms": "^2.1.3", "multer": "^1.4.2", "mysql": "^2.18.1", "node-fetch": "^2.6.7", @@ -52,6 +55,7 @@ "promise-tools": "^2.1.0", "response-time": "^2.3.2", "swagger-ui-express": "^4.3.0", + "url-join": "^5.0.0", "winston": "^3.2.1", "yamljs": "^0.3.0", "zod": "^3.21.4" @@ -77,12 +81,14 @@ "@types/express-serve-static-core": "4.16.7", "@types/jsonwebtoken": "^8.3.2", "@types/lodash": "^4.14.134", + "@types/memoizee": "^0.4.11", "@types/migrate-mongo": "^7.0.0", "@types/mocha": "^9.1.1", "@types/mongodb": "^3.1.28", "@types/mongoose": "^5.11.97", "@types/mongoose-sequence": "^3.0.3", "@types/morgan": "^1.7.35", + "@types/ms": "^0.7.34", "@types/multer": "^1.3.7", "@types/mysql": "^2.15.9", "@types/node": "^12.0.10", @@ -542,11 +548,6 @@ "node": ">=12" } }, - "node_modules/@apollo/server/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, "node_modules/@apollo/server/node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1072,11 +1073,6 @@ "triple-beam": "^1.3.0" } }, - "node_modules/@overturebio-stack/lectern-client/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, "node_modules/@overturebio-stack/lectern-client/node_modules/node-worker-threads-pool": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.5.1.tgz", @@ -1570,6 +1566,12 @@ "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, + "node_modules/@types/memoizee": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@types/memoizee/-/memoizee-0.4.11.tgz", + "integrity": "sha512-2gyorIBZu8GoDr9pYjROkxWWcFtHCquF7TVbN2I+/OvgZhnIGQS0vX5KJz4lXNKb8XOSfxFOSG5OLru1ESqLUg==", + "dev": true + }, "node_modules/@types/migrate-mongo": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/@types/migrate-mongo/-/migrate-mongo-7.0.0.tgz", @@ -1650,6 +1652,12 @@ "@types/express": "*" } }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "dev": true + }, "node_modules/@types/multer": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.3.7.tgz", @@ -2370,6 +2378,29 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, + "node_modules/axios": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -3731,6 +3762,18 @@ "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", "integrity": "sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw==" }, + "node_modules/d": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.2.tgz", + "integrity": "sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==", + "dependencies": { + "es5-ext": "^0.10.64", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -3763,6 +3806,11 @@ "ms": "2.0.0" } }, + "node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/decamelize": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", @@ -3943,12 +3991,6 @@ "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", "dev": true }, - "node_modules/docker-modem/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, "node_modules/docker-modem/node_modules/readable-stream": { "version": "1.0.34", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", @@ -4102,6 +4144,54 @@ "node": ">= 0.8" } }, + "node_modules/es5-ext": { + "version": "0.10.64", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz", + "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==", + "hasInstallScript": true, + "dependencies": { + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.3", + "esniff": "^2.0.1", + "next-tick": "^1.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, + "node_modules/es6-symbol": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.4.tgz", + "integrity": "sha512-U9bFFjX8tFiATgtkJ1zg25+KviIXpgRvRHS8sau3GfhVzThRQrOeksPeT0BWW2MNZs1OEWJ1DPXOQMn0KKRkvg==", + "dependencies": { + "d": "^1.0.2", + "ext": "^1.7.0" + }, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/es6-weak-map": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", + "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.46", + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.1" + } + }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -4134,6 +4224,20 @@ "node": ">=0.8.0" } }, + "node_modules/esniff": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz", + "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==", + "dependencies": { + "d": "^1.0.1", + "es5-ext": "^0.10.62", + "event-emitter": "^0.3.5", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -4164,6 +4268,15 @@ "node": ">= 0.6" } }, + "node_modules/event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", + "dependencies": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, "node_modules/execa": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", @@ -4279,6 +4392,14 @@ "node": ">= 8.0.0" } }, + "node_modules/ext": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", + "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", + "dependencies": { + "type": "^2.7.2" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -4402,6 +4523,25 @@ "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -5133,6 +5273,11 @@ "node": ">=8" } }, + "node_modules/is-promise": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" + }, "node_modules/is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -5291,11 +5436,6 @@ "npm": ">=1.4.28" } }, - "node_modules/jsonwebtoken/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, "node_modules/jsprim": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", @@ -5557,11 +5697,6 @@ "triple-beam": "^1.3.0" } }, - "node_modules/logform/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, "node_modules/loglevel": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.8.1.tgz", @@ -5604,6 +5739,14 @@ "yallist": "^2.1.2" } }, + "node_modules/lru-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", + "integrity": "sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==", + "dependencies": { + "es5-ext": "~0.10.2" + } + }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -5641,6 +5784,24 @@ "node": ">= 0.6" } }, + "node_modules/memoizee": { + "version": "0.4.17", + "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.17.tgz", + "integrity": "sha512-DGqD7Hjpi/1or4F/aYAspXKNm5Yili0QDAFAY4QYvpqpgiY6+1jOfqpmByzjxbWd/T9mChbCArXAbDAsTm5oXA==", + "dependencies": { + "d": "^1.0.2", + "es5-ext": "^0.10.64", + "es6-weak-map": "^2.0.3", + "event-emitter": "^0.3.5", + "is-promise": "^2.2.2", + "lru-queue": "^0.1.0", + "next-tick": "^1.1.0", + "timers-ext": "^0.1.7" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", @@ -5926,12 +6087,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/mocha/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, "node_modules/mocha/node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -6275,9 +6430,9 @@ } }, "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/multer": { "version": "1.4.2", @@ -6386,6 +6541,11 @@ "node": ">= 0.6" } }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" + }, "node_modules/nice-try": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", @@ -6470,6 +6630,11 @@ "ms": "2.0.0" } }, + "node_modules/node-vault/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/node-worker-threads-pool": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.2.2.tgz", @@ -6521,12 +6686,6 @@ "ms": "^2.1.1" } }, - "node_modules/nodemon/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, "node_modules/nodemon/node_modules/semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -7131,6 +7290,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -7958,12 +8122,6 @@ "ms": "^2.1.1" } }, - "node_modules/superagent/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -8143,6 +8301,18 @@ "xtend": "~4.0.1" } }, + "node_modules/timers-ext": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.8.tgz", + "integrity": "sha512-wFH7+SEAcKfJpfLPkrgMPvvwnEtj8W4IurvEyrKsDleXnKLCDw71w8jltvfLa8Rm4qQxxT4jmDBYbJG/z7qoww==", + "dependencies": { + "es5-ext": "^0.10.64", + "next-tick": "^1.1.0" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/to-buffer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", @@ -8341,6 +8511,11 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, + "node_modules/type": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/type/-/type-2.7.3.tgz", + "integrity": "sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==" + }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -8595,6 +8770,15 @@ "punycode": "^2.1.0" } }, + "node_modules/url-join": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-5.0.0.tgz", + "integrity": "sha512-n2huDr9h9yzd6exQVnH/jU5mr+Pfx08LRXXZhkLLetAMESRj+anQsTAh940iMrIetKAmry9coFuZQ2jY8/p3WA==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, "node_modules/url-parse-lax": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", diff --git a/package.json b/package.json index 33d736e77..12b7880f3 100644 --- a/package.json +++ b/package.json @@ -58,12 +58,14 @@ "@types/express-serve-static-core": "4.16.7", "@types/jsonwebtoken": "^8.3.2", "@types/lodash": "^4.14.134", + "@types/memoizee": "^0.4.11", "@types/migrate-mongo": "^7.0.0", "@types/mocha": "^9.1.1", "@types/mongodb": "^3.1.28", "@types/mongoose": "^5.11.97", "@types/mongoose-sequence": "^3.0.3", "@types/morgan": "^1.7.35", + "@types/ms": "^0.7.34", "@types/multer": "^1.3.7", "@types/mysql": "^2.15.9", "@types/node": "^12.0.10", @@ -105,6 +107,7 @@ "adm-zip": "^0.4.16", "apollo-server-core": "^3.12.0", "async": "^3.0.1", + "axios": "^1.6.7", "bcrypt-nodejs": "^0.0.3", "bluebird": "^3.5.5", "body-parser": "^1.19.0", @@ -124,6 +127,7 @@ "kafkajs": "^1.12.0", "kind-of": "^6.0.3", "lodash": "^4.17.21", + "memoizee": "^0.4.17", "migrate-mongo": "^8.2.3", "mock-http-server": "^1.4.1", "mongoose": "^5.13.14", @@ -131,6 +135,7 @@ "mongoose-sequence": "^5.2.2", "morgan": "^1.10.0", "mquery": ">=3.2.3", + "ms": "^2.1.3", "multer": "^1.4.2", "mysql": "^2.18.1", "node-fetch": "^2.6.7", @@ -140,6 +145,7 @@ "promise-tools": "^2.1.0", "response-time": "^2.3.2", "swagger-ui-express": "^4.3.0", + "url-join": "^5.0.0", "winston": "^3.2.1", "yamljs": "^0.3.0", "zod": "^3.21.4" diff --git a/src/clinical/donor-repo.ts b/src/clinical/donor-repo.ts index a72525511..e3bca42a0 100644 --- a/src/clinical/donor-repo.ts +++ b/src/clinical/donor-repo.ts @@ -24,6 +24,7 @@ import mongoose, { PaginateModel } from 'mongoose'; import mongoosePaginate from 'mongoose-paginate-v2'; import { DeepReadonly } from 'deep-freeze'; import { F, MongooseUtils, notEmpty } from '../utils'; +import { setEntityIds, setEntityIdsForDonors } from './id-generator'; export const SUBMITTER_ID = 'submitterId'; export const SPECIMEN_SUBMITTER_ID = 'specimen.submitterId'; @@ -109,20 +110,22 @@ export interface DonorRepository { ): Promise | undefined>; iterateAllByProgramId(programId: string): AsyncIterable>; create(donor: DeepReadonly>): Promise>; - update(donor: DeepReadonly): Promise>; - updateAll(donors: DeepReadonly[]): Promise[]>; + update(donor: Donor): Promise>; + updateAll(donors: Donor[]): Promise[]>; countBy(filter: any): Promise; } // Mongoose implementation of the DonorRepository export const donorDao: DonorRepository = { async insertDonors(donors: Donor[]) { - await mongoose.connection.db.collection('donors').insertMany(donors); + const donorsWithIds = await setEntityIdsForDonors(donors); + await mongoose.connection.db.collection('donors').insertMany(donorsWithIds); }, async updateDonor(donor: Donor) { + const donorsWithIds = await setEntityIds(donor); await mongoose.connection.db .collection('donors') - .findOneAndUpdate({ donorId: donor.donorId }, { $set: donor }); + .findOneAndUpdate({ donorId: donor.donorId }, { $set: donorsWithIds }); }, async countBy(filter: any) { return await DonorModel.count(filter).exec(); @@ -413,27 +416,41 @@ export const donorDao: DonorRepository = { return iterateAllByProgramId(programId); }, - async update(donor: DeepReadonly) { - const newDonor = new DonorModel(donor); + async update(donor: Donor) { + const dnr = await setEntityIds(donor); + const newDonor = new DonorModel(await setEntityIds(donor)); unsetIsNewFlagForUpdate(newDonor); - await newDonor.save(); return F(MongooseUtils.toPojo(newDonor) as Donor); }, - async updateAll(donors: DeepReadonly[]) { - const newDonors = donors.map((donor) => { - const newDonor = new DonorModel(donor); + async updateAll(donors: Donor[]) { + const newDonors = donors.map(async (donor) => { + const newDonor = new DonorModel(await setEntityIds(donor)); unsetIsNewFlagForUpdate(newDonor); return newDonor; }); - const results = await Promise.all(newDonors.map((donor) => donor.save())); - return newDonors.map((donor) => F(MongooseUtils.toPojo(donor) as Donor)); + await Promise.all( + newDonors.map((donor) => { + donor.then(async (d) => { + await d.save(); + }); + }), + ); + const result = await Promise.all( + newDonors.map((donor) => + donor.then((d) => { + return F(MongooseUtils.toPojo(d) as Donor); + }), + ), + ); + + return result; }, - async create(donor: DeepReadonly) { - const newDonor = new DonorModel(donor); + async create(donor: Partial) { + const newDonor = new DonorModel(await setEntityIds(donor)); const doc = await newDonor.save(); return F(MongooseUtils.toPojo(newDonor) as Donor); }, @@ -656,58 +673,9 @@ DonorSchema.index({ 'specimens.samples.submitterId': 1, programId: 1 }, { unique * multiple times, and that makes them hard to test because tests depend * on resetting the config and bootstraping but global variables keep their state. */ -DonorSchema.plugin(AutoIncrement, { - inc_field: 'donorId', - start_seq: process.env.DONOR_ID_SEED || 250000, -}); DonorSchema.plugin(mongoosePaginate); -SpecimenSchema.plugin(AutoIncrement, { - inc_field: 'specimenId', - start_seq: process.env.SPECIMEN_ID_SEED || 210000, -}); - -SampleSchema.plugin(AutoIncrement, { - inc_field: 'sampleId', - start_seq: process.env.SAMPLE_ID_SEED || 610000, -}); - -FollowUpSchema.plugin(AutoIncrement, { - inc_field: 'followUpId', - start_seq: 1, -}); - -PrimaryDiagnosisSchema.plugin(AutoIncrement, { - inc_field: 'primaryDiagnosisId', - start_seq: 1, -}); - -FamilyHistorySchema.plugin(AutoIncrement, { - inc_field: 'familyHistoryId', - start_seq: 1, -}); - -ExposureSchema.plugin(AutoIncrement, { - inc_field: 'exposureId', - start_seq: 1, -}); - -BiomarkerSchema.plugin(AutoIncrement, { - inc_field: 'biomarkerId', - start_seq: 1, -}); - -ComorbiditySchema.plugin(AutoIncrement, { - inc_field: 'comorbidityId', - start_seq: 1, -}); - -TreatmentSchema.plugin(AutoIncrement, { - inc_field: 'treatmentId', - start_seq: 1, -}); - export let DonorModel = mongoose.model('Donor', DonorSchema) as PaginateModel< DonorDocument >; diff --git a/src/clinical/id-generator.ts b/src/clinical/id-generator.ts new file mode 100644 index 000000000..2680de2db --- /dev/null +++ b/src/clinical/id-generator.ts @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import axios from 'axios'; +import memoize from 'memoizee'; +import ms from 'ms'; +import { Donor } from './clinical-entities'; +import { ClinicalEntitySchemaNames } from '../common-model/entities'; +import { config } from '../config'; +import { Errors } from '../utils'; +import urlJoin from 'url-join'; + +interface IdGenerationRequest { + programId: string | undefined; + submitterSpecimenId: string | number | boolean | string[] | number[] | boolean[] | undefined; + submitterSampleId: string | undefined; + submitterPrimaryDiagnosisId: + | string + | number + | boolean + | string[] + | number[] + | boolean[] + | undefined; + submitterFollowUpId: string | number | boolean | string[] | number[] | boolean[] | undefined; + submitterTreatmentId: string | undefined; + submitterDonorId: string | undefined; + testInterval: string | undefined; + family_relative_id: string | number | boolean | string[] | number[] | boolean[] | undefined; + comorbidityTypeCode: string | number | boolean | string[] | number[] | boolean[] | undefined; + entityType: string | undefined; +} + +const request = { + programId: '-', + submitterSpecimenId: '-', + submitterSampleId: '-', + submitterPrimaryDiagnosisId: '-', + submitterFollowUpId: '-', + submitterTreatmentId: '-', + submitterDonorId: '-', + testInterval: '-', + family_relative_id: '-', + comorbidityTypeCode: '-', + entityType: '-', +}; + + +export async function getId(req: IdGenerationRequest) { + console.log('getId function called: ' + req.submitterDonorId); + const token = await getToken(); + const headers = { + headers: { + Authorization: 'Bearer ' + token, + }, + }; + + let response; + try { + const url = + config.getConfig().idServiceUrl() + + `${req.programId}/${req.submitterDonorId}/${req.submitterSpecimenId}/${req.submitterSampleId}/${req.submitterPrimaryDiagnosisId}/${req.submitterFollowUpId}/${req.submitterTreatmentId}/${req.testInterval}/${req.family_relative_id}/${req.comorbidityTypeCode}/${req.entityType}`; + + response = await axios.get(url, headers); + console.log('getId response: ' + response.data.entityId + ' - ' + response.data.entityType); + } catch (e) { + throw new Errors.NetworkError('Error sending request to ID service. Caused by: ' + e); + } + if (!response.data.entityId) { + throw new Errors.IdGenerationError('Error generating entity ids.'); + } + return parseInt(response.data.entityId); +} + +export async function setEntityIdsForDonors(donors: Donor[]) { + const donorsWithIds = donors.map(async (donor) => { + return await setEntityIds(donor); + }); + return await Promise.all(donorsWithIds); +} + +export async function setEntityIds(donor: Partial) { + const submitterDonorId = donor.submitterId; + // -- DONOR -- + const donorId = await getId({ + ...request, + programId: donor.programId, + submitterDonorId: donor.submitterId, + entityType: ClinicalEntitySchemaNames.DONOR, + }); + donor.donorId = donorId; + console.log(donor.donorId); + + // -- SPECIMEN -- + if (donor.specimens && donor.specimens.length > 0) { + for (const specimen of donor.specimens) { + const submitterId = !specimen.clinicalInfo.submitter_specimen_id + ? specimen.submitterId + : specimen.clinicalInfo.submitter_specimen_id; + const id = await getId({ + ...request, + programId: donor.programId, + submitterSpecimenId: submitterId, + submitterDonorId, + entityType: ClinicalEntitySchemaNames.SPECIMEN, + }); + specimen.specimenId = id; + + // -- SAMPLE -- + const samples = specimen.samples; + for (const sample of samples) { + const id = await getId({ + ...request, + programId: donor.programId, + submitterSampleId: sample.submitterId, + submitterSpecimenId: specimen.submitterId, + submitterDonorId, + entityType: ClinicalEntitySchemaNames.REGISTRATION, + }); + sample.sampleId = id; + } + } + } + + // -- BIOMARKER -- + if (donor.biomarker && donor.biomarker.length > 0) { + for (const biomarker of donor.biomarker) { + const id = await getId({ + ...request, + programId: donor.programId, + submitterSpecimenId: biomarker.clinicalInfo.submitter_specimen_id?.toString() || '-', + submitterPrimaryDiagnosisId: + biomarker.clinicalInfo.submitter_primary_diagnosis_id?.toString() || '-', + submitterFollowUpId: biomarker.clinicalInfo.submitter_follow_up_id?.toString() || '-', + submitterTreatmentId: biomarker.clinicalInfo.submitter_treatment_id?.toString() || '-', + submitterDonorId, + testInterval: biomarker.clinicalInfo.test_interval?.toString() || '-', + entityType: ClinicalEntitySchemaNames.BIOMARKER, + }); + biomarker.biomarkerId = id; + } + } + + // -- COMORBIDITY -- + if (donor.comorbidity && donor.comorbidity.length > 0) { + for (const comorbidity of donor.comorbidity) { + const id = await getId({ + ...request, + programId: donor.programId, + submitterDonorId, + comorbidityTypeCode: comorbidity.clinicalInfo.comorbidity_type_code, + entityType: ClinicalEntitySchemaNames.COMORBIDITY, + }); + comorbidity.comorbidityId = id; + } + } + + // -- PRIMARY DIAGNOSIS -- + if (donor.primaryDiagnoses && donor.primaryDiagnoses.length > 0) { + for (const primaryDiagnosis of donor.primaryDiagnoses) { + const id = await getId({ + ...request, + programId: donor.programId, + submitterPrimaryDiagnosisId: primaryDiagnosis.clinicalInfo.submitter_primary_diagnosis_id, + submitterDonorId, + entityType: ClinicalEntitySchemaNames.PRIMARY_DIAGNOSIS, + }); + primaryDiagnosis.primaryDiagnosisId = id; + } + } + + // -- TREATMENT -- + if (donor.treatments && donor.treatments.length > 0) { + for (const treatment of donor.treatments) { + const submitter_treatment_id = treatment.clinicalInfo?.submitter_treatment_id?.toString(); + const therapy_submitter_treatment_id = treatment.therapies[0]?.clinicalInfo?.submitter_treatment_id?.toString(); + const id = await getId({ + ...request, + programId: donor.programId, + submitterTreatmentId: !submitter_treatment_id + ? therapy_submitter_treatment_id + : submitter_treatment_id, + submitterDonorId, + entityType: ClinicalEntitySchemaNames.TREATMENT, + }); + treatment.treatmentId = id; + } + } + + // -- FAMILY HISTORY -- + if (donor.familyHistory && donor.familyHistory.length > 0) { + for (const familyHistory of donor.familyHistory) { + const id = await getId({ + ...request, + programId: donor.programId, + submitterDonorId, + family_relative_id: familyHistory.clinicalInfo.family_relative_id, + entityType: ClinicalEntitySchemaNames.FAMILY_HISTORY, + }); + familyHistory.familyHistoryId = id; + } + } + + // -- FOLLOW UP -- + if (donor.followUps && donor.followUps.length > 0) { + for (const followUp of donor.followUps) { + const id = await getId({ + ...request, + programId: donor.programId, + submitterFollowUpId: followUp.clinicalInfo.submitter_follow_up_id, + submitterDonorId, + entityType: ClinicalEntitySchemaNames.FOLLOW_UP, + }); + followUp.followUpId = id; + } + } + + // -- EXPOSURE -- + if (donor.exposure && donor.exposure.length > 0) { + for (const exposure of donor.exposure) { + const id = await getId({ + ...request, + programId: donor.programId, + submitterDonorId, + entityType: ClinicalEntitySchemaNames.EXPOSURE, + }); + exposure.exposureId = id; + } + } + + return donor; +} + +const getToken = memoize( + async () => { + const headers = { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }; + const data = { + grant_type: 'client_credentials', + client_id: config.getConfig().egoClientId(), + client_secret: config.getConfig().egoClientSecret(), + }; + try { + const url = config.getConfig().tokenUrl(); + const response = await axios.post(config.getConfig().tokenUrl(), data, headers); + return response.data.access_token; + } catch (e) { + throw new Errors.NetworkError('Error fetching ego token. Caused by: ' + e); + } + }, + { + maxAge: ms('1d'), + preFetch: true, + }, +); diff --git a/src/config.ts b/src/config.ts index df21ef044..5ad93f87a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -40,6 +40,8 @@ export interface AppConfig { egoUrl(): string; egoClientId(): string; egoClientSecret(): string; + tokenUrl(): string; + idServiceUrl(): string; } class ConfigManager { diff --git a/src/exception/missing-entity-exceptions/service.ts b/src/exception/missing-entity-exceptions/service.ts index 3b139ab05..bcc4e6ad7 100644 --- a/src/exception/missing-entity-exceptions/service.ts +++ b/src/exception/missing-entity-exceptions/service.ts @@ -138,7 +138,7 @@ const recalculateAndUpdateDonors = async (props: { const updatedDonors = await updateDonorsCompletionStats( (donorDocuments || []) as DeepReadonly[], ); - const savedDonors = await donorRepo.updateAll(updatedDonors.map((dto) => deepFreeze(dto))); + const savedDonors = await donorRepo.updateAll(updatedDonors); return success(savedDonors); } catch (error) { const message = 'Error thrown while updating donor core completion data'; diff --git a/src/server.ts b/src/server.ts index 6e3c9f16c..16d78292a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -136,6 +136,12 @@ let server: Server; egoUrl(): string { return process.env.EGO_URL || secrets.EGO_URL || ''; }, + tokenUrl(): string { + return process.env.TOKEN_URL || ''; + }, + idServiceUrl(): string { + return process.env.ID_SERVICE_URL || ''; + }, egoClientId(): string { return process.env.EGO_CLIENT_ID || secrets.EGO_CLIENT_ID || ''; }, diff --git a/src/submission/submission-to-clinical/submission-to-clinical.ts b/src/submission/submission-to-clinical/submission-to-clinical.ts index 0bd30de17..7eb0bd1ad 100644 --- a/src/submission/submission-to-clinical/submission-to-clinical.ts +++ b/src/submission/submission-to-clinical/submission-to-clinical.ts @@ -139,7 +139,7 @@ const performCommitSubmission = async ( try { // write each updated donor to the db - await donorDao.updateAll(updatedDonors.map((dto) => F(dto))); + await donorDao.updateAll(updatedDonors); // If the save completed without error, we can delete the active registration submissionRepository.deleteByProgramId(activeSubmission.programId); diff --git a/src/utils.ts b/src/utils.ts index 99dbbcd5a..6219feee2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -125,6 +125,10 @@ export namespace ControllerUtils { return res.status(422).send({ message }); }; + export const unableToGenerateId = (res: Response, message: string) => { + return res.status(500).send({ message }); + }; + export const invalidBatch = ( res: Response, batchErrors: SubmissionBatchError | SubmissionBatchError[], @@ -203,6 +207,17 @@ export namespace Errors { super(msg || `TSV file is formatted incorrectly`); } } + + export class IdGenerationError extends Error { + constructor(msg: string) { + super(msg); + } + } + export class NetworkError extends Error { + constructor(msg: string) { + super(msg); + } + } } export namespace MongooseUtils { diff --git a/test/integration/clinical/clinical.spec.ts b/test/integration/clinical/clinical.spec.ts index 6a6fded2c..91fe13085 100644 --- a/test/integration/clinical/clinical.spec.ts +++ b/test/integration/clinical/clinical.spec.ts @@ -169,6 +169,12 @@ describe('clinical Api', () => { egoUrl() { return ''; }, + tokenUrl(): string { + return process.env.TOKEN_URL || ''; + }, + idServiceUrl(): string { + return process.env.ID_SERVICE_URL || ''; + }, egoClientId() { return ''; }, diff --git a/test/integration/submission/submission-schema-migration.spec.ts b/test/integration/submission/submission-schema-migration.spec.ts index 357a61cd4..48b39e0f8 100644 --- a/test/integration/submission/submission-schema-migration.spec.ts +++ b/test/integration/submission/submission-schema-migration.spec.ts @@ -215,6 +215,12 @@ describe('schema migration api', () => { egoUrl() { return ''; }, + tokenUrl(): string { + return process.env.TOKEN_URL || ''; + }, + idServiceUrl(): string { + return process.env.ID_SERVICE_URL || ''; + }, egoClientId() { return ''; }, diff --git a/test/integration/submission/submission.spec.ts b/test/integration/submission/submission.spec.ts index 499a8cf36..9ee553177 100644 --- a/test/integration/submission/submission.spec.ts +++ b/test/integration/submission/submission.spec.ts @@ -169,6 +169,12 @@ describe('Submission Api', () => { egoUrl() { return ''; }, + tokenUrl(): string { + return process.env.TOKEN_URL || ''; + }, + idServiceUrl(): string { + return process.env.ID_SERVICE_URL || ''; + }, egoClientId() { return ''; }, diff --git a/test/performance-test/submission.spec.ts b/test/performance-test/submission.spec.ts index 6d9da9280..69fa657cb 100644 --- a/test/performance-test/submission.spec.ts +++ b/test/performance-test/submission.spec.ts @@ -157,6 +157,12 @@ describe('Submission Api', () => { egoUrl() { return ''; }, + tokenUrl(): string { + return process.env.TOKEN_URL || ''; + }, + idServiceUrl(): string { + return process.env.ID_SERVICE_URL || ''; + }, egoClientId() { return ''; }, diff --git a/test/unit/decorator/decorator.spec.ts b/test/unit/decorator/decorator.spec.ts index 3b5942933..4e646cd9d 100644 --- a/test/unit/decorator/decorator.spec.ts +++ b/test/unit/decorator/decorator.spec.ts @@ -86,6 +86,12 @@ describe('decorator', () => { egoUrl() { return ''; }, + idServiceUrl(): string { + return process.env.ID_SERVICE_URL || ''; + }, + tokenUrl(): string { + return process.env.TOKEN_URL || ''; + }, egoClientId() { return ''; }, diff --git a/test/unit/submission/submission-to-clinical.spec.ts b/test/unit/submission/submission-to-clinical.spec.ts index 6c22e5544..b8d9272b5 100644 --- a/test/unit/submission/submission-to-clinical.spec.ts +++ b/test/unit/submission/submission-to-clinical.spec.ts @@ -133,10 +133,7 @@ describe('submission-to-clinical', () => { [deepFreeze.DeepReadonly>], Promise> >; - let updateDonorStub: sinon.SinonStub< - [deepFreeze.DeepReadonly], - Promise> - >; + let updateDonorStub: sinon.SinonStub<[Donor], Promise>>; let missingEntityRepoGetByProgramStub: sinon.SinonStub< [string], Promise>