From 0091f90498f1c7224b36baa40e6ee9071ff4f4ab Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Thu, 21 Mar 2019 23:04:10 +0100 Subject: [PATCH 01/10] add typescript (with very permissive settings) --- .envrc | 1 + .npmignore | 2 + package.json | 118 +++++++++++++++++++---------------- tsconfig.json | 27 ++++++++ yarn.lock | 169 +++++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 263 insertions(+), 54 deletions(-) create mode 100644 .envrc create mode 100644 tsconfig.json diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..a8a7606 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +layout node diff --git a/.npmignore b/.npmignore index 7f7c70d..d56fa9d 100644 --- a/.npmignore +++ b/.npmignore @@ -5,7 +5,9 @@ perf/ sasl-ssl-example/ ssl-example/ test/ +.idea .editorconfig +.envrc .eslintignore .eslintrc.js .gitignore diff --git a/package.json b/package.json index af24fd2..64fea71 100644 --- a/package.json +++ b/package.json @@ -1,71 +1,85 @@ { "name": "sinek", "version": "7.29.3", - "description": "Node.js kafka client, consumer, producer polite out of the box", - "main": "index.js", - "typings": "index.d.ts", - "engines": { - "node": ">=8.11.3" - }, - "scripts": { - "lint": "eslint .", - "fix": "eslint --fix .", - "kafka:start": "./kafka-setup/start.sh", - "kafka:stop": "./kafka-setup/stop.sh", - "kafka:logs": "docker-compose --file ./kafka-setup/docker-compose.yml logs -f", - "kafka:console": "./kafka-setup/kafka-console.sh", - "test": "_mocha --recursive --timeout 32500 --exit -R spec test/int", - "yarn:openssl": "LDFLAGS='-L/usr/local/opt/openssl/lib' CPPFLAGS='-I/usr/local/opt/openssl/include' yarn" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/nodefluent/node-sinek.git" - }, - "keywords": [ - "polite", - "kafka", - "client", - "sinek", - "simon", - "nice", - "easy", - "producer", - "consumer", - "backpressure", - "control", - "flow", - "queue", - "ssl", - "secure", - "sasl", - "kerberos", - "librdkafka", - "stream", - "batch" - ], - "author": "Christian Fröhlingsdorf", - "license": "MIT", - "bugs": { - "url": "https://github.com/nodefluent/node-sinek/issues" - }, - "homepage": "https://github.com/nodefluent/node-sinek#readme", - "dependencies": { + "description": "Node.js kafka client, consumer, producer polite out of the box", + "main": "index.js", + "typings": "index.d.ts", + "engines": { + "node": ">=8.11.3" + }, + "scripts": { + "lint": "eslint .", + "fix": "eslint --fix .", + "kafka:start": "./kafka-setup/start.sh", + "kafka:stop": "./kafka-setup/stop.sh", + "kafka:logs": "docker-compose --file ./kafka-setup/docker-compose.yml logs -f", + "kafka:console": "./kafka-setup/kafka-console.sh", + "test": "_mocha --recursive --timeout 32500 --exit -R spec test/int", + "yarn:openssl": "LDFLAGS='-L/usr/local/opt/openssl/lib' CPPFLAGS='-I/usr/local/opt/openssl/include' yarn" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/nodefluent/node-sinek.git" + }, + "keywords": [ + "polite", + "kafka", + "client", + "sinek", + "simon", + "nice", + "easy", + "producer", + "consumer", + "backpressure", + "control", + "flow", + "queue", + "ssl", + "secure", + "sasl", + "kerberos", + "librdkafka", + "stream", + "batch" + ], + "author": "Christian Fröhlingsdorf", + "license": "MIT", + "bugs": { + "url": "https://github.com/nodefluent/node-sinek/issues" + }, + "homepage": "https://github.com/nodefluent/node-sinek#readme", + "dependencies": { + "@types/async": "^2.4.0", + "@types/bluebird": "^3.5.25", + "@types/debug": "^4.1.3", + "@types/kafka-node": "^2.0.7", + "@types/lodash.merge": "^4.6.4", + "@types/murmurhash": "^0.0.1", + "@types/node": "11.13.0", + "@types/uuid": "^3.4.4", "async": "~2.6.2", "bluebird": "~3.5.4", "debug": "~4.1.1", "lodash.merge": "~4.6.1", "murmur2-partitioner": "~1.0.0", "murmurhash": "~0.0.2", - "uuid": "~3.3.2", - "node-rdkafka": "~2.6.1" + "node-rdkafka": "~2.6.1", + "uuid": "~3.3.2" }, "devDependencies": { + "@types/expect.js": "^0.3.29", + "@types/express": "^4.16.1", + "@types/mocha": "^5.2.5", + "@types/sinon": "^7.0.11", "eslint": "~5.16.0", "expect.js": "~0.3.1", "express": "~4.16.4", "istanbul": "~0.4.5", "mocha": "~6.1.4", - "sinon": "~7.3.2" + "sinon": "~7.3.2", + "ts-node": "^8.0.2", + "typescript": "^3.4.1" }, "optionalDependencies": { "kafka-node": "~4.1.3" diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..35c1a0e --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "module": "commonjs", + "esModuleInterop": true, + "target": "es2016", + "sourceMap": true, + "noEmit": true, + + "noImplicitReturns": false, + "noFallthroughCasesInSwitch": true, + "noUnusedParameters": false, + "noUnusedLocals": false, + + "strict": true, + + "noImplicitAny": false, + "strictBindCallApply": true, + "strictNullChecks": false + }, + "exclude": [ + "./dist", + "./perf", + "./node_modules", + "./sasl-ssl-example", + "./ssl-example" + ] +} diff --git a/yarn.lock b/yarn.lock index a9bc8b0..8b5cd8c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -47,6 +47,127 @@ resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz#8da5c6530915653f3a1f38fd5f101d8c3f8079c5" integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ== +"@types/async@^2.4.0": + version "2.4.1" + resolved "https://registry.yarnpkg.com/@types/async/-/async-2.4.1.tgz#43c3b2c60eab41c25ca0009c07ca7d619d943119" + integrity sha512-C09BK/wXzbW+/JK9zckhe+FeSbg7NmvVjUWwApnw7ksRpUq3ecGLiq2Aw1LlY4Z/VmtdhSaIs7jO5/MWRYMcOA== + +"@types/bluebird@^3.5.25": + version "3.5.26" + resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.26.tgz#a38c438ae84fa02431d6892edf86e46edcbca291" + integrity sha512-aj2mrBLn5ky0GmAg6IPXrQjnN0iB/ulozuJ+oZdrHRAzRbXjGmu4UXsNCjFvPbSaaPZmniocdOzsM392qLOlmQ== + +"@types/body-parser@*": + version "1.17.0" + resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.17.0.tgz#9f5c9d9bd04bb54be32d5eb9fc0d8c974e6cf58c" + integrity sha512-a2+YeUjPkztKJu5aIF2yArYFQQp8d51wZ7DavSHjFuY1mqVgidGyzEQ41JIVNy82fXj8yPgy2vJmfIywgESW6w== + dependencies: + "@types/connect" "*" + "@types/node" "*" + +"@types/connect@*": + version "3.4.32" + resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.32.tgz#aa0e9616b9435ccad02bc52b5b454ffc2c70ba28" + integrity sha512-4r8qa0quOvh7lGD0pre62CAb1oni1OO6ecJLGCezTmhQ8Fz50Arx9RUszryR8KlgK6avuSXvviL6yWyViQABOg== + dependencies: + "@types/node" "*" + +"@types/debug@^4.1.3": + version "4.1.4" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.4.tgz#56eec47706f0fd0b7c694eae2f3172e6b0b769da" + integrity sha512-D9MyoQFI7iP5VdpEyPZyjjqIJ8Y8EDNQFIFVLOmeg1rI1xiHOChyUPMPRUVfqFCerxfE+yS3vMyj37F6IdtOoQ== + +"@types/expect.js@^0.3.29": + version "0.3.29" + resolved "https://registry.yarnpkg.com/@types/expect.js/-/expect.js-0.3.29.tgz#28dd359155b84b8ecb094afc3f4b74c3222dca3b" + integrity sha1-KN01kVW4S47LCUr8P0t0wyItyjs= + +"@types/express-serve-static-core@*": + version "4.16.4" + resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.16.4.tgz#56bb8be4559401d68af4a3624ae9dd3166103e60" + integrity sha512-x/8h6FHm14rPWnW2HP5likD/rsqJ3t/77OWx2PLxym0hXbeBWQmcPyHmwX+CtCQpjIfgrUdEoDFcLPwPZWiqzQ== + dependencies: + "@types/node" "*" + "@types/range-parser" "*" + +"@types/express@^4.16.1": + version "4.16.1" + resolved "https://registry.yarnpkg.com/@types/express/-/express-4.16.1.tgz#d756bd1a85c34d87eaf44c888bad27ba8a4b7cf0" + integrity sha512-V0clmJow23WeyblmACoxbHBu2JKlE5TiIme6Lem14FnPW9gsttyHtk6wq7njcdIWH1njAaFgR8gW09lgY98gQg== + dependencies: + "@types/body-parser" "*" + "@types/express-serve-static-core" "*" + "@types/serve-static" "*" + +"@types/kafka-node@^2.0.7": + version "2.0.8" + resolved "https://registry.yarnpkg.com/@types/kafka-node/-/kafka-node-2.0.8.tgz#f45d96611d9fa349041f56f370095edc56426d0c" + integrity sha512-WMhZLv75LorppurHDLb/ghOwX8TCBRWs4C5CpuuXa4lFyKcbahzkgUKmwLRYy4ixdjEsNANrjiDrybTzVPu6YQ== + dependencies: + "@types/node" "*" + +"@types/lodash.merge@^4.6.4": + version "4.6.6" + resolved "https://registry.yarnpkg.com/@types/lodash.merge/-/lodash.merge-4.6.6.tgz#b84b403c1d31bc42d51772d1cd5557fa008cd3d6" + integrity sha512-IB90krzMf7YpfgP3u/EvZEdXVvm4e3gJbUvh5ieuI+o+XqiNEt6fCzqNRaiLlPVScLI59RxIGZMQ3+Ko/DJ8vQ== + dependencies: + "@types/lodash" "*" + +"@types/lodash@*": + version "4.14.124" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.124.tgz#16fb067a8fc4be42f044c505d8b5316c6f54de93" + integrity sha512-6bKEUVbHJ8z34jisA7lseJZD2g31SIvee3cGX2KEZCS4XXWNbjPZpmO1/2rGNR9BhGtaYr6iYXPl1EzRrDAFTA== + +"@types/mime@*": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d" + integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw== + +"@types/mocha@^5.2.5": + version "5.2.6" + resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-5.2.6.tgz#b8622d50557dd155e9f2f634b7d68fd38de5e94b" + integrity sha512-1axi39YdtBI7z957vdqXI4Ac25e7YihYQtJa+Clnxg1zTJEaIRbndt71O3sP4GAMgiAm0pY26/b9BrY4MR/PMw== + +"@types/murmurhash@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@types/murmurhash/-/murmurhash-0.0.1.tgz#564a85100cb16eeec43218a09b08cd87da076afc" + integrity sha512-jiYuTCJ60WxEDOkJFs2192G/nH0PY8mtyHeZkv5fOQQ5DkSkmEgY7YLzCxyM8Y8jibd8Dprb9bGth4EO34mAjw== + +"@types/node@*": + version "12.0.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-12.0.0.tgz#d11813b9c0ff8aaca29f04cbc12817f4c7d656e5" + integrity sha512-Jrb/x3HT4PTJp6a4avhmJCDEVrPdqLfl3e8GGMbpkGGdwAV5UGlIs4vVEfsHHfylZVOKZWpOqmqFH8CbfOZ6kg== + +"@types/node@11.13.0": + version "11.13.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-11.13.0.tgz#b0df8d6ef9b5001b2be3a94d909ce3c29a80f9e1" + integrity sha512-rx29MMkRdVmzunmiA4lzBYJNnXsW/PhG4kMBy2ATsYaDjGGR75dCFEVVROKpNwlVdcUX3xxlghKQOeDPBJobng== + +"@types/range-parser@*": + version "1.2.3" + resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c" + integrity sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA== + +"@types/serve-static@*": + version "1.13.2" + resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.2.tgz#f5ac4d7a6420a99a6a45af4719f4dcd8cd907a48" + integrity sha512-/BZ4QRLpH/bNYgZgwhKEh+5AsboDBcUdlBYgzoLX0fpj3Y2gp6EApyOlM3bK53wQS/OE1SrdSYBAbux2D1528Q== + dependencies: + "@types/express-serve-static-core" "*" + "@types/mime" "*" + +"@types/sinon@^7.0.11": + version "7.0.11" + resolved "https://registry.yarnpkg.com/@types/sinon/-/sinon-7.0.11.tgz#6f28f005a36e779b7db0f1359b9fb9eef72aae88" + integrity sha512-6ee09Ugx6GyEr0opUIakmxIWFNmqYPjkqa3/BuxCBokA0klsOLPgMD5K4q40lH7/yZVuJVzOfQpd7pipwjngkQ== + +"@types/uuid@^3.4.4": + version "3.4.4" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-3.4.4.tgz#7af69360fa65ef0decb41fd150bf4ca5c0cefdf5" + integrity sha512-tPIgT0GUmdJQNSHxp0X2jnpQfBSTfGxUMc/2CXBU2mnyTFVYVa2ojpoQ74w0U2yn2vw3jnC640+77lkFFpdVDw== + dependencies: + "@types/node" "*" + abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" @@ -135,6 +256,11 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" +arg@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.0.tgz#583c518199419e0037abb74062c37f8519e575f0" + integrity sha512-ZWc51jO3qegGkVh8Hwpv636EkbesNV5ZNQPCtRa+0qytRYPEs9IYT9qITY9buezqUH5uqyzlWLcufrzU2rffdg== + argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -262,6 +388,11 @@ buffer-fill@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" integrity sha1-+PeLdniYiO858gXNY39o5wISKyw= +buffer-from@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" + integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== + buffermaker@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/buffermaker/-/buffermaker-1.2.0.tgz#bb73252ec0882b7639e9b556b829dabfc2cae1ba" @@ -479,7 +610,7 @@ detect-libc@^1.0.3: resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= -diff@3.5.0, diff@^3.5.0: +diff@3.5.0, diff@^3.1.0, diff@^3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== @@ -1233,6 +1364,11 @@ long@1.1.2: resolved "https://registry.yarnpkg.com/long/-/long-1.1.2.tgz#eaef5951ca7551d96926b82da242db9d6b28fb53" integrity sha1-6u9ZUcp1UdlpJrgtokLbnWso+1M= +make-error@^1.1.1: + version "1.3.5" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.5.tgz#efe4e81f6db28cadd605c70f29c831b58ef776c8" + integrity sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g== + map-age-cleaner@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a" @@ -1957,7 +2093,15 @@ snappy@^6.0.1: nan "^2.12.1" prebuild-install "^5.2.2" -source-map@^0.6.1, source-map@~0.6.1: +source-map-support@^0.5.6: + version "0.5.12" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.12.tgz#b4f3b10d51857a5af0138d3ce8003b201613d599" + integrity sha512-4h2Pbvyy15EE02G+JOZpUCmqWJuqrs+sEkzewTm++BPi7Hvn/HwcqLAcNxYAyI0x13CpPPn+kMjl+hplXMHITQ== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + +source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== @@ -2129,6 +2273,17 @@ to-buffer@^1.1.1: resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk= +ts-node@^8.0.2: + version "8.1.0" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.1.0.tgz#8c4b37036abd448577db22a061fd7a67d47e658e" + integrity sha512-34jpuOrxDuf+O6iW1JpgTRDFynUZ1iEqtYruBqh35gICNjN8x+LpVcPAcwzLPi9VU6mdA3ym+x233nZmZp445A== + dependencies: + arg "^4.1.0" + diff "^3.1.0" + make-error "^1.1.1" + source-map-support "^0.5.6" + yn "^3.0.0" + tslib@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.3.tgz#d7e4dd79245d85428c4d7e4822a79917954ca286" @@ -2161,6 +2316,11 @@ type-is@~1.6.16: media-typer "0.3.0" mime-types "~2.1.24" +typescript@^3.4.1: + version "3.4.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.4.5.tgz#2d2618d10bb566572b8d7aad5180d84257d70a99" + integrity sha512-YycBxUb49UUhdNMU5aJ7z5Ej2XGmaIBL0x34vZ82fn3hGvD+bgrMrVDpatgz2f7YxUMJxMkbWxJZeAvDxVe7Vw== + uglify-js@^3.1.4: version "3.5.11" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.5.11.tgz#833442c0aa29b3a7d34344c7c63adaa3f3504f6a" @@ -2332,3 +2492,8 @@ yargs@^12.0.5: which-module "^2.0.0" y18n "^3.2.1 || ^4.0.0" yargs-parser "^11.1.1" + +yn@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.0.tgz#fcbe2db63610361afcc5eb9e0ac91e976d046114" + integrity sha512-kKfnnYkbTfrAdd0xICNFw7Atm8nKpLcLv9AZGEt+kczL/WQVai4e2V6ZN8U/O+iI6WrNuJjNNOyu4zfhl9D3Hg== From c844f0a59a4c33b8cf0c505ace057ad907bea900 Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Wed, 3 Apr 2019 18:23:06 +0200 Subject: [PATCH 02/10] turn current (manual) type-definitions into interfaces --- index.d.ts | 382 ---------------------------------------------- src/interfaces.ts | 376 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 376 insertions(+), 382 deletions(-) delete mode 100644 index.d.ts create mode 100644 src/interfaces.ts diff --git a/index.d.ts b/index.d.ts deleted file mode 100644 index aa92c25..0000000 --- a/index.d.ts +++ /dev/null @@ -1,382 +0,0 @@ -// declare type Buffer = any; - -declare module "sinek" { - - export interface KafkaHealthConfig { - thresholds?: { - consumer?: { - errors?: number; - lag?: number; - stallLag?: number; - minMessages?: number; - }; - producer?: { - errors?: number; - minMessages?: number; - }; - }; - } - - export interface NCommonKafkaOptions { - "builtin.features"?: string; - "client.id"?: string; - "metadata.broker.list": string; - "message.max.bytes"?: number; - "message.copy.max.bytes"?: number; - "receive.message.max.bytes"?: number; - "max.in.flight.requests.per.connection"?: number; - "metadata.request.timeout.ms"?: number; - "topic.metadata.refresh.interval.ms"?: number; - "metadata.max.age.ms"?: number; - "topic.metadata.refresh.fast.interval.ms"?: number; - "topic.metadata.refresh.fast.cnt"?: number; - "topic.metadata.refresh.sparse"?: boolean; - "topic.blacklist"?: string; - "debug"?: string; - "socket.timeout.ms"?: number; - "socket.blocking.max.ms"?: number; - "socket.send.buffer.bytes"?: number; - "socket.receive.buffer.bytes"?: number; - "socket.keepalive.enable"?: boolean; - "socket.nagle.disable"?: boolean; - "socket.max.fails"?: number; - "broker.address.ttl"?: number; - "broker.address.family"?: "any" | "v4" | "v6"; - "reconnect.backoff.jitter.ms"?: number; - "statistics.interval.ms"?: number; - "enabled_events"?: number; - "log_level"?: number; - "log.queue"?: boolean; - "log.thread.name"?: boolean; - "log.connection.close"?: boolean; - "internal.termination.signal"?: number; - "api.version.request"?: boolean; - "api.version.fallback.ms"?: number; - "broker.version.fallback"?: string; - "security.protocol"?: "plaintext" | "ssl" | "sasl_plaintext" | "sasl_ssl"; - "ssl.cipher.suites"?: string; - "ssl.key.location"?: string; - "ssl.key.password"?: string; - "ssl.certificate.location"?: string; - "ssl.ca.location"?: string; - "ssl.crl.location"?: string; - "sasl.mechanisms"?: string; - "sasl.kerberos.service.name"?: string; - "sasl.kerberos.principal"?: string; - "sasl.kerberos.kinit.cmd"?: string; - "sasl.kerberos.keytab"?: string; - "sasl.kerberos.min.time.before.relogin"?: number; - "sasl.username"?: string; - "sasl.password"?: string; - "partition.assignment.strategy"?: string; - "session.timeout.ms"?: number; - "heartbeat.interval.ms"?: number; - "group.protocol.type"?: string; - "coordinator.query.interval.ms"?: number; - "group.id"?: string; - "event_cb"?: boolean; - "dr_cb"?: boolean; - } - - export interface NConsumerKafkaOptions extends NCommonKafkaOptions { - "group.id": string; - "enable.auto.commit"?: boolean; - "auto.commit.interval.ms"?: number; - "enable.auto.offset.store"?: boolean; - "queued.min.messages"?: number; - "queued.max.messages.kbytes"?: number; - "fetch.wait.max.ms"?: number; - "fetch.message.max.bytes"?: number; - "fetch.min.bytes"?: number; - "fetch.error.backoff.ms"?: number; - "offset.store.method"?: "none" | "file" | "broker"; - "enable.partition.eof"?: boolean; - "check.crcs"?: boolean; - } - - export interface NProducerKafkaOptions extends NCommonKafkaOptions { - "queue.buffering.max.messages"?: number; - "queue.buffering.max.kbytes"?: number; - "queue.buffering.max.ms"?: number; - "message.send.max.retries"?: number; - "retry.backoff.ms"?: number; - "compression.codec"?: "none" | "gzip" | "snappy" | "lz4"; - "batch.num.messages"?: number; - "delivery.report.only.error"?: boolean; - } - - export interface KafkaConsumerConfig { - kafkaHost?: string; - groupId?: string; - workerPerPartition?: number; - options?: { - sessionTimeout?: number; - protocol?: [string]; - fromOffset?: string; - fetchMaxBytes?: number; - fetchMinBytes?: number; - fetchMaxWaitMs?: number; - heartbeatInterval?: number; - retryMinTimeout?: number; - autoCommit?: boolean; - autoCommitIntervalMs?: number; - requireAcks?: number; - ackTimeoutMs?: number; - }; - health?: KafkaHealthConfig; - tconf?: { - "auto.commit.enable"?: boolean; - "auto.commit.interval.ms"?: number; - "auto.offset.reset"?: "smallest" | "earliest" | "beginning" | "largest" | "latest" | "end" | "error"; - "offset.store.path"?: string; - "offset.store.sync.interval.ms"?: number; - "offset.store.method"?: "file" | "broker"; - "consume.callback.max.messages"?: number; - }; - noptions?: NConsumerKafkaOptions; - logger?: KafkaLogger; - } - - export interface KafkaProducerConfig { - kafkaHost?: string; - clientName?: string; - workerPerPartition?: number; - options?: { - sessionTimeout?: number; - protocol?: [string]; - fromOffset?: string; - fetchMaxBytes?: number; - fetchMinBytes?: number; - fetchMaxWaitMs?: number; - heartbeatInterval?: number; - retryMinTimeout?: number; - requireAcks?: number; - ackTimeoutMs?: number; - partitionerType?: number; - }; - health?: KafkaHealthConfig; - tconf?: { - "request.required.acks"?: number; - "request.timeout.ms"?: number; - "message.timeout.ms"?: number; - "produce.offset.report"?: boolean; - }; - noptions?: NProducerKafkaOptions; - logger?: KafkaLogger; - } - - export interface KafkaMessage { - topic: string; - partition: number; - offset: number; - key: Buffer | string; - value: Buffer | string | any; - size: number; - timestamp: number; - } - - export interface SortedMessageBatch { - [topic: string]: { - [partition: string]: KafkaMessage[]; - }; - } - - export interface BatchConfig { - batchSize?: number; - commitEveryNBatch?: number; - concurrency?: number; - commitSync?: boolean; - noBatchCommits?: boolean; - manualBatching?: boolean; - sortedManualBatch?: boolean; - } - - export interface ConsumerStats { - totalIncoming: number; - lastMessage: number; - receivedFirstMsg: boolean; - totalProcessed: number; - lastProcessed: number; - queueSize: null; - isPaused: boolean; - drainStats: null; - omittingQueue: boolean; - autoComitting: boolean; - consumedSinceCommit: number; - batch: { - current: number; - committs: number; - total: number; - config: BatchConfig; - currentEmptyFetches: number; - avgProcessingTime: number; - }; - lag: any; - totalErrors: number; - } - - export interface LagStatus { - topic: string; - partition: number; - lowDistance: number; - highDistance: number; - detail: { - lowOffset: number; - highOffset: number; - comittedOffset: number; - }; - } - - export interface ProducerStats { - totalPublished: number; - last: number; - isPaused: boolean; - totalErrors: number; - } - - export interface MessageReturn { - key: string; - partition: number; - offset?: number | null; - } - - export class NConsumer { - constructor(topic: Array | string, config: KafkaConsumerConfig); - on(eventName: "message", callback: (message: KafkaMessage) => any): void; - on(eventName: "error", callback: (error: any) => any): void; - on(eventName: "ready", callback: () => any): void; - on(eventName: "analytics", callback: (analytics: object) => void): void; - on(eventName: "batch", callback: (batch: Array) => void): void; - on(eventName: "first-drain-message", callback: () => void): void; - connect(asStream?: boolean, opts?: {asString?: boolean, asJSON?: boolean}): Promise; - - consume(syncEvent?: (message: KafkaMessage | KafkaMessage[] | SortedMessageBatch, callback: (error?: any) => void) => void, - asString?: boolean, asJSON?: boolean, options?: BatchConfig): Promise; - - pause(topics: Array): void; - resume(topics: Array): void; - getStats(): ConsumerStats; - close(commit?: boolean): object; - enableAnalytics(options: object): void; - haltAnalytics(): void; - addSubscriptions(topics: Array): Array; - adjustSubscription(topics: Array): Array; - commit(async: boolean): boolean; - commitMessage(async: boolean, message: KafkaMessage): boolean; - commitOffsetHard(topic: string, partition: number, offset: number, async: boolean): boolean; - commitLocalOffsetsForTopic(topic: string): any; - getOffsetForTopicPartition(topic: string, partition: number, timeout: number): Promise; - getComittedOffsets(timeout: number): Promise>; - getAssignedPartitions(): Array; - static findPartitionOffset(topic: string, partition: number, offsets: Array): object; - getLagStatus(noCache: boolean): Promise>; - getAnalytics(): object; - checkHealth(): object; - getTopicMetadata(topic: string, timeout: number): Promise; - getMetadata(timeout: number): Promise; - getTopicList(): Promise>; - } - - export class NProducer { - constructor(config: KafkaProducerConfig, _?: null, defaultPartitionCount?: number | "auto") - on(eventName: "error", callback: (error: any) => any): void; - on(eventName: "ready", callback: () => any): void; - connect(): Promise; - - send(topicName: string, message: string | Buffer, _partition?: number, _key?: string | Buffer, - _partitionKey?: string, _opaqueKey?: string): Promise; - - buffer(topic: string, identifier: string | undefined, payload: object, partition?: number, - version?: number, partitionKey?: string): Promise; - - bufferFormat(topic: string, identifier: string | undefined, payload: object, version?: number, - compressionType?: number, partitionKey?: string): Promise; - - bufferFormatPublish(topic: string, identifier: string | undefined, _payload: object, version?: number, - _?: null, partitionKey?: string, partition?: number): Promise; - - bufferFormatUpdate(topic: string, identifier: string | undefined, _payload: object, version?: number, - _?: null, partitionKey?: string, partition?: number): Promise; - - bufferFormatUnpublish(topic: string, identifier: string | undefined, _payload: object, version?: number, - _?: null, partitionKey?: string, partition?: number): Promise; - - pause(): void; - resume(): void; - getStats(): ProducerStats; - refreshMetadata(topics: Array): void; - close(): object; - enableAnalytics(options: object): void; - haltAnalytics(): void; - getAnalytics(): object; - checkHealth(): object; - getTopicMetadata(topic: string, timeout: number): Promise; - getMetadata(timeout: number): Promise; - getTopicList(): Promise>; - getPartitionCountOfTopic(topic: string): Promise; - getStoredPartitionCounts(): object; - tombstone(topic: string, key: string | Buffer, _partition?: number | null): Promise; - } - - export class Consumer { - constructor(topic: String, config: KafkaConsumerConfig); - on(eventName: "message", callback: (message: object) => any): void; - on(eventName: "error", callback: (error: any) => any): void; - connect(backpressure?: boolean): Promise; - consume(syncEvent?: object): Promise; - consumeOnce(syncEvent?: object, drainThreshold?: number, timeout?: number): object; - pause(): void; - resume(): void; - getStats(): object; - close(commit?: boolean): object; - } - - export class Producer { - constructor(config: KafkaProducerConfig, topic: Array, defaultPartitionCount: number); - on(eventName: "error", callback: (error: any) => any): void; - connect(): Promise; - send(topic: string, message: string | string[]): Promise; - buffer(topic: string, identifier?: string, payload?: object, compressionType?: number): Promise; - bufferFormat(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; - bufferFormatPublish(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; - bufferFormatUpdate(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; - bufferFormatUnpublish(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; - pause(): void; - resume(): void; - getStats(): object; - refreshMetadata(topics: Array): void; - close(): object; - } - - export class Kafka { - constructor(conString: String, logger: object, connectDirectlyToBroker: boolean) - } - - export class Drainer { - constructor(consumer: object, asyncLimit: number, autoJsonParsing: boolean, omitQueue: boolean, commitOnDrain: boolean) - } - - export class Publisher { - constructor(producer: object, partitionCount: number, autoFlushBuffer: number, flushPeriod: number) - } - - export class PartitionDrainer { - constructor(consumer: object, asyncLimit: number, commitOnDrain: boolean, autoJsonParsing: boolean) - } - - export class PartitionQueue { - constructor(partition: object, drainEvent: object, drainer: object, asyncLimit: number, queueDrain: object) - } - - export interface MessageType { - key: String; - value: String; - } - - export interface KafkaLogger { - debug(message: string): void; - info(message: string): void; - warn(message: string, error?: Error): void; - error(error: string | Error): void; - } -} diff --git a/src/interfaces.ts b/src/interfaces.ts new file mode 100644 index 0000000..36761cf --- /dev/null +++ b/src/interfaces.ts @@ -0,0 +1,376 @@ +export interface KafkaHealthConfig { + thresholds?: { + consumer?: { + errors?: number; + lag?: number; + stallLag?: number; + minMessages?: number; + }; + producer?: { + errors?: number; + minMessages?: number; + }; + }; +} + +export interface NCommonKafkaOptions { + "builtin.features"?: string; + "client.id"?: string; + "metadata.broker.list": string; + "message.max.bytes"?: number; + "message.copy.max.bytes"?: number; + "receive.message.max.bytes"?: number; + "max.in.flight.requests.per.connection"?: number; + "metadata.request.timeout.ms"?: number; + "topic.metadata.refresh.interval.ms"?: number; + "metadata.max.age.ms"?: number; + "topic.metadata.refresh.fast.interval.ms"?: number; + "topic.metadata.refresh.fast.cnt"?: number; + "topic.metadata.refresh.sparse"?: boolean; + "topic.blacklist"?: string; + "debug"?: string; + "socket.timeout.ms"?: number; + "socket.blocking.max.ms"?: number; + "socket.send.buffer.bytes"?: number; + "socket.receive.buffer.bytes"?: number; + "socket.keepalive.enable"?: boolean; + "socket.nagle.disable"?: boolean; + "socket.max.fails"?: number; + "broker.address.ttl"?: number; + "broker.address.family"?: "any" | "v4" | "v6"; + "reconnect.backoff.jitter.ms"?: number; + "statistics.interval.ms"?: number; + "enabled_events"?: number; + "log_level"?: number; + "log.queue"?: boolean; + "log.thread.name"?: boolean; + "log.connection.close"?: boolean; + "internal.termination.signal"?: number; + "api.version.request"?: boolean; + "api.version.fallback.ms"?: number; + "broker.version.fallback"?: string; + "security.protocol"?: "plaintext" | "ssl" | "sasl_plaintext" | "sasl_ssl"; + "ssl.cipher.suites"?: string; + "ssl.key.location"?: string; + "ssl.key.password"?: string; + "ssl.certificate.location"?: string; + "ssl.ca.location"?: string; + "ssl.crl.location"?: string; + "sasl.mechanisms"?: string; + "sasl.kerberos.service.name"?: string; + "sasl.kerberos.principal"?: string; + "sasl.kerberos.kinit.cmd"?: string; + "sasl.kerberos.keytab"?: string; + "sasl.kerberos.min.time.before.relogin"?: number; + "sasl.username"?: string; + "sasl.password"?: string; + "partition.assignment.strategy"?: string; + "session.timeout.ms"?: number; + "heartbeat.interval.ms"?: number; + "group.protocol.type"?: string; + "coordinator.query.interval.ms"?: number; + "group.id"?: string; + "event_cb"?: boolean; + "dr_cb"?: boolean; +} + +export interface NConsumerKafkaOptions extends NCommonKafkaOptions { + "group.id": string; + "enable.auto.commit"?: boolean; + "auto.commit.interval.ms"?: number; + "enable.auto.offset.store"?: boolean; + "queued.min.messages"?: number; + "queued.max.messages.kbytes"?: number; + "fetch.wait.max.ms"?: number; + "fetch.message.max.bytes"?: number; + "fetch.min.bytes"?: number; + "fetch.error.backoff.ms"?: number; + "offset.store.method"?: "none" | "file" | "broker"; + "enable.partition.eof"?: boolean; + "check.crcs"?: boolean; +} + +export interface NProducerKafkaOptions extends NCommonKafkaOptions { + "queue.buffering.max.messages"?: number; + "queue.buffering.max.kbytes"?: number; + "queue.buffering.max.ms"?: number; + "message.send.max.retries"?: number; + "retry.backoff.ms"?: number; + "compression.codec"?: "none" | "gzip" | "snappy" | "lz4"; + "batch.num.messages"?: number; + "delivery.report.only.error"?: boolean; +} + +export interface KafkaConsumerConfig { + kafkaHost?: string; + groupId?: string; + workerPerPartition?: number; + options?: { + sessionTimeout?: number; + protocol?: [string]; + fromOffset?: string; + fetchMaxBytes?: number; + fetchMinBytes?: number; + fetchMaxWaitMs?: number; + heartbeatInterval?: number; + retryMinTimeout?: number; + autoCommit?: boolean; + autoCommitIntervalMs?: number; + requireAcks?: number; + ackTimeoutMs?: number; + }; + health?: KafkaHealthConfig; + tconf?: { + "auto.commit.enable"?: boolean; + "auto.commit.interval.ms"?: number; + "auto.offset.reset"?: "smallest" | "earliest" | "beginning" | "largest" | "latest" | "end" | "error"; + "offset.store.path"?: string; + "offset.store.sync.interval.ms"?: number; + "offset.store.method"?: "file" | "broker"; + "consume.callback.max.messages"?: number; + }; + noptions?: NConsumerKafkaOptions; + logger?: KafkaLogger; +} + +export interface KafkaProducerConfig { + kafkaHost?: string; + clientName?: string; + workerPerPartition?: number; + options?: { + sessionTimeout?: number; + protocol?: [string]; + fromOffset?: string; + fetchMaxBytes?: number; + fetchMinBytes?: number; + fetchMaxWaitMs?: number; + heartbeatInterval?: number; + retryMinTimeout?: number; + requireAcks?: number; + ackTimeoutMs?: number; + partitionerType?: number; + }; + health?: KafkaHealthConfig; + tconf?: { + "request.required.acks"?: number; + "request.timeout.ms"?: number; + "message.timeout.ms"?: number; + "produce.offset.report"?: boolean; + }; + noptions?: NProducerKafkaOptions; + logger?: KafkaLogger; +} + +export interface KafkaMessage { + topic: string; + partition: number; + offset: number; + key: Buffer | string; + value: Buffer | string | any; + size: number; + timestamp: number; +} + +export interface SortedMessageBatch { + [topic: string]: { + [partition: string]: KafkaMessage[]; + }; +} + +export interface BatchConfig { + batchSize?: number; + commitEveryNBatch?: number; + concurrency?: number; + commitSync?: boolean; + noBatchCommits?: boolean; + manualBatching?: boolean; + sortedManualBatch?: boolean; +} + +export interface ConsumerStats { + totalIncoming: number; + lastMessage: number; + receivedFirstMsg: boolean; + totalProcessed: number; + lastProcessed: number; + queueSize: null; + isPaused: boolean; + drainStats: null; + omittingQueue: boolean; + autoComitting: boolean; + consumedSinceCommit: number; + batch: { + current: number; + committs: number; + total: number; + config: BatchConfig; + currentEmptyFetches: number; + avgProcessingTime: number; + }; + lag: any; + totalErrors: number; +} + +export interface LagStatus { + topic: string; + partition: number; + lowDistance: number; + highDistance: number; + detail: { + lowOffset: number; + highOffset: number; + comittedOffset: number; + }; +} + +export interface ProducerStats { + totalPublished: number; + last: number; + isPaused: boolean; + totalErrors: number; +} + +export interface MessageReturn { + key: string; + partition: number; + offset?: number | null; +} + +export interface INConsumer { + constructor(topic: Array | string, config: KafkaConsumerConfig); + on(eventName: "message", callback: (message: KafkaMessage) => any): void; + on(eventName: "error", callback: (error: any) => any): void; + on(eventName: "ready", callback: () => any): void; + on(eventName: "analytics", callback: (analytics: object) => void): void; + on(eventName: "batch", callback: (batch: Array) => void): void; + on(eventName: "first-drain-message", callback: () => void): void; + connect(asStream?: boolean, opts?: {asString?: boolean, asJSON?: boolean}): Promise; + + consume(syncEvent?: (message: KafkaMessage | KafkaMessage[] | SortedMessageBatch, callback: (error?: any) => void) => void, + asString?: boolean, asJSON?: boolean, options?: BatchConfig): Promise; + + pause(topics: Array): void; + resume(topics: Array): void; + getStats(): ConsumerStats; + close(commit?: boolean): object; + enableAnalytics(options: object): void; + haltAnalytics(): void; + addSubscriptions(topics: Array): Array; + adjustSubscription(topics: Array): Array; + commit(async: boolean): boolean; + commitMessage(async: boolean, message: KafkaMessage): boolean; + commitOffsetHard(topic: string, partition: number, offset: number, async: boolean): boolean; + commitLocalOffsetsForTopic(topic: string): any; + getOffsetForTopicPartition(topic: string, partition: number, timeout: number): Promise; + getComittedOffsets(timeout: number): Promise>; + getAssignedPartitions(): Array; + getLagStatus(noCache: boolean): Promise>; + getAnalytics(): object; + checkHealth(): object; + getTopicMetadata(topic: string, timeout: number): Promise; + getMetadata(timeout: number): Promise; + getTopicList(): Promise>; +} + +export interface INProducer { + constructor(config: KafkaProducerConfig, _?: null, defaultPartitionCount?: number | "auto") + on(eventName: "error", callback: (error: any) => any): void; + on(eventName: "ready", callback: () => any): void; + connect(): Promise; + + send(topicName: string, message: string | Buffer, _partition?: number, _key?: string | Buffer, + _partitionKey?: string, _opaqueKey?: string): Promise; + + buffer(topic: string, identifier: string | undefined, payload: object, partition?: number, + version?: number, partitionKey?: string): Promise; + + bufferFormat(topic: string, identifier: string | undefined, payload: object, version?: number, + compressionType?: number, partitionKey?: string): Promise; + + bufferFormatPublish(topic: string, identifier: string | undefined, _payload: object, version?: number, + _?: null, partitionKey?: string, partition?: number): Promise; + + bufferFormatUpdate(topic: string, identifier: string | undefined, _payload: object, version?: number, + _?: null, partitionKey?: string, partition?: number): Promise; + + bufferFormatUnpublish(topic: string, identifier: string | undefined, _payload: object, version?: number, + _?: null, partitionKey?: string, partition?: number): Promise; + + pause(): void; + resume(): void; + getStats(): ProducerStats; + refreshMetadata(topics: Array): void; + close(): object; + enableAnalytics(options: object): void; + haltAnalytics(): void; + getAnalytics(): object; + checkHealth(): object; + getTopicMetadata(topic: string, timeout: number): Promise; + getMetadata(timeout: number): Promise; + getTopicList(): Promise>; + getPartitionCountOfTopic(topic: string): Promise; + getStoredPartitionCounts(): object; + tombstone(topic: string, key: string | Buffer, _partition?: number | null): Promise; +} + +export interface IConsumer { + constructor(topic: string, config: KafkaConsumerConfig); + on(eventName: "message", callback: (message: object) => any): void; + on(eventName: "error", callback: (error: any) => any): void; + connect(backpressure?: boolean): Promise; + consume(syncEvent?: object): Promise; + consumeOnce(syncEvent?: object, drainThreshold?: number, timeout?: number): object; + pause(): void; + resume(): void; + getStats(): object; + close(commit?: boolean): object; +} + +export interface IProducer { + constructor(config: KafkaProducerConfig, topic: Array, defaultPartitionCount: number); + on(eventName: "error", callback: (error: any) => any): void; + connect(): Promise; + send(topic: string, message: string | string[]): Promise; + buffer(topic: string, identifier?: string, payload?: object, compressionType?: number): Promise; + bufferFormat(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; + bufferFormatPublish(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; + bufferFormatUpdate(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; + bufferFormatUnpublish(topic: string, identifier?: string, payload?: object, version?: number, compressionType?: number, partitionKey?: string): Promise; + pause(): void; + resume(): void; + getStats(): object; + refreshMetadata(topics: Array): void; + close(): object; +} + +export interface IKafka { + constructor(conString: string, logger: object, connectDirectlyToBroker: boolean) +} + +export interface IDrainer { + constructor(consumer: object, asyncLimit: number, autoJsonParsing: boolean, omitQueue: boolean, commitOnDrain: boolean) +} + +export interface IPublisher { + constructor(producer: object, partitionCount: number, autoFlushBuffer: number, flushPeriod: number) +} + +export interface IPartitionDrainer { + constructor(consumer: object, asyncLimit: number, commitOnDrain: boolean, autoJsonParsing: boolean) +} + +export interface IPartitionQueue { + constructor(partition: object, drainEvent: object, drainer: object, asyncLimit: number, queueDrain: object) +} + +export interface MessageType { + key: string; + value: string; +} + +export interface KafkaLogger { + debug(message: string): void; + info(message: string): void; + warn(message: string, error?: Error): void; + error(error: string | Error): void; +} From 17b1e0715b90b02ca69e6522b95304d13ff12607 Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Thu, 21 Mar 2019 23:04:10 +0100 Subject: [PATCH 03/10] add typescript distribution settings --- .gitignore | 3 +++ .npmignore | 3 ++- package.json | 7 ++++--- tsconfig.dist.json | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 tsconfig.dist.json diff --git a/.gitignore b/.gitignore index e35139e..ab7f666 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ logs .vscode certs + +# generated files +dist diff --git a/.npmignore b/.npmignore index d56fa9d..e7f57d9 100644 --- a/.npmignore +++ b/.npmignore @@ -4,6 +4,7 @@ kafka-setup/ perf/ sasl-ssl-example/ ssl-example/ +src test/ .idea .editorconfig @@ -12,4 +13,4 @@ test/ .eslintrc.js .gitignore .jshintrc -.travis.yml \ No newline at end of file +.travis.yml diff --git a/package.json b/package.json index 64fea71..01c94d9 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,8 @@ "name": "sinek", "version": "7.29.3", "description": "Node.js kafka client, consumer, producer polite out of the box", - "main": "index.js", - "typings": "index.d.ts", + "main": "dist/index.js", + "types": "dist/index.d.ts", "engines": { "node": ">=8.11.3" }, @@ -15,7 +15,8 @@ "kafka:logs": "docker-compose --file ./kafka-setup/docker-compose.yml logs -f", "kafka:console": "./kafka-setup/kafka-console.sh", "test": "_mocha --recursive --timeout 32500 --exit -R spec test/int", - "yarn:openssl": "LDFLAGS='-L/usr/local/opt/openssl/lib' CPPFLAGS='-I/usr/local/opt/openssl/include' yarn" + "yarn:openssl": "LDFLAGS='-L/usr/local/opt/openssl/lib' CPPFLAGS='-I/usr/local/opt/openssl/include' yarn", + "prepublishOnly": "tsc -p tsconfig.dist.json" }, "repository": { "type": "git", diff --git a/tsconfig.dist.json b/tsconfig.dist.json new file mode 100644 index 0000000..8272192 --- /dev/null +++ b/tsconfig.dist.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig", + "compilerOptions": { + "rootDir": "src/", + "noEmit": false, + "outDir": "dist", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": [ + "src" + ] +} From 8417b2877b885b76c533dde8053b0088c1b6cffd Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Thu, 21 Mar 2019 23:04:10 +0100 Subject: [PATCH 04/10] rename js-files to typescript --- lib/Sinek.js => src/Sinek.ts | 0 .../Consumer.js => src/connect/Consumer.ts | 0 .../Producer.js => src/connect/Producer.ts | 0 {lib => src}/connect/README.md | 0 index.js => src/index.ts | 0 lib/kafka/Drainer.js => src/kafka/Drainer.ts | 0 lib/kafka/Kafka.js => src/kafka/Kafka.ts | 0 .../kafka/PartitionDrainer.ts | 0 .../kafka/PartitionQueue.ts | 0 .../Publisher.js => src/kafka/Publisher.ts | 0 .../librdkafka/Analytics.ts | 0 .../Health.js => src/librdkafka/Health.ts | 0 .../Metadata.js => src/librdkafka/Metadata.ts | 0 .../librdkafka/NConsumer.ts | 32 +++++++++---------- .../librdkafka/NProducer.ts | 30 ++++++++--------- {lib => src}/librdkafka/README.md | 0 .../tools/CompressionTypes.ts | 0 lib/tools/index.js => src/tools/index.ts | 0 18 files changed, 31 insertions(+), 31 deletions(-) rename lib/Sinek.js => src/Sinek.ts (100%) rename lib/connect/Consumer.js => src/connect/Consumer.ts (100%) rename lib/connect/Producer.js => src/connect/Producer.ts (100%) rename {lib => src}/connect/README.md (100%) rename index.js => src/index.ts (100%) rename lib/kafka/Drainer.js => src/kafka/Drainer.ts (100%) rename lib/kafka/Kafka.js => src/kafka/Kafka.ts (100%) rename lib/kafka/PartitionDrainer.js => src/kafka/PartitionDrainer.ts (100%) rename lib/kafka/PartitionQueue.js => src/kafka/PartitionQueue.ts (100%) rename lib/kafka/Publisher.js => src/kafka/Publisher.ts (100%) rename lib/librdkafka/Analytics.js => src/librdkafka/Analytics.ts (100%) rename lib/librdkafka/Health.js => src/librdkafka/Health.ts (100%) rename lib/librdkafka/Metadata.js => src/librdkafka/Metadata.ts (100%) rename lib/librdkafka/NConsumer.js => src/librdkafka/NConsumer.ts (98%) rename lib/librdkafka/NProducer.js => src/librdkafka/NProducer.ts (96%) rename {lib => src}/librdkafka/README.md (100%) rename lib/tools/CompressionTypes.js => src/tools/CompressionTypes.ts (100%) rename lib/tools/index.js => src/tools/index.ts (100%) diff --git a/lib/Sinek.js b/src/Sinek.ts similarity index 100% rename from lib/Sinek.js rename to src/Sinek.ts diff --git a/lib/connect/Consumer.js b/src/connect/Consumer.ts similarity index 100% rename from lib/connect/Consumer.js rename to src/connect/Consumer.ts diff --git a/lib/connect/Producer.js b/src/connect/Producer.ts similarity index 100% rename from lib/connect/Producer.js rename to src/connect/Producer.ts diff --git a/lib/connect/README.md b/src/connect/README.md similarity index 100% rename from lib/connect/README.md rename to src/connect/README.md diff --git a/index.js b/src/index.ts similarity index 100% rename from index.js rename to src/index.ts diff --git a/lib/kafka/Drainer.js b/src/kafka/Drainer.ts similarity index 100% rename from lib/kafka/Drainer.js rename to src/kafka/Drainer.ts diff --git a/lib/kafka/Kafka.js b/src/kafka/Kafka.ts similarity index 100% rename from lib/kafka/Kafka.js rename to src/kafka/Kafka.ts diff --git a/lib/kafka/PartitionDrainer.js b/src/kafka/PartitionDrainer.ts similarity index 100% rename from lib/kafka/PartitionDrainer.js rename to src/kafka/PartitionDrainer.ts diff --git a/lib/kafka/PartitionQueue.js b/src/kafka/PartitionQueue.ts similarity index 100% rename from lib/kafka/PartitionQueue.js rename to src/kafka/PartitionQueue.ts diff --git a/lib/kafka/Publisher.js b/src/kafka/Publisher.ts similarity index 100% rename from lib/kafka/Publisher.js rename to src/kafka/Publisher.ts diff --git a/lib/librdkafka/Analytics.js b/src/librdkafka/Analytics.ts similarity index 100% rename from lib/librdkafka/Analytics.js rename to src/librdkafka/Analytics.ts diff --git a/lib/librdkafka/Health.js b/src/librdkafka/Health.ts similarity index 100% rename from lib/librdkafka/Health.js rename to src/librdkafka/Health.ts diff --git a/lib/librdkafka/Metadata.js b/src/librdkafka/Metadata.ts similarity index 100% rename from lib/librdkafka/Metadata.js rename to src/librdkafka/Metadata.ts diff --git a/lib/librdkafka/NConsumer.js b/src/librdkafka/NConsumer.ts similarity index 98% rename from lib/librdkafka/NConsumer.js rename to src/librdkafka/NConsumer.ts index 9d479b6..a25e525 100644 --- a/lib/librdkafka/NConsumer.js +++ b/src/librdkafka/NConsumer.ts @@ -63,7 +63,7 @@ class NConsumer extends EventEmitter { this.topics = Array.isArray(topics) ? topics : [topics]; this.config = config; - this._health = new ConsumerHealth(this, this.config.health || {}); + this._health = new ConsumerHealth(this, this.config.health || {}); this.consumer = null; this._resume = true; @@ -121,7 +121,7 @@ class NConsumer extends EventEmitter { // Make analytics available immediately return this._runAnalytics() - .then(() => this._runLagCheck()); + .then(() => this._runLagCheck()); } /** @@ -534,7 +534,7 @@ class NConsumer extends EventEmitter { if(this._isAutoCommitting !== null && typeof this._isAutoCommitting !== "undefined"){ this.config.logger.warn("enable.auto.commit has no effect in 1:n consume-mode, set to null or undefined to remove this message." + - "You can pass 'noBatchCommits' as true via options to .consume(), if you want to commit manually."); + "You can pass 'noBatchCommits' as true via options to .consume(), if you want to commit manually."); } if(this._isAutoCommitting){ @@ -545,7 +545,7 @@ class NConsumer extends EventEmitter { this._batchConfig = options; //store for stats if(manualBatching) { - + this.config.logger.info("Batching manually.."); super.on("batch", (messages) => { @@ -553,12 +553,12 @@ class NConsumer extends EventEmitter { const startBPT = Date.now(); this._totalIncomingMessages += messages.length; this._lastReceived = Date.now(); - + const mappedMessages = messages.map((message) => { this.config.logger.debug(message); message.value = this._convertMessageValue(message.value, asString, asJSON); - + if(!this._firstMessageConsumed){ this._firstMessageConsumed = true; super.emit("first-drain-message", message); @@ -572,7 +572,7 @@ class NConsumer extends EventEmitter { const sortedBatch = {}; if(sortedManualBatch) { mappedMessages.forEach((mappedMessage) => { - + if(typeof sortedBatch[mappedMessage.topic] === "undefined"){ sortedBatch[mappedMessage.topic] = {}; } @@ -595,7 +595,7 @@ class NConsumer extends EventEmitter { this._totalProcessedMessages += mappedMessages.length; this._lastProcessed = Date.now(); - + //when all messages from the batch are processed this._avgBatchProcessingTime = (this._avgBatchProcessingTime + (Date.now() - startBPT)) / 2; @@ -633,7 +633,7 @@ class NConsumer extends EventEmitter { } }); }); - + } else { // TODO: refactor this monstrousity this.config.logger.info("Batching automatically.."); @@ -853,7 +853,7 @@ class NConsumer extends EventEmitter { /** * commits all local stored offsets for assigned partitions of a topic - * @param {string} topic + * @param {string} topic */ async commitLocalOffsetsForTopic(topic) { @@ -861,12 +861,12 @@ class NConsumer extends EventEmitter { this.config.logger.debug(`Committing local offsets for topic ${topic}`); } - const assignedPartitionsOfTopic = - this.getAssignedPartitions() - .filter((topicPartition) => topicPartition.topic === topic); + const assignedPartitionsOfTopic = + this.getAssignedPartitions() + .filter((topicPartition) => topicPartition.topic === topic); const currentLocalOffsets = this.consumer.position(assignedPartitionsOfTopic) - .filter((topicPartition) => typeof topicPartition.offset !== "undefined"); + .filter((topicPartition) => typeof topicPartition.offset !== "undefined"); if (this.config && this.config.logger && this.config.logger.debug) { this.config.logger.debug(`Committing local offsets for topic ${topic} as`, currentLocalOffsets); @@ -1151,8 +1151,8 @@ class NConsumer extends EventEmitter { } return this._analytics.run() - .then(res => super.emit("analytics", res)) - .catch(error => super.emit("error", error)); + .then(res => super.emit("analytics", res)) + .catch(error => super.emit("error", error)); } /** diff --git a/lib/librdkafka/NProducer.js b/src/librdkafka/NProducer.ts similarity index 96% rename from lib/librdkafka/NProducer.js rename to src/librdkafka/NProducer.ts index 14dd66b..922239d 100644 --- a/lib/librdkafka/NProducer.js +++ b/src/librdkafka/NProducer.ts @@ -74,7 +74,7 @@ class NProducer extends EventEmitter { } this.config = config; - this._health = new ProducerHealth(this, this.config.health || {}); + this._health = new ProducerHealth(this, this.config.health || {}); this.paused = false; this.producer = null; @@ -92,16 +92,16 @@ class NProducer extends EventEmitter { this.config.logger.info(`using murmur ${this._murmurHashVersion} partitioner.`); switch (this._murmurHashVersion) { - case "2": - this._murmur = (key, partitionCount) => murmur2Partitioner.partition(key, partitionCount); - break; + case "2": + this._murmur = (key, partitionCount) => murmur2Partitioner.partition(key, partitionCount); + break; - case "3": - this._murmur = (key, partitionCount) => murmur.v3(key) % partitionCount; - break; + case "3": + this._murmur = (key, partitionCount) => murmur.v3(key) % partitionCount; + break; - default: - throw new Error(`${this._murmurHashVersion} is not a supported murmur hash version. Choose '2' or '3'.`); + default: + throw new Error(`${this._murmurHashVersion} is not a supported murmur hash version. Choose '2' or '3'.`); } this._errors = 0; @@ -310,7 +310,7 @@ class NProducer extends EventEmitter { maxPartitions = await this.getPartitionCountOfTopic(topicName); if (maxPartitions === -1) { throw new Error("defaultPartition set to 'auto', but was not able to resolve partition count for topic" + - topicName + ", please make sure the topic exists before starting the producer in auto mode."); + topicName + ", please make sure the topic exists before starting the producer in auto mode."); } } else { maxPartitions = this.defaultPartitionCount; @@ -330,7 +330,7 @@ class NProducer extends EventEmitter { partition, key })); - + const producedAt = Date.now(); this._lastProcessed = producedAt; @@ -493,7 +493,7 @@ class NProducer extends EventEmitter { * on a key compacted topic/partition this will delete all occurances of the key * @param {string} topic - name of the topic * @param {string} key - key - * @param {number|null} _partition - optional partition + * @param {number|null} _partition - optional partition */ tombstone(topic, key, _partition = null){ @@ -577,7 +577,7 @@ class NProducer extends EventEmitter { /** * returns a list of available kafka topics on the connected brokers - * @param {number} timeout + * @param {number} timeout */ async getTopicList(timeout = 2500){ const metadata = await this.getMetadata(timeout); @@ -675,8 +675,8 @@ class NProducer extends EventEmitter { } this._analytics.run() - .then(res => super.emit("analytics", res)) - .catch(error => super.emit("error", error)); + .then(res => super.emit("analytics", res)) + .catch(error => super.emit("error", error)); } /** diff --git a/lib/librdkafka/README.md b/src/librdkafka/README.md similarity index 100% rename from lib/librdkafka/README.md rename to src/librdkafka/README.md diff --git a/lib/tools/CompressionTypes.js b/src/tools/CompressionTypes.ts similarity index 100% rename from lib/tools/CompressionTypes.js rename to src/tools/CompressionTypes.ts diff --git a/lib/tools/index.js b/src/tools/index.ts similarity index 100% rename from lib/tools/index.js rename to src/tools/index.ts From 5fb3be39f173c54abb0779afa94fad207d989808 Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Thu, 21 Mar 2019 23:04:10 +0100 Subject: [PATCH 05/10] switch imports/exports --- src/Sinek.ts | 73 +++++++++++++++++++++-------------- src/connect/Consumer.ts | 14 +++---- src/connect/Producer.ts | 16 +++----- src/index.ts | 2 +- src/kafka/Drainer.ts | 12 ++---- src/kafka/Kafka.ts | 17 +++----- src/kafka/PartitionDrainer.ts | 11 ++---- src/kafka/PartitionQueue.ts | 8 +--- src/kafka/Publisher.ts | 18 ++++----- src/librdkafka/Analytics.ts | 11 +----- src/librdkafka/Health.ts | 13 ++----- src/librdkafka/Metadata.ts | 6 +-- src/librdkafka/NConsumer.ts | 20 ++++------ src/librdkafka/NProducer.ts | 29 +++++--------- src/tools/CompressionTypes.ts | 5 +-- src/tools/index.ts | 7 +--- 16 files changed, 106 insertions(+), 156 deletions(-) diff --git a/src/Sinek.ts b/src/Sinek.ts index 02cc1ff..7d4bf61 100644 --- a/src/Sinek.ts +++ b/src/Sinek.ts @@ -1,30 +1,43 @@ -"use strict"; - -const util = require("util"); - -const Kafka = require("./kafka/Kafka.js"); -const Drainer = require("./kafka/Drainer.js"); -const Publisher = require("./kafka/Publisher.js"); -const PartitionDrainer = require("./kafka/PartitionDrainer.js"); - -const Consumer = require("./connect/Consumer.js"); -const Producer = require("./connect/Producer.js"); - -const NConsumer = require("./librdkafka/NConsumer.js"); -const NProducer = require("./librdkafka/NProducer.js"); - -const Health = require("./librdkafka/Health.js"); -const Analytics = require("./librdkafka/Analytics.js"); - -module.exports = { - Kafka: util.deprecate(Kafka, "Kafka is deprecated, please use 'NConsumer' if possible."), - Drainer: util.deprecate(Drainer, "Drainer is deprecated, please use 'NConsumer' if possible."), - PartitionDrainer: util.deprecate(PartitionDrainer, "PartitionDrainer is deprecated, please use 'NConsumer' if possible."), - Publisher: util.deprecate(Publisher, "Publisher is deprecated, please use 'NProducer' if possible."), - Consumer: util.deprecate(Consumer, "Consumer is deprecated, please use (noptions) 'NConsumer' if possible."), - Producer: util.deprecate(Producer, "Producer is deprecated, please use (noptions) 'NProducer' if possible."), - NConsumer, - NProducer, - Health, - Analytics -}; \ No newline at end of file +import util from "util"; + +import {default as DeprecatedKafka} from "./kafka/Kafka"; +import {default as DeprecatedDrainer} from "./kafka/Drainer"; +import {default as DeprecatedPublisher} from "./kafka/Publisher"; +import {default as DeprecatedPartitionDrainer} from "./kafka/PartitionDrainer"; + +import {default as DeprecatedConsumer} from "./connect/Consumer"; +import {default as DeprecatedProducer} from "./connect/Producer"; + +export {default as NConsumer} from "./librdkafka/NConsumer"; +export {default as NProducer} from "./librdkafka/NProducer"; + +import {ProducerHealth, ConsumerHealth} from "./librdkafka/Health"; +import {ProducerAnalytics, ConsumerAnalytics} from "./librdkafka/Analytics"; + + +const Kafka = util.deprecate(DeprecatedKafka, "Kafka is deprecated, please use 'NConsumer' if possible."); +const Drainer = util.deprecate(DeprecatedDrainer, "Drainer is deprecated, please use 'NConsumer' if possible."); +const PartitionDrainer = util.deprecate(DeprecatedPartitionDrainer, "PartitionDrainer is deprecated, please use 'NConsumer' if possible."); +const Publisher = util.deprecate(DeprecatedPublisher, "Publisher is deprecated, please use 'NProducer' if possible."); +const Consumer = util.deprecate(DeprecatedConsumer, "Consumer is deprecated, please use (noptions) 'NConsumer' if possible."); +const Producer = util.deprecate(DeprecatedProducer, "Producer is deprecated, please use (noptions) 'NProducer' if possible."); + +const Health = { + ProducerHealth, + ConsumerHealth, +}; + +const Analytics = { + ProducerAnalytics, + ConsumerAnalytics, +}; + + +export { + Kafka, Drainer, PartitionDrainer, Publisher, + + Consumer, Producer, + + Health, + Analytics +} diff --git a/src/connect/Consumer.ts b/src/connect/Consumer.ts index 488c746..cde37ed 100644 --- a/src/connect/Consumer.ts +++ b/src/connect/Consumer.ts @@ -1,12 +1,10 @@ -"use strict"; +import Promise from "bluebird"; +import EventEmitter from "events"; -const Promise = require("bluebird"); -const EventEmitter = require("events"); +import Kafka from "./../kafka/Kafka"; +import Drainer from "./../kafka/Drainer"; -const Kafka = require("./../kafka/Kafka.js"); -const Drainer = require("./../kafka/Drainer.js"); - -class Consumer extends EventEmitter { +export default class Consumer extends EventEmitter { constructor(topic, config = { options: {} }) { super(); @@ -118,5 +116,3 @@ class Consumer extends EventEmitter { } } } - -module.exports = Consumer; diff --git a/src/connect/Producer.ts b/src/connect/Producer.ts index 7b6d53d..1e495f2 100644 --- a/src/connect/Producer.ts +++ b/src/connect/Producer.ts @@ -1,13 +1,11 @@ -"use strict"; +import EventEmitter from "events"; +import Promise from "bluebird"; +import uuid from "uuid"; -const EventEmitter = require("events"); -const Promise = require("bluebird"); -const uuid = require("uuid"); +import Kafka from "./../kafka/Kafka"; +import Publisher from "./../kafka/Publisher"; -const Kafka = require("./../kafka/Kafka.js"); -const Publisher = require("./../kafka/Publisher.js"); - -class Producer extends EventEmitter { +export default class Producer extends EventEmitter { constructor(config, topic = [], defaultPartitionCount = 1) { super(); @@ -191,5 +189,3 @@ class Producer extends EventEmitter { return Math.floor(Math.random() * (max - min + 1)) + min; } } - -module.exports = Producer; diff --git a/src/index.ts b/src/index.ts index eb7aee2..88b1e58 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1 @@ -module.exports = require("./lib/Sinek.js"); \ No newline at end of file +export * from "./Sinek"; diff --git a/src/kafka/Drainer.ts b/src/kafka/Drainer.ts index fdb54c7..bc0b62e 100644 --- a/src/kafka/Drainer.ts +++ b/src/kafka/Drainer.ts @@ -1,13 +1,11 @@ -"use strict"; - -const Kafka = require("./Kafka.js"); -const async = require("async"); -const Promise = require("bluebird"); +import Kafka from "./Kafka"; +import async from "async"; +import Promise from "bluebird"; const DEFAULT_DRAIN_INTV = 3000; const NOOP = () => {}; -class Drainer { +export default class Drainer { constructor(consumer = null, asyncLimit = 1, autoJsonParsing = true, omitQueue = false, commitOnDrain = false) { @@ -465,5 +463,3 @@ class Drainer { this.consumer.emit(...args); } } - -module.exports = Drainer; diff --git a/src/kafka/Kafka.ts b/src/kafka/Kafka.ts index 1ae8ccb..7b5057c 100644 --- a/src/kafka/Kafka.ts +++ b/src/kafka/Kafka.ts @@ -1,9 +1,7 @@ -"use strict"; - -const EventEmitter = require("events"); -const {ConsumerGroup, Offset, Client, KafkaClient, HighLevelProducer} = require("kafka-node"); -const Promise = require("bluebird"); -const debug = require("debug"); +import EventEmitter from "events"; +import {ConsumerGroup, Offset, KafkaClient, HighLevelProducer} from "kafka-node"; +import Promise from "bluebird"; +import debug from "debug"; const NOOPL = { debug: debug("sinek:debug"), @@ -22,9 +20,8 @@ const DEFAULT_RETRY_OPTIONS = { unref: false }; -class Kafka extends EventEmitter { - - constructor(conString, logger = null, connectDirectlyToBroker = false){ +export default class Kafka extends EventEmitter { + constructor(private conString: string, logger = null, connectDirectlyToBroker = false){ super(); @@ -426,5 +423,3 @@ class Kafka extends EventEmitter { return null; } } - -module.exports = Kafka; diff --git a/src/kafka/PartitionDrainer.ts b/src/kafka/PartitionDrainer.ts index fc5fd51..a8dcd60 100644 --- a/src/kafka/PartitionDrainer.ts +++ b/src/kafka/PartitionDrainer.ts @@ -1,13 +1,11 @@ -"use strict"; +import Kafka from "./Kafka"; +import Promise from "bluebird"; -const Kafka = require("./Kafka.js"); -const Promise = require("bluebird"); - -const PartitionQueue = require("./PartitionQueue.js"); +import PartitionQueue from "./PartitionQueue"; const DEFAULT_DRAIN_INTV = 3000; -class PartitionDrainer { +export default class PartitionDrainer { constructor(consumer = null, asyncLimit = 1, commitOnDrain = false, autoJsonParsing = true) { @@ -536,4 +534,3 @@ class PartitionDrainer { } } -module.exports = PartitionDrainer; diff --git a/src/kafka/PartitionQueue.ts b/src/kafka/PartitionQueue.ts index 507cbc3..a43f58c 100644 --- a/src/kafka/PartitionQueue.ts +++ b/src/kafka/PartitionQueue.ts @@ -1,8 +1,6 @@ -"use strict"; +import async from "async"; -const async = require("async"); - -class PartitionQueue { +export default class PartitionQueue { constructor(partition, drainEvent, drainer, asyncLimit = 1, queueDrain = null){ @@ -163,5 +161,3 @@ class PartitionQueue { this._getLogger().info("queue closed."); } } - -module.exports = PartitionQueue; \ No newline at end of file diff --git a/src/kafka/Publisher.ts b/src/kafka/Publisher.ts index da37a81..2477f4b 100644 --- a/src/kafka/Publisher.ts +++ b/src/kafka/Publisher.ts @@ -1,12 +1,10 @@ -"use strict"; +import Kafka from "./Kafka"; +import Promise from "bluebird"; +import {v3 as murmur} from "murmurhash"; +import uuid from "uuid"; +import {KeyedMessage} from "kafka-node"; -const Kafka = require("./Kafka.js"); -const Promise = require("bluebird"); -const murmur = require("murmurhash").v3; -const uuid = require("uuid"); -const {KeyedMessage} = require("kafka-node"); - -const {CompressionTypes} = require("./../tools/index.js"); +import {CompressionTypes} from "./../tools/index"; const MESSAGE_TYPES = { PUBLISH: "-published", @@ -14,7 +12,7 @@ const MESSAGE_TYPES = { UPDATE: "-updated" }; -class Publisher { +export default class Publisher { constructor(producer = null, partitionCount = 1, autoFlushBuffer = 0, flushPeriod = 100) { @@ -511,5 +509,3 @@ class Publisher { this.producer.emit(...args); } } - -module.exports = Publisher; diff --git a/src/librdkafka/Analytics.ts b/src/librdkafka/Analytics.ts index 41c2ded..94f3332 100644 --- a/src/librdkafka/Analytics.ts +++ b/src/librdkafka/Analytics.ts @@ -1,5 +1,3 @@ -"use strict"; - const INTERESTING_DISTANCE = 10; /** @@ -61,7 +59,7 @@ class Analytics { /** * outsourced analytics for nconsumers */ -class ConsumerAnalytics extends Analytics { +export class ConsumerAnalytics extends Analytics { /** * creates a new instance @@ -268,7 +266,7 @@ class ConsumerAnalytics extends Analytics { /** * outsourced analytics for nproducers */ -class ProducerAnalytics extends Analytics { +export class ProducerAnalytics extends Analytics { /** * creates a new instance @@ -335,8 +333,3 @@ class ProducerAnalytics extends Analytics { return this._lastRes; } } - -module.exports = { - ConsumerAnalytics, - ProducerAnalytics -}; diff --git a/src/librdkafka/Health.ts b/src/librdkafka/Health.ts index fa600b7..fb3605b 100644 --- a/src/librdkafka/Health.ts +++ b/src/librdkafka/Health.ts @@ -1,6 +1,4 @@ -"use strict"; - -const merge = require("lodash.merge"); +import merge from "lodash.merge"; const defaultConfig = { thresholds: { @@ -109,7 +107,7 @@ class Health { * health check adapted for NConsumers * @extends Health */ -class ConsumerHealth extends Health { +export class ConsumerHealth extends Health { /** * creates a new instance @@ -182,7 +180,7 @@ class ConsumerHealth extends Health { * health check adapted for NProducers * @extends Health */ -class ProducerHealth extends Health { +export class ProducerHealth extends Health { /** * creates a new instance @@ -237,8 +235,3 @@ class ProducerHealth extends Health { return check; } } - -module.exports = { - ConsumerHealth, - ProducerHealth -}; diff --git a/src/librdkafka/Metadata.ts b/src/librdkafka/Metadata.ts index 1a7cd4b..baf3e17 100644 --- a/src/librdkafka/Metadata.ts +++ b/src/librdkafka/Metadata.ts @@ -1,9 +1,7 @@ -"use strict"; - /** * wrapper arround node-librdkafka metadata object */ -class Metadata { +export default class Metadata { /** * creates a new instance @@ -142,5 +140,3 @@ class Metadata { }); } } - -module.exports = Metadata; diff --git a/src/librdkafka/NConsumer.ts b/src/librdkafka/NConsumer.ts index a25e525..6bbf254 100644 --- a/src/librdkafka/NConsumer.ts +++ b/src/librdkafka/NConsumer.ts @@ -1,13 +1,11 @@ -"use strict"; +import Promise from "bluebird"; +import EventEmitter from "events"; +import debug from "debug"; +import async from "async"; -const Promise = require("bluebird"); -const EventEmitter = require("events"); -const debug = require("debug"); -const async = require("async"); - -const {ConsumerAnalytics} = require("./Analytics.js"); -const {ConsumerHealth} = require("./Health.js"); -const Metadata = require("./Metadata.js"); +import {ConsumerAnalytics} from "./Analytics"; +import {ConsumerHealth} from "./Health"; +import Metadata from "./Metadata"; //@OPTIONAL let BlizzKafka = null; @@ -30,7 +28,7 @@ const DEFAULT_LOGGER = { * native consumer wrapper for node-librdkafka * @extends EventEmitter */ -class NConsumer extends EventEmitter { +export default class NConsumer extends EventEmitter { /** * creates a new consumer instance @@ -1236,5 +1234,3 @@ class NConsumer extends EventEmitter { return metadata.asTopicList(); } } - -module.exports = NConsumer; diff --git a/src/librdkafka/NProducer.ts b/src/librdkafka/NProducer.ts index 922239d..1ecd41d 100644 --- a/src/librdkafka/NProducer.ts +++ b/src/librdkafka/NProducer.ts @@ -1,21 +1,14 @@ -"use strict"; +import EventEmitter from "events"; +import Promise from "bluebird"; +import uuid from "uuid"; +import murmur from "murmurhash"; +import debug from "debug"; +import murmur2Partitioner from "murmur2-partitioner"; -const EventEmitter = require("events"); -const Promise = require("bluebird"); -const uuid = require("uuid"); -const murmur = require("murmurhash"); -const debug = require("debug"); -const murmur2Partitioner = require("murmur2-partitioner"); +import Metadata from "./Metadata"; -const Metadata = require("./Metadata.js"); - -const { - ProducerAnalytics -} = require("./Analytics.js"); - -const { - ProducerHealth -} = require("./Health.js"); +import {ProducerAnalytics} from "./Analytics"; +import {ProducerHealth} from "./Health"; //@OPTIONAL let BlizzKafka = null; @@ -41,7 +34,7 @@ const DEFAULT_LOGGER = { * native producer wrapper for node-librdkafka * @extends EventEmitter */ -class NProducer extends EventEmitter { +export default class NProducer extends EventEmitter { /** * creates a new producer instance @@ -702,5 +695,3 @@ class NProducer extends EventEmitter { return this._health.check(); } } - -module.exports = NProducer; diff --git a/src/tools/CompressionTypes.ts b/src/tools/CompressionTypes.ts index 543291b..e3a8844 100644 --- a/src/tools/CompressionTypes.ts +++ b/src/tools/CompressionTypes.ts @@ -1,5 +1,3 @@ -"use strict"; - const TYPES = [0,1,2]; class CompressionTypes { @@ -20,4 +18,5 @@ class CompressionTypes { } } -module.exports = new CompressionTypes(); +export default new CompressionTypes(); + diff --git a/src/tools/index.ts b/src/tools/index.ts index 7e9f018..703d275 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -1,5 +1,2 @@ -"use strict"; - -const CompressionTypes = require("./CompressionTypes.js"); - -module.exports = {CompressionTypes}; \ No newline at end of file +import CompressionTypes from "./CompressionTypes"; +export {CompressionTypes} From 3be3451dbd986602c4042855b3bfa187f8689137 Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Wed, 3 Apr 2019 19:16:22 +0200 Subject: [PATCH 06/10] export interfaces --- src/Sinek.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/Sinek.ts b/src/Sinek.ts index 7d4bf61..e75f1eb 100644 --- a/src/Sinek.ts +++ b/src/Sinek.ts @@ -41,3 +41,20 @@ export { Health, Analytics } + +export { + KafkaHealthConfig, + NCommonKafkaOptions, + NConsumerKafkaOptions, + NProducerKafkaOptions, + KafkaConsumerConfig, + KafkaProducerConfig, + KafkaMessage, + SortedMessageBatch, + BatchConfig, + ConsumerStats, + LagStatus, + ProducerStats, + MessageReturn, + KafkaLogger, +} from "./interfaces"; From 9dc70f40c21c8ca72bf9047aabbccb37c32cd10c Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Thu, 21 Mar 2019 23:04:10 +0100 Subject: [PATCH 07/10] simple file convetion - introduce fields - introduce basic types - all compilation errors are being ignored via @ts-ignore - this is more honest than some quirky any-type/casting - @ts-ignore may be fixed in a later step --- src/Sinek.ts | 9 +- src/connect/Consumer.ts | 150 +++---- src/connect/Producer.ts | 18 +- src/interfaces.ts | 40 +- src/kafka/Drainer.ts | 650 +++++++++++++++--------------- src/kafka/Kafka.ts | 678 ++++++++++++++++--------------- src/kafka/PartitionDrainer.ts | 730 +++++++++++++++++----------------- src/kafka/PartitionQueue.ts | 280 ++++++------- src/kafka/Publisher.ts | 616 ++++++++++++++-------------- src/librdkafka/Analytics.ts | 582 +++++++++++++-------------- src/librdkafka/Health.ts | 349 ++++++++-------- src/librdkafka/Metadata.ts | 212 +++++----- src/librdkafka/NConsumer.ts | 81 +++- src/librdkafka/NProducer.ts | 60 +-- src/librdkafka/index.ts | 8 + src/tools/CompressionTypes.ts | 3 + 16 files changed, 2294 insertions(+), 2172 deletions(-) create mode 100644 src/librdkafka/index.ts diff --git a/src/Sinek.ts b/src/Sinek.ts index e75f1eb..cb51ba2 100644 --- a/src/Sinek.ts +++ b/src/Sinek.ts @@ -13,12 +13,13 @@ export {default as NProducer} from "./librdkafka/NProducer"; import {ProducerHealth, ConsumerHealth} from "./librdkafka/Health"; import {ProducerAnalytics, ConsumerAnalytics} from "./librdkafka/Analytics"; +import {DrainerConstructor, KafkaConstructor, PartitionDrainerConstructor, PublisherConstructor} from "./interfaces"; -const Kafka = util.deprecate(DeprecatedKafka, "Kafka is deprecated, please use 'NConsumer' if possible."); -const Drainer = util.deprecate(DeprecatedDrainer, "Drainer is deprecated, please use 'NConsumer' if possible."); -const PartitionDrainer = util.deprecate(DeprecatedPartitionDrainer, "PartitionDrainer is deprecated, please use 'NConsumer' if possible."); -const Publisher = util.deprecate(DeprecatedPublisher, "Publisher is deprecated, please use 'NProducer' if possible."); +const Kafka: KafkaConstructor = util.deprecate(DeprecatedKafka, "Kafka is deprecated, please use 'NConsumer' if possible."); +const Drainer: DrainerConstructor = util.deprecate(DeprecatedDrainer, "Drainer is deprecated, please use 'NConsumer' if possible."); +const PartitionDrainer: PartitionDrainerConstructor = util.deprecate(DeprecatedPartitionDrainer, "PartitionDrainer is deprecated, please use 'NConsumer' if possible."); +const Publisher: PublisherConstructor = util.deprecate(DeprecatedPublisher, "Publisher is deprecated, please use 'NProducer' if possible."); const Consumer = util.deprecate(DeprecatedConsumer, "Consumer is deprecated, please use (noptions) 'NConsumer' if possible."); const Producer = util.deprecate(DeprecatedProducer, "Producer is deprecated, please use (noptions) 'NProducer' if possible."); diff --git a/src/connect/Consumer.ts b/src/connect/Consumer.ts index cde37ed..5cd54fb 100644 --- a/src/connect/Consumer.ts +++ b/src/connect/Consumer.ts @@ -1,118 +1,118 @@ -import Promise from "bluebird"; import EventEmitter from "events"; import Kafka from "./../kafka/Kafka"; import Drainer from "./../kafka/Drainer"; +import {IConsumer} from "../interfaces"; -export default class Consumer extends EventEmitter { +export default class Consumer extends EventEmitter implements IConsumer { + private kafkaConsumerClient; + private consumer; - constructor(topic, config = { options: {} }) { - super(); - - this.topic = topic; - this.config = config; + constructor(private topic, private config = {options: {}}) { + super(); + this.kafkaConsumerClient = null; + this.consumer = null; + } - this.kafkaConsumerClient = null; - this.consumer = null; - } + connect(backpressure = false): Promise { + return new Promise(resolve => { - connect(backpressure = false) { - return new Promise(resolve => { + // @ts-ignore + const {zkConStr, kafkaHost, logger, groupId, workerPerPartition, options} = this.config; - const { zkConStr, kafkaHost, logger, groupId, workerPerPartition, options } = this.config; + let conStr = null; - let conStr = null; + if (typeof kafkaHost === "string") { + conStr = kafkaHost; + } - if(typeof kafkaHost === "string"){ - conStr = kafkaHost; - } + if (typeof zkConStr === "string") { + conStr = zkConStr; + } - if(typeof zkConStr === "string"){ - conStr = zkConStr; - } + if (conStr === null) { + throw new Error("One of the following: zkConStr or kafkaHost must be defined."); + } - if(conStr === null){ - throw new Error("One of the following: zkConStr or kafkaHost must be defined."); - } + this.kafkaConsumerClient = new Kafka(conStr, logger, conStr === kafkaHost); - this.kafkaConsumerClient = new Kafka(conStr, logger, conStr === kafkaHost); + this.kafkaConsumerClient.on("ready", () => resolve()); + this.kafkaConsumerClient.on("error", error => super.emit("error", error)); - this.kafkaConsumerClient.on("ready", () => resolve()); - this.kafkaConsumerClient.on("error", error => super.emit("error", error)); + this.kafkaConsumerClient.becomeConsumer([this.topic], groupId, options); - this.kafkaConsumerClient.becomeConsumer([this.topic], groupId, options); + // @ts-ignore + const commitManually = !options.autoCommit; + this.consumer = new Drainer(this.kafkaConsumerClient, workerPerPartition, false, !backpressure, commitManually); + }); + } - const commitManually = !options.autoCommit; - this.consumer = new Drainer(this.kafkaConsumerClient, workerPerPartition, false, !backpressure, commitManually); - }); - } + consume(syncEvent = null): Promise { + return new Promise(resolve => { - consume(syncEvent = null) { - return new Promise(resolve => { + this.consumer.drain((message, done) => { - this.consumer.drain((message, done) => { + super.emit("message", message); - super.emit("message", message); + if (!syncEvent) { + return done(); + } - if (!syncEvent) { - return done(); - } + syncEvent(message, () => { - syncEvent(message, () => { + /* ### sync event callback does not handle errors ### */ - /* ### sync event callback does not handle errors ### */ + done(); + }); + }); - done(); + this.consumer.once("first-drain-message", () => resolve()); }); - }); - - this.consumer.once("first-drain-message", () => resolve()); - }); - } + } - consumeOnce(syncEvent = null, drainThreshold = 10000, timeout = 0) { - return this.consumer.drainOnce((message, done) => { + consumeOnce(syncEvent = null, drainThreshold = 10000, timeout = 0) { + return this.consumer.drainOnce((message, done) => { - super.emit("message", message); + super.emit("message", message); - if (!syncEvent) { - return done(); - } + if (!syncEvent) { + return done(); + } - syncEvent(message, () => { + syncEvent(message, () => { - /* ### sync event callback does not handle errors ### */ + /* ### sync event callback does not handle errors ### */ - done(); - }); + done(); + }); - }, drainThreshold, timeout); - } + }, drainThreshold, timeout); + } - pause() { + pause() { - if (this.consumer) { - this.consumer.pause(); + if (this.consumer) { + this.consumer.pause(); + } } - } - resume() { + resume() { - if (this.consumer) { - this.consumer.resume(); + if (this.consumer) { + this.consumer.resume(); + } } - } - getStats() { - return this.consumer ? this.consumer.getStats() : {}; - } + getStats() { + return this.consumer ? this.consumer.getStats() : {}; + } - close(commit = false) { + close(commit = false) { - if (this.consumer) { - const result = this.consumer.close(commit); - this.consumer = null; - return result; + if (this.consumer) { + const result = this.consumer.close(commit); + this.consumer = null; + return result; + } } - } } diff --git a/src/connect/Producer.ts b/src/connect/Producer.ts index 1e495f2..f6a6eba 100644 --- a/src/connect/Producer.ts +++ b/src/connect/Producer.ts @@ -1,26 +1,24 @@ import EventEmitter from "events"; -import Promise from "bluebird"; import uuid from "uuid"; - import Kafka from "./../kafka/Kafka"; import Publisher from "./../kafka/Publisher"; +import {IProducer, KafkaProducerConfig} from "../interfaces"; -export default class Producer extends EventEmitter { +export default class Producer extends EventEmitter implements IProducer { + private kafkaProducerClient; + private producer; + private targetTopics: string[]; - constructor(config, topic = [], defaultPartitionCount = 1) { + constructor(private config: KafkaProducerConfig, topic: string[] = [], private defaultPartitionCount = 1) { super(); this.targetTopics = Array.isArray(topic) ? topic : [topic]; - this.config = config; - - this.kafkaProducerClient = null; - this.producer = null; - this.defaultPartitionCount = defaultPartitionCount; } - connect() { + connect(): Promise { return new Promise(resolve => { + // @ts-ignore const { zkConStr, kafkaHost, logger, clientName, options } = this.config; let conStr = null; diff --git a/src/interfaces.ts b/src/interfaces.ts index 36761cf..1ee40e7 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -1,3 +1,8 @@ +import Kafka from "./kafka/Kafka"; +import Drainer from "./kafka/Drainer"; +import Publisher from "./kafka/Publisher"; +import PartitionDrainer from "./kafka/PartitionDrainer"; + export interface KafkaHealthConfig { thresholds?: { consumer?: { @@ -237,7 +242,6 @@ export interface MessageReturn { } export interface INConsumer { - constructor(topic: Array | string, config: KafkaConsumerConfig); on(eventName: "message", callback: (message: KafkaMessage) => any): void; on(eventName: "error", callback: (error: any) => any): void; on(eventName: "ready", callback: () => any): void; @@ -252,7 +256,7 @@ export interface INConsumer { pause(topics: Array): void; resume(topics: Array): void; getStats(): ConsumerStats; - close(commit?: boolean): object; + close(commit?: boolean): void; enableAnalytics(options: object): void; haltAnalytics(): void; addSubscriptions(topics: Array): Array; @@ -273,7 +277,6 @@ export interface INConsumer { } export interface INProducer { - constructor(config: KafkaProducerConfig, _?: null, defaultPartitionCount?: number | "auto") on(eventName: "error", callback: (error: any) => any): void; on(eventName: "ready", callback: () => any): void; connect(): Promise; @@ -300,7 +303,7 @@ export interface INProducer { resume(): void; getStats(): ProducerStats; refreshMetadata(topics: Array): void; - close(): object; + close(): void; enableAnalytics(options: object): void; haltAnalytics(): void; getAnalytics(): object; @@ -314,7 +317,6 @@ export interface INProducer { } export interface IConsumer { - constructor(topic: string, config: KafkaConsumerConfig); on(eventName: "message", callback: (message: object) => any): void; on(eventName: "error", callback: (error: any) => any): void; connect(backpressure?: boolean): Promise; @@ -327,7 +329,6 @@ export interface IConsumer { } export interface IProducer { - constructor(config: KafkaProducerConfig, topic: Array, defaultPartitionCount: number); on(eventName: "error", callback: (error: any) => any): void; connect(): Promise; send(topic: string, message: string | string[]): Promise; @@ -340,32 +341,23 @@ export interface IProducer { resume(): void; getStats(): object; refreshMetadata(topics: Array): void; - close(): object; -} - -export interface IKafka { - constructor(conString: string, logger: object, connectDirectlyToBroker: boolean) + close(): void; } -export interface IDrainer { - constructor(consumer: object, asyncLimit: number, autoJsonParsing: boolean, omitQueue: boolean, commitOnDrain: boolean) +export interface KafkaConstructor { + new (conString: string, logger: object, connectDirectlyToBroker: boolean): Kafka; } -export interface IPublisher { - constructor(producer: object, partitionCount: number, autoFlushBuffer: number, flushPeriod: number) +export interface DrainerConstructor { + new (consumer: object, asyncLimit: number, autoJsonParsing: boolean, omitQueue: boolean, commitOnDrain: boolean): Drainer; } -export interface IPartitionDrainer { - constructor(consumer: object, asyncLimit: number, commitOnDrain: boolean, autoJsonParsing: boolean) +export interface PublisherConstructor { + new (producer: object, partitionCount: number, autoFlushBuffer: number, flushPeriod: number): Publisher; } -export interface IPartitionQueue { - constructor(partition: object, drainEvent: object, drainer: object, asyncLimit: number, queueDrain: object) -} - -export interface MessageType { - key: string; - value: string; +export interface PartitionDrainerConstructor { + new (consumer: object, asyncLimit: number, commitOnDrain: boolean, autoJsonParsing: boolean): PartitionDrainer; } export interface KafkaLogger { diff --git a/src/kafka/Drainer.ts b/src/kafka/Drainer.ts index bc0b62e..852393b 100644 --- a/src/kafka/Drainer.ts +++ b/src/kafka/Drainer.ts @@ -3,143 +3,147 @@ import async from "async"; import Promise from "bluebird"; const DEFAULT_DRAIN_INTV = 3000; -const NOOP = () => {}; +const NOOP = () => { +}; export default class Drainer { + private _drainEvent; + private _q; + private _lastProcessed: number; + private _lastReceived: number; + private _totalIncomingMessages: number; + private _totalProcessedMessages: number; + private _messagesSinceLastDrain: number; + private _receivedFirst: boolean; + private _drainStart; + private _stats: {}; + private _lastMessageHandlerRef: ((msg: unknown) => void) | null; + private DRAIN_INTV: number; + private raw; + + constructor(private consumer = null, private asyncLimit = 1, private autoJsonParsing = true, private omitQueue = false, private commitOnDrain = false) { + // @ts-ignore + if (!consumer || !(consumer instanceof Kafka) || !consumer.isConsumer) { + throw new Error("Consumer is not a valid Sinek Kafka(Consumer)"); + } - constructor(consumer = null, asyncLimit = 1, autoJsonParsing = true, omitQueue = false, commitOnDrain = false) { - - if (!consumer || !(consumer instanceof Kafka) || - !consumer.isConsumer) { - throw new Error("Consumer is not a valid Sinek Kafka(Consumer)"); - } - - if (omitQueue && commitOnDrain) { - throw new Error("Cannot run drain commits when queue is omitted. Please either run: " + + if (omitQueue && commitOnDrain) { + throw new Error("Cannot run drain commits when queue is omitted. Please either run: " + " a manual committer with backpressure OR an auto-commiter without backpressure."); - } - - this.consumer = consumer; - this.raw = consumer.consumer; - - this.asyncLimit = asyncLimit; - this.commitOnDrain = commitOnDrain; - this.autoJsonParsing = autoJsonParsing; - this.omitQueue = omitQueue; + } - this._drainEvent = null; - this._q = null; + this.raw = consumer.consumer; - this._lastProcessed = Date.now(); - this._lastReceived = Date.now(); + this._lastProcessed = Date.now(); + this._lastReceived = Date.now(); - this._totalIncomingMessages = 0; - this._totalProcessedMessages = 0; + this._totalIncomingMessages = 0; + this._totalProcessedMessages = 0; - this._messagesSinceLastDrain = 0; - this._receivedFirst = false; - this._drainStart = null; + this._messagesSinceLastDrain = 0; + this._receivedFirst = false; + this._drainStart = null; - this._stats = {}; + this._stats = {}; - this._lastMessageHandlerRef = null; + this._lastMessageHandlerRef = null; - this.DRAIN_INTV = DEFAULT_DRAIN_INTV; - } + this.DRAIN_INTV = DEFAULT_DRAIN_INTV; + } - _getLogger() { - return this.consumer._getLogger(); - } + _getLogger() { + return this.consumer._getLogger(); + } - /** + /** * stops any active drain process * closes the consumer and its client */ - close() { + close() { - if (this._lastMessageHandlerRef) { - this.raw.removeListener("message", this._lastMessageHandlerRef); - this._lastMessageHandlerRef = null; - } else { - this._getLogger().warn("message handler ref not present during close, could not remove listener."); - } + if (this._lastMessageHandlerRef) { + this.raw.removeListener("message", this._lastMessageHandlerRef); + this._lastMessageHandlerRef = null; + } else { + this._getLogger().warn("message handler ref not present during close, could not remove listener."); + } - this._getLogger().info("[Drainer] closed."); - return this.consumer.close(); - } + this._getLogger().info("[Drainer] closed."); + return this.consumer.close(); + } - /** + /** * returns a few insights * @returns {{totalIncoming: number, last: (number|*), isPaused: *}} */ - getStats() { - return { - totalIncoming: this._totalIncomingMessages, - lastMessage: this._lastReceived, + getStats() { + return { + totalIncoming: this._totalIncomingMessages, + lastMessage: this._lastReceived, - receivedFirstMsg: this._receivedFirst, + receivedFirstMsg: this._receivedFirst, - totalProcessed: this._totalProcessedMessages, - lastProcessed: this._lastProcessed, + totalProcessed: this._totalProcessedMessages, + lastProcessed: this._lastProcessed, - queueSize: this._q ? this._q.length() : null, + queueSize: this._q ? this._q.length() : null, - isPaused: this.consumer && this.consumer.isConsumer ? this.isPaused() : null, + isPaused: this.consumer && this.consumer.isConsumer ? this.isPaused() : null, - drainStats: this._stats, - omittingQueue: this.omitQueue - }; - } + drainStats: this._stats, + omittingQueue: this.omitQueue + }; + } - /** + /** * resets all offsets and starts from being * also un-pauses consumer if necessary * @param topics * @returns {Promise.} */ - resetConsumer() { - return Promise.reject("resetConsumer has been removed, due to supporting bad kafka consumer behaviour."); - } + resetConsumer() { + return Promise.reject("resetConsumer has been removed, due to supporting bad kafka consumer behaviour."); + } - /** + /** * resets all offsets and makes sure the consumer is paused * @param topics * @returns {Promise.} */ - resetOffset() { - return Promise.reject("resetOffset has been removed, due to supporting bad kafka consumer behaviour."); - } + resetOffset() { + return Promise.reject("resetOffset has been removed, due to supporting bad kafka consumer behaviour."); + } - /** + /** * main reg. function, pass it a function to receive messages * under flow control * @param drainEvent */ - drain(drainEvent = null) { + drain(drainEvent = null) { - if (!drainEvent || typeof drainEvent !== "function") { - throw new Error("drainEvent must be a valid function"); - } + if (!drainEvent || typeof drainEvent !== "function") { + throw new Error("drainEvent must be a valid function"); + } - if (this._drainEvent) { - throw new Error("a drain process is currently active."); - } + if (this._drainEvent) { + throw new Error("a drain process is currently active."); + } - //reset - this._lastProcessed = Date.now(); - this._lastReceived = Date.now(); - this._stats = {}; + //reset + this._lastProcessed = Date.now(); + this._lastReceived = Date.now(); + this._stats = {}; - this._messagesSinceLastDrain = this._totalIncomingMessages; - this._drainEvent = drainEvent; - this._startToReceiveMessages(); + this._messagesSinceLastDrain = this._totalIncomingMessages; + this._drainEvent = drainEvent; + this._startToReceiveMessages(); - if (this.isPaused()) { - this.resume(); + if (this.isPaused()) { + this.resume(); + } } - } - /** + /** * main req. function, pass it a function to receive messages * under flow control, until they are stall for a certain amount * of time (e.g. when all messages on the queue are consumed) @@ -148,318 +152,318 @@ export default class Drainer { * @param drainThreshold * @param timeout */ - drainOnce(drainEvent = null, drainThreshold = 10000, timeout = 0) { - return new Promise((resolve, reject) => { + drainOnce(drainEvent = null, drainThreshold = 10000, timeout = 0) { + return new Promise((resolve, reject) => { - if (!drainEvent || typeof drainEvent !== "function") { - return reject("drainEvent must be a valid function"); - } + if (!drainEvent || typeof drainEvent !== "function") { + return reject("drainEvent must be a valid function"); + } - if (this._drainEvent) { - return reject("a drain process is currently active."); - } + if (this._drainEvent) { + return reject("a drain process is currently active."); + } - if (timeout !== 0 && timeout < this.DRAIN_INTV) { - return reject(`timeout must be either 0 or > ${this.DRAIN_INTV}.`); - } + if (timeout !== 0 && timeout < this.DRAIN_INTV) { + return reject(`timeout must be either 0 or > ${this.DRAIN_INTV}.`); + } - if (timeout !== 0 && timeout <= drainThreshold) { - return reject(`timeout ${timeout} must be greater than the drainThreshold ${drainThreshold}.`); - } + if (timeout !== 0 && timeout <= drainThreshold) { + return reject(`timeout ${timeout} must be greater than the drainThreshold ${drainThreshold}.`); + } - let t = null; - let intv = null; + let t = null; + let intv = null; - intv = setInterval(() => { + intv = setInterval(() => { - const spanProcessed = Date.now() - this._lastProcessed; - const spanReceived = Date.now() - this._lastReceived; + const spanProcessed = Date.now() - this._lastProcessed; + const spanReceived = Date.now() - this._lastReceived; - this._getLogger().debug("drainOnce interval running, current span-rec: " + + this._getLogger().debug("drainOnce interval running, current span-rec: " + `${spanReceived} / span-proc: ${spanProcessed} ms.`); - //set stats - this._countStats("intv-cycle"); - this._stats["last-proc-since"] = spanProcessed; - this._stats["last-rec-since"] = spanReceived; - - //choose the smaller span - const span = spanProcessed < spanReceived ? spanProcessed : spanReceived; + //set stats + this._countStats("intv-cycle"); + this._stats["last-proc-since"] = spanProcessed; + this._stats["last-rec-since"] = spanReceived; + + //choose the smaller span + const span = spanProcessed < spanReceived ? spanProcessed : spanReceived; + + if (span >= drainThreshold) { + this._getLogger().info(`drainOnce span ${span} hit threshold ${drainThreshold}.`); + clearInterval(intv); + clearTimeout(t); + this.stopDrain(); + resolve(this._totalIncomingMessages - this._messagesSinceLastDrain); + } + }, this.DRAIN_INTV); + + if (timeout !== 0) { + this._getLogger().info(`drainOnce timeout active: ${timeout} ms.`); + t = setTimeout(() => { + this._getLogger().warn(`drainOnce timeout hit after ${timeout} ms.`); + clearInterval(intv); + this.stopDrain(); + reject("drainOnce ran into timeout."); + }, timeout); + } + + //start the drain process + this.drain(drainEvent); + }); + } - if (span >= drainThreshold) { - this._getLogger().info(`drainOnce span ${span} hit threshold ${drainThreshold}.`); - clearInterval(intv); - clearTimeout(t); - this.stopDrain(); - resolve(this._totalIncomingMessages - this._messagesSinceLastDrain); - } - }, this.DRAIN_INTV); - - if (timeout !== 0) { - this._getLogger().info(`drainOnce timeout active: ${timeout} ms.`); - t = setTimeout(() => { - this._getLogger().warn(`drainOnce timeout hit after ${timeout} ms.`); - clearInterval(intv); - this.stopDrain(); - reject("drainOnce ran into timeout."); - }, timeout); - } - - //start the drain process - this.drain(drainEvent); - }); - } - - /** + /** * stops any active drain process */ - stopDrain() { + stopDrain() { - if (!this._drainEvent) { - throw new Error("there is no drain active."); - } + if (!this._drainEvent) { + throw new Error("there is no drain active."); + } - //reset - if (this._lastMessageHandlerRef) { - this.raw.removeListener("message", this._lastMessageHandlerRef); - this._lastMessageHandlerRef = null; - } else { - this._getLogger().warn("message handler ref not present during close, could not remove listener."); - } + //reset + if (this._lastMessageHandlerRef) { + this.raw.removeListener("message", this._lastMessageHandlerRef); + this._lastMessageHandlerRef = null; + } else { + this._getLogger().warn("message handler ref not present during close, could not remove listener."); + } - this._drainEvent = null; - this._q = null; - this._receivedFirst = false; + this._drainEvent = null; + this._q = null; + this._receivedFirst = false; - const duration = (Date.now() - this._drainStart) / 1000; - this._getLogger().info(`[Drainer] stopped drain process, had been open for ${duration} seconds.`); - } + const duration = (Date.now() - this._drainStart) / 1000; + this._getLogger().info(`[Drainer] stopped drain process, had been open for ${duration} seconds.`); + } - /** + /** * removes kafka topics (if broker allows this action) * @param topics */ - removeTopics(topics = []) { - return new Promise((resolve, reject) => { - this._getLogger().info(`deleting topics ${JSON.stringify(topics)}.`); - this.raw.client.removeTopicMetadata(topics, (err, data) => { + removeTopics(topics = []) { + return new Promise((resolve, reject) => { + this._getLogger().info(`deleting topics ${JSON.stringify(topics)}.`); + this.raw.client.removeTopicMetadata(topics, (err, data) => { + + if (err) { + return reject(err); + } + + resolve(data); + }); + }); + } - if (err) { - return reject(err); + pause() { + + if (!this.isPaused()) { + this._countStats("paused"); } - resolve(data); - }); - }); - } + return this.consumer.pause(); + } - pause() { + resume() { - if (!this.isPaused()) { - this._countStats("paused"); + if (this.isPaused()) { + this._countStats("resumed"); + } + + return this.consumer.resume(); } - return this.consumer.pause(); - } + isPaused() { + return this.consumer.isPaused(); + } - resume() { + _startToReceiveMessages() { - if (this.isPaused()) { - this._countStats("resumed"); + if (!this.omitQueue) { + this._startToReceiveMessagesThroughQueue(); + } else { + this._startToReceiveMessagesWithoutQueue(); + } } - return this.consumer.resume(); - } + _startToReceiveMessagesThroughQueue() { + + this._q = async.queue((msg, done) => { + if (this._drainEvent) { + setImmediate(() => this._drainEvent(msg, err => { + this._lastProcessed = Date.now(); + this._totalProcessedMessages++; + done(err); + })); + } else { + this._getLogger().debug("drainEvent not present, message is dropped."); + } + }, this.asyncLimit); + + this._q.drain = () => { + + if (!this.commitOnDrain) { + return this.resume(); + } + + //commit state first, before resuming + this._getLogger().debug("committing manually, reason: drain event."); + this._commit().then(() => { + this._getLogger().debug("committed successfully, resuming."); + this.resume(); + }).catch(error => { + this._getLogger().error("failed to commit offsets, resuming anyway after: " + error); + this.resume(); + }); + }; + + this._q.error(err => { + if (err) { + this._countStats("msg-process-fail"); + this._getLogger().warn("error was passed back to consumer queue, dropping it silently: " + JSON.stringify(err)); + } + }); + + this._lastMessageHandlerRef = this._onMessageForQueue.bind(this); + this.raw.on("message", this._lastMessageHandlerRef); + this._getLogger().info("[Drainer] started drain process."); + this._drainStart = Date.now(); + } - isPaused() { - return this.consumer.isPaused(); - } + _commit() { + return this.consumer.commitCurrentOffsets(); + } - _startToReceiveMessages() { + _startToReceiveMessagesWithoutQueue() { - if (!this.omitQueue) { - this._startToReceiveMessagesThroughQueue(); - } else { - this._startToReceiveMessagesWithoutQueue(); + this._lastMessageHandlerRef = this._onMessageNoQueue.bind(this); + this.raw.on("message", this._lastMessageHandlerRef); + this._getLogger().info("[Drainer] started drain process."); + this._drainStart = Date.now(); } - } - - _startToReceiveMessagesThroughQueue() { - - this._q = async.queue((msg, done) => { - if (this._drainEvent) { - setImmediate(() => this._drainEvent(msg, err => { - this._lastProcessed = Date.now(); - this._totalProcessedMessages++; - done(err); - })); - } else { - this._getLogger().debug("drainEvent not present, message is dropped."); - } - }, this.asyncLimit); - - this._q.drain = () => { - - if (!this.commitOnDrain) { - return this.resume(); - } - - //commit state first, before resuming - this._getLogger().debug("committing manually, reason: drain event."); - this._commit().then(() => { - this._getLogger().debug("committed successfully, resuming."); - this.resume(); - }).catch(error => { - this._getLogger().error("failed to commit offsets, resuming anyway after: " + error); - this.resume(); - }); - }; - - this._q.error(err => { - if (err) { - this._countStats("msg-process-fail"); - this._getLogger().warn("error was passed back to consumer queue, dropping it silently: " + JSON.stringify(err)); - } - }); - - this._lastMessageHandlerRef = this._onMessageForQueue.bind(this); - this.raw.on("message", this._lastMessageHandlerRef); - this._getLogger().info("[Drainer] started drain process."); - this._drainStart = Date.now(); - } - - _commit() { - return this.consumer.commitCurrentOffsets(); - } - - _startToReceiveMessagesWithoutQueue() { - - this._lastMessageHandlerRef = this._onMessageNoQueue.bind(this); - this.raw.on("message", this._lastMessageHandlerRef); - this._getLogger().info("[Drainer] started drain process."); - this._drainStart = Date.now(); - } - - /** + + /** * with backpressure * @param {*} message */ - _onMessageForQueue(message) { + _onMessageForQueue(message) { - this._getLogger().debug("received kafka message => length: " + (message.value && message.value.length) + ", offset: " + + this._getLogger().debug("received kafka message => length: " + (message.value && message.value.length) + ", offset: " + message.offset + ", partition: " + message.partition + ", on topic: " + message.topic); - if (this.autoJsonParsing) { - try { - message.value = JSON.parse(message.value); - } catch (e) { - this._countStats("msg-parse-fail"); - return this.emit("error", "failed to json parse message value: " + message); - } - - if (!message.value) { - this._countStats("msg-empty"); - return this.emit("error", "message value is empty: " + message); - } - } + if (this.autoJsonParsing) { + try { + message.value = JSON.parse(message.value); + } catch (e) { + this._countStats("msg-parse-fail"); + return this.emit("error", "failed to json parse message value: " + message); + } + + if (!message.value) { + this._countStats("msg-empty"); + return this.emit("error", "message value is empty: " + message); + } + } - this._q.push(message); - //error handling happens directly on the queue object initialisation + this._q.push(message); + //error handling happens directly on the queue object initialisation - this.pause(); + this.pause(); - this._totalIncomingMessages++; - this._lastReceived = Date.now(); + this._totalIncomingMessages++; + this._lastReceived = Date.now(); - if (!this._receivedFirst) { - this._receivedFirst = true; - this._getLogger().info("consumer received first message."); - this.emit("first-drain-message", message); + if (!this._receivedFirst) { + this._receivedFirst = true; + this._getLogger().info("consumer received first message."); + this.emit("first-drain-message", message); + } } - } - /** + /** * no backpressure * @param {*} message */ - _onMessageNoQueue(message) { + _onMessageNoQueue(message) { - this._getLogger().debug("received kafka message => length: " + (message.value && message.value.length) + ", offset: " + + this._getLogger().debug("received kafka message => length: " + (message.value && message.value.length) + ", offset: " + message.offset + ", partition: " + message.partition + ", on topic: " + message.topic); - if (this.autoJsonParsing) { - try { - message.value = JSON.parse(message.value); - } catch (e) { - this._countStats("msg-parse-fail"); - return this.emit("error", "failed to json parse message value: " + message); - } - - if (!message.value) { - this._countStats("msg-empty"); - return this.emit("error", "message value is empty: " + message); - } - } + if (this.autoJsonParsing) { + try { + message.value = JSON.parse(message.value); + } catch (e) { + this._countStats("msg-parse-fail"); + return this.emit("error", "failed to json parse message value: " + message); + } + + if (!message.value) { + this._countStats("msg-empty"); + return this.emit("error", "message value is empty: " + message); + } + } - this._totalIncomingMessages++; - this._lastReceived = Date.now(); + this._totalIncomingMessages++; + this._lastReceived = Date.now(); - if (this._drainEvent) { - this._drainEvent(message, NOOP); - this._lastProcessed = Date.now(); - this._totalProcessedMessages++; - } + if (this._drainEvent) { + this._drainEvent(message, NOOP); + this._lastProcessed = Date.now(); + this._totalProcessedMessages++; + } - if (!this._receivedFirst) { - this._receivedFirst = true; - this._getLogger().info("consumer received first message."); - this.emit("first-drain-message", message); + if (!this._receivedFirst) { + this._receivedFirst = true; + this._getLogger().info("consumer received first message."); + this.emit("first-drain-message", message); + } } - } - _countStats(key) { + _countStats(key) { - if (!this._stats) { - return; - } + if (!this._stats) { + return; + } - if (!this._stats[key]) { - this._stats[key] = 1; - return; - } + if (!this._stats[key]) { + this._stats[key] = 1; + return; + } - this._stats[key]++; - } + this._stats[key]++; + } - /** + /** * consumer proxy * @param args */ - on(...args) { - this.consumer.on(...args); - } + on(...args) { + this.consumer.on(...args); + } - /** + /** * consumer proxy * @param args */ - once(...args) { - this.consumer.once(...args); - } + once(...args) { + this.consumer.once(...args); + } - /** + /** * consumer proxy * @param args */ - removeListener(...args) { - this.consumer.removeListener(...args); - } + removeListener(...args) { + this.consumer.removeListener(...args); + } - /** + /** * consumer proxy * @param args */ - emit(...args) { - this.consumer.emit(...args); - } + emit(...args) { + this.consumer.emit(...args); + } } diff --git a/src/kafka/Kafka.ts b/src/kafka/Kafka.ts index 7b5057c..4519931 100644 --- a/src/kafka/Kafka.ts +++ b/src/kafka/Kafka.ts @@ -1,425 +1,441 @@ import EventEmitter from "events"; -import {ConsumerGroup, Offset, KafkaClient, HighLevelProducer} from "kafka-node"; +import {ConsumerGroup, Offset, KafkaClient, HighLevelProducer, ConsumerGroupOptions} from "kafka-node"; import Promise from "bluebird"; import debug from "debug"; const NOOPL = { - debug: debug("sinek:debug"), - info: debug("sinek:info"), - warn: debug("sinek:warn"), - error: debug("sinek:error") + debug: debug("sinek:debug"), + info: debug("sinek:info"), + warn: debug("sinek:warn"), + error: debug("sinek:error") }; const DEFAULT_RETRY_OPTIONS = { - retries: 1000, // overwritten by forever - factor: 3, - minTimeout: 1000, // 1 sec - maxTimeout: 30000, // 30 secs - randomize: true, - forever: true, - unref: false + retries: 1000, // overwritten by forever + factor: 3, + minTimeout: 1000, // 1 sec + maxTimeout: 30000, // 30 secs + randomize: true, + forever: true, + unref: false }; export default class Kafka extends EventEmitter { - constructor(private conString: string, logger = null, connectDirectlyToBroker = false){ - - super(); + private client; + public consumer; + private offset; + public isConsumer: boolean; + private _autoCommitEnabled; + private _isManual: boolean; + public isProducer: boolean; + public producer; + private _producerReadyFired: boolean; + private targetTopics: string[]; + + constructor(private conString: string, logger: object, private connectDirectlyToBroker: boolean = false) { + + super(); + + this.client = null; + + //consumer + this.consumer = null; + this.offset = null; + this.isConsumer = false; + this._autoCommitEnabled = null; + this._isManual = false; + + //producer + this.isProducer = false; + this.producer = null; + this.targetTopics = []; + + this._producerReadyFired = false; + } - this.conString = conString; - this.connectDirectlyToBroker = connectDirectlyToBroker; - this.client = null; + getPartitions(topic) { + return new Promise((resolve, reject) => { - //consumer - this.consumer = null; - this.offset = null; - this.isConsumer = false; - this._autoCommitEnabled = null; - this._isManual = false; + if (!this.client) { + return reject("client is not defined yet, cannot create offset to gather partitions."); + } - //producer - this.isProducer = false; - this.producer = null; - this.targetTopics = []; + const offset = new Offset(this.client); + offset.fetchEarliestOffsets([topic], (err, data) => { - this._logger = logger; - this._producerReadyFired = false; - } + if (err || !data[topic]) { + return reject("failed to get offsets of topic: " + topic + "; " + err); + } - getPartitions(topic){ - return new Promise((resolve, reject) => { + resolve(Object.keys(data[topic]).map(key => key)); + }); + }); + } - if(!this.client){ - return reject("client is not defined yet, cannot create offset to gather partitions."); - } + getEarliestOffsets(topic) { + return new Promise((resolve, reject) => { - const offset = new Offset(this.client); - offset.fetchEarliestOffsets([topic], (err, data) => { + if (!this.client) { + return reject("client is not defined yet, cannot create offset to reset."); + } - if(err || !data[topic]){ - return reject("failed to get offsets of topic: " + topic + "; " + err); - } + const offset = new Offset(this.client); + offset.fetchEarliestOffsets([topic], (err, data) => { - resolve(Object.keys(data[topic]).map(key => key)); - }); - }); - } + if (err || !data[topic]) { + return reject("failed to get offsets of topic: " + topic + "; " + err); + } - getEarliestOffsets(topic){ - return new Promise((resolve, reject) => { + resolve(data[topic]); + }); + }); + } - if(!this.client){ - return reject("client is not defined yet, cannot create offset to reset."); - } + getOffsets(topic) { + return new Promise((resolve, reject) => { - const offset = new Offset(this.client); - offset.fetchEarliestOffsets([topic], (err, data) => { + if (!this.client) { + return reject("client is not defined yet, cannot create offset to reset."); + } - if(err || !data[topic]){ - return reject("failed to get offsets of topic: " + topic + "; " + err); - } + const offset = new Offset(this.client); + offset.fetchLatestOffsets([topic], (err, data) => { - resolve(data[topic]); - }); - }); - } + if (err || !data[topic]) { + return reject("failed to get offsets of topic: " + topic + "; " + err); + } - getOffsets(topic){ - return new Promise((resolve, reject) => { + resolve(data[topic]); + }); + }); + } - if(!this.client){ - return reject("client is not defined yet, cannot create offset to reset."); - } + getTopics() { + return this.targetTopics; + } - const offset = new Offset(this.client); - offset.fetchLatestOffsets([topic], (err, data) => { + hardOffsetReset() { + return Promise.reject("hardOffsetReset has been removed, as it was supporting bad kafka consumer behaviour."); + } - if(err || !data[topic]){ - return reject("failed to get offsets of topic: " + topic + "; " + err); + _getLogger() { + // @ts-ignore + if (this._logger) { + // @ts-ignore + return this._logger; } - resolve(data[topic]); - }); - }); - } + return NOOPL; + } - getTopics(){ - return this.targetTopics; - } + setConsumerOffset(topic = "t", partition = 0, offset = 0) { + this._getLogger().debug("adjusting offset for topic: " + topic + " on partition: " + partition + " to " + offset); + this.consumer.setOffset(topic, partition, offset); + } - hardOffsetReset(){ - return Promise.reject("hardOffsetReset has been removed, as it was supporting bad kafka consumer behaviour."); - } + commitCurrentOffsets() { + return new Promise((resolve, reject) => { + this.consumer.commit((err, data) => { - _getLogger(){ + if (err) { + return reject(err); + } - if(this._logger){ - return this._logger; + resolve(data); + }); + }); } - return NOOPL; - } - - setConsumerOffset(topic = "t", partition = 0, offset = 0){ - this._getLogger().debug("adjusting offset for topic: " + topic + " on partition: " + partition + " to " + offset); - this.consumer.setOffset(topic, partition, offset); - } + becomeManualConsumer(topics, groupId, options, dontListenForSIGINT) { + this._isManual = true; + return this.becomeConsumer(topics, groupId, options, dontListenForSIGINT, false); + } - commitCurrentOffsets(){ - return new Promise((resolve, reject) => { - this.consumer.commit((err, data) => { + becomeConsumer(topics = ["t"], groupId = "kafka-node-group", _options = {}, dontListenForSIGINT = false, autoCommit = true) { - if(err){ - return reject(err); + if (!Array.isArray(topics) || topics.length <= 0) { + throw new Error("becomeConsumer requires a valid topics array, with at least a single topic."); } - resolve(data); - }); - }); - } + if (this.isConsumer) { + throw new Error("this kafka instance has already been initialised as consumer."); + } - becomeManualConsumer(topics, groupId, options, dontListenForSIGINT){ - this._isManual = true; - return this.becomeConsumer(topics, groupId, options, dontListenForSIGINT, false); - } + if (this.isProducer) { + throw new Error("this kafka instance has already been initialised as producer."); + } - becomeConsumer(topics = ["t"], groupId = "kafka-node-group", _options = {}, dontListenForSIGINT = false, autoCommit = true){ + if (!groupId) { + throw new Error("missing groupId or consumer configuration."); + } - if(!Array.isArray(topics) || topics.length <= 0){ - throw new Error("becomeConsumer requires a valid topics array, with at least a single topic."); + const options: ConsumerGroupOptions = { + // @ts-ignore + host: this.connectDirectlyToBroker ? undefined : this.conString, + kafkaHost: this.connectDirectlyToBroker ? this.conString : undefined, + //zk: undefined, + //batch: undefined, + ssl: false, + groupId: groupId, + sessionTimeout: 30000, + protocol: ["roundrobin"], + fromOffset: "earliest", // latest + migrateHLC: false, + migrateRolling: false, + fetchMaxBytes: 1024 * 100, + fetchMinBytes: 1, + fetchMaxWaitMs: 100, + autoCommit: autoCommit, + autoCommitIntervalMs: 5000, + // @ts-ignore + connectRetryOptions: this.connectDirectlyToBroker ? DEFAULT_RETRY_OPTIONS : undefined, + encoding: "buffer", + keyEncoding: "buffer" + }; + + //overwrite default options + _options = _options || {}; + Object.keys(_options).forEach(key => options[key] = _options[key]); + + this._autoCommitEnabled = options.autoCommit; + + this.consumer = new ConsumerGroup(options, topics); + this.client = this.consumer.client; + this.isConsumer = true; + this.pause(); + + this.targetTopics = topics; + this._getLogger().info("starting ConsumerGroup for topic: " + JSON.stringify(topics)); + + this._attachConsumerListeners(dontListenForSIGINT); } - if(this.isConsumer){ - throw new Error("this kafka instance has already been initialised as consumer."); - } + becomeProducer(targetTopics = ["t"], clientId = "kafka-node-client", _options = {}) { - if(this.isProducer){ - throw new Error("this kafka instance has already been initialised as producer."); - } + if (this.isConsumer) { + throw new Error("this kafka instance has already been initialised as consumer."); + } - if(!groupId){ - throw new Error("missing groupId or consumer configuration."); - } + if (this.isProducer) { + throw new Error("this kafka instance has already been initialised as producer."); + } - const options = { - host: this.connectDirectlyToBroker ? undefined : this.conString, - kafkaHost: this.connectDirectlyToBroker ? this.conString : undefined, - //zk: undefined, - //batch: undefined, - ssl: false, - groupId: groupId, - sessionTimeout: 30000, - protocol: ["roundrobin"], - fromOffset: "earliest", // latest - migrateHLC: false, - migrateRolling: false, - fetchMaxBytes: 1024 * 100, - fetchMinBytes: 1, - fetchMaxWaitMs: 100, - autoCommit: autoCommit, - autoCommitIntervalMs: 5000, - connectRetryOptions: this.connectDirectlyToBroker ? DEFAULT_RETRY_OPTIONS : undefined, - encoding: "buffer", - keyEncoding: "buffer" - }; - - //overwrite default options - _options = _options || {}; - Object.keys(_options).forEach(key => options[key] = _options[key]); - - this._autoCommitEnabled = options.autoCommit; - - this.consumer = new ConsumerGroup(options, topics); - this.client = this.consumer.client; - this.isConsumer = true; - this.pause(); - - this.targetTopics = topics; - this._getLogger().info("starting ConsumerGroup for topic: " + JSON.stringify(topics)); - - this._attachConsumerListeners(dontListenForSIGINT); - } - - becomeProducer(targetTopics = ["t"], clientId = "kafka-node-client", _options = {}){ - - if(this.isConsumer){ - throw new Error("this kafka instance has already been initialised as consumer."); - } + const options = { + requireAcks: 1, + ackTimeoutMs: 100, + partitionerType: 3 + }; + + //overwrite default options + _options = _options || {}; + Object.keys(_options).forEach(key => options[key] = _options[key]); + + this.client = null; + if (this.connectDirectlyToBroker) { + + const kafkaOptions = { + kafkaHost: this.conString, + // @ts-ignore + ssl: !!_options.sslOptions, + // @ts-ignore + sslOptions: _options.sslOptions, + connectTimeout: 1000, + requestTimeout: 30000, + // @ts-ignore + autoConnect: _options.autoConnect || true, + connectRetryOptions: DEFAULT_RETRY_OPTIONS + }; + + this.client = new KafkaClient(kafkaOptions); + } else { + // @ts-ignore + this.client = new Client(this.conString, clientId, {}, _options.sslOptions || {}); + } - if(this.isProducer){ - throw new Error("this kafka instance has already been initialised as producer."); - } + this.producer = new HighLevelProducer(this.client, _options); + this.isProducer = true; - const options = { - requireAcks: 1, - ackTimeoutMs: 100, - partitionerType: 3 - }; - - //overwrite default options - _options = _options || {}; - Object.keys(_options).forEach(key => options[key] = _options[key]); - - this.client = null; - if(this.connectDirectlyToBroker){ - - const kafkaOptions = { - kafkaHost: this.conString, - ssl: !!_options.sslOptions, - sslOptions: _options.sslOptions, - connectTimeout: 1000, - requestTimeout: 30000, - autoConnect: _options.autoConnect || true, - connectRetryOptions: DEFAULT_RETRY_OPTIONS - }; - - this.client = new KafkaClient(kafkaOptions); - } else { - this.client = new Client(this.conString, clientId, {}, _options.sslOptions || {}); + this._getLogger().info("starting Producer."); + this.targetTopics = targetTopics; + this._attachProducerListeners(); } - this.producer = new HighLevelProducer(this.client, _options); - this.isProducer = true; - - this._getLogger().info("starting Producer."); - this.targetTopics = targetTopics; - this._attachProducerListeners(); - } - - _attachProducerListeners(){ - - this.client.on("connect", () => { - this._getLogger().info("producer is connected."); - }); - - this.producer.on("ready", () => { - - this._getLogger().debug("producer ready fired."); - if(this._producerReadyFired){ - return; - } - - this._producerReadyFired = true; - this._getLogger().info("producer is ready."); - - //prevents key-partition errors - this.refreshMetadata(this.targetTopics).then(() => { - this.emit("ready"); - }); - }); - - this.producer.on("error", error => { - //dont log these, they emit very often - this.emit("error", error); - }); - } - - _attachConsumerListeners(dontListenForSIGINT = false, commitOnSIGINT = false){ - - this.consumer.once("connect", () => { - this._getLogger().info("consumer is connected / ready."); - this.emit("connect"); - this.emit("ready"); - }); - - //do not listen for "message" here - - this.consumer.on("error", error => { - //dont log these, they emit very often - this.emit("error", error); - }); - - this.consumer.on("offsetOutOfRange", error => { - //dont log these, they emit very often - this.emit("error", error); - }); - - //prevents re-balance errors - if(!dontListenForSIGINT){ - process.on("SIGINT", () => { - if(this.consumer){ - this.consumer.close(commitOnSIGINT, () => { - process.exit(); - }); - } - }); - } - } + _attachProducerListeners() { + + this.client.on("connect", () => { + this._getLogger().info("producer is connected."); + }); - _resetConsumer(){ - this.isConsumer = false; - this.client = null; - this.consumer = null; - } + this.producer.on("ready", () => { - _resetProducer(){ - this.isProducer = false; - this.client = null; - this.producer = null; - this._producerReadyFired = false; - } + this._getLogger().debug("producer ready fired."); + if (this._producerReadyFired) { + return; + } - _closeConsumer(commit) { - return new Promise((resolve, reject) => { + this._producerReadyFired = true; + this._getLogger().info("producer is ready."); - this._getLogger().info("trying to close consumer."); + //prevents key-partition errors + // @ts-ignore + this.refreshMetadata(this.targetTopics).then(() => { + this.emit("ready"); + }); + }); - if(!this.consumer){ - return reject("consumer is null"); - } + this.producer.on("error", error => { + //dont log these, they emit very often + this.emit("error", error); + }); + } - if(!commit){ + _attachConsumerListeners(dontListenForSIGINT = false, commitOnSIGINT = false) { - this.consumer.close(() => { - this._resetConsumer(); - resolve(); + this.consumer.once("connect", () => { + this._getLogger().info("consumer is connected / ready."); + this.emit("connect"); + this.emit("ready"); }); - return; - } + //do not listen for "message" here - this._getLogger().info("trying to commit kafka consumer before close."); + this.consumer.on("error", error => { + //dont log these, they emit very often + this.emit("error", error); + }); - this.consumer.commit((err, data) => { + this.consumer.on("offsetOutOfRange", error => { + //dont log these, they emit very often + this.emit("error", error); + }); - if(err){ - return reject(err); + //prevents re-balance errors + if (!dontListenForSIGINT) { + process.on("SIGINT", () => { + if (this.consumer) { + this.consumer.close(commitOnSIGINT, () => { + process.exit(); + }); + } + }); } + } - this.consumer.close(() => { - this._resetConsumer(); - resolve(data); - }); - }); - }); - } + _resetConsumer() { + this.isConsumer = false; + this.client = null; + this.consumer = null; + } + + _resetProducer() { + this.isProducer = false; + this.client = null; + this.producer = null; + this._producerReadyFired = false; + } + + _closeConsumer(commit) { + return new Promise((resolve, reject) => { - _closeProducer(){ - return new Promise((resolve, reject) => { + this._getLogger().info("trying to close consumer."); - this._getLogger().info("trying to close producer."); + if (!this.consumer) { + return reject("consumer is null"); + } - if(!this.producer){ - return reject("producer is null"); - } + if (!commit) { - this.producer.close(() => { - this._resetProducer(); - resolve(true); - }); - }); - } + this.consumer.close(() => { + this._resetConsumer(); + resolve(); + }); - refreshMetadata(topics = []){ + return; + } - if(!topics || topics.length <= 0){ - return Promise.resolve(); + this._getLogger().info("trying to commit kafka consumer before close."); + + this.consumer.commit((err, data) => { + + if (err) { + return reject(err); + } + + this.consumer.close(() => { + this._resetConsumer(); + resolve(data); + }); + }); + }); } - return new Promise(resolve => { - this.client.refreshMetadata(topics, () => { - this._getLogger().info("meta-data refreshed."); - resolve(); - }); - }); - } + _closeProducer() { + return new Promise((resolve, reject) => { - isPaused(){ + this._getLogger().info("trying to close producer."); - if(this.isConsumer){ - return this.consumer.paused; + if (!this.producer) { + return reject("producer is null"); + } + + this.producer.close(() => { + this._resetProducer(); + resolve(true); + }); + }); } - return false; - } + refreshMetadata(topics = []) { - pause(){ + if (!topics || topics.length <= 0) { + return Promise.resolve(); + } - if(this.isConsumer){ - return this.consumer.pause(); + return new Promise(resolve => { + this.client.refreshMetadata(topics, () => { + this._getLogger().info("meta-data refreshed."); + resolve(); + }); + }); } - return false; - } + isPaused() { - resume(){ + if (this.isConsumer) { + return this.consumer.paused; + } - if(this.isConsumer){ - return this.consumer.resume(); + return false; } - return false; - } + pause() { - close(commit = false){ + if (this.isConsumer) { + return this.consumer.pause(); + } - if(this.isConsumer){ - return this._closeConsumer(commit); + return false; } - if(this.isProducer){ - return this._closeProducer(); + resume() { + + if (this.isConsumer) { + return this.consumer.resume(); + } + + return false; } - return null; - } + close(commit = false) { + + if (this.isConsumer) { + return this._closeConsumer(commit); + } + + if (this.isProducer) { + return this._closeProducer(); + } + + return null; + } } diff --git a/src/kafka/PartitionDrainer.ts b/src/kafka/PartitionDrainer.ts index a8dcd60..a45586c 100644 --- a/src/kafka/PartitionDrainer.ts +++ b/src/kafka/PartitionDrainer.ts @@ -6,46 +6,60 @@ import PartitionQueue from "./PartitionQueue"; const DEFAULT_DRAIN_INTV = 3000; export default class PartitionDrainer { - - constructor(consumer = null, asyncLimit = 1, commitOnDrain = false, autoJsonParsing = true) { - - if (!consumer || !(consumer instanceof Kafka) || + private raw; + private _queueMap; + private _drainMap; + private _totalIncomingMessages: number; + private _incomingSinceLastDrain: number; + private _lastReceived: number; + private _receivedFirst: boolean; + private _drainStart; + private _lastMessageHandlerRef; + private _stats; + private DRAIN_INTV: number; + private _drainTargetTopic; + private disablePauseResume: boolean; + private _drainEvent; + + constructor(private consumer = null, private asyncLimit = 1, private commitOnDrain = false, private autoJsonParsing = true) { + + if (!consumer || !(consumer instanceof Kafka) || !consumer.isConsumer) { - throw new Error("consumer is not a valid Sinek Kafka(Consumer)"); - } + throw new Error("consumer is not a valid Sinek Kafka(Consumer)"); + } - this.consumer = consumer; - this.raw = consumer.consumer; + this.consumer = consumer; + this.raw = consumer.consumer; - this.asyncLimit = asyncLimit; - this.commitOnDrain = commitOnDrain; - this.autoJsonParsing = autoJsonParsing; + this.asyncLimit = asyncLimit; + this.commitOnDrain = commitOnDrain; + this.autoJsonParsing = autoJsonParsing; - this._queueMap = null; - this._drainMap = {}; + this._queueMap = null; + this._drainMap = {}; - this._totalIncomingMessages = 0; - this._incomingSinceLastDrain = 0; + this._totalIncomingMessages = 0; + this._incomingSinceLastDrain = 0; - this._lastReceived = Date.now(); - this._receivedFirst = false; - this._drainStart = null; + this._lastReceived = Date.now(); + this._receivedFirst = false; + this._drainStart = null; - this._lastMessageHandlerRef = null; + this._lastMessageHandlerRef = null; - this._stats = {}; + this._stats = {}; - this.DRAIN_INTV = DEFAULT_DRAIN_INTV; + this.DRAIN_INTV = DEFAULT_DRAIN_INTV; - this._drainTargetTopic = null; - this.disablePauseResume = false; - } + this._drainTargetTopic = null; + this.disablePauseResume = false; + } - _getLogger(){ - return this.consumer._getLogger(); - } + _getLogger() { + return this.consumer._getLogger(); + } - /** + /** * gets all partitions of the given topic * and builds a map of async.queues (PartionQueue) * with a single queue for each partition @@ -58,234 +72,234 @@ export default class PartitionDrainer { * @returns {*} * @private */ - _buildOffsetMap(topic, drainEvent, asyncLimit = 1){ + _buildOffsetMap(topic, drainEvent, asyncLimit = 1) { - if(typeof topic !== "string"){ - return Promise.reject("offset map can only be build for a single topic."); - } + if (typeof topic !== "string") { + return Promise.reject("offset map can only be build for a single topic."); + } - if(typeof drainEvent !== "function"){ - return Promise.reject("drainEvent must be a valid function."); - } + if (typeof drainEvent !== "function") { + return Promise.reject("drainEvent must be a valid function."); + } - if(this.consumer.getTopics().indexOf(topic) === -1){ - return Promise.reject(topic + " is not a supported topic, it has to be set during becomeConsumer()."); - } + if (this.consumer.getTopics().indexOf(topic) === -1) { + return Promise.reject(topic + " is not a supported topic, it has to be set during becomeConsumer()."); + } - return this.consumer.getPartitions(topic).then(partitions => { + return this.consumer.getPartitions(topic).then(partitions => { - if(!partitions || partitions.length <= 0){ - return Promise.reject(`partitions request for topic ${topic} returned empty.`); - } + if (!partitions || partitions.length <= 0) { + return Promise.reject(`partitions request for topic ${topic} returned empty.`); + } - const queueMap = {}; - const drainMap = {}; + const queueMap = {}; + const drainMap = {}; - partitions.forEach(partition => { - //build a parition queue for each partition - queueMap[partition] = new PartitionQueue(partition, drainEvent, this, asyncLimit, - this._onPartitionQueueDrain.bind(this)).build(); - }); + partitions.forEach(partition => { + //build a parition queue for each partition + queueMap[partition] = new PartitionQueue(partition, drainEvent, this, asyncLimit, + this._onPartitionQueueDrain.bind(this)).build(); + }); - partitions.forEach(partition => { - //drain map is build to check if all queues have been drained - drainMap[partition] = false; - }); + partitions.forEach(partition => { + //drain map is build to check if all queues have been drained + drainMap[partition] = false; + }); - return { - queueMap, - drainMap - }; - }); - } + return { + queueMap, + drainMap + }; + }); + } - /** + /** * partiton queue drain callback that makes sure to resume the consumer if * all queues have drained * @param partition * @param offset * @private */ - _onPartitionQueueDrain(partition){ + _onPartitionQueueDrain(partition) { - if(typeof this._drainMap[partition] === "undefined"){ - this._getLogger().warn(`partition queue drain called but ${partition} is not a present key.`); - return; - } + if (typeof this._drainMap[partition] === "undefined") { + this._getLogger().warn(`partition queue drain called but ${partition} is not a present key.`); + return; + } - this._drainMap[partition] = true; + this._drainMap[partition] = true; - //this queue drained, lets commit the latest offset - if(this.commitOnDrain && (this.consumer._isManual || !this.consumer._autoCommitEnabled)){ + //this queue drained, lets commit the latest offset + if (this.commitOnDrain && (this.consumer._isManual || !this.consumer._autoCommitEnabled)) { - if(this.consumer._autoCommitEnabled){ - throw new Error("you have started a consumer with auto commit enabled, but requested partition drainer" + + if (this.consumer._autoCommitEnabled) { + throw new Error("you have started a consumer with auto commit enabled, but requested partition drainer" + "to run commits manually for you - both cannot work at the same time."); - } - - //running setConsumerOffset while commit manually is a bad idea - //message offset is already hold in the client, only committing is needed - //this.consumer.setConsumerOffset(this._drainTargetTopic, partition, offset); - } + } - if(Object.keys(this._drainMap).map(key => this._drainMap[key]).filter(v => !v).length){ - this._getLogger().debug("not all partition queues have drained yet."); - } else { - this._getLogger().debug("all partition queues have drained."); - - // reset drain map - Object.keys(this._drainMap).forEach(key => { - this._drainMap[key] = false; - }); - - if(!this.commitOnDrain){ - if(!this.disablePauseResume){ - this.resume(); - } - return; //do not execute commit logic^ - } - - //resume consumer, which will cause new message to be pushed into the queues - //but make sure to commit current offsets first - this.consumer.commitCurrentOffsets().then(() => { - if(!this.disablePauseResume){ - this.resume(); + //running setConsumerOffset while commit manually is a bad idea + //message offset is already hold in the client, only committing is needed + //this.consumer.setConsumerOffset(this._drainTargetTopic, partition, offset); } - }).catch(e => { - this._getLogger().error(`failed to commit offsets after all partitions have been drained. ${e}.`); - if(!this.disablePauseResume){ - this.resume(); + + if (Object.keys(this._drainMap).map(key => this._drainMap[key]).filter(v => !v).length) { + this._getLogger().debug("not all partition queues have drained yet."); + } else { + this._getLogger().debug("all partition queues have drained."); + + // reset drain map + Object.keys(this._drainMap).forEach(key => { + this._drainMap[key] = false; + }); + + if (!this.commitOnDrain) { + if (!this.disablePauseResume) { + this.resume(); + } + return; //do not execute commit logic^ + } + + //resume consumer, which will cause new message to be pushed into the queues + //but make sure to commit current offsets first + this.consumer.commitCurrentOffsets().then(() => { + if (!this.disablePauseResume) { + this.resume(); + } + }).catch(e => { + this._getLogger().error(`failed to commit offsets after all partitions have been drained. ${e}.`); + if (!this.disablePauseResume) { + this.resume(); + } + }); } - }); } - } - _resetQueueMaps(){ + _resetQueueMaps() { - this._getLogger().info("resetting queue maps."); + this._getLogger().info("resetting queue maps."); - if(this._queueMap){ - Object.keys(this._queueMap).forEach(key => { - this._queueMap[key].close(); - }); - this._queueMap = null; - this._drainMap = {}; + if (this._queueMap) { + Object.keys(this._queueMap).forEach(key => { + this._queueMap[key].close(); + }); + this._queueMap = null; + this._drainMap = {}; + } } - } - /** + /** * stops any active drain process * closes the consumer and its client */ - close() { + close() { - if(this._lastMessageHandlerRef){ - this.raw.removeListener("message", this._lastMessageHandlerRef); - this._lastMessageHandlerRef = null; - } else { - this._getLogger().warn("message handler ref not present during close, could not remove listener."); - } + if (this._lastMessageHandlerRef) { + this.raw.removeListener("message", this._lastMessageHandlerRef); + this._lastMessageHandlerRef = null; + } else { + this._getLogger().warn("message handler ref not present during close, could not remove listener."); + } - this._resetQueueMaps(); + this._resetQueueMaps(); - this._getLogger().info("[Drainer] closed."); - return this.consumer.close(); - } + this._getLogger().info("[Drainer] closed."); + return this.consumer.close(); + } - /** + /** * returns a few insights * @returns {{totalIncoming: number, last: (number|*), isPaused: *}} */ - getStats() { - return { - totalIncoming: this._totalIncomingMessages, - lastMessage: this._lastReceived, + getStats() { + return { + totalIncoming: this._totalIncomingMessages, + lastMessage: this._lastReceived, - receivedFirstMsg: this._receivedFirst, + receivedFirstMsg: this._receivedFirst, - isPaused: this.consumer && this.consumer.isConsumer ? this.isPaused() : null, + isPaused: this.consumer && this.consumer.isConsumer ? this.isPaused() : null, - drainStats: this._stats, - partitions: this._queueMap ? Object.keys(this._queueMap).length : null, - queues: this._queueMap ? Object.keys(this._queueMap).map(key => this._queueMap[key].getStats()) : null - }; - } + drainStats: this._stats, + partitions: this._queueMap ? Object.keys(this._queueMap).length : null, + queues: this._queueMap ? Object.keys(this._queueMap).map(key => this._queueMap[key].getStats()) : null + }; + } - /** + /** * resets all offsets and starts from being * also un-pauses consumer if necessary * @param topics * @returns {Promise.} */ - resetConsumer() { - return Promise.reject("resetConsumer has been removed, due to supporting bad kafka consumer behaviour."); - } + resetConsumer() { + return Promise.reject("resetConsumer has been removed, due to supporting bad kafka consumer behaviour."); + } - /** + /** * resets all offsets and makes sure the consumer is paused * @param topics * @returns {Promise.} */ - resetOffset(){ - return Promise.reject("resetOffset has been removed, due to supporting bad kafka consumer behaviour."); - } + resetOffset() { + return Promise.reject("resetOffset has been removed, due to supporting bad kafka consumer behaviour."); + } - /** + /** * main reg. function, pass it a function to receive messages * under flow control, returns a promise * @param topic * @param drainEvent */ - drain(topic = "t", drainEvent = null) { + drain(topic = "t", drainEvent = null) { - this._drainTargetTopic = topic; + this._drainTargetTopic = topic; - if(!drainEvent || typeof drainEvent !== "function"){ - throw new Error("drainEvent must be a valid function"); - } + if (!drainEvent || typeof drainEvent !== "function") { + throw new Error("drainEvent must be a valid function"); + } - if(this._drainEvent){ - throw new Error("a drain process is currently active."); - } + if (this._drainEvent) { + throw new Error("a drain process is currently active."); + } - this._incomingSinceLastDrain = this._totalIncomingMessages; - this._drainEvent = drainEvent; - this._lastReceived = Date.now(); - this._stats = {}; + this._incomingSinceLastDrain = this._totalIncomingMessages; + this._drainEvent = drainEvent; + this._lastReceived = Date.now(); + this._stats = {}; - return this._buildOffsetMap(topic, drainEvent, this.asyncLimit).then(maps => { + return this._buildOffsetMap(topic, drainEvent, this.asyncLimit).then(maps => { - this._queueMap = maps.queueMap; - this._drainMap = maps.drainMap; + this._queueMap = maps.queueMap; + this._drainMap = maps.drainMap; - this._startToReceiveMessages(); + this._startToReceiveMessages(); - if(this.isPaused()){ - this.resume(); - } - }); - } + if (this.isPaused()) { + this.resume(); + } + }); + } - _getEarliestProcessedOnQueues(){ + _getEarliestProcessedOnQueues() { - //error prevention - if(!this._queueMap){ - return this._lastReceived; - } + //error prevention + if (!this._queueMap) { + return this._lastReceived; + } - let earliest = this._queueMap[Object.keys(this._queueMap)[0]].getLastProcessed(); - let ne = null; - Object.keys(this._queueMap).forEach(key => { - ne = this._queueMap[key].getLastProcessed(); - if(ne < earliest){ - earliest = ne; - } - }); + let earliest = this._queueMap[Object.keys(this._queueMap)[0]].getLastProcessed(); + let ne = null; + Object.keys(this._queueMap).forEach(key => { + ne = this._queueMap[key].getLastProcessed(); + if (ne < earliest) { + earliest = ne; + } + }); - return earliest; - } + return earliest; + } - /** + /** * main req. function, pass it a function to receive messages * under flow control, until they are stall for a certain amount * of time (e.g. when all messages on the queue are consumed) @@ -295,242 +309,242 @@ export default class PartitionDrainer { * @param drainThreshold * @param timeout */ - drainOnce(topic = "t", drainEvent = null, drainThreshold = 10000, timeout = 0){ - return new Promise((resolve, reject) => { + drainOnce(topic = "t", drainEvent = null, drainThreshold = 10000, timeout = 0) { + return new Promise((resolve, reject) => { - if(!drainEvent || typeof drainEvent !== "function"){ - return reject("drainEvent must be a valid function"); - } + if (!drainEvent || typeof drainEvent !== "function") { + return reject("drainEvent must be a valid function"); + } - if(this._drainEvent){ - return reject("a drain process is currently active."); - } + if (this._drainEvent) { + return reject("a drain process is currently active."); + } - if(timeout !== 0 && timeout < this.DRAIN_INTV){ - return reject(`timeout must be either 0 or > ${this.DRAIN_INTV}.`); - } + if (timeout !== 0 && timeout < this.DRAIN_INTV) { + return reject(`timeout must be either 0 or > ${this.DRAIN_INTV}.`); + } - if(timeout !== 0 && timeout <= drainThreshold){ - return reject(`timeout ${timeout} must be greater than the drainThreshold ${drainThreshold}.`); - } + if (timeout !== 0 && timeout <= drainThreshold) { + return reject(`timeout ${timeout} must be greater than the drainThreshold ${drainThreshold}.`); + } - let t = null; - let intv = null; + let t = null; + let intv = null; - intv = setInterval(() => { + intv = setInterval(() => { - const spanProcessed = Date.now() - this._getEarliestProcessedOnQueues(); - const spanReceived = Date.now() - this._lastReceived; + const spanProcessed = Date.now() - this._getEarliestProcessedOnQueues(); + const spanReceived = Date.now() - this._lastReceived; - this._getLogger().debug("drainOnce interval running, current span-rec: " + + this._getLogger().debug("drainOnce interval running, current span-rec: " + `${spanReceived} / span-proc: ${spanProcessed} ms.`); - //set stats - this._countStats("intv-cycle"); - this._stats["last-proc-since"] = spanProcessed; - this._stats["last-rec-since"] = spanReceived; - - //choose the smaller span - const span = spanProcessed < spanReceived ? spanProcessed : spanReceived; + //set stats + this._countStats("intv-cycle"); + this._stats["last-proc-since"] = spanProcessed; + this._stats["last-rec-since"] = spanReceived; + + //choose the smaller span + const span = spanProcessed < spanReceived ? spanProcessed : spanReceived; + + if (span >= drainThreshold) { + this._getLogger().info(`drainOnce span ${span} hit threshold ${drainThreshold}.`); + clearInterval(intv); + clearTimeout(t); + this.stopDrain(); + resolve(this._totalIncomingMessages - this._incomingSinceLastDrain); + } + }, this.DRAIN_INTV); + + if (timeout !== 0) { + this._getLogger().info(`drainOnce timeout active: ${timeout} ms.`); + t = setTimeout(() => { + this._getLogger().warn(`drainOnce timeout hit after ${timeout} ms.`); + clearInterval(intv); + this.stopDrain(); + reject("drainOnce ran into timeout."); + }, timeout); + } + + //start the drain process + this.drain(topic, drainEvent).then(() => { + this._getLogger().info("drain process of drainOnce has started."); + }).catch(e => { + reject(`failed to start drain process of drainOnce, because: ${e}.`); + }); + }); + } - if(span >= drainThreshold){ - this._getLogger().info(`drainOnce span ${span} hit threshold ${drainThreshold}.`); - clearInterval(intv); - clearTimeout(t); - this.stopDrain(); - resolve(this._totalIncomingMessages - this._incomingSinceLastDrain); - } - }, this.DRAIN_INTV); - - if(timeout !== 0){ - this._getLogger().info(`drainOnce timeout active: ${timeout} ms.`); - t = setTimeout(() => { - this._getLogger().warn(`drainOnce timeout hit after ${timeout} ms.`); - clearInterval(intv); - this.stopDrain(); - reject("drainOnce ran into timeout."); - }, timeout); - } - - //start the drain process - this.drain(topic, drainEvent).then(() => { - this._getLogger().info("drain process of drainOnce has started."); - }).catch(e => { - reject(`failed to start drain process of drainOnce, because: ${e}.`); - }); - }); - } - - /** + /** * stops any active drain process */ - stopDrain(){ + stopDrain() { - if(!this._drainEvent){ - throw new Error("there is no drain active."); - } + if (!this._drainEvent) { + throw new Error("there is no drain active."); + } - this._drainTargetTopic = null; + this._drainTargetTopic = null; - //reset - if(this._lastMessageHandlerRef){ - this.raw.removeListener("message", this._lastMessageHandlerRef); - this._lastMessageHandlerRef = null; - } else { - this._getLogger().warn("message handler ref not present during close, could not remove listener."); - } + //reset + if (this._lastMessageHandlerRef) { + this.raw.removeListener("message", this._lastMessageHandlerRef); + this._lastMessageHandlerRef = null; + } else { + this._getLogger().warn("message handler ref not present during close, could not remove listener."); + } - this._drainEvent = null; - this._receivedFirst = false; + this._drainEvent = null; + this._receivedFirst = false; - this._resetQueueMaps(); + this._resetQueueMaps(); - const duration = (Date.now() - this._drainStart) / 1000; - this._getLogger().info(`[Drainer] stopped drain process, had been open for ${duration} seconds.`); - } + const duration = (Date.now() - this._drainStart) / 1000; + this._getLogger().info(`[Drainer] stopped drain process, had been open for ${duration} seconds.`); + } - /** + /** * removes kafka topics (if broker allows this action) * @param topics */ - removeTopics(topics = []){ - return new Promise((resolve, reject) => { - this._getLogger().info(`deleting topics ${JSON.stringify(topics)}.`); - this.raw.client.removeTopicMetadata(topics, (err, data) => { - - if(err){ - return reject(err); - } + removeTopics(topics = []) { + return new Promise((resolve, reject) => { + this._getLogger().info(`deleting topics ${JSON.stringify(topics)}.`); + this.raw.client.removeTopicMetadata(topics, (err, data) => { + + if (err) { + return reject(err); + } + + resolve(data); + }); + }); + } - resolve(data); - }); - }); - } + pause() { - pause(){ + if (!this.isPaused()) { + this._countStats("paused"); + } - if(!this.isPaused()){ - this._countStats("paused"); + return this.consumer.pause(); } - return this.consumer.pause(); - } + resume() { - resume(){ + if (this.isPaused()) { + this._countStats("resumed"); + } - if(this.isPaused()){ - this._countStats("resumed"); + return this.consumer.resume(); } - return this.consumer.resume(); - } - - isPaused(){ - return this.consumer.isPaused(); - } + isPaused() { + return this.consumer.isPaused(); + } - _startToReceiveMessages(){ - this._lastMessageHandlerRef = this._onMessage.bind(this); - this.raw.on("message", this._lastMessageHandlerRef); - this._getLogger().info("[Drainer] started drain process."); - this._drainStart = Date.now(); - } + _startToReceiveMessages() { + this._lastMessageHandlerRef = this._onMessage.bind(this); + this.raw.on("message", this._lastMessageHandlerRef); + this._getLogger().info("[Drainer] started drain process."); + this._drainStart = Date.now(); + } - _onMessage(message){ + _onMessage(message) { - this._getLogger().debug("received kafka message => length: " + message.value.length + ", offset: " + + this._getLogger().debug("received kafka message => length: " + message.value.length + ", offset: " + message.offset + ", partition: " + message.partition + ", on topic: " + message.topic); - if(this.autoJsonParsing){ - try { - message.value = JSON.parse(message.value); - } catch(e){ - this._countStats("msg-parse-fail"); - return this.emit("error", "failed to json parse message value: " + message); - } - - if(!message.value){ - this._countStats("msg-empty"); - return this.emit("error", "message value is empty: " + message); - } - } + if (this.autoJsonParsing) { + try { + message.value = JSON.parse(message.value); + } catch (e) { + this._countStats("msg-parse-fail"); + return this.emit("error", "failed to json parse message value: " + message); + } + + if (!message.value) { + this._countStats("msg-empty"); + return this.emit("error", "message value is empty: " + message); + } + } - //we only want to drain messages that belong to our topic - if(message.topic === this._drainTargetTopic) { - - //identify queue for this topic - if (!this._queueMap) { - this._countStats("queue-map-missing"); - this._getLogger().warn("received message, but queue map is missing."); - } else if (!this._queueMap[message.partition]) { - this._countStats("queue-partition-missing"); - this._getLogger().warn("received message, but queue partition is missing for partition: " + message.partition); - } else { - //and push message into queue - this._queueMap[message.partition].push(message); - } - } else { - this._getLogger().warn(`receiving messages from other topic ${message.topic} only expecting to receive from ${this._drainTargetTopic} for this instance.`); - } + //we only want to drain messages that belong to our topic + if (message.topic === this._drainTargetTopic) { + + //identify queue for this topic + if (!this._queueMap) { + this._countStats("queue-map-missing"); + this._getLogger().warn("received message, but queue map is missing."); + } else if (!this._queueMap[message.partition]) { + this._countStats("queue-partition-missing"); + this._getLogger().warn("received message, but queue partition is missing for partition: " + message.partition); + } else { + //and push message into queue + this._queueMap[message.partition].push(message); + } + } else { + this._getLogger().warn(`receiving messages from other topic ${message.topic} only expecting to receive from ${this._drainTargetTopic} for this instance.`); + } - if(!this.disablePauseResume){ - this.pause(); - } + if (!this.disablePauseResume) { + this.pause(); + } - this._totalIncomingMessages++; - this._lastReceived = Date.now(); + this._totalIncomingMessages++; + this._lastReceived = Date.now(); - if(!this._receivedFirst){ - this._receivedFirst = true; - this._getLogger().info("consumer received first message."); - this.emit("first-drain-message", message); + if (!this._receivedFirst) { + this._receivedFirst = true; + this._getLogger().info("consumer received first message."); + this.emit("first-drain-message", message); + } } - } - _countStats(key){ + _countStats(key) { - if(!this._stats){ - return; - } + if (!this._stats) { + return; + } - if(!this._stats[key]){ - this._stats[key] = 1; - return; - } + if (!this._stats[key]) { + this._stats[key] = 1; + return; + } - this._stats[key]++; - } + this._stats[key]++; + } - /** + /** * consumer proxy * @param args */ - on(...args) { - this.consumer.on(...args); - } + on(...args) { + this.consumer.on(...args); + } - /** + /** * consumer proxy * @param args */ - once(...args) { - this.consumer.once(...args); - } + once(...args) { + this.consumer.once(...args); + } - /** + /** * consumer proxy * @param args */ - removeListener(...args){ - this.consumer.removeListener(...args); - } + removeListener(...args) { + this.consumer.removeListener(...args); + } - /** + /** * consumer proxy * @param args */ - emit(...args){ - this.consumer.emit(...args); - } + emit(...args) { + this.consumer.emit(...args); + } } diff --git a/src/kafka/PartitionQueue.ts b/src/kafka/PartitionQueue.ts index a43f58c..bb87268 100644 --- a/src/kafka/PartitionQueue.ts +++ b/src/kafka/PartitionQueue.ts @@ -1,163 +1,171 @@ import async from "async"; export default class PartitionQueue { - - constructor(partition, drainEvent, drainer, asyncLimit = 1, queueDrain = null){ - - if(typeof drainEvent !== "function"){ - throw new Error("drainEvent must be a function."); + private _onQueueDrain: Function; + private _q; + private _lastProcessed: number; + private _lastPushed: number; + private _totalPushed: number; + private _totalProcessedMessages: number; + private _totalMessageProcessFails: number; + private _lastOffset: number; + private _drainCheckIntv; + + constructor(public partition, private _drainEvent, private _drainer, private asyncLimit = 1, queueDrain = null) { + + if (typeof _drainEvent !== "function") { + throw new Error("drainEvent must be a function."); + } + + if (typeof queueDrain !== "function") { + throw new Error("queueDrain must be a function."); + } + + this.partition = partition; + this._onQueueDrain = queueDrain; + this._q = null; + + this._lastProcessed = Date.now(); + this._lastPushed = Date.now(); + + this._totalPushed = 0; + this._totalProcessedMessages = 0; + this._totalMessageProcessFails = 0; + this._lastOffset = -1; + + this._drainCheckIntv = null; } - if(typeof queueDrain !== "function"){ - throw new Error("queueDrain must be a function."); + _getLogger() { + return this._drainer._getLogger(); } - this.partition = partition; - this._onQueueDrain = queueDrain; - this._drainEvent = drainEvent; - this._drainer = drainer; - this.asyncLimit = asyncLimit; - this._q = null; - - this._lastProcessed = Date.now(); - this._lastPushed = Date.now(); - - this._totalPushed = 0; - this._totalProcessedMessages = 0; - this._totalMessageProcessFails = 0; - this._lastOffset = -1; - - this._drainCheckIntv = null; - } - - _getLogger(){ - return this._drainer._getLogger(); - } - - _emitDrain(){ - if(this._onQueueDrain){ - process.nextTick(() => { //await potential writing of lastOffset - this._onQueueDrain(this.partition, this._lastOffset); - }); + _emitDrain() { + if (this._onQueueDrain) { + process.nextTick(() => { //await potential writing of lastOffset + this._onQueueDrain(this.partition, this._lastOffset); + }); + } } - } - _runDrainCheckIntv(ms = 500, drainSpan = 2500){ + _runDrainCheckIntv(ms = 500, drainSpan = 2500) { - if(this._drainCheckIntv){ - throw new Error("drain check interval already active for partition queue."); - } + if (this._drainCheckIntv) { + throw new Error("drain check interval already active for partition queue."); + } - this._drainCheckIntv = setInterval(() => { + this._drainCheckIntv = setInterval(() => { - if(!this._q){ - return; - } + if (!this._q) { + return; + } - //queue size is greater than 0, will emit own drain event soon anyway - if(this._q.length() > 0){ - return; - } + //queue size is greater than 0, will emit own drain event soon anyway + if (this._q.length() > 0) { + return; + } - if(Date.now() - this._lastPushed > drainSpan){ - this._getLogger().debug(`partition ${this.partition} received no messages, flushing queue anyway.`); - this._emitDrain(); - } - }, ms); - } + if (Date.now() - this._lastPushed > drainSpan) { + this._getLogger().debug(`partition ${this.partition} received no messages, flushing queue anyway.`); + this._emitDrain(); + } + }, ms); + } - _closeDrainCheckIntv(){ - if(this._drainCheckIntv){ - clearInterval(this._drainCheckIntv); + _closeDrainCheckIntv() { + if (this._drainCheckIntv) { + clearInterval(this._drainCheckIntv); + } } - } - push(message){ - if(this._q){ - this._totalPushed++; - this._lastPushed = Date.now(); - this._q.push(message); + push(message) { + if (this._q) { + this._totalPushed++; + this._lastPushed = Date.now(); + this._q.push(message); + } } - } - - getLastProcessed(){ - return this._lastProcessed; - } - - getStats(){ - return { - partition: this.partition, - lastProcessed: this._lastProcessed, - totalProcessed: this._totalProcessedMessages, - totalProcessFails: this._totalMessageProcessFails, - queueSize: this._q ? this._q.length() : null, - workers: this.asyncLimit, - lastPushed: this._lastPushed, - totalPushed: this._totalPushed, - lastProcessedOffset: this._lastOffset - }; - } - - build(){ - - if(this._q){ - throw new Error("this queue has already been build."); + + getLastProcessed() { + return this._lastProcessed; } - this._q = async.queue((msg, done) => { - if(this._drainEvent){ - setImmediate(() => this._drainEvent(msg, err => { + getStats() { + return { + partition: this.partition, + lastProcessed: this._lastProcessed, + totalProcessed: this._totalProcessedMessages, + totalProcessFails: this._totalMessageProcessFails, + queueSize: this._q ? this._q.length() : null, + workers: this.asyncLimit, + lastPushed: this._lastPushed, + totalPushed: this._totalPushed, + lastProcessedOffset: this._lastOffset + }; + } - try { - if(typeof msg.offset === "undefined"){ - if(!err){ - err = new Error("missing offset on message: " + JSON.stringify(msg)); - } - this._getLogger().error("missing offset on message: " + JSON.stringify(msg)); + build() { + + if (this._q) { + throw new Error("this queue has already been build."); + } + + this._q = async.queue((msg, done) => { + if (this._drainEvent) { + setImmediate(() => this._drainEvent(msg, err => { + + try { + // @ts-ignore + if (typeof msg.offset === "undefined") { + if (!err) { + err = new Error("missing offset on message: " + JSON.stringify(msg)); + } + this._getLogger().error("missing offset on message: " + JSON.stringify(msg)); + } else { + // @ts-ignore + this._lastOffset = msg.offset; + } + } catch (e) { + if (!err) { + err = new Error("failed to parse message offset: " + e); + } + this._getLogger().error("failed to parse message offset: " + e); + } + + this._lastProcessed = Date.now(); + this._totalProcessedMessages++; + done(err); + })); } else { - this._lastOffset = msg.offset; + this._getLogger().debug("drainEvent not present, message is dropped."); } - } catch(e){ - if(!err){ - err = new Error("failed to parse message offset: " + e); + }, this.asyncLimit); + + this._q.drain = () => { + this._emitDrain(); + }; + + this._q.error(err => { + if (err) { + this._totalMessageProcessFails++; + this._getLogger().warn("error was passed back to consumer queue, dropping it silently: " + JSON.stringify(err)); } - this._getLogger().error("failed to parse message offset: " + e); - } - - this._lastProcessed = Date.now(); - this._totalProcessedMessages++; - done(err); - })); - } else { - this._getLogger().debug("drainEvent not present, message is dropped."); - } - }, this.asyncLimit); - - this._q.drain = () => { - this._emitDrain(); - }; - - this._q.error(err => { - if (err) { - this._totalMessageProcessFails++; - this._getLogger().warn("error was passed back to consumer queue, dropping it silently: " + JSON.stringify(err)); - } - }); - - this._getLogger().info(`partition queue has been build for partition: ${this.partition}.`); - this._runDrainCheckIntv(); - return this; //chain? - } - - close(){ - - this._closeDrainCheckIntv(); - - if(this._q){ - this._q.kill(); - this._q = null; + }); + + this._getLogger().info(`partition queue has been build for partition: ${this.partition}.`); + this._runDrainCheckIntv(); + return this; //chain? } - this._getLogger().info("queue closed."); - } + close() { + + this._closeDrainCheckIntv(); + + if (this._q) { + this._q.kill(); + this._q = null; + } + + this._getLogger().info("queue closed."); + } } diff --git a/src/kafka/Publisher.ts b/src/kafka/Publisher.ts index 2477f4b..9212a3d 100644 --- a/src/kafka/Publisher.ts +++ b/src/kafka/Publisher.ts @@ -7,129 +7,132 @@ import {KeyedMessage} from "kafka-node"; import {CompressionTypes} from "./../tools/index"; const MESSAGE_TYPES = { - PUBLISH: "-published", - UNPUBLISH: "-unpublished", - UPDATE: "-updated" + PUBLISH: "-published", + UNPUBLISH: "-unpublished", + UPDATE: "-updated" }; export default class Publisher { + private raw; + private _lastProcessed: number; + private _totalSentMessages: number; + private _paused: boolean; + private _buffer; + private _flushIntv; + private CompressionTypes; + private _bufferDisabled: boolean; + + constructor(public producer = null, public partitionCount = 1, private autoFlushBuffer = 0, private flushPeriod = 100) { + + if (!producer || !(producer instanceof Kafka) || !producer.isProducer) { + throw new Error("producer is not a valid Sinek Kafka(Producer)"); + } - constructor(producer = null, partitionCount = 1, autoFlushBuffer = 0, flushPeriod = 100) { - - if (!producer || !(producer instanceof Kafka) || !producer.isProducer) { - throw new Error("producer is not a valid Sinek Kafka(Producer)"); - } - - this.producer = producer; - this.raw = producer.producer; - this.partitionCount = partitionCount; - - this._lastProcessed = Date.now(); - this._totalSentMessages = 0; + this.raw = producer.producer; - this._paused = false; + this._lastProcessed = Date.now(); + this._totalSentMessages = 0; - this._buffer = {}; - this._flushIntv = null; + this._paused = false; - this.CompressionTypes = CompressionTypes; + this._buffer = {}; + this._flushIntv = null; + this.CompressionTypes = CompressionTypes; - this.autoFlushBuffer = autoFlushBuffer; - this.flushPeriod = flushPeriod; - this._bufferDisabled = false; - this.disableBuffer(); - } + this._bufferDisabled = false; + this.disableBuffer(); + } - /** + /** * default behaviour */ - disableBuffer(){ - this._getLogger().info("[Publisher] buffer disabled."); - this._stopAutoBufferFlushInterval(); - this._bufferDisabled = true; - } + disableBuffer() { + this._getLogger().info("[Publisher] buffer disabled."); + this._stopAutoBufferFlushInterval(); + this._bufferDisabled = true; + } - /** + /** * BETA */ - enableBuffer(){ + enableBuffer() { - this._getLogger().info("[Publisher] buffer enabled."); + this._getLogger().info("[Publisher] buffer enabled."); - if(this.autoFlushBuffer > 0){ - this.setAutoFlushBuffer(this.autoFlushBuffer, this.flushPeriod); + if (this.autoFlushBuffer > 0) { + this.setAutoFlushBuffer(this.autoFlushBuffer, this.flushPeriod); + } } - } - setAutoFlushBuffer(minBufferSize = 0, period = 100){ + setAutoFlushBuffer(minBufferSize = 0, period = 100) { + + if (typeof minBufferSize !== "number" || minBufferSize < 0) { + throw new Error("minBufferSize must be a number and higher or equal to 0."); + } + + if (typeof period !== "number" || period < 5 || period > 60000) { + throw new Error("period must be a number and > 5 and < 60000."); + } - if(typeof minBufferSize !== "number" || minBufferSize < 0){ - throw new Error("minBufferSize must be a number and higher or equal to 0."); + this._getLogger().info(`[Publisher] Adjusting auto flush buffer size: ${minBufferSize} and period: ${period}.`); + this._runAutoBufferFlushInterval(minBufferSize, period); } - if(typeof period !== "number" || period < 5 || period > 60000){ - throw new Error("period must be a number and > 5 and < 60000."); + stopAutoFlushBuffer() { + this._stopAutoBufferFlushInterval(); } - this._getLogger().info(`[Publisher] Adjusting auto flush buffer size: ${minBufferSize} and period: ${period}.`); - this._runAutoBufferFlushInterval(minBufferSize, period); - } - - stopAutoFlushBuffer(){ - this._stopAutoBufferFlushInterval(); - } - - _runAutoBufferFlushInterval(minSize, ms){ - this._flushIntv = setInterval(() => { - - Promise.all(Object - .keys(this._buffer) - .filter(k => this._buffer[k].length >= minSize) - .map(topic => this.flushBuffer(topic))) - .then(() => { - this._getLogger().debug("[Publisher] flushed buffer."); - }, e => { - this._getLogger().error(`[Publisher] failed to flush buffer: ${e}.`); - }); - }, ms); - } + _runAutoBufferFlushInterval(minSize, ms) { + this._flushIntv = setInterval(() => { + + Promise.all(Object + .keys(this._buffer) + .filter(k => this._buffer[k].length >= minSize) + .map(topic => this.flushBuffer(topic))) + .then(() => { + this._getLogger().debug("[Publisher] flushed buffer."); + }, e => { + this._getLogger().error(`[Publisher] failed to flush buffer: ${e}.`); + }); + }, ms); + } - _stopAutoBufferFlushInterval(){ + _stopAutoBufferFlushInterval() { - if(this._flushIntv){ - this._getLogger().debug("[Publisher] stopping auto-buffer flush interval."); - clearInterval(this._flushIntv); + if (this._flushIntv) { + this._getLogger().debug("[Publisher] stopping auto-buffer flush interval."); + clearInterval(this._flushIntv); + } } - } - _getLogger() { - return this.producer._getLogger(); - } + _getLogger() { + return this.producer._getLogger(); + } - /** + /** * closes the publisher (and the underlying producer/client) */ - close() { - this._getLogger().info("[Publisher] closed."); - this._stopAutoBufferFlushInterval(); - return this.producer.close(); - } + close() { + this._getLogger().info("[Publisher] closed."); + this._stopAutoBufferFlushInterval(); + return this.producer.close(); + } - /** + /** * returns a few insights * @returns {{totalPublished: (number|*), last: (number|*), isPaused: *}} */ - getStats() { - return { - totalPublished: this._totalSentMessages, - last: this._lastProcessed, - isPaused: this.producer && this.producer.isProducer ? this.isPaused() : null - }; - } - - /** + getStats() { + return { + totalPublished: this._totalSentMessages, + last: this._lastProcessed, + isPaused: this.producer && this.producer.isProducer ? this.isPaused() : null + }; + } + + /** * uses the partition count to identify * a partition in range using a hashed representation * of the key's string value @@ -137,57 +140,58 @@ export default class Publisher { * @param partitionCount * @returns {Promise} */ - getPartitionForKey(key, partitionCount = 0){ + getPartitionForKey(key, partitionCount = 0) { - if(typeof key !== "string"){ - return Promise.reject("key must be a valid string"); - } + if (typeof key !== "string") { + return Promise.reject("key must be a valid string"); + } - if(partitionCount === 0){ - partitionCount = this.partitionCount; - } + if (partitionCount === 0) { + partitionCount = this.partitionCount; + } - return Promise.resolve(murmur(key) % partitionCount); - } + // @ts-ignore + return Promise.resolve(murmur(key) % partitionCount); + } - getRandomPartition(partitionCount = 0){ - return this.getPartitionForKey(uuid.v4(), partitionCount); - } + getRandomPartition(partitionCount = 0) { + return this.getPartitionForKey(uuid.v4(), partitionCount); + } - /** + /** * create topics (be aware that this requires * certain settings in your broker to be active) * @param topics */ - createTopics(topics = ["t"]){ - return new Promise((resolve, reject) => { - this._getLogger().info(`[Publisher] creating topics ${JSON.stringify(topics)}.`); - this.raw.createTopics(topics, true, (err, data) => { + createTopics(topics = ["t"]) { + return new Promise((resolve, reject) => { + this._getLogger().info(`[Publisher] creating topics ${JSON.stringify(topics)}.`); + this.raw.createTopics(topics, true, (err, data) => { - if(err){ - return reject(err); - } + if (err) { + return reject(err); + } - resolve(data); - }); - }); - } + resolve(data); + }); + }); + } - /** + /** * returns a default message type object * @returns {{topic: string, messages: Array, key: null, partition: number, attributes: number}} */ - static getKafkaBaseMessage(){ - return { - topic: "", - messages: [], - key: null, - partition: 0, - attributes: 0 - }; - } - - /** + static getKafkaBaseMessage() { + return { + topic: "", + messages: [], + key: null, + partition: 0, + attributes: 0 + }; + } + + /** * returns a kafka producer payload ready to be sent * identifies partition of message by using identifier * @param topic @@ -197,25 +201,25 @@ export default class Publisher { * @param {string | null} partitionKey base string for partition determination * @returns {*} */ - getKeyedPayload(topic = "t", identifier = "", object = {}, compressionType = 0, partitionKey = null) { + getKeyedPayload(topic = "t", identifier = "", object = {}, compressionType = 0, partitionKey = null) { - if(!this.CompressionTypes.isValid(compressionType)){ - return Promise.reject("compressionType is not valid checkout publisher.CompressionTypes."); - } + if (!this.CompressionTypes.isValid(compressionType)) { + return Promise.reject("compressionType is not valid checkout publisher.CompressionTypes."); + } - partitionKey = typeof partitionKey === "string" ? partitionKey : identifier; + partitionKey = typeof partitionKey === "string" ? partitionKey : identifier; - return this.getPartitionForKey(partitionKey).then(partition => { - return { - topic, - partition, - messages: new KeyedMessage(identifier, JSON.stringify(object)), - attributes: compressionType - }; - }); - } + return this.getPartitionForKey(partitionKey).then(partition => { + return { + topic, + partition, + messages: new KeyedMessage(identifier, JSON.stringify(object)), + attributes: compressionType + }; + }); + } - /** + /** * easy access to compliant kafka topic api * this will create a store a message payload describing a "CREATE" event * @param topic @@ -226,34 +230,34 @@ export default class Publisher { * @param {string | null} partitionKey base string for partition determination * @returns {*} */ - bufferPublishMessage(topic, identifier, object, version = 1, compressionType = 0, partitionKey = null){ + bufferPublishMessage(topic, identifier, object, version = 1, compressionType = 0, partitionKey = null) { - if(typeof identifier !== "string"){ - return Promise.reject("expecting identifier to be of type string."); - } + if (typeof identifier !== "string") { + return Promise.reject("expecting identifier to be of type string."); + } - if(typeof object !== "object"){ - return Promise.reject("expecting object to be of type object."); - } + if (typeof object !== "object") { + return Promise.reject("expecting object to be of type object."); + } - if(!object.id){ - object.id = identifier; - } + if (!object.id) { + object.id = identifier; + } - if(typeof object.version === "undefined"){ - object.version = version; - } + if (typeof object.version === "undefined") { + object.version = version; + } - return this.appendBuffer(topic, identifier, { - payload: object, - key: identifier, - id: uuid.v4(), - time: (new Date()).toISOString(), - type: topic + MESSAGE_TYPES.PUBLISH - }, compressionType, partitionKey); - } + return this.appendBuffer(topic, identifier, { + payload: object, + key: identifier, + id: uuid.v4(), + time: (new Date()).toISOString(), + type: topic + MESSAGE_TYPES.PUBLISH + }, compressionType, partitionKey); + } - /** + /** * easy access to compliant kafka topic api * this will create a store a message payload describing a "DELETE" event * @param topic @@ -264,34 +268,38 @@ export default class Publisher { * @param partitionKey * @returns {*} */ - bufferUnpublishMessage(topic, identifier, object = {}, version = 1, compressionType = 0, partitionKey = null){ + bufferUnpublishMessage(topic, identifier, object = {}, version = 1, compressionType = 0, partitionKey = null) { - if(typeof identifier !== "string"){ - return Promise.reject("expecting identifier to be of type string."); - } + if (typeof identifier !== "string") { + return Promise.reject("expecting identifier to be of type string."); + } - if(typeof object !== "object"){ - return Promise.reject("expecting object to be of type object."); - } + if (typeof object !== "object") { + return Promise.reject("expecting object to be of type object."); + } - if(!object.id){ - object.id = identifier; - } + // @ts-ignore + if (!object.id) { + // @ts-ignore + object.id = identifier; + } - if(typeof object.version === "undefined"){ - object.version = version; - } + // @ts-ignore + if (typeof object.version === "undefined") { + // @ts-ignore + object.version = version; + } - return this.appendBuffer(topic, identifier, { - payload: object, - key: identifier, - id: uuid.v4(), - time: (new Date()).toISOString(), - type: topic + MESSAGE_TYPES.UNPUBLISH - }, compressionType, partitionKey); - } + return this.appendBuffer(topic, identifier, { + payload: object, + key: identifier, + id: uuid.v4(), + time: (new Date()).toISOString(), + type: topic + MESSAGE_TYPES.UNPUBLISH + }, compressionType, partitionKey); + } - /** + /** * easy access to compliant kafka topic api * this will create a store a message payload describing an "UPDATE" event * @param topic @@ -302,34 +310,34 @@ export default class Publisher { * @param partitionKey * @returns {*} */ - bufferUpdateMessage(topic, identifier, object, version = 1, compressionType = 0, partitionKey = null){ + bufferUpdateMessage(topic, identifier, object, version = 1, compressionType = 0, partitionKey = null) { - if(typeof identifier !== "string"){ - return Promise.reject("expecting identifier to be of type string."); - } + if (typeof identifier !== "string") { + return Promise.reject("expecting identifier to be of type string."); + } - if(typeof object !== "object"){ - return Promise.reject("expecting object to be of type object."); - } + if (typeof object !== "object") { + return Promise.reject("expecting object to be of type object."); + } - if(!object.id){ - object.id = identifier; - } + if (!object.id) { + object.id = identifier; + } - if(typeof object.version === "undefined"){ - object.version = version; - } + if (typeof object.version === "undefined") { + object.version = version; + } - return this.appendBuffer(topic, identifier, { - payload: object, - key: identifier, - id: uuid.v4(), - time: (new Date()).toISOString(), - type: topic + MESSAGE_TYPES.UPDATE - }, compressionType, partitionKey); - } + return this.appendBuffer(topic, identifier, { + payload: object, + key: identifier, + id: uuid.v4(), + time: (new Date()).toISOString(), + type: topic + MESSAGE_TYPES.UPDATE + }, compressionType, partitionKey); + } - /** + /** * build a buffer per topic for message payloads * if autoBufferFlush is > 0 flushBuffer might be called * @param topic @@ -339,43 +347,43 @@ export default class Publisher { * @param {string | null} partitionKey base string for partition determination * @returns {Promise.} */ - appendBuffer(topic, identifier, object, compressionType = 0, partitionKey = null){ + appendBuffer(topic, identifier, object, compressionType = 0, partitionKey = null) { - return this.getKeyedPayload(topic, identifier, object, compressionType, partitionKey).then(payload => { + return this.getKeyedPayload(topic, identifier, object, compressionType, partitionKey).then(payload => { - //if buffer is disbaled, this message will be send instantly - if(this._bufferDisabled){ - return this.batch([payload]); - } + //if buffer is disbaled, this message will be send instantly + if (this._bufferDisabled) { + return this.batch([payload]); + } - if(!this._buffer[topic]){ - this._buffer[topic] = []; - } + if (!this._buffer[topic]) { + this._buffer[topic] = []; + } - this._buffer[topic].push(payload); - }); - } + this._buffer[topic].push(payload); + }); + } - /** + /** * send all message payloads in buffer for a topic * in a single batch request * @param topic * @param skipBlock * @returns {*} */ - flushBuffer(topic){ + flushBuffer(topic) { - if(!this._buffer[topic]){ - return Promise.reject(`topic ${topic} has no buffer, you should call appendBuffer() first.`); - } + if (!this._buffer[topic]) { + return Promise.reject(`topic ${topic} has no buffer, you should call appendBuffer() first.`); + } - const batch = this._buffer[topic]; - this._buffer[topic] = []; + const batch = this._buffer[topic]; + this._buffer[topic] = []; - return this.batch(batch); - } + return this.batch(batch); + } - /** + /** * appends and sends the message payloads in the buffer * (you can also use this so send a single message immediately) * @param topic @@ -384,13 +392,13 @@ export default class Publisher { * @param compressionType * @returns {Promise.} */ - appendAndFlushBuffer(topic, identifier, object, compressionType = 0){ - return this.appendBuffer(topic, identifier, object, compressionType).then(() => { - return this.flushBuffer(topic); - }); - } + appendAndFlushBuffer(topic, identifier, object, compressionType = 0) { + return this.appendBuffer(topic, identifier, object, compressionType).then(() => { + return this.flushBuffer(topic); + }); + } - /** + /** * most versatile function to produce a message on a topic(s) * you can send multiple messages at once (but keep them to the same topic!) * if you need full flexibility on payload (message definition) basis @@ -402,110 +410,112 @@ export default class Publisher { * @param compressionType * @returns {*} */ - send(topic = "t", messages = [], partitionKey = null, partition = null, compressionType = 0){ + send(topic = "t", messages = [], partitionKey = null, partition = null, compressionType = 0) { - if(!this.CompressionTypes.isValid(compressionType)){ - return Promise.reject("compressionType is not valid checkout publisher.CompressionTypes."); - } + if (!this.CompressionTypes.isValid(compressionType)) { + return Promise.reject("compressionType is not valid checkout publisher.CompressionTypes."); + } - const payload = { - topic, - messages, - attributes: compressionType - }; + const payload = { + topic, + messages, + attributes: compressionType + }; - if(partitionKey !== null){ - payload.key = partitionKey; - } + if (partitionKey !== null) { + // @ts-ignore + payload.key = partitionKey; + } - if(partition !== null){ - payload.partition = partition; - } + if (partition !== null) { + // @ts-ignore + payload.partition = partition; + } - return this.batch([ payload ]); - } + return this.batch([payload]); + } - /** + /** * leaves full flexibility when sending different message definitions (e.g. mulitple topics) * at once use with care: https://www.npmjs.com/package/kafka-node#sendpayloads-cb * @param payloads * @returns {Promise.<{}>} */ - batch(payloads){ - - if(this._paused){ - return Promise.resolve({}); - } - - return new Promise((resolve, reject) => { - this.raw.send(payloads, (err, data) => { + batch(payloads) { - if(err){ - return reject(err); + if (this._paused) { + return Promise.resolve({}); } - //update stats - this._lastProcessed = Date.now(); - payloads.forEach(p => { - if(p && p.messages){ - if(Array.isArray(p.messages)){ - this._totalSentMessages += p.messages.length; - } else { - this._totalSentMessages++; - } - } + return new Promise((resolve, reject) => { + this.raw.send(payloads, (err, data) => { + + if (err) { + return reject(err); + } + + //update stats + this._lastProcessed = Date.now(); + payloads.forEach(p => { + if (p && p.messages) { + if (Array.isArray(p.messages)) { + this._totalSentMessages += p.messages.length; + } else { + this._totalSentMessages++; + } + } + }); + + resolve(data); + }); }); + } - resolve(data); - }); - }); - } - - pause(){ - this._paused = true; - } + pause() { + this._paused = true; + } - resume(){ - this._paused = false; - } + resume() { + this._paused = false; + } - isPaused(){ - return this._paused; - } + isPaused() { + return this._paused; + } - refreshMetadata(topics = []){ - return this.producer.refreshMetadata(topics); - } + refreshMetadata(topics = []) { + return this.producer.refreshMetadata(topics); + } - /** + /** * producer proxy * @param args */ - on(...args) { - this.producer.on(...args); - } + on(...args) { + this.producer.on(...args); + } - /** + /** * producer proxy * @param args */ - once(...args) { - this.producer.once(...args); - } + once(...args) { + this.producer.once(...args); + } - /** + /** * producer proxy * @param args */ - removeListener(...args){ - this.producer.removeListener(...args); - } + removeListener(...args) { + this.producer.removeListener(...args); + } - /** + /** * producer proxy * @param args */ - emit(...args){ - this.producer.emit(...args); - } + emit(...args) { + this.producer.emit(...args); + } } diff --git a/src/librdkafka/Analytics.ts b/src/librdkafka/Analytics.ts index 94f3332..5a5ac23 100644 --- a/src/librdkafka/Analytics.ts +++ b/src/librdkafka/Analytics.ts @@ -4,332 +4,338 @@ const INTERESTING_DISTANCE = 10; * parent analytics class */ class Analytics { + private _lastErrors: number = 0; + + /** + * creates a new instance + * @param {NConsumer|NProducer} client + * @param {object} config + * @param {object} logger + */ + constructor(protected client, protected config, protected logger) { + } + + /** + * @private + * returns occured errors in interval + * @param {object} stats - getStats() client result + * @returns {number} + */ + _errorsInInterval(stats) { + const diff = (stats.totalErrors || 0) - this._lastErrors; + this._lastErrors = stats.totalErrors || 0; + return diff; + } + + /** + * @static + * @param {Array} offsets + */ + static statusArrayToKeyedObject(offsets = []) { + + const obj = {}; - /** - * creates a new instance - * @param {NConsumer|NProducer} client - * @param {object} config - * @param {object} logger - */ - constructor(client, config, logger){ - this.client = client; - this.config = config; - this.logger = logger; - - this._lastErrors = 0; - } - - /** - * @private - * returns occured errors in interval - * @param {object} stats - getStats() client result - * @returns {number} - */ - _errorsInInterval(stats){ - const diff = (stats.totalErrors || 0) - this._lastErrors; - this._lastErrors = stats.totalErrors || 0; - return diff; - } - - /** - * @static - * @param {Array} offsets - */ - static statusArrayToKeyedObject(offsets = []){ - - const obj = {}; - - offsets.forEach(offset => { - - if(!obj[offset.topic]){ - obj[offset.topic] = {}; - } - - obj[offset.topic][offset.partition] = { - lowDistance: offset.lowDistance, - highDistance: offset.highDistance, - detail: offset.detail - }; - }); - - return obj; - } + offsets.forEach(offset => { + + if (!obj[offset.topic]) { + obj[offset.topic] = {}; + } + + obj[offset.topic][offset.partition] = { + lowDistance: offset.lowDistance, + highDistance: offset.highDistance, + detail: offset.detail + }; + }); + + return obj; + } } /** * outsourced analytics for nconsumers */ export class ConsumerAnalytics extends Analytics { - - /** - * creates a new instance - * @param {NConsumer} nconsumer - * @param {object} config - * @param {object} logger - */ - constructor(nconsumer, config, logger){ - super(nconsumer, config, logger); - - this._lastRes = null; - this._consumedCount = 0; - } - - /** - * resolves a comparison between lag states - * @private - * @returns {Promise.} - */ - async _checkLagChanges(){ - - const last = this.client._lastLagStatus; - await this.client.getLagStatus(); //await potential refresh - const newest = this.client._lagCache; - - if(!last || !newest){ - return { - error: "No lag status fetched yet." - }; - } - - if(!last){ - return { - error: "Only newest status fetched yet." - }; + private _lastRes = null; + private _consumedCount: number = 0; + + /** + * creates a new instance + * @param {NConsumer} nconsumer + * @param {object} config + * @param {object} logger + */ + constructor(nconsumer, config, logger) { + super(nconsumer, config, logger); } - if(!newest){ - return { - error: "Only last status fetched yet." - }; - } + /** + * resolves a comparison between lag states + * @private + * @returns {Promise.} + */ + async _checkLagChanges() { + + const last = this.client._lastLagStatus; + await this.client.getLagStatus(); //await potential refresh + const newest = this.client._lagCache; + + if (!last || !newest) { + return { + error: "No lag status fetched yet." + }; + } - const newLags = {}; - const changedLags = {}; - const resolvedLags = {}; - const stallLags = {}; + if (!last) { + return { + error: "Only newest status fetched yet." + }; + } - const lastKeyed = Analytics.statusArrayToKeyedObject(last.status); + if (!newest) { + return { + error: "Only last status fetched yet." + }; + } - newest.status.forEach(offset => { + const newLags = {}; + const changedLags = {}; + const resolvedLags = {}; + const stallLags = {}; + + const lastKeyed = Analytics.statusArrayToKeyedObject(last.status); + + newest.status.forEach(offset => { + + //didnt exist in last check + if (!lastKeyed[offset.topic] || !lastKeyed[offset.topic][offset.partition]) { + //distance is interesting + if (offset.highDistance >= INTERESTING_DISTANCE) { + if (!newLags[offset.topic]) { + newLags[offset.topic] = {}; + } + + //store new lag for this partition + newLags[offset.topic][offset.partition] = offset.highDistance; + } + return; + } + //did exist in last check + + //distance decreased + if (offset.highDistance < INTERESTING_DISTANCE) { + + if (!resolvedLags[offset.topic]) { + resolvedLags[offset.topic] = {}; + } + + resolvedLags[offset.topic][offset.partition] = offset.highDistance; + return; + } + + //distance equals + if (offset.highDistance === lastKeyed[offset.topic][offset.partition].highDistance) { + + if (!stallLags[offset.topic]) { + stallLags[offset.topic] = {}; + } + + stallLags[offset.topic][offset.partition] = offset.highDistance; + return; + } + + //distance changed (but did not decrease enough) + if (!changedLags[offset.topic]) { + changedLags[offset.topic] = {}; + } + + changedLags[offset.topic][offset.partition] = offset.highDistance; + }); + + return { + timelyDifference: newest.at - last.at, + fetchPerformance: last.took - newest.took, + newLags, + changedLags, + resolvedLags, + stallLags + }; + } - //didnt exist in last check - if(!lastKeyed[offset.topic] || !lastKeyed[offset.topic][offset.partition]){ - //distance is interesting - if(offset.highDistance >= INTERESTING_DISTANCE){ - if(!newLags[offset.topic]){ - newLags[offset.topic] = {}; - } + /** + * gets the largest lag in all assigned offsets + * @private + * @returns {object} + */ + _identifyLargestLag() { - //store new lag for this partition - newLags[offset.topic][offset.partition] = offset.highDistance; - } - return; - } - //did exist in last check + let lag = { + highDistance: -1 + }; - //distance decreased - if(offset.highDistance < INTERESTING_DISTANCE){ + const newest = this.client._lagCache; - if(!resolvedLags[offset.topic]){ - resolvedLags[offset.topic] = {}; + if (!newest) { + return { + error: "Only last status fetched yet." + }; } - resolvedLags[offset.topic][offset.partition] = offset.highDistance; - return; - } + newest.status.forEach(offset => { + if (offset.highDistance > lag.highDistance) { + lag = offset; + } + }); - //distance equals - if(offset.highDistance === lastKeyed[offset.topic][offset.partition].highDistance){ + return lag; + } - if(!stallLags[offset.topic]){ - stallLags[offset.topic] = {}; + /** + * returns consumed amount of messages in interval + * @private + * @param {object} stats - getStats() client result + * @returns {number} + */ + _consumed(stats) { + const diff = (stats.totalIncoming || 0) - this._consumedCount; + this._consumedCount = stats.totalIncoming || 0; + return diff; + } + + /** + * @async + * called in interval + * @returns {object} + */ + async run() { + + const res = { + generatedAt: Date.now(), + interval: this.config.analyticsInterval + }; + + try { + // @ts-ignore + res.lagChange = await this._checkLagChanges(); + } catch (error) { + this.logger.error(`Failed to calculate lag changes ${error.message}.`); + // @ts-ignore + res.lagChange = null; } - stallLags[offset.topic][offset.partition] = offset.highDistance; - return; - } - - //distance changed (but did not decrease enough) - if(!changedLags[offset.topic]){ - changedLags[offset.topic] = {}; - } - - changedLags[offset.topic][offset.partition] = offset.highDistance; - }); - - return { - timelyDifference: newest.at - last.at, - fetchPerformance: last.took - newest.took, - newLags, - changedLags, - resolvedLags, - stallLags - }; - } - - /** - * gets the largest lag in all assigned offsets - * @private - * @returns {object} - */ - _identifyLargestLag(){ - - let lag = { - highDistance: -1 - }; - - const newest = this.client._lagCache; - - if(!newest){ - return { - error: "Only last status fetched yet." - }; - } + try { + // @ts-ignore + res.largestLag = this._identifyLargestLag(); + } catch (error) { + this.logger.error(`Failed to calculate largest lag ${error.message}.`); + // @ts-ignore + res.largestLag = null; + } - newest.status.forEach(offset => { - if(offset.highDistance > lag.highDistance){ - lag = offset; - } - }); - - return lag; - } - - /** - * returns consumed amount of messages in interval - * @private - * @param {object} stats - getStats() client result - * @returns {number} - */ - _consumed(stats){ - const diff = (stats.totalIncoming || 0) - this._consumedCount; - this._consumedCount = stats.totalIncoming || 0; - return diff; - } - - /** - * @async - * called in interval - * @returns {object} - */ - async run(){ - - const res = { - generatedAt: Date.now(), - interval: this.config.analyticsInterval - }; - - try { - res.lagChange = await this._checkLagChanges(); - } catch(error){ - this.logger.error(`Failed to calculate lag changes ${error.message}.`); - res.lagChange = null; - } + const stats = this.client.getStats(); - try { - res.largestLag = this._identifyLargestLag(); - } catch(error){ - this.logger.error(`Failed to calculate largest lag ${error.message}.`); - res.largestLag = null; - } + try { + // @ts-ignore + res.consumed = this._consumed(stats); + } catch (error) { + this.logger.error(`Failed to get consumed count ${error.message}.`); + // @ts-ignore + res.consumed = null; + } - const stats = this.client.getStats(); + try { + // @ts-ignore + res.errors = this._errorsInInterval(stats); + } catch (error) { + this.logger.error(`Failed to get error count ${error.message}.`); + // @ts-ignore + res.errors = null; + } - try { - res.consumed = this._consumed(stats); - } catch(error){ - this.logger.error(`Failed to get consumed count ${error.message}.`); - res.consumed = null; + this.logger.debug(res); + this._lastRes = res; + return res; } - try { - res.errors = this._errorsInInterval(stats); - } catch(error){ - this.logger.error(`Failed to get error count ${error.message}.`); - res.errors = null; + /** + * returns the last result of run() + * @returns {object} + */ + getLastResult() { + return this._lastRes; } - - this.logger.debug(res); - this._lastRes = res; - return res; - } - - /** - * returns the last result of run() - * @returns {object} - */ - getLastResult(){ - return this._lastRes; - } } /** * outsourced analytics for nproducers */ export class ProducerAnalytics extends Analytics { + private _lastRes = null; + private _producedCount: number = 0; + + /** + * creates a new instance + * @param {NProducer} nproducer + * @param {object} config + * @param {object} logger + */ + constructor(nproducer, config, logger) { + super(nproducer, config, logger); + } - /** - * creates a new instance - * @param {NProducer} nproducer - * @param {object} config - * @param {object} logger - */ - constructor(nproducer, config, logger){ - super(nproducer, config, logger); - - this._lastRes = null; - this._producedCount = 0; - } - - /** - * returns produced amount of messages in interval - * @private - * @param {object} stats - getStats() client result - * @returns {number} - */ - _produced(stats){ - const diff = (stats.totalPublished || 0) - this._producedCount; - this._producedCount = stats.totalPublished || 0; - return diff; - } - - /** - * called in interval - * @returns {object} - */ - async run(){ - - const res = { - generatedAt: Date.now(), - interval: this.config.analyticsInterval - }; - - const stats = this.client.getStats(); - - try { - res.produced = this._produced(stats); - } catch(error){ - this.logger.error(`Failed to get produced count ${error.message}.`); - res.produced = null; + /** + * returns produced amount of messages in interval + * @private + * @param {object} stats - getStats() client result + * @returns {number} + */ + _produced(stats) { + const diff = (stats.totalPublished || 0) - this._producedCount; + this._producedCount = stats.totalPublished || 0; + return diff; } - try { - res.errors = this._errorsInInterval(stats); - } catch(error){ - this.logger.error(`Failed to get error count ${error.message}.`); - res.errors = null; + /** + * called in interval + * @returns {object} + */ + async run() { + + const res = { + generatedAt: Date.now(), + interval: this.config.analyticsInterval + }; + + const stats = this.client.getStats(); + + try { + // @ts-ignore + res.produced = this._produced(stats); + } catch (error) { + this.logger.error(`Failed to get produced count ${error.message}.`); + // @ts-ignore + res.produced = null; + } + + try { + // @ts-ignore + res.errors = this._errorsInInterval(stats); + } catch (error) { + this.logger.error(`Failed to get error count ${error.message}.`); + // @ts-ignore + res.errors = null; + } + + this.logger.debug(res); + this._lastRes = res; + return res; } - this.logger.debug(res); - this._lastRes = res; - return res; - } - - /** - * returns the last result of run() - * @returns {object} - */ - getLastResult(){ - return this._lastRes; - } + /** + * returns the last result of run() + * @returns {object} + */ + getLastResult() { + return this._lastRes; + } } diff --git a/src/librdkafka/Health.ts b/src/librdkafka/Health.ts index fb3605b..53db5a2 100644 --- a/src/librdkafka/Health.ts +++ b/src/librdkafka/Health.ts @@ -1,106 +1,107 @@ import merge from "lodash.merge"; const defaultConfig = { - thresholds: { - consumer: { - errors: 5, - lag: 1000, - stallLag: 10, - minMessages: 1 - }, - producer: { - errors: 4, - minMessages: 1 + thresholds: { + consumer: { + errors: 5, + lag: 1000, + stallLag: 10, + minMessages: 1 + }, + producer: { + errors: 4, + minMessages: 1 + } } - } }; const STATES = { - DIS_ANALYTICS: -4, - NO_ANALYTICS: -3, - UNKNOWN: -2, - UNCONNECTED: -1, - HEALTHY: 0, - RISK: 1, - WARNING: 2, - CRITICAL: 3 + DIS_ANALYTICS: -4, + NO_ANALYTICS: -3, + UNKNOWN: -2, + UNCONNECTED: -1, + HEALTHY: 0, + RISK: 1, + WARNING: 2, + CRITICAL: 3 }; const MESSAGES = { - DIS_ANALYTICS: "Analytics are disabled, cannot measure required parameters. Please enable.", - NO_ANALYTICS: "Analytics have not yet run, checks will be available after first run.", - UNKNOWN: "State is unknown.", - UNCONNECTED: "The client is not connected.", - HEALTHY: "No problems detected, client is healthy." + DIS_ANALYTICS: "Analytics are disabled, cannot measure required parameters. Please enable.", + NO_ANALYTICS: "Analytics have not yet run, checks will be available after first run.", + UNKNOWN: "State is unknown.", + UNCONNECTED: "The client is not connected.", + HEALTHY: "No problems detected, client is healthy." }; /** * little pojso class around the check object */ class Check { + public messages: string[]; - /** - * creates a new instance - * @param {number} status - status code - * @param {Array|string} message - message/s, pass an empty array to initialise clean - */ - constructor(status = STATES.HEALTHY, message = MESSAGES.HEALTHY){ - this.status = status; - this.messages = Array.isArray(message) ? message : [message]; - } - - /** - * - * @param {number} status - new status code - * @returns {boolean} - */ - changeStatus(status = STATES.UNKNOWN){ - - if(status > this.status){ - this.status = status; - return true; + /** + * creates a new instance + * @param {number} status - status code + * @param {Array|string} message - message/s, pass an empty array to initialise clean + */ + constructor(public status = STATES.HEALTHY, message: string | string[] = MESSAGES.HEALTHY) { + this.messages = Array.isArray(message) ? message : [message]; } - return false; - } - - /** - * adds a message to the check - * @param {string} message - string message to attach - * @returns {number} - */ - add(message = MESSAGES.UNKNOWN){ - return this.messages.push(message); - } + /** + * + * @param {number} status - new status code + * @returns {boolean} + */ + changeStatus(status = STATES.UNKNOWN) { + + if (status > this.status) { + this.status = status; + return true; + } + + return false; + } + + /** + * adds a message to the check + * @param {string} message - string message to attach + * @returns {number} + */ + add(message = MESSAGES.UNKNOWN) { + return this.messages.push(message); + } } /** * health parent class */ class Health { - - /** - * creates a new instance - * @param {NConsumer|NProducer} client - */ - constructor(client, config){ - this.client = client; - - this.config = merge({}, defaultConfig, config); - - //make them accessable - this.STATES = STATES; - this.MESSAGES = MESSAGES; - } - - /** - * returns a new check instance - * @param {number} status - * @param {Array|string} message - */ - createCheck(status, message){ - return new Check(status, message); - } + protected config; + private STATES: { DIS_ANALYTICS: number; NO_ANALYTICS: number; UNKNOWN: number; UNCONNECTED: number; HEALTHY: number; RISK: number; WARNING: number; CRITICAL: number }; + private MESSAGES: { DIS_ANALYTICS: string; NO_ANALYTICS: string; UNKNOWN: string; UNCONNECTED: string; HEALTHY: string }; + + /** + * creates a new instance + * @param {NConsumer|NProducer} client + */ + constructor(protected client, config) { + this.config = merge({}, defaultConfig, config); + + //make them accessable + this.STATES = STATES; + this.MESSAGES = MESSAGES; + } + + /** + * returns a new check instance + * @param {number} status + * @param {Array|string} message + */ + createCheck(status, message) { + return new Check(status, message); + } } /** @@ -109,71 +110,72 @@ class Health { */ export class ConsumerHealth extends Health { - /** - * creates a new instance - * @param {NConsumer} nconsumer - */ - constructor(nconsumer, config){ - super(nconsumer, config); - } - - /** - * runs the health check - * @async - * @returns {Promise.} - */ - async check(){ - - /* ### preparation ### */ - - if(!this.client.consumer){ - return super.createCheck(STATES.UNCONNECTED, MESSAGES.UNCONNECTED); + /** + * creates a new instance + * @param {NConsumer} nconsumer + */ + constructor(nconsumer, config) { + super(nconsumer, config); } - if(!this.client._analytics){ - return super.createCheck(STATES.DIS_ANALYTICS, MESSAGES.DIS_ANALYTICS); - } + /** + * runs the health check + * @async + * @returns {Promise.} + */ + async check() { - const analytics = this.client._analytics.getLastResult(); + /* ### preparation ### */ - if(!analytics){ - return super.createCheck(STATES.NO_ANALYTICS, MESSAGES.NO_ANALYTICS); - } + if (!this.client.consumer) { + return super.createCheck(STATES.UNCONNECTED, MESSAGES.UNCONNECTED); + } - /* ### eof preparation ### */ + if (!this.client._analytics) { + return super.createCheck(STATES.DIS_ANALYTICS, MESSAGES.DIS_ANALYTICS); + } - const check = new Check(STATES.HEALTHY, []); + const analytics = this.client._analytics.getLastResult(); - if(analytics.errors !== null && analytics.errors >= this.config.thresholds.consumer.errors){ - check.changeStatus(STATES.CRITICAL); - check.add(MESSAGES.ERRORS); - } + if (!analytics) { + return super.createCheck(STATES.NO_ANALYTICS, MESSAGES.NO_ANALYTICS); + } - if(analytics.largestLag !== null && analytics.largestLag.highDistance && - analytics.largestLag.highDistance > this.config.thresholds.consumer.lag){ - check.changeStatus(STATES.WARNING); - check.add(`Lag exceeds threshold with a lag of ${analytics.largestLag.highDistance}` + - ` on ${analytics.largestLag.topic}:${analytics.largestLag.partition}.`); - } + /* ### eof preparation ### */ - if(analytics.lagChange !== null && typeof analytics.lagChange.stallLags === "object" && - Object.keys(analytics.lagChange.stallLags).length > this.config.thresholds.consumer.stallLag){ - check.changeStatus(STATES.RISK); - check.add(`Amount of stall lags exceeds threshold with ${Object.keys(analytics.lagChange.stallLags).length} unchanged lagging offsets.`); - } + const check = new Check(STATES.HEALTHY, []); - if(analytics.consumed !== null && analytics.consumed < this.config.thresholds.consumer.minMessages){ - check.changeStatus(STATES.RISK); - check.add(`Amount of consumed messages is low ${analytics.consumed}.`); - } + if (analytics.errors !== null && analytics.errors >= this.config.thresholds.consumer.errors) { + check.changeStatus(STATES.CRITICAL); + // @ts-ignore + check.add(MESSAGES.ERRORS); + } - if(check.status === STATES.HEALTHY){ - check.add(MESSAGES.HEALTHY); - check.add(`Consumed ${analytics.consumed} message/s in the last interval, with ${analytics.errors} errors.`); - } + if (analytics.largestLag !== null && analytics.largestLag.highDistance && + analytics.largestLag.highDistance > this.config.thresholds.consumer.lag) { + check.changeStatus(STATES.WARNING); + check.add(`Lag exceeds threshold with a lag of ${analytics.largestLag.highDistance}` + + ` on ${analytics.largestLag.topic}:${analytics.largestLag.partition}.`); + } - return check; - } + if (analytics.lagChange !== null && typeof analytics.lagChange.stallLags === "object" && + Object.keys(analytics.lagChange.stallLags).length > this.config.thresholds.consumer.stallLag) { + check.changeStatus(STATES.RISK); + check.add(`Amount of stall lags exceeds threshold with ${Object.keys(analytics.lagChange.stallLags).length} unchanged lagging offsets.`); + } + + if (analytics.consumed !== null && analytics.consumed < this.config.thresholds.consumer.minMessages) { + check.changeStatus(STATES.RISK); + check.add(`Amount of consumed messages is low ${analytics.consumed}.`); + } + + if (check.status === STATES.HEALTHY) { + check.add(MESSAGES.HEALTHY); + check.add(`Consumed ${analytics.consumed} message/s in the last interval, with ${analytics.errors} errors.`); + } + + return check; + } } /** @@ -182,56 +184,57 @@ export class ConsumerHealth extends Health { */ export class ProducerHealth extends Health { - /** - * creates a new instance - * @param {NProducer} nproducer - */ - constructor(nproducer, config){ - super(nproducer, config); - } - - /** - * runs the health check - * @async - * @returns {Promise.} - */ - async check(){ - - /* ### preparation ### */ - - if(!this.client.producer){ - return super.createCheck(STATES.UNCONNECTED, MESSAGES.UNCONNECTED); + /** + * creates a new instance + * @param {NProducer} nproducer + */ + constructor(nproducer, config) { + super(nproducer, config); } - if(!this.client._analytics){ - return super.createCheck(STATES.DIS_ANALYTICS, MESSAGES.DIS_ANALYTICS); - } + /** + * runs the health check + * @async + * @returns {Promise.} + */ + async check() { - const analytics = this.client._analytics.getLastResult(); + /* ### preparation ### */ - if(!analytics){ - return super.createCheck(STATES.NO_ANALYTICS, MESSAGES.NO_ANALYTICS); - } + if (!this.client.producer) { + return super.createCheck(STATES.UNCONNECTED, MESSAGES.UNCONNECTED); + } - /* ### eof preparation ### */ + if (!this.client._analytics) { + return super.createCheck(STATES.DIS_ANALYTICS, MESSAGES.DIS_ANALYTICS); + } - const check = new Check(STATES.HEALTHY, []); + const analytics = this.client._analytics.getLastResult(); - if(analytics.errors !== null && analytics.errors >= this.config.thresholds.producer.errors){ - check.changeStatus(STATES.CRITICAL); - check.add(MESSAGES.ERRORS); - } + if (!analytics) { + return super.createCheck(STATES.NO_ANALYTICS, MESSAGES.NO_ANALYTICS); + } - if(analytics.produced !== null && analytics.produced < this.config.thresholds.producer.minMessages){ - check.changeStatus(STATES.RISK); - check.add(`Amount of produced messages is low ${analytics.produced}.`); - } + /* ### eof preparation ### */ - if(check.status === STATES.HEALTHY){ - check.add(MESSAGES.HEALTHY); - check.add(`Produced ${analytics.produced} message/s in the last interval, with ${analytics.errors} errors.`); - } + const check = new Check(STATES.HEALTHY, []); + + if (analytics.errors !== null && analytics.errors >= this.config.thresholds.producer.errors) { + check.changeStatus(STATES.CRITICAL); + // @ts-ignore + check.add(MESSAGES.ERRORS); + } - return check; - } + if (analytics.produced !== null && analytics.produced < this.config.thresholds.producer.minMessages) { + check.changeStatus(STATES.RISK); + check.add(`Amount of produced messages is low ${analytics.produced}.`); + } + + if (check.status === STATES.HEALTHY) { + check.add(MESSAGES.HEALTHY); + check.add(`Produced ${analytics.produced} message/s in the last interval, with ${analytics.errors} errors.`); + } + + return check; + } } diff --git a/src/librdkafka/Metadata.ts b/src/librdkafka/Metadata.ts index baf3e17..95af212 100644 --- a/src/librdkafka/Metadata.ts +++ b/src/librdkafka/Metadata.ts @@ -3,32 +3,30 @@ */ export default class Metadata { - /** - * creates a new instance - * @param {object} raw - metadata object response of node-librdkafka client - */ - constructor(raw){ - this.raw = raw; - } + /** + * creates a new instance + * @param {object} raw - metadata object response of node-librdkafka client + */ + constructor(private raw) { + } - /** - * @throws - * returns the count of partitions of the given topic - * @param {string} topicName - name of the kafka topic - * @returns {number} - */ - getPartitionCountOfTopic(topicName){ + /** + * @throws + * returns the count of partitions of the given topic + * @param {string} topicName - name of the kafka topic + * @returns {number} + */ + getPartitionCountOfTopic(topicName) { + const topic = this.raw.topics.filter(topic => topic.name === topicName)[0]; - const topic = this.raw.topics.filter(topic => topic.name === topicName)[0]; + if (!topic) { + throw new Error(topicName + " does not exist in fetched metadata."); + } - if(!topic){ - throw new Error(topicName + " does not exist in fetched metadata."); + return topic.partitions.length; } - return topic.partitions.length; - } - - /** + /** * @throws * returns a partition (id) array of the given topic * @param {string} topicName - name of the kafka topic @@ -46,97 +44,101 @@ export default class Metadata { } /** - * @throws - * returns a list of topic names - */ - asTopicList(){ - return this.raw.topics.map(topic => topic.name).filter(topic => { - return topic !== "__consumer_offsets"; - }); - } - - /** - * @throws - * gets formatted metadata information about give topic - * @param {string} topicName - name of the kafka topic - * @returns {object} - */ - asTopicDescription(topicName){ - - if(!this.raw.topics || !this.raw.topics.length){ - return {}; + * @throws + * returns a list of topic names + */ + asTopicList() { + return this.raw.topics.map(topic => topic.name).filter(topic => { + return topic !== "__consumer_offsets"; + }); } - let topic = null; - for(let i = 0; i < this.raw.topics.length; i++){ - if(this.raw.topics[i].name === topicName){ - topic = this.raw.topics[i]; - break; - } + /** + * @throws + * gets formatted metadata information about give topic + * @param {string} topicName - name of the kafka topic + * @returns {object} + */ + asTopicDescription(topicName) { + + if (!this.raw.topics || !this.raw.topics.length) { + return {}; + } + + let topic = null; + for (let i = 0; i < this.raw.topics.length; i++) { + if (this.raw.topics[i].name === topicName) { + topic = this.raw.topics[i]; + break; + } + } + + if (!topic) { + return {}; + } + + return { + name: topic.name, + configs: null, + partitions: Metadata.formatPartitions(topic.partitions) + }; } - if(!topic){ - return {}; - } - - return { - name: topic.name, - configs: null, - partitions: Metadata.formatPartitions(topic.partitions) - }; - } - - /** - * @throws - * gets a list of formatted partition info for topic - * @param {string} topicName - name of the kafka topic - * @returns {Array} - */ - asTopicPartitions(topicName){ - - if(!this.raw.topics || !this.raw.topics.length){ - return {}; + /** + * @throws + * gets a list of formatted partition info for topic + * @param {string} topicName - name of the kafka topic + * @returns {Array} + */ + asTopicPartitions(topicName) { + + if (!this.raw.topics || !this.raw.topics.length) { + return {}; + } + + let topic = null; + for (let i = 0; i < this.raw.topics.length; i++) { + if (this.raw.topics[i].name === topicName) { + topic = this.raw.topics[i]; + break; + } + } + + if (!topic) { + return {}; + } + + return Metadata.formatPartitions(topic.partitions); } - let topic = null; - for(let i = 0; i < this.raw.topics.length; i++){ - if(this.raw.topics[i].name === topicName){ - topic = this.raw.topics[i]; - break; - } + /** + * @throws + * gets a broker object (list of broker ids) + * @returns {object} + */ + asBrokers() { + return { + brokers: this.raw.brokers.map(broker => broker.id) + }; } - if(!topic){ - return {}; + /** + * @throws + * maps partitions into kafka-rest format + * @param {Array} partitions - array of partitions + * @returns {Array} + */ + static formatPartitions(partitions) { + return partitions.map((p) => { + p.partition = p.id; + p.replicas = p.replicas.map((r) => ({ + broker: r, + in_sync: p.isrs.indexOf(r) !== -1, + leader: r === p.leader + })); + delete p.id; + delete p.isrs; + return p; + }); } - - return Metadata.formatPartitions(topic.partitions); - } - - /** - * @throws - * gets a broker object (list of broker ids) - * @returns {object} - */ - asBrokers(){ - return { - brokers: this.raw.brokers.map(broker => broker.id) - }; - } - - /** - * @throws - * maps partitions into kafka-rest format - * @param {Array} partitions - array of partitions - * @returns {Array} - */ - static formatPartitions(partitions){ - return partitions.map((p)=> { - p.partition=p.id; - p.replicas = p.replicas.map((r)=>({ broker: r, in_sync: p.isrs.indexOf(r) !== -1, leader: r === p.leader })); - delete p.id; - delete p.isrs; - return p; - }); - } } diff --git a/src/librdkafka/NConsumer.ts b/src/librdkafka/NConsumer.ts index 6bbf254..ec72572 100644 --- a/src/librdkafka/NConsumer.ts +++ b/src/librdkafka/NConsumer.ts @@ -1,4 +1,3 @@ -import Promise from "bluebird"; import EventEmitter from "events"; import debug from "debug"; import async from "async"; @@ -6,6 +5,8 @@ import async from "async"; import {ConsumerAnalytics} from "./Analytics"; import {ConsumerHealth} from "./Health"; import Metadata from "./Metadata"; +import {Logger} from "./index"; +import {BatchConfig, INConsumer, KafkaConsumerConfig, KafkaMessage, SortedMessageBatch} from "../interfaces"; //@OPTIONAL let BlizzKafka = null; @@ -17,25 +18,53 @@ const FETCH_ERROR_GRACE_TIME_MS = 1500; const ASYNC_COMMIT_REQ_TIME_MS = 250; const MESSAGE_CHARSET = "utf8"; -const DEFAULT_LOGGER = { - debug: debug("sinek:nconsumer:debug"), - info: debug("sinek:nconsumer:info"), - warn: debug("sinek:nconsumer:warn"), - error: debug("sinek:nconsumer:error") +const DEFAULT_LOGGER: Logger = { + debug: debug("sinek:nconsumer:debug"), + info: debug("sinek:nconsumer:info"), + warn: debug("sinek:nconsumer:warn"), + error: debug("sinek:nconsumer:error") }; /** * native consumer wrapper for node-librdkafka * @extends EventEmitter */ -export default class NConsumer extends EventEmitter { +export default class NConsumer extends EventEmitter implements INConsumer { + private _health: ConsumerHealth; + private topics: Array; + private consumer; + private _resume: boolean; + private _inClosing: boolean; + private _firstMessageConsumed: boolean; + private _totalIncomingMessages: number; + private _lastReceived; + private _totalProcessedMessages: number; + private _lastProcessed; + private _isAutoCommitting; + private _stream; + private _asStream; + private _batchCount: number; + private _batchCommitts: number; + private _totalBatches: number; + private _batchConfig; + private _analyticsIntv; + private _lagCheckIntv; + private _lagCache; + private _analyticsOptions; + private _analytics; + private _lastLagStatus; + private _consumedSinceCommit: number; + private _emptyFetches: number; + private _avgBatchProcessingTime: number; + private _extCommitCallback; + private _errors: number; /** * creates a new consumer instance * @param {string|Array} topics - topic or topics to subscribe to * @param {object} config - configuration object */ - constructor(topics, config = { options: {}, health: {} }) { + constructor(topics: string | string[], private config: KafkaConsumerConfig = {options: {}, health: {}}) { super(); if(!config){ @@ -105,10 +134,8 @@ export default class NConsumer extends EventEmitter { throw new Error("analytics intervals are already running."); } - let { - analyticsInterval, - lagFetchInterval - } = options; + // @ts-ignore + let {analyticsInterval, lagFetchInterval} = options; this._analyticsOptions = options; analyticsInterval = analyticsInterval || 1000 * 150; // 150 sec @@ -142,10 +169,13 @@ export default class NConsumer extends EventEmitter { * @param {object} opts - optional, options asString, asJSON (booleans) * @returns {Promise.<*>} */ - connect(asStream = false, opts = {}) { + connect(asStream = false, opts = {}): Promise { + // @ts-ignore let { zkConStr, kafkaHost, logger, groupId, options, noptions, tconf } = this.config; + // @ts-ignore const { autoCommit } = options; + // @ts-ignore const {asString = false, asJSON = false} = opts; let conStr = null; @@ -183,20 +213,25 @@ export default class NConsumer extends EventEmitter { this._extCommitCallback = noptions["offset_commit_cb"]; } + // @ts-ignore noptions = noptions || {}; noptions = Object.assign({}, config, noptions, overwriteConfig); + // @ts-ignore logger.debug(noptions); this._isAutoCommitting = noptions["enable.auto.commit"]; tconf = tconf || undefined; + // @ts-ignore logger.debug(tconf); this._asStream = asStream; if(asStream){ + // @ts-ignore return this._connectAsStream(logger, noptions, tconf, {asString, asJSON}); } + // @ts-ignore return this._connectInFlow(logger, noptions, tconf); } @@ -287,6 +322,7 @@ export default class NConsumer extends EventEmitter { _connectAsStream(logger, noptions, tconf = {}, opts = {}){ return new Promise(resolve => { + // @ts-ignore const {asString = false, asJSON = false} = opts; const topics = this.topics; @@ -397,6 +433,7 @@ export default class NConsumer extends EventEmitter { //retry asap this._emptyFetches++; + // @ts-ignore let graceTime = (this.config.options.consumeGraceMs || SINGLE_CONSUME_GRACE_TIME_MS) * GRACE_TIME_FACTOR; graceTime = graceTime * (this._emptyFetches > MAX_EMPTY_FETCH_COUNT ? MAX_EMPTY_FETCH_COUNT : this._emptyFetches); @@ -468,7 +505,8 @@ export default class NConsumer extends EventEmitter { * @param {object} options - optional object containing options for 1:n mode: * @returns {Promise.<*>} */ - consume(syncEvent = null, asString = true, asJSON = false, options = {}) { + consume(syncEvent?: (message: KafkaMessage | KafkaMessage[] | SortedMessageBatch, callback: (error?: any) => void) => void, + asString?: boolean, asJSON?: boolean, options?: BatchConfig): Promise { let { batchSize, @@ -539,6 +577,7 @@ export default class NConsumer extends EventEmitter { return reject(new Error("Please disable enable.auto.commit when using 1:n consume-mode.")); } + // @ts-ignore this.config.logger.info("running in", `1:${batchSize}`, "mode"); this._batchConfig = options; //store for stats @@ -609,6 +648,7 @@ export default class NConsumer extends EventEmitter { return this._singleConsumeRecursive(batchSize); } + // @ts-ignore this.config.logger.debug("committing after", this._batchCount, "batches, messages: " + this._consumedSinceCommit); super.emit("commit", this._consumedSinceCommit); this._batchCount = 0; @@ -638,7 +678,7 @@ export default class NConsumer extends EventEmitter { //we do not listen to "data" here //we have to grab the whole batch that is delivered via consume(count) - super.on("batch", messages => { + super.on("batch", (messages: KafkaMessage[]) => { const startBPT = Date.now(); this._totalIncomingMessages += messages.length; @@ -646,6 +686,7 @@ export default class NConsumer extends EventEmitter { async.eachLimit(messages, concurrency, (message, _callback) => { + // @ts-ignore this.config.logger.debug(message); message.value = this._convertMessageValue(message.value, asString, asJSON); @@ -687,6 +728,7 @@ export default class NConsumer extends EventEmitter { return this._singleConsumeRecursive(batchSize); } + // @ts-ignore this.config.logger.debug("committing after", this._batchCount, "batches, messages: " + this._consumedSinceCommit); super.emit("commit", this._consumedSinceCommit); this._batchCount = 0; @@ -867,6 +909,7 @@ export default class NConsumer extends EventEmitter { .filter((topicPartition) => typeof topicPartition.offset !== "undefined"); if (this.config && this.config.logger && this.config.logger.debug) { + // @ts-ignore this.config.logger.debug(`Committing local offsets for topic ${topic} as`, currentLocalOffsets); } @@ -1013,7 +1056,7 @@ export default class NConsumer extends EventEmitter { * @param {number} timeout - optional, default is 2500 * @returns {Promise.} */ - getComittedOffsets(timeout = 2500){ + getComittedOffsets(timeout = 2500): Promise { if(!this.consumer){ return Promise.resolve([]); @@ -1099,10 +1142,14 @@ export default class NConsumer extends EventEmitter { return { topic: topicPartition.topic, partition: topicPartition.partition, + // @ts-ignore lowDistance: comittedOffset - brokerState.lowOffset, + // @ts-ignore highDistance: brokerState.highOffset - comittedOffset, detail: { + // @ts-ignore lowOffset: brokerState.lowOffset, + // @ts-ignore highOffset: brokerState.highOffset, comittedOffset } @@ -1191,7 +1238,7 @@ export default class NConsumer extends EventEmitter { * @param {number} timeout - optional, default is 2500 * @returns {Promise.} */ - getTopicMetadata(topic, timeout = 2500) { + getTopicMetadata(topic, timeout = 2500): Promise { return new Promise((resolve, reject) => { if (!this.consumer) { diff --git a/src/librdkafka/NProducer.ts b/src/librdkafka/NProducer.ts index 1ecd41d..a8dd1ae 100644 --- a/src/librdkafka/NProducer.ts +++ b/src/librdkafka/NProducer.ts @@ -1,5 +1,4 @@ import EventEmitter from "events"; -import Promise from "bluebird"; import uuid from "uuid"; import murmur from "murmurhash"; import debug from "debug"; @@ -9,6 +8,8 @@ import Metadata from "./Metadata"; import {ProducerAnalytics} from "./Analytics"; import {ProducerHealth} from "./Health"; +import {Logger} from "./index"; +import {INProducer, KafkaProducerConfig, MessageReturn} from "../interfaces"; //@OPTIONAL let BlizzKafka = null; @@ -23,7 +24,7 @@ const MAX_PART_AGE_MS = 1e3 * 60 * 5; //5 minutes const MAX_PART_STORE_SIZE = 1e4; const DEFAULT_MURMURHASH_VERSION = "3"; -const DEFAULT_LOGGER = { +const DEFAULT_LOGGER: Logger = { debug: debug("sinek:nproducer:debug"), info: debug("sinek:nproducer:info"), warn: debug("sinek:nproducer:warn"), @@ -34,7 +35,21 @@ const DEFAULT_LOGGER = { * native producer wrapper for node-librdkafka * @extends EventEmitter */ -export default class NProducer extends EventEmitter { +export default class NProducer extends EventEmitter implements INProducer { + private _health: ProducerHealth; + private paused: boolean; + private producer; + private _producerPollIntv; + private _partitionCounts; + private _inClosing: boolean; + private _totalSentMessages: number; + private _lastProcessed; + private _analyticsOptions; + private _analyticsIntv; + private _analytics; + private _murmurHashVersion: string; + private _murmur: (key, partitionCount) => any; + private _errors: number; /** * creates a new producer instance @@ -42,7 +57,7 @@ export default class NProducer extends EventEmitter { * @param {*} _ - ignore this param (api compatability) * @param {number|string} defaultPartitionCount - amount of default partitions for the topics to produce to */ - constructor(config = { options: {}, health: {} }, _, defaultPartitionCount = 1) { + constructor(private config: KafkaProducerConfig = {options: {}, health: {}}, _?: null, private defaultPartitionCount: number | "auto" = 1) { super(); if (!config) { @@ -66,13 +81,11 @@ export default class NProducer extends EventEmitter { config.options = {}; } - this.config = config; this._health = new ProducerHealth(this, this.config.health || {}); this.paused = false; this.producer = null; this._producerPollIntv = null; - this.defaultPartitionCount = defaultPartitionCount; this._partitionCounts = {}; this._inClosing = false; this._totalSentMessages = 0; @@ -81,6 +94,7 @@ export default class NProducer extends EventEmitter { this._analyticsIntv = null; this._analytics = null; + // @ts-ignore this._murmurHashVersion = this.config.options.murmurHashVersion || DEFAULT_MURMURHASH_VERSION; this.config.logger.info(`using murmur ${this._murmurHashVersion} partitioner.`); @@ -112,9 +126,8 @@ export default class NProducer extends EventEmitter { throw new Error("analytics intervals are already running."); } - let { - analyticsInterval - } = options; + // @ts-ignore + let {analyticsInterval} = options; this._analyticsOptions = options; analyticsInterval = analyticsInterval || 1000 * 150; // 150 sec @@ -136,21 +149,14 @@ export default class NProducer extends EventEmitter { * connects to the broker * @returns {Promise.<*>} */ - connect() { + connect(): Promise { return new Promise((resolve, reject) => { - let { - zkConStr, - kafkaHost, - logger, - options, - noptions, - tconf - } = this.config; + // @ts-ignore + let {zkConStr, kafkaHost, logger, options, noptions, tconf} = this.config; - const { - pollIntervalMs - } = options; + // @ts-ignore + const {pollIntervalMs} = options; let conStr = null; @@ -175,13 +181,16 @@ export default class NProducer extends EventEmitter { "dr_cb": true }; + // @ts-ignore noptions = noptions || {}; noptions = Object.assign({}, config, noptions); + // @ts-ignore logger.debug(noptions); tconf = tconf ? tconf : { "request.required.acks": 1 }; + // @ts-ignore logger.debug(tconf); this.producer = new BlizzKafka.HighLevelProducer(noptions, tconf); @@ -213,7 +222,7 @@ export default class NProducer extends EventEmitter { this.producer.setKeySerializer((value) => { return Promise.resolve(value); }); - + this.producer.setValueSerializer((value) => { return value; }); @@ -278,7 +287,7 @@ export default class NProducer extends EventEmitter { * @param {*} _opaqueKey - optional opaque token, which gets passed along to your delivery reports (deprecated) * @returns {Promise.} */ - async send(topicName, message, _partition = null, _key = null, _partitionKey = null, _opaqueKey = null) { + async send(topicName, message, _partition = null, _key = null, _partitionKey = null, _opaqueKey = null): Promise { if (!this.producer) { throw new Error("You must call and await .connect() before trying to produce messages."); @@ -306,6 +315,7 @@ export default class NProducer extends EventEmitter { topicName + ", please make sure the topic exists before starting the producer in auto mode."); } } else { + // @ts-ignore maxPartitions = this.defaultPartitionCount; } @@ -488,7 +498,7 @@ export default class NProducer extends EventEmitter { * @param {string} key - key * @param {number|null} _partition - optional partition */ - tombstone(topic, key, _partition = null){ + tombstone(topic, key, _partition = null): Promise { if(!key){ return Promise.reject(new Error("Tombstone messages only work on a key compacted topic, please provide a key.")); @@ -538,7 +548,7 @@ export default class NProducer extends EventEmitter { * @param {number} timeout - optional, default is 2500 * @returns {Promise.} */ - getTopicMetadata(topic, timeout = 2500) { + getTopicMetadata(topic, timeout = 2500): Promise { return new Promise((resolve, reject) => { if (!this.producer) { diff --git a/src/librdkafka/index.ts b/src/librdkafka/index.ts new file mode 100644 index 0000000..fb657b8 --- /dev/null +++ b/src/librdkafka/index.ts @@ -0,0 +1,8 @@ +import debug from "debug"; + +export type Logger = { + debug: debug.IDebugger, + info: debug.IDebugger, + warn: debug.IDebugger, + error: debug.IDebugger, +}; diff --git a/src/tools/CompressionTypes.ts b/src/tools/CompressionTypes.ts index e3a8844..85522bb 100644 --- a/src/tools/CompressionTypes.ts +++ b/src/tools/CompressionTypes.ts @@ -1,6 +1,9 @@ const TYPES = [0,1,2]; class CompressionTypes { + public NONE: number; + public GZIP: number; + public SNAPPY: number; constructor(){ this.NONE = 0; From 73d6de16709dfe5d9adce10a4eec6810305e56a8 Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Thu, 21 Mar 2019 23:04:10 +0100 Subject: [PATCH 08/10] fix type-clash for declaration-generation --- src/librdkafka/Health.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/librdkafka/Health.ts b/src/librdkafka/Health.ts index 53db5a2..923597f 100644 --- a/src/librdkafka/Health.ts +++ b/src/librdkafka/Health.ts @@ -37,7 +37,7 @@ const MESSAGES = { /** * little pojso class around the check object */ -class Check { +export class Check { public messages: string[]; /** @@ -121,7 +121,7 @@ export class ConsumerHealth extends Health { /** * runs the health check * @async - * @returns {Promise.} + * @returns {BluebirdPromise.} */ async check() { @@ -195,7 +195,7 @@ export class ProducerHealth extends Health { /** * runs the health check * @async - * @returns {Promise.} + * @returns {BluebirdPromise.} */ async check() { From 838084682f70e3962cb005829052e538ca2afc8a Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Thu, 21 Mar 2019 23:04:10 +0100 Subject: [PATCH 09/10] enable eslint for typescript --- .eslintrc.js | 6 +----- package.json | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 29194d4..f537b33 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,13 +1,9 @@ module.exports = { "env": { - "es6": true, "node": true, "mocha": true }, - "parserOptions": { - "sourceType": "module", - "ecmaVersion": 2017 - }, + "parser": "@typescript-eslint/parser", "rules": { "indent": [ "error", diff --git a/package.json b/package.json index 01c94d9..0270116 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "@types/express": "^4.16.1", "@types/mocha": "^5.2.5", "@types/sinon": "^7.0.11", + "@typescript-eslint/parser": "^1.5.0", "eslint": "~5.16.0", "expect.js": "~0.3.1", "express": "~4.16.4", From 641ed1dddb00bd4d6c251a643ccd22b7d1377423 Mon Sep 17 00:00:00 2001 From: Elmar Athmer Date: Wed, 3 Apr 2019 12:25:06 +0200 Subject: [PATCH 10/10] convert tests to ts --- package.json | 14 ++++++++- src/librdkafka/Health.ts | 4 +-- test/{config.js => config.ts} | 23 ++++----------- test/int/{Health.test.js => Health.test.ts} | 13 ++------- test/int/{JSSinek.test.js => JSSinek.test.ts} | 7 ++--- test/int/{NSinek.test.js => NSinek.test.ts} | 12 ++++---- test/mocha.opts | 1 - yarn.lock | 29 ++++++++++++++++++- 8 files changed, 58 insertions(+), 45 deletions(-) rename test/{config.js => config.ts} (74%) rename test/int/{Health.test.js => Health.test.ts} (96%) rename test/int/{JSSinek.test.js => JSSinek.test.ts} (92%) rename test/int/{NSinek.test.js => NSinek.test.ts} (90%) delete mode 100644 test/mocha.opts diff --git a/package.json b/package.json index 0270116..232366b 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "kafka:stop": "./kafka-setup/stop.sh", "kafka:logs": "docker-compose --file ./kafka-setup/docker-compose.yml logs -f", "kafka:console": "./kafka-setup/kafka-console.sh", - "test": "_mocha --recursive --timeout 32500 --exit -R spec test/int", + "test": "_mocha -R spec test/int", "yarn:openssl": "LDFLAGS='-L/usr/local/opt/openssl/lib' CPPFLAGS='-I/usr/local/opt/openssl/include' yarn", "prepublishOnly": "tsc -p tsconfig.dist.json" }, @@ -85,5 +85,17 @@ }, "optionalDependencies": { "kafka-node": "~4.1.3" + }, + "mocha": { + "extension": [ + "ts" + ], + "exit": true, + "timeout": 32500, + "recursive": true, + "require": [ + "ts-node/register", + "source-map-support/register" + ] } } diff --git a/src/librdkafka/Health.ts b/src/librdkafka/Health.ts index 923597f..24d5b35 100644 --- a/src/librdkafka/Health.ts +++ b/src/librdkafka/Health.ts @@ -114,7 +114,7 @@ export class ConsumerHealth extends Health { * creates a new instance * @param {NConsumer} nconsumer */ - constructor(nconsumer, config) { + constructor(nconsumer, config = {}) { super(nconsumer, config); } @@ -188,7 +188,7 @@ export class ProducerHealth extends Health { * creates a new instance * @param {NProducer} nproducer */ - constructor(nproducer, config) { + constructor(nproducer, config = {}) { super(nproducer, config); } diff --git a/test/config.js b/test/config.ts similarity index 74% rename from test/config.js rename to test/config.ts index 478d885..56bbc6d 100644 --- a/test/config.js +++ b/test/config.ts @@ -1,5 +1,3 @@ -"use strict"; - const config = { /*logger: { debug: console.log, @@ -14,7 +12,7 @@ const config = { }, }; -const producerConfig = Object.assign({}, config, { +export const producerConfig = Object.assign({}, config, { noptions: { "metadata.broker.list": "localhost:9092", "client.id": "n-test-producer", @@ -28,7 +26,7 @@ const producerConfig = Object.assign({}, config, { }, }); -const consumerConfig = Object.assign({}, config, { +export const consumerConfig = Object.assign({}, config, { noptions: { "metadata.broker.list": "localhost:9092", "group.id": "n-test-group", @@ -41,13 +39,13 @@ const consumerConfig = Object.assign({}, config, { }, }); -const jsProducerConfig = { +export const jsProducerConfig = { kafkaHost: "localhost:9092", clientName: "n-test-producer-js", options: {}, }; -const jsConsumerConfig = { +export const jsConsumerConfig = { kafkaHost: "localhost:9092", groupId: "n-test-group-js", options: { @@ -55,19 +53,10 @@ const jsConsumerConfig = { }, }; -const topic = "n-test-topic"; +export const topic = "n-test-topic"; -const batchOptions = { +export const batchOptions = { batchSize: 1000, commitEveryNBatch: 1, manualBatching: true, }; - -module.exports = { - topic, - producerConfig, - consumerConfig, - batchOptions, - jsProducerConfig, - jsConsumerConfig, -}; diff --git a/test/int/Health.test.js b/test/int/Health.test.ts similarity index 96% rename from test/int/Health.test.js rename to test/int/Health.test.ts index 94cd140..a32cf47 100644 --- a/test/int/Health.test.js +++ b/test/int/Health.test.ts @@ -1,14 +1,5 @@ -"use strict"; - -const assert = require("assert"); - -const { - Health -} = require("./../../index.js"); -const { - ConsumerHealth, - ProducerHealth -} = Health; +import assert from "assert"; +import {ConsumerHealth, ProducerHealth} from "../../src/librdkafka/Health"; describe("Health UNIT", () => { diff --git a/test/int/JSSinek.test.js b/test/int/JSSinek.test.ts similarity index 92% rename from test/int/JSSinek.test.js rename to test/int/JSSinek.test.ts index 8d05da8..5991092 100644 --- a/test/int/JSSinek.test.js +++ b/test/int/JSSinek.test.ts @@ -1,8 +1,5 @@ -"use strict"; - -const assert = require("assert"); -const {Consumer, Producer} = require("./../../index.js"); -const {jsProducerConfig, jsConsumerConfig, topic} = require("../config"); +import assert from "assert"; +import {topic} from "../config"; describe("Javascript Client INT", () => { diff --git a/test/int/NSinek.test.js b/test/int/NSinek.test.ts similarity index 90% rename from test/int/NSinek.test.js rename to test/int/NSinek.test.ts index 15ffba5..586e766 100644 --- a/test/int/NSinek.test.js +++ b/test/int/NSinek.test.ts @@ -1,8 +1,6 @@ -"use strict"; - -const assert = require("assert"); -const { NConsumer, NProducer } = require("./../../index.js"); -const { producerConfig, consumerConfig, topic, batchOptions } = require("../config"); +import assert from "assert"; +import {batchOptions, consumerConfig, producerConfig, topic} from "../config"; +import {NConsumer, NProducer} from "../../src"; describe("Native Client INT", () => { @@ -15,8 +13,8 @@ describe("Native Client INT", () => { before(done => { - producer = new NProducer(producerConfig, null, 1); - consumer = new NConsumer([topic], consumerConfig); + producer = new NProducer(producerConfig as any, null, 1); + consumer = new NConsumer([topic], consumerConfig as any); producer.on("error", error => console.error(error)); consumer.on("error", error => console.error(error)); diff --git a/test/mocha.opts b/test/mocha.opts deleted file mode 100644 index e306495..0000000 --- a/test/mocha.opts +++ /dev/null @@ -1 +0,0 @@ ---exit diff --git a/yarn.lock b/yarn.lock index 8b5cd8c..bf3cdd9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -168,6 +168,23 @@ dependencies: "@types/node" "*" +"@typescript-eslint/parser@^1.5.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-1.7.0.tgz#c3ea0d158349ceefbb6da95b5b09924b75357851" + integrity sha512-1QFKxs2V940372srm12ovSE683afqc1jB6zF/f8iKhgLz1yoSjYeGHipasao33VXKI+0a/ob9okeogGdKGvvlg== + dependencies: + "@typescript-eslint/typescript-estree" "1.7.0" + eslint-scope "^4.0.0" + eslint-visitor-keys "^1.0.0" + +"@typescript-eslint/typescript-estree@1.7.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-1.7.0.tgz#59ec02f5371964da1cc679dba7b878a417bc8c60" + integrity sha512-K5uedUxVmlYrVkFbyV3htDipvLqTE3QMOUQEHYJaKtgzxj6r7c5Ca/DG1tGgFxX+fsbi9nDIrf4arq7Ib7H/Yw== + dependencies: + lodash.unescape "4.0.1" + semver "5.5.0" + abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" @@ -687,7 +704,7 @@ escodegen@1.8.x: optionalDependencies: source-map "~0.2.0" -eslint-scope@^4.0.3: +eslint-scope@^4.0.0, eslint-scope@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-4.0.3.tgz#ca03833310f6889a3264781aa82e63eb9cfe7848" integrity sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg== @@ -1337,6 +1354,11 @@ lodash.merge@~4.6.1: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.1.tgz#adc25d9cb99b9391c59624f379fbba60d7111d54" integrity sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ== +lodash.unescape@4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/lodash.unescape/-/lodash.unescape-4.0.1.tgz#bf2249886ce514cda112fae9218cdc065211fc9c" + integrity sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw= + lodash@^4.17.11, lodash@^4.17.4: version "4.17.11" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" @@ -1987,6 +2009,11 @@ safe-buffer@5.1.2, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, s resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +semver@5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" + integrity sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA== + semver@^5.4.1, semver@^5.5.0, semver@^5.5.1, semver@^5.7.0: version "5.7.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.0.tgz#790a7cf6fea5459bac96110b29b60412dc8ff96b"