diff --git a/.gitignore b/.gitignore index ecb60e8..6483d41 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,4 @@ npm-debug.log .idea .vscode tmp -lib/tests/ -lib/**/*.map +lib diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js deleted file mode 100644 index 33e3046..0000000 --- a/lib/assets/sitemapper.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var _xml2js=require("xml2js"),_got=_interopRequireDefault(require("got")),_zlib=_interopRequireDefault(require("zlib")),_pLimit=_interopRequireDefault(require("p-limit")),_isGzip=_interopRequireDefault(require("is-gzip"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}class Sitemapper{constructor(a){var b=a||{requestHeaders:{}};this.url=b.url,this.timeout=b.timeout||15e3,this.timeoutTable={},this.lastmod=b.lastmod||0,this.requestHeaders=b.requestHeaders,this.debug=b.debug,this.concurrency=b.concurrency||10,this.retries=b.retries||0,this.rejectUnauthorized=!1!==b.rejectUnauthorized}fetch(){var a=arguments,b=this;return _asyncToGenerator(function*(){var c=0b.cancel(),this.timeout)}crawl(a){var b=arguments,c=this;return _asyncToGenerator(function*(){var d=1{if(0===c.lastmod)return!0;if(void 0===a.lastmod)return!1;var b=new Date(a.lastmod[0]).getTime();return b>=c.lastmod}).map(a=>a.loc&&a.loc[0]);return{sites:m,errors:[]}}if(l&&l.sitemapindex){c.debug&&console.debug("Additional sitemap found during \"crawl('".concat(a,"')\""));var e=l.sitemapindex.sitemap.map(a=>a.loc&&a.loc[0]),f=(0,_pLimit.default)(c.concurrency),g=e.map(a=>f(()=>c.crawl(a))),h=yield Promise.all(g),i=h.filter(a=>0===a.errors.length).reduce((a,b)=>{var{sites:c}=b;return[...a,...c]},[]),j=h.filter(a=>0!==a.errors.length).reduce((a,b)=>{var{errors:c}=b;return[...a,...c]},[]);return{sites:i,errors:j}}return d{var d=Buffer.from(a);_zlib.default.gunzip(d,(a,d)=>{a?c(a):b(d)})})}}exports.default=Sitemapper,module.exports=exports.default,module.exports.default=exports.default; -//# sourceMappingURL=sitemapper.js.map \ No newline at end of file diff --git a/lib/examples/google.js b/lib/examples/google.js deleted file mode 100644 index ea2caa7..0000000 --- a/lib/examples/google.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict";var _sitemapper=_interopRequireDefault(require("../assets/sitemapper.js"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}var Google=new _sitemapper.default({url:"https://www.google.com/work/sitemap.xml",debug:!1,timeout:15e3});Google.fetch().then(a=>console.log(a.sites)).catch(a=>console.log(a)); -//# sourceMappingURL=google.js.map \ No newline at end of file diff --git a/lib/examples/index.js b/lib/examples/index.js deleted file mode 100644 index 64b6198..0000000 --- a/lib/examples/index.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict";var _sitemapper=_interopRequireDefault(require("../assets/sitemapper"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}var exampleURL="https://www.walmart.com/sitemap_topic.xml",sitemapper=new _sitemapper.default({url:"https://www.walmart.com/sitemap_topic.xml",debug:!1,timeout:1e4,concurrency:10,retries:0});_asyncToGenerator(function*(){try{var a=yield sitemapper.fetch();console.log(a)}catch(a){console.error(a)}})(); -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 24a9a48..15c7aa6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "sitemapper", "version": "3.2.10", "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 142248c..c97cabf 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,8 @@ "test": "npm run build && mocha ./lib/tests/*.js && npm run lint", "lint": "eslint src", "clean": "rm -rf lib", - "docs": "documentation build ./src/assets/sitemapper.js -f md > docs.md" + "docs": "documentation build ./src/assets/sitemapper.js -f md > docs.md", + "prepack": "npm run build" }, "maintainers": [ {