diff --git a/README.md b/README.md index 1e2da7e..727f96a 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,14 @@ You can add options on the initial Sitemapper object when instantiating it. + `debug`: (Boolean) - Enables/Disables debug console logging. Default: False + `concurrency`: (Number) - Sets the maximum number of concurrent sitemap crawling threads. Default: 10 + `retries`: (Number) - Sets the maximum number of retries to attempt in case of an error response (e.g. 404 or Timeout). Default: 0 ++ `requestHeaders`: (Object) - Additional Request Headers ++ `insecure`: (Boolean) - Ignore invalid certificates ```javascript const sitemapper = new Sitemapper({ url: 'https://art-works.community/sitemap.xml', + insecure: true, timeout: 15000, requestHeaders: { 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0' diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js index fdb12da..01835e1 100644 --- a/lib/assets/sitemapper.js +++ b/lib/assets/sitemapper.js @@ -1,2 +1,2 @@ "use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var _xml2js=require("xml2js"),_got=_interopRequireDefault(require("got")),_zlib=_interopRequireDefault(require("zlib")),_pLimit=_interopRequireDefault(require("p-limit")),_isGzip=_interopRequireDefault(require("is-gzip"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}class Sitemapper{constructor(a){var b=a||{requestHeaders:{}};this.url=b.url,this.timeout=b.timeout||15e3,this.timeoutTable={},this.requestHeaders=b.requestHeaders,this.debug=b.debug,this.concurrency=b.concurrency||10,this.retries=b.retries||0}fetch(){var a=arguments,b=this;return _asyncToGenerator(function*(){var c=0b.cancel(),this.timeout)}crawl(a){var b=arguments,c=this;return _asyncToGenerator(function*(){var d=1a.loc&&a.loc[0]);return{sites:m,errors:[]}}if(l&&l.sitemapindex){c.debug&&console.debug("Additional sitemap found during \"crawl('".concat(a,"')\""));var e=l.sitemapindex.sitemap.map(a=>a.loc&&a.loc[0]),f=(0,_pLimit.default)(c.concurrency),g=e.map(a=>f(()=>c.crawl(a))),h=yield Promise.all(g),i=h.filter(a=>0===a.errors.length).reduce((a,b)=>{var{sites:c}=b;return[...a,...c]},[]),j=h.filter(a=>0!==a.errors.length).reduce((a,b)=>{var{errors:c}=b;return[...a,...c]},[]);return{sites:i,errors:j}}return d{var d=Buffer.from(a);_zlib.default.gunzip(d,(a,d)=>{a?c(a):b(d)})})}}exports.default=Sitemapper,module.exports=exports.default,module.exports.default=exports.default; -//# sourceMappingURL=sitemapper.js.map \ No newline at end of file +//# sourceMappingURL=sitemapper.js.map diff --git a/src/assets/sitemapper.js b/src/assets/sitemapper.js index cf1cb7e..164a588 100644 --- a/src/assets/sitemapper.js +++ b/src/assets/sitemapper.js @@ -40,6 +40,7 @@ export default class Sitemapper { this.debug = settings.debug; this.concurrency = settings.concurrency || 10; this.retries = settings.retries || 0; + this.insecure = settings.insecure || false; } /** @@ -149,6 +150,9 @@ export default class Sitemapper { gzip: true, responseType: 'buffer', headers: this.requestHeaders, + https: { + rejectUnauthorized: Boolean(this.insecure) + } }; try {