diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js index 8128c63..48fec18 100644 --- a/lib/assets/sitemapper.js +++ b/lib/assets/sitemapper.js @@ -1,2 +1,2 @@ -"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var _xml2js=require("xml2js"),_got=_interopRequireDefault(require("got")),_zlib=_interopRequireDefault(require("zlib")),_url=_interopRequireDefault(require("url")),_path=_interopRequireDefault(require("path")),_pLimit=_interopRequireDefault(require("p-limit"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}class Sitemapper{constructor(a){var b=a||{requestHeaders:{}};this.url=b.url,this.timeout=b.timeout||15e3,this.timeoutTable={},this.requestHeaders=b.requestHeaders,this.debug=b.debug,this.concurrency=b.concurrency||10,this.retries=b.retries||0}fetch(){var a=arguments,b=this;return _asyncToGenerator(function*(){var c=0b.cancel(),this.timeout)}crawl(a){var b=arguments,c=this;return _asyncToGenerator(function*(){var d=1a.loc&&a.loc[0]);return{sites:m,errors:[]}}if(l&&l.sitemapindex){c.debug&&console.debug("Additional sitemap found during \"crawl('".concat(a,"')\""));var e=l.sitemapindex.sitemap.map(a=>a.loc&&a.loc[0]),f=(0,_pLimit.default)(c.concurrency),g=e.map(a=>f(()=>c.crawl(a))),h=yield Promise.all(g),i=h.filter(a=>0==a.errors.length).reduce((a,b)=>a.concat(b.sites),[]),j=h.filter(a=>a.errors).reduce((a,b)=>a.concat(b.errors),[]);return{sites:i,errors:j}}return d{var d=Buffer.from(a);_zlib.default.gunzip(d,(a,d)=>{a?c(a):b(d)})})}}exports.default=Sitemapper,module.exports=exports.default,module.exports.default=exports.default; +"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var _xml2js=require("xml2js"),_got=_interopRequireDefault(require("got")),_zlib=_interopRequireDefault(require("zlib")),_url=_interopRequireDefault(require("url")),_path=_interopRequireDefault(require("path")),_pLimit=_interopRequireDefault(require("p-limit"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}class Sitemapper{constructor(a){var b=a||{requestHeaders:{}};this.url=b.url,this.timeout=b.timeout||15e3,this.timeoutTable={},this.requestHeaders=b.requestHeaders,this.debug=b.debug,this.concurrency=b.concurrency||10,this.retries=b.retries||0}fetch(){var a=arguments,b=this;return _asyncToGenerator(function*(){var c=0b.cancel(),this.timeout)}crawl(a){var b=arguments,c=this;return _asyncToGenerator(function*(){var d=1a.loc&&a.loc[0]);return{sites:m,errors:[]}}if(l&&l.sitemapindex){c.debug&&console.debug("Additional sitemap found during \"crawl('".concat(a,"')\""));var e=l.sitemapindex.sitemap.map(a=>a.loc&&a.loc[0]),f=(0,_pLimit.default)(c.concurrency),g=e.map(a=>f(()=>c.crawl(a))),h=yield Promise.all(g),i=h.filter(a=>0==a.errors.length).reduce((a,b)=>a.concat(b.sites),[]),j=h.filter(a=>a.errors.length).reduce((a,b)=>a.concat(b.errors),[]);return{sites:i,errors:j}}return d{var d=Buffer.from(a);_zlib.default.gunzip(d,(a,d)=>{a?c(a):b(d)})})}}exports.default=Sitemapper,module.exports=exports.default,module.exports.default=exports.default; //# sourceMappingURL=sitemapper.js.map \ No newline at end of file diff --git a/sitemapper.d.ts b/sitemapper.d.ts index 7c4c521..b0be714 100644 --- a/sitemapper.d.ts +++ b/sitemapper.d.ts @@ -1,7 +1,7 @@ export interface SitemapperResponse { - url: string; - sites: string[]; - errors: SitemapperErrorData[]; + url: string; + sites: string[]; + errors: SitemapperErrorData[]; } export interface SitemapperErrorData { @@ -11,26 +11,26 @@ export interface SitemapperErrorData { } export interface SitemapperOptions { - url?: string; - timeout?: number; - requestHeaders?: {[name: string]: string}; - debug?: boolean; - concurrency?: number; - retries?: number; + url?: string; + timeout?: number; + requestHeaders?: {[name: string]: string}; + debug?: boolean; + concurrency?: number; + retries?: number; } declare class Sitemapper { - timeout: number; + timeout: number; - constructor(options: SitemapperOptions) + constructor(options: SitemapperOptions) - /** - * Gets the sites from a sitemap.xml with a given URL - * - * @param url URL to the sitemap.xml file - */ - fetch(url?: string): Promise; + /** + * Gets the sites from a sitemap.xml with a given URL + * + * @param url URL to the sitemap.xml file + */ + fetch(url?: string): Promise; } export default Sitemapper; diff --git a/src/assets/sitemapper.js b/src/assets/sitemapper.js index c6d86f0..56e9b5f 100644 --- a/src/assets/sitemapper.js +++ b/src/assets/sitemapper.js @@ -192,7 +192,7 @@ export default class Sitemapper { // Otherwise notify of another error return { - error: error.error, + error: `Error occurred: ${error.name}`, data: error }; } @@ -279,7 +279,7 @@ export default class Sitemapper { .filter(result => (result.errors.length == 0)) .reduce((prev, curr) => prev.concat(curr.sites), []); const errors = results - .filter(result => result.errors) + .filter(result => result.errors.length) .reduce((prev, curr) => prev.concat(curr.errors), []); return { diff --git a/src/tests/test.es5.js b/src/tests/test.es5.js index 00985c5..e5e535b 100644 --- a/src/tests/test.es5.js +++ b/src/tests/test.es5.js @@ -79,6 +79,7 @@ describe('Sitemapper', function () { sitemapper.fetch(url) .then(data => { data.sites.should.be.Array; + data.errors.should.be.Array; done(); }) .catch(error => { diff --git a/src/tests/test.js b/src/tests/test.js index 3d89cf5..9ad20f2 100644 --- a/src/tests/test.js +++ b/src/tests/test.js @@ -77,9 +77,12 @@ describe('Sitemapper', function () { it('gibberish.gibberish should fail silently with an empty array', function (done) { this.timeout(30000); const url = 'http://gibberish.gibberish'; + sitemapper.debug = true; sitemapper.fetch(url) .then(data => { data.sites.should.be.Array; + data.errors.should.be.Array; + console.log(data); done(); }) .catch(error => { @@ -130,6 +133,8 @@ describe('Sitemapper', function () { sitemapper.fetch(url) .then(data => { data.sites.should.be.Array; + data.errors.should.be.Array; + console.log(data); done(); }) .catch(error => { @@ -187,6 +192,7 @@ describe('Sitemapper', function () { sitemapper.fetch(url) .then(data => { data.sites.should.be.Array; + data.errors.should.be.Array; data.sites.length.should.be.greaterThan(0); done(); })