cleanup (#76)

* feat: support gzip sitemaps * refactor: simplify code and change method name * chore: cleanup * chore: cleanup * Adding comments * auto generated Co-authored-by: Jason Ibrahim <[email protected]> Co-authored-by: Sean Thomas Burke <[email protected]>
seantomburke · Jan 13, 2021 · 68bf930 · 68bf930
1 parent 6a72c7a
commit 68bf930
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 4 deletions.
diff --git a/cspell.json b/cspell.json
@@ -14,7 +14,8 @@
     ],
     "words": [
         "sitemapper",
-        "esmodules"
+        "esmodules",
+        "gzipped"
     ],
     "allowCompoundWords": true,
     "flagWords": [],

diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js
diff --git a/src/assets/sitemapper.js b/src/assets/sitemapper.js
@@ -255,8 +255,8 @@ export default class Sitemapper {
 
 
   /**
-   * /**
    * Gets the sites from a sitemap.xml with a given URL
+   *
    * @deprecated
    * @param {string} url - url to query
    * @param {getSitesCallback} callback - callback for sites and error
@@ -278,16 +278,28 @@ export default class Sitemapper {
     return callback(err, sites);
   }
 
+  /**
+   * Check to see if the url is a gzipped url
+   *
+   * @param {string} url - url to query
+   * @returns {Boolean}
+   */
   isGzip(url) {
     const parsed = Url.parse(url);
     const ext = path.extname(parsed.path);
     return ext === '.gz';
   }
 
+  /**
+   * Decompress the gzipped response body using zlib.gunzip
+   *
+   * @param {Buffer} body - body of the gzipped file
+   * @returns {Boolean}
+   */
   decompressResponseBody(body) {
     return new Promise((resolve, reject) => {
       const buffer = Buffer.from(body);
-      zlib.gunzip(buffer, function (err, result) {
+      zlib.gunzip(buffer, (err, result) => {
         if (err) {
           reject(err);
         } else {

diff --git a/src/tests/test.js b/src/tests/test.js
@@ -155,6 +155,32 @@ describe('Sitemapper', function () {
     });
   });
 
+  describe('gzipped sitemaps', function () {
+    beforeEach(() => {
+      sitemapper = new Sitemapper({
+        requestHeaders: {
+          'Accept-Encoding': 'gzip,deflate,sdch',
+        }
+      });
+    });
+
+    it('https://www.banggood.com/sitemap/products-Toys-Hobbies-and-Robot-5-hu-HU.xml.gz gzip should be a non-empty array', function (done) {
+      this.timeout(30000);
+      const url = 'https://www.banggood.com/sitemap/products-Toys-Hobbies-and-Robot-5-hu-HU.xml.gz';
+      sitemapper.timeout = 10000;
+      sitemapper.fetch(url)
+        .then(data => {
+          data.sites.should.be.Array;
+          data.sites.length.should.be.greaterThan(0);
+          done();
+        })
+        .catch(error => {
+          console.error('Test failed');
+          done(error);
+        });
+    });
+  });
+
   describe('getSites method', function () {
     it('getSites should be backwards compatible', function (done) {
       this.timeout(30000);