app review changes: trailing slashes removed, error fixed in worker catch block, comments updated

Author: imk13

README.md

@@ -4,12 +4,12 @@ Crawl medium.com urls recursively.
 Local setup
 * make sure node version is > 8.x. To install node version nvm `nvm install 8.x`
 * install mongodb | follow this link  https://www.digitalocean.com/community/tutorials/how-to-install-mongodb-on-ubuntu-18-04
+* make sure `mongod` service is running `sudo service mongod status` (for ubunutu)
 * `npm start` to start app in dev env
 
 
 Dev Setup
 * make sure `docker` and `docker-compose` is installed. If now, follow link https://docs.docker.com/install/
-* make sure `mongod` service is running `sudo service mongod status` (for ubunutu)
 * `npm run app:test` to run
 
 Prod setup

components/controllers/params.controller.js

@@ -4,6 +4,7 @@
 const config = require('../../config');
 const Params = require('../models/params');
 module.exports = {
+    // To save query params data in DB
     save: async function(data) {
         let urlParams = new Params(data);
         let err, result;
@@ -14,6 +15,7 @@ module.exports = {
         }
         return [err, result];
     },
+    // To get record from DB based on queryParams, e.g. find record by md5
     getOneby: async function(queryParams) {
         let err, result;
         try {

components/controllers/url.controller.js

@@ -4,6 +4,7 @@
 const config = require('../../config');
 const Urls = require('../models/urls');
 module.exports = {
+    // To save url data in DB
     save: async function(data) {
         let url = new Urls(data);
         let err, result;
@@ -14,6 +15,7 @@ module.exports = {
         }
         return [err, result];
     },
+    // To get record from DB based on queryParams, e.g. find by md5
     getOneby: async function(params) {
         let err, result;
         try {

lib/index.js

@@ -28,10 +28,20 @@ function fetch(url) {
     return response;
 }
 
+function stripEndSlash(href) {
+    let urlInfo = href.length ? href.split('?') : ['', ''];
+    let url = urlInfo[0];
+    if (url[url.length - 1] == '/') {
+        return url.substr(0, url.length - 1) + '?' + urlInfo[1];
+    }
+    return href;
+}
+
 function filterDuplicates(arr) {
     var uniqueLinks = {};
     var results = [];
-    arr.forEach(element => {
+    arr.forEach(item => {
+        let element = stripEndSlash(item);
         if (!uniqueLinks[element]) {
             uniqueLinks[element] = true;
             results.push(element);
@@ -46,5 +56,6 @@ module.exports = {
     generateMD5: generateMD5,
     filterDuplicates: filterDuplicates,
     fetch: fetch,
+    stripEndSlash: stripEndSlash,
     To: awaitHandler
 }

worker/index.js

@@ -1,5 +1,6 @@
 /**
  * Worker to intiate crawling of medium.com urls
+ *
  */
 const cheerio = require('cheerio'),
     config = require('../config'),
@@ -11,45 +12,55 @@ let runningProcess = 0;
 let urlCounter = 0;
 let mediumUrl = 'https://medium.com/';
 
+// Max number of process allowed
 const MAX_PROCESS = config.MAX_PROCESS || 5;
 // Experimental config
 const MAX_URLS = config.MAX_URLS || 8;
 const DELAY = config.DELAY || 1000;
 
 
 queue.push(mediumUrl);
-
+/**
+ * This function crawls medium urls recursively in batches of 5.
+ * If encountered url is new after filtering unique url then 
+ * that url is saved in DB,similar process happens for query params 
+ */
 function crawlProcess() {
     let href = queue.shift();
     console.log("Fethching from " + href + "...");
-    //  Number of URLs to track
-    //    if (href && urlCounter <= (MAX_URLS - MAX_PROCESS)) {
     runningProcess++;
+    // Fetching page with url
     lib.fetch(href).then(async(res) => {
         let htmlRes = res.body;
         const $ = cheerio.load(htmlRes);
         let querySelectorString = `a[href^="${mediumUrl}"]`;
         let anchorUrls = [];
+        // Running query selector to get all anchore nodes
+        // which have href starting with ${mediumUrl}
         $(querySelectorString).filter((key, node) => {
             anchorUrls.push(node.attribs.href);
         });
+        //Stripping end slashes and filtering duplicate Urls
         anchorUrls = lib.filterDuplicates(anchorUrls);
         queue = queue.concat(anchorUrls);
+
         let urlInfo = href ? href.split("?") : "";
         let url = urlInfo[0];
+        // Building params for db to store
         let data = {
             url: url,
             md5: lib.generateMD5(url)
         };
-        //console.log("RUNNING PROCESS: ", runningProcess);
+        // Parsing url query params
         let urlParams = urlInfo[1] ? lib.parseQueryParams(urlInfo[1]) : null;
         let urlRes, queryRes;
+
         try {
-            // See if url data already exist or not;
+            // See if url data already exist in DB or not;
             urlRes = await controllers.urlCtrl.getOneby({ md5: data.md5 });
             if (!urlRes[1] || !urlRes[1]._id) {
+                // Saving new url data in DB
                 urlRes = await controllers.urlCtrl.save(data);
-                // update urlCounter on saving new url
                 urlCounter++;
                 console.log("URL COUNTER: ", urlCounter);
             }
@@ -67,39 +78,32 @@ function crawlProcess() {
                     },
                     md5: lib.generateMD5(href)
                 };
-                // See if query data already exist or not;
+                // See if query data already exist in DB or not;
                 queryRes = await controllers.paramsCtrl.getOneby({ md5: queryData.md5 });
-                //console.log("queryRes", queryRes);
                 if (!queryRes[1] || !queryRes[1]._id) {
+                    // Saving new query params data in DB
                     queryRes = await controllers.paramsCtrl.save(queryData);
                 }
             }
-            //console.log("response ", urlRes, queryRes);
         } catch (ex) {
             console.debug(ex);
         }
         if (runningProcess < MAX_PROCESS && queue.length) {
-            //console.log("sync run");
-            // linear multiplier can be  added to provide processing delay e.g. runningProcess*100
             crawlProcess();
         }
         runningProcess--;
     }, (reject) => {
-        console.log("Fetching error ", reject);
-        queue.push(url);
+        console.debug("Fetching error ", reject);
+        queue.push(href);
         setTimeout(() => {
             crawlProcess();
         }, (runningProcess + 1) * 1000);
     });
+    // Setting up MAX_PROCESS (5) asynchronous crawcallslProcess
+    // for the first after fetching landing page, otherwise it will always process 1-1;
     if (runningProcess < MAX_PROCESS && queue.length) {
-        //console.log("async run");
-        // linear multiplier can be  added to provide processing delay e.g. runningProcess*100
         crawlProcess();
     }
-    //}
-    // else {
-    //     console.log("I AM DONE !");
-    // }
 }
 
 module.exports = crawlProcess;