Allow to pass multiple sitemaps

Author: samiahmedsiddiqui

README.md

@@ -39,7 +39,10 @@ checkHttpStatus({
       'Accept': 'text/html',
     },
   },
-  'sitemap': 'https://www.trunkcode.com/page-sitemap.xml',
+  'sitemap':  [
+    'https://www.trunkcode.com/page-sitemap.xml',
+    'https://www.trunkcode.com/post-sitemap.xml'
+  ],
   'skip200': true, // Do not report the URLs having HTTP code 200.
 });
 ```

index.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const consoleColumns = require('./lib/console-columns');
-const fetchFromSitemap = require('./lib/fetch-from-sitemap');
+const fetchAllSitemaps = require('./lib/fetch-from-sitemap');
 const fs = require('fs');
 const generateExcel = require('./lib/generate-excel');
 const httpList = require('./lib/http-list');
@@ -30,7 +30,7 @@ async function checkHttpStatus(config) {
   }
 
   if (config.sitemap) {
-    urlsList = await fetchFromSitemap(config.sitemap);
+    urlsList = await fetchAllSitemaps(config.sitemap);
   } else if (config.urls && Array.isArray(config.urls)) {
     urlsList = config.urls;
   }

lib/fetch-from-sitemap.js

@@ -7,7 +7,7 @@ const xml2js = require('xml2js');
  * Fetch Sitemap URL and dumb all the URLs from the XML to a variable.
  */
 function fetchFromSitemap(sitemapUrl) {
-  var errorMessage = 'Error: Sitemap URL returns with status code ';
+  var errorMessage = 'Error: ' + sitemapUrl + ' returns with status code ';
   var fetchedUrls = [];
 
   return new Promise((resolve) => {
@@ -42,4 +42,27 @@ function fetchFromSitemap(sitemapUrl) {
   });
 }
 
-module.exports = fetchFromSitemap;
+async function fetchAllSitemaps(sitemapUrls) {
+  const sitemapUrlsLists = [];
+  const urlsList = [];
+
+  var mergeLists = [];
+  for (const sitemapUrl of sitemapUrls) {
+    sitemapUrlsLists.push(fetchFromSitemap(sitemapUrl));
+  }
+
+  const allUrls = await Promise.all(sitemapUrlsLists);
+  for (const singleList of allUrls) {
+    mergeLists = mergeLists.concat(singleList);
+  }
+
+  for (const singleUrl of mergeLists) {
+    if (!urlsList.includes(singleUrl)) {
+      urlsList.push(singleUrl);
+    }
+  }
+
+  return urlsList;
+}
+
+module.exports = fetchAllSitemaps;

test/sitemap.js

@@ -16,6 +16,9 @@ checkHttpStatus({
       'Accept': 'text/html',
     },
   },
-  'sitemap': 'https://www.trunkcode.com/page-sitemap.xml',
+  'sitemap': [
+    'https://www.trunkcode.com/page-sitemap.xml',
+    'https://www.trunkcode.com/post-sitemap.xml'
+  ],
   'skip200': true,
 });