diff --git a/src/parser.js b/src/parser.js index a7c8f37..a75bccd 100644 --- a/src/parser.js +++ b/src/parser.js @@ -20,7 +20,11 @@ const cleanSpaces = (rawString) => rawString.replace(whitespace, '').trim(); const splitOnLines = (string) => string.split(lineEndings); const robustSplit = (string) => { - return !string.includes('') ? [...string.match(recordSlices)].map(cleanSpaces) : []; + const matches = (null === string || undefined === string) ? null : string.match(recordSlices); + if (!matches) { + return []; + } + return !string.includes('') ? [...matches].map(cleanSpaces) : []; }; const parseRecord = (line) => { diff --git a/test/parser/can-parse-test-files.js b/test/parser/can-parse-test-files.js index ba37c89..10972cc 100644 --- a/test/parser/can-parse-test-files.js +++ b/test/parser/can-parse-test-files.js @@ -4,6 +4,7 @@ const exampleRobotsBcc = require('../test-data/example-robots-txt-bcc.js'); const exampleRobotsKarwei = require('../test-data/example-robots-txt-karwei.js'); const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); const exampleRobotsZalando = require('../test-data/example-robots-txt-zalando.js'); +const exampleRobotsEmpty = require('../test-data/example-robots-txt-empty.js'); const parse = require('../../src/parser.js'); const { expect } = chai; @@ -66,4 +67,12 @@ describe('can-parse-test-files', () => { expect(parseResult['screaming frog seo spider'].disallow).to.have.lengthOf(1); expect(parseResult.sitemaps).to.have.lengthOf(0); }); + + it('Should completely parse robots-txt-empty', () => { + const parseResult = parse(exampleRobotsEmpty); + const userAgents = Object.keys(parseResult).filter((val) => val !== 'sitemaps' && val !== 'host'); + expect(userAgents).to.have.lengthOf(0); + expect(parseResult).to.have.keys(['sitemaps']); + expect(parseResult.sitemaps).to.have.lengthOf(0); + }); }); diff --git a/test/test-data/example-robots-txt-empty.js b/test/test-data/example-robots-txt-empty.js new file mode 100644 index 0000000..38b08af --- /dev/null +++ b/test/test-data/example-robots-txt-empty.js @@ -0,0 +1 @@ +module.exports = "";