Feature: callback and event for comments

src/N3Lexer.js

@@ -68,7 +68,7 @@ export default class N3Lexer {
       this._n3Mode = options.n3 !== false;
     }
     // Don't output comment tokens by default
-    this._comments = !!options.comments;
+    this.comments = !!options.comments;
     // Cache the last tested closing position of long literals
     this._literalClosingPos = 0;
   }
@@ -85,7 +85,7 @@ export default class N3Lexer {
       let whiteSpaceMatch, comment;
       while (whiteSpaceMatch = this._newline.exec(input)) {
         // Try to find a comment
-        if (this._comments && (comment = this._comment.exec(whiteSpaceMatch[0])))
+        if (this.comments && (comment = this._comment.exec(whiteSpaceMatch[0])))
           emitToken('comment', comment[1], '', this._line, whiteSpaceMatch[0].length);
         // Advance the input
         input = input.substr(whiteSpaceMatch[0].length, input.length);
@@ -101,7 +101,7 @@ export default class N3Lexer {
         // If the input is finished, emit EOF
         if (inputFinished) {
           // Try to find a final comment
-          if (this._comments && (comment = this._comment.exec(input)))
+          if (this.comments && (comment = this._comment.exec(input)))
             emitToken('comment', comment[1], '', this._line, input.length);
           input = null;
           emitToken('eof', '', '', this._line, 0);

src/N3Parser.js

@@ -1010,21 +1010,33 @@ export default class N3Parser {
 
   // ## Public methods
 
-  // ### `parse` parses the N3 input and emits each parsed quad through the callback
+  // ### `parse` parses the N3 input and emits each parsed quad through the onQuad callback.
   parse(input, quadCallback, prefixCallback) {
+    // The second parameter accepts an object { onQuad: ..., onPrefix: ..., onComment: ...}
+    // As a second and third parameter it still accepts a separate quadCallback and prefixCallback for backward compatibility as well
+    let onQuad, onPrefix, onComment;
+    if (quadCallback && (quadCallback.onQuad || quadCallback.onPrefix || quadCallback.onComment)) {
+      onQuad = quadCallback.onQuad;
+      onPrefix = quadCallback.onPrefix;
+      onComment = quadCallback.onComment;
+    }
+    else {
+      onQuad = quadCallback;
+      onPrefix = prefixCallback;
+    }
     // The read callback is the next function to be executed when a token arrives.
     // We start reading in the top context.
     this._readCallback = this._readInTopContext;
     this._sparqlStyle = false;
     this._prefixes = Object.create(null);
     this._prefixes._ = this._blankNodePrefix ? this._blankNodePrefix.substr(2)
                                              : `b${blankNodePrefix++}_`;
-    this._prefixCallback = prefixCallback || noop;
+    this._prefixCallback = onPrefix || noop;
     this._inversePredicate = false;
     this._quantified = Object.create(null);
 
     // Parse synchronously if no quad callback is given
-    if (!quadCallback) {
+    if (!onQuad) {
       const quads = [];
       let error;
       this._callback = (e, t) => { e ? (error = e) : t && quads.push(t); };
@@ -1035,14 +1047,33 @@ export default class N3Parser {
       return quads;
     }
 
-    // Parse asynchronously otherwise, executing the read callback when a token arrives
-    this._callback = quadCallback;
-    this._lexer.tokenize(input, (error, token) => {
+    let processNextToken = (error, token) => {
       if (error !== null)
         this._callback(error), this._callback = noop;
       else if (this._readCallback)
         this._readCallback = this._readCallback(token);
-    });
+    };
+
+    // Enable checking for comments on every token when a commentCallback has been set
+    if (onComment) {
+      // Enable the lexer to return comments as tokens first (disabled by default)
+      this._lexer.comments = true;
+      // Patch the processNextToken function
+      processNextToken = (error, token) => {
+        if (error !== null)
+          this._callback(error), this._callback = noop;
+        else if (this._readCallback) {
+          if (token.type === 'comment')
+            onComment(token.value);
+          else
+            this._readCallback = this._readCallback(token);
+        }
+      };
+    }
+
+    // Parse asynchronously otherwise, executing the read callback when a token arrives
+    this._callback = onQuad;
+    this._lexer.tokenize(input, processNextToken);
   }
 }
 

src/N3StreamParser.js

@@ -11,19 +11,25 @@ export default class N3StreamParser extends Transform {
     // Set up parser with dummy stream to obtain `data` and `end` callbacks
     const parser = new N3Parser(options);
     let onData, onEnd;
+
+    const callbacks = {
+        // Handle quads by pushing them down the pipeline
+      onQuad: (error, quad) => { error && this.emit('error', error) || quad && this.push(quad); },
+        // Emit prefixes through the `prefix` event
+      onPrefix: (prefix, uri) => { this.emit('prefix', prefix, uri); },
+    };
+
+    if (options && options.comments)
+      callbacks.onComment = comment => { this.emit('comment', comment); };
+
     parser.parse({
       on: (event, callback) => {
         switch (event) {
         case 'data': onData = callback; break;
         case 'end':   onEnd = callback; break;
         }
       },
-    },
-      // Handle quads by pushing them down the pipeline
-      (error, quad) => { error && this.emit('error', error) || quad && this.push(quad); },
-      // Emit prefixes through the `prefix` event
-      (prefix, uri) => { this.emit('prefix', prefix, uri); },
-    );
+    }, callbacks);
 
     // Implement Transform methods through parser callbacks
     this._transform = (chunk, encoding, done) => { onData(chunk); done(); };

test/N3Parser-test.js

@@ -41,6 +41,20 @@ describe('Parser', () => {
                   ['g', 'h', 'i']),
     );
 
+    it(
+      'should parse three triples with comments if no comment callback is set',
+      shouldParse('<a> <b> #comment2\n <c> . \n<d> <e> <f>.\n<g> <h> <i>.',
+                  ['a', 'b', 'c'],
+                  ['d', 'e', 'f'],
+                  ['g', 'h', 'i']),
+    );
+
+    it(
+      'should callback comments when a comment allback is set',
+      shouldCallbackComments('#comment1\n<a> <b> #comment2\n <c> . \n<d> <e> <f>.\n<g> <h> <i>.',
+                  'comment1', 'comment2'),
+    );
+
     it('should parse a triple with a literal', shouldParse('<a> <b> "string".',
                 ['a', 'b', '"string"']));
 
@@ -1632,6 +1646,12 @@ describe('Parser', () => {
       shouldNotParse(parser, '<<_:a <http://ex.org/b> _:b <http://ex.org/b>>> <http://ex.org/b> "c" .',
         'Expected >> to follow "_:b0_b" on line 1.'),
     );
+
+    it(
+      'should not parse nested quads with comments',
+      shouldNotParseWithComments(parser, '#comment1\n<<_:a <http://ex.org/b> _:b <http://ex.org/b>>> <http://ex.org/b> "c" .',
+        'Expected >> to follow "_:b0_b" on line 2.'),
+    );
   });
 
   describe('A Parser instance for the TriG format', () => {
@@ -3038,6 +3058,31 @@ function shouldParse(parser, input) {
   };
 }
 
+
+function shouldCallbackComments(parser, input) {
+  const expected = Array.prototype.slice.call(arguments, 1);
+  // Shift parameters as necessary
+  if (parser.call)
+    expected.shift();
+  else
+    input = parser, parser = Parser;
+
+  return function (done) {
+    const items = expected;
+    const comments = [];
+    new parser({ baseIRI: BASE_IRI }).parse(input, {
+      onQuad: (error, triple) => {
+        if (!triple) {
+          // Marks the end
+          expect(JSON.stringify(comments)).toBe(JSON.stringify(items));
+          done();
+        }
+      },
+      onComment: comment => { comments.push(comment); },
+    });
+  };
+}
+
 function mapToQuad(item) {
   item = item.map(t => {
     // don't touch if it's already an object
@@ -3082,6 +3127,30 @@ function shouldNotParse(parser, input, expectedError, expectedContext) {
   };
 }
 
+function shouldNotParseWithComments(parser, input, expectedError, expectedContext) {
+  // Shift parameters if necessary
+  if (!parser.call)
+    expectedContext = expectedError, expectedError = input, input = parser, parser = Parser;
+
+  return function (done) {
+    new parser({ baseIRI: BASE_IRI }).parse(input, {
+      onQuad: (error, triple) => {
+        if (error) {
+          expect(triple).toBeFalsy();
+          expect(error).toBeInstanceOf(Error);
+          expect(error.message).toEqual(expectedError);
+          if (expectedContext) expect(error.context).toEqual(expectedContext);
+          done();
+        }
+        else if (!triple)
+          done(new Error(`Expected error ${expectedError}`));
+      },
+      // Enables comment mode
+      onComment: () => {},
+    });
+  };
+}
+
 function itShouldResolve(baseIRI, relativeIri, expected) {
   let result;
   describe(`resolving <${relativeIri}> against <${baseIRI}>`, () => {

test/N3StreamParser-test.js

@@ -64,6 +64,21 @@ describe('StreamParser', () => {
                          { a: new NamedNode('http://a.org/#'), b: new NamedNode('http://b.org/#') }),
     );
 
+    it(
+      'parses two triples with comments',
+      shouldParse(['#comment1\n<a> <b> #comment2\n#comment3\n <c>. <d> <e> <f>.'], 2),
+    );
+
+    it(
+      'emits "comment" events',
+      shouldEmitComments(['#comment1\n<a> <b> #comment2\n#comment3\n <c>. <d> <e> <f>.'], ['comment1', 'comment2', 'comment3']),
+    );
+
+    it(
+      'emits "comment" events',
+      shouldNotEmitCommentsWhenNotEnabled(['#comment1\n<a> <b> #comment2\n#comment3\n <c>. <d> <e> <f>.'], ['comment1', 'comment2', 'comment3']),
+    );
+
     it('passes an error', () => {
       const input = new Readable(), parser = new StreamParser();
       let error = null;
@@ -126,6 +141,36 @@ function shouldEmitPrefixes(chunks, expectedPrefixes) {
   };
 }
 
+function shouldEmitComments(chunks, expectedComments) {
+  return function (done) {
+    const comments = [],
+        parser = new StreamParser({ comments: true }),
+        inputStream = new ArrayReader(chunks);
+    inputStream.pipe(parser);
+    parser.on('data', () => {});
+    parser.on('comment', comment => { comments.push(comment); });
+    parser.on('error', done);
+    parser.on('end', error => {
+      expect(comments).toEqual(expectedComments);
+      done(error);
+    });
+  };
+}
+
+function shouldNotEmitCommentsWhenNotEnabled(chunks, expectedComments) {
+  return function (done) {
+    const parser = new StreamParser(),
+        inputStream = new ArrayReader(chunks);
+    inputStream.pipe(parser);
+    parser.on('data', () => {});
+    parser.on('comment', comment => { done(new Error('Should not emit comments but it did')); });
+    parser.on('error', done);
+    parser.on('end', error => {
+      done();
+    });
+  };
+}
+
 function ArrayReader(items) {
   const reader = new Readable();
   reader._read = function () { this.push(items.shift() || null); };