Skip to content

Commit f9fd287

Browse files
raydacraydac
committed
refactoring of comments processing during read
1 parent 21c8b2a commit f9fd287

File tree

4 files changed

+139
-52
lines changed

4 files changed

+139
-52
lines changed

‎src/main/java/com/igormaznitsa/prologparser/PrologParser.java

Lines changed: 95 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -187,58 +187,84 @@ public ParserContext getContext() {
187187
return context;
188188
}
189189

190-
public boolean hasNext() {
191-
return this.tokenizer.peek() != null;
190+
private PrologTermReadResult deferredReadTerm;
191+
192+
private static boolean isComment(final TokenizerResult tokenizerResult) {
193+
return tokenizerResult != null
194+
&& tokenizerResult.getResult().getType() == TermType.ATOM
195+
&& (tokenizerResult.getResult().getQuotation() == Quotation.COMMENT_LINE
196+
|| tokenizerResult.getResult().getQuotation() == Quotation.COMMENT_BLOCK);
197+
}
198+
199+
private void notifyCommentTokenListeners(final TokenizerResult commentToken) {
200+
for (final TokenizedCommentListener listener : this.commentTokenListeners) {
201+
listener.onCommentToken(this, commentToken);
202+
}
192203
}
193204

194-
private TokenizerResult readNextTokenCommentAware() {
205+
private TokenizerResult extractNextTokenCommentAware() {
195206
TokenizerResult result;
196-
if (this.commentsAsAtoms) {
197-
while (true) {
198-
result = this.tokenizer.readNextToken();
199-
if (result != null
200-
&& (result.getResult().getQuotation() == Quotation.COMMENT_BLOCK ||
201-
result.getResult().getQuotation() == Quotation.COMMENT_LINE)) {
202-
for (final TokenizedCommentListener listener : this.commentTokenListeners) {
203-
listener.onCommentToken(this, result);
204-
}
205-
} else {
206-
break;
207-
}
208-
}
209-
} else {
207+
while (true) {
210208
result = this.tokenizer.readNextToken();
209+
if (isComment(result)) {
210+
this.notifyCommentTokenListeners(result);
211+
} else {
212+
break;
213+
}
211214
}
212215
return result;
213216
}
214217

215-
public PrologTerm next() {
216-
final PrologTerm found = readBlock(OPERATORS_PHRASE);
217-
if (found == null) {
218-
throw new NoSuchElementException("No terms in source");
219-
} else {
220-
final TokenizerResult endAtom = this.readNextTokenCommentAware();
221-
if (endAtom == null || !endAtom.getResult().getText().equals(OPERATOR_DOT.getText())) {
222-
throw new PrologParserException("End operator is not found",
223-
this.tokenizer.getLine(),
224-
this.tokenizer.getPos());
218+
private PrologTermReadResult extractNextBlockAndWrapError() {
219+
try {
220+
final PrologTerm found = this.readBlock(OPERATORS_PHRASE);
221+
if (found == null) {
222+
throw new NoSuchElementException("No terms in source");
223+
} else {
224+
final TokenizerResult endAtom = this.extractNextTokenCommentAware();
225+
if (endAtom == null || !endAtom.getResult().getText().equals(OPERATOR_DOT.getText())) {
226+
throw new PrologParserException("End operator is not found",
227+
this.tokenizer.getLine(),
228+
this.tokenizer.getPos());
229+
}
225230
}
231+
return new PrologTermReadResult(found, null);
232+
} catch (RuntimeException ex) {
233+
return new PrologTermReadResult(null, ex);
234+
}
235+
}
236+
237+
public boolean hasNext() {
238+
if (this.deferredReadTerm == null) {
239+
this.deferredReadTerm = this.extractNextBlockAndWrapError();
226240
}
227-
return found;
241+
return this.deferredReadTerm.isPresented();
242+
}
243+
244+
public PrologTerm next() {
245+
if (this.deferredReadTerm == null) {
246+
this.deferredReadTerm = this.extractNextBlockAndWrapError();
247+
}
248+
final PrologTerm result;
249+
try {
250+
result = this.deferredReadTerm.getResult();
251+
} finally {
252+
this.deferredReadTerm = null;
253+
}
254+
return result;
228255
}
229256

230257
private PrologStruct readStruct(final PrologTerm functor) {
231258
final List<PrologTerm> listOfAtoms = new ArrayList<>();
232259
PrologStruct result;
233260
boolean active = true;
234261
while (active) {
235-
final PrologTerm block = readBlock(OPERATORS_INSIDE_STRUCT);
236-
262+
final PrologTerm block = this.readBlock(OPERATORS_INSIDE_STRUCT);
237263
if (block == null) {
238264
return null;
239265
}
240266

241-
final TokenizerResult nextAtom = this.readNextTokenCommentAware();
267+
final TokenizerResult nextAtom = this.extractNextTokenCommentAware();
242268
if (nextAtom == null) {
243269
throw new PrologParserException("Can't read next token in block", this.tokenizer.getLine(),
244270
this.tokenizer.getPos());
@@ -277,7 +303,7 @@ private PrologTerm readList(final TokenizerResult openingBracket) {
277303
while (continueReading) {
278304
final PrologTerm block = readBlock(OPERATORS_INSIDE_LIST);
279305

280-
final TokenizerResult nextAtom = this.readNextTokenCommentAware();
306+
final TokenizerResult nextAtom = this.extractNextTokenCommentAware();
281307
if (nextAtom == null) {
282308
throw new PrologParserException("Can't read next token in list", this.tokenizer.getLine(),
283309
this.tokenizer.getPos());
@@ -316,7 +342,7 @@ private PrologTerm readList(final TokenizerResult openingBracket) {
316342
tokenizer.getLastTokenPos(), null);
317343
}
318344

319-
final TokenizerResult nextAtomTwo = this.readNextTokenCommentAware();
345+
final TokenizerResult nextAtomTwo = this.extractNextTokenCommentAware();
320346
if (nextAtomTwo == null) {
321347
throw new PrologParserException("Can't find expected token in list",
322348
this.tokenizer.getLine(), this.tokenizer.getPos());
@@ -374,21 +400,14 @@ private PrologTerm readList(final TokenizerResult openingBracket) {
374400
return leftPartFirst;
375401
}
376402

377-
private void checkForNull(final Object obj, final String message,
378-
final TokenizerResult startTerm) {
379-
if (obj == null) {
380-
throw new PrologParserException(message, startTerm.getLine(), startTerm.getPos());
381-
}
382-
}
383-
384403
private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
385404
// the variable will contain last processed tree item contains either
386405
// atom or operator
387406
AstItem currentTreeItem = null;
388407

389408
while (true) {
390409
// read next atom from tokenizer
391-
TokenizerResult readAtomContainer = this.readNextTokenCommentAware();
410+
TokenizerResult readAtomContainer = this.extractNextTokenCommentAware();
392411

393412
if (readAtomContainer == null) {
394413
if (currentTreeItem == null) {
@@ -505,7 +524,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
505524
readAtomContainer.getLine(), readAtomContainer.getPos());
506525
}
507526

508-
final TokenizerResult token = this.readNextTokenCommentAware();
527+
final TokenizerResult token = this.extractNextTokenCommentAware();
509528

510529
final PrologTerm closingAtom;
511530
if (token == null) {
@@ -534,7 +553,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
534553
}
535554
} else {
536555
if (readAtom.getType() != TermType.VAR || (this.parserFlags & FLAG_VAR_AS_FUNCTOR) != 0) {
537-
TokenizerResult nextToken = this.readNextTokenCommentAware();
556+
TokenizerResult nextToken = this.extractNextTokenCommentAware();
538557

539558
if (nextToken == null) {
540559
throw new PrologParserException("Non-closed clause", this.tokenizer.getLastTokenLine(),
@@ -666,6 +685,19 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
666685
}
667686
}
668687

688+
private void checkForNull(final Object obj, final String message,
689+
final TokenizerResult startTerm) {
690+
if (obj == null) {
691+
throw new PrologParserException(message, startTerm.getLine(), startTerm.getPos());
692+
}
693+
}
694+
695+
@Override
696+
public void close() throws IOException {
697+
this.deferredReadTerm = null;
698+
this.tokenizer.close(this.autoCloseReaderFlag);
699+
}
700+
669701
@Override
670702
public Iterator<PrologTerm> iterator() {
671703
return new Iterator<>() {
@@ -690,9 +722,26 @@ public Stream<PrologTerm> stream() {
690722
);
691723
}
692724

693-
@Override
694-
public void close() throws IOException {
695-
this.tokenizer.close(this.autoCloseReaderFlag);
725+
private static final class PrologTermReadResult {
726+
private final PrologTerm result;
727+
private final RuntimeException exception;
728+
729+
private PrologTermReadResult(final PrologTerm result, final RuntimeException error) {
730+
this.result = result;
731+
this.exception = error;
732+
}
733+
734+
boolean isPresented() {
735+
return this.exception == null || !(this.exception instanceof NoSuchElementException);
736+
}
737+
738+
PrologTerm getResult() {
739+
if (this.exception == null) {
740+
return this.result;
741+
} else {
742+
throw this.exception;
743+
}
744+
}
696745
}
697746

698747
}

‎src/main/java/com/igormaznitsa/prologparser/tokenizer/Tokenizer.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,8 @@ public void push(final TokenizerResult object) {
250250
public TokenizerResult peek() {
251251
TokenizerResult result;
252252
if (this.lastPushedTerm == null) {
253-
result = readNextToken();
254-
push(result);
253+
result = this.readNextToken();
254+
this.push(result);
255255
} else {
256256
result = this.lastPushedTerm;
257257
}
@@ -332,7 +332,7 @@ public TokenizerResult pop() {
332332
public TokenizerResult readNextToken() {
333333

334334
if (this.lastPushedTerm != null) {
335-
return pop();
335+
return this.pop();
336336
}
337337

338338
Quotation quoting = Quotation.NONE;

‎src/test/java/com/igormaznitsa/prologparser/IntegrationTest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,13 @@ void testReadingClausesTokensAndCharsFromSameParser() throws Exception {
7373
assertEquals("lsome(a)", term1.toString());
7474
assertTrue(parser.hasNext());
7575
final TokenizerResult token1 = parser.getInternalTokenizer().readNextToken();
76-
assertEquals("alone", Objects.requireNonNull(token1).getResult().toString());
76+
assertEquals("next", Objects.requireNonNull(token1).getResult().toString());
7777
assertTrue(parser.hasNext());
7878
final TokenizerResult token2 = parser.getInternalTokenizer().readNextToken();
79-
assertEquals(".", Objects.requireNonNull(token2).getResult().getText());
79+
assertEquals("(", Objects.requireNonNull(token2).getResult().getText());
8080
assertTrue(parser.hasNext());
8181
final PrologTerm term2 = parser.next();
82-
assertEquals("next(b)", term2.toString());
82+
assertEquals("alone", term2.toString());
8383
assertTrue(parser.hasNext());
8484
assertThrows(PrologParserException.class, parser::next);
8585
}

‎src/test/java/com/igormaznitsa/prologparser/exceptions/PrologParserExceptionTest.java

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@
44
import static org.junit.jupiter.api.Assertions.assertFalse;
55
import static org.junit.jupiter.api.Assertions.assertTrue;
66

7+
import com.igormaznitsa.prologparser.DefaultParserContext;
8+
import com.igormaznitsa.prologparser.GenericPrologParser;
9+
import com.igormaznitsa.prologparser.ParserContext;
10+
import com.igormaznitsa.prologparser.PrologParser;
11+
import com.igormaznitsa.prologparser.terms.PrologTerm;
12+
import com.igormaznitsa.prologparser.tokenizer.Op;
13+
import com.igormaznitsa.prologparser.tokenizer.TokenizerResult;
14+
import java.io.StringReader;
15+
import java.util.ArrayList;
16+
import java.util.List;
717
import org.junit.jupiter.api.Test;
818

919
public class PrologParserExceptionTest {
@@ -72,4 +82,32 @@ public void testContainsRightPositionData() {
7282
public void testToString() {
7383
assertEquals("Hello World[1:10]", new PrologParserException("Hello World", 1, 10).toString());
7484
}
85+
86+
@Test
87+
public void testNotificationAboutComments() throws Exception {
88+
final List<TokenizerResult> foundComments = new ArrayList<>();
89+
final List<PrologTerm> foundTerms = new ArrayList<>();
90+
91+
try (final PrologParser parser = new GenericPrologParser(
92+
new StringReader("% Line1\nhello(/*HHH*/world/*End*/).% Ending comment\n[1,2,3]."),
93+
DefaultParserContext.of(
94+
ParserContext.FLAG_BLOCK_COMMENTS | ParserContext.FLAG_COMMENTS_AS_ATOMS, Op.ISO),
95+
(parser1, comment) -> {
96+
foundComments.add(comment);
97+
}
98+
)) {
99+
parser.forEach(foundTerms::add);
100+
}
101+
102+
assertEquals(4, foundComments.size());
103+
assertEquals(" Line1", foundComments.get(0).getResult().getText());
104+
assertEquals("HHH", foundComments.get(1).getResult().getText());
105+
assertEquals("End", foundComments.get(2).getResult().getText());
106+
assertEquals(" Ending comment", foundComments.get(3).getResult().getText());
107+
108+
assertEquals(2, foundTerms.size());
109+
assertEquals("hello(world)", foundTerms.get(0).toString());
110+
assertEquals("[1, 2, 3]", foundTerms.get(1).toString());
111+
112+
}
75113
}

0 commit comments

Comments
 (0)