Skip to content

Commit 24ae7da

Browse files
aNNiMONaNNiMON
committed
Добавлены тесты, обновлён лексер
1 parent 5aae3e2 commit 24ae7da

File tree

11 files changed

+444
-6
lines changed

11 files changed

+444
-6
lines changed

‎nbproject/project.properties

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@ javac.source=1.8
4848
javac.target=1.8
4949
javac.test.classpath=\
5050
${javac.classpath}:\
51-
${build.classes.dir}
51+
${build.classes.dir}:\
52+
${libs.junit_4.classpath}:\
53+
${libs.hamcrest.classpath}
5254
javac.test.processorpath=\
5355
${javac.test.classpath}
5456
javadoc.additionalparam=

‎src/com/annimon/ownlang/Main.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public static void main(String[] args) throws IOException {
6464
private static void run(String input, boolean showTokens, boolean showAst, boolean showMeasurements) {
6565
final TimeMeasurement measurement = new TimeMeasurement();
6666
measurement.start("Tokenize time");
67-
final List<Token> tokens = new Lexer(input).tokenize();
67+
final List<Token> tokens = Lexer.tokenize(input);
6868
measurement.stop("Tokenize time");
6969
if (showTokens) {
7070
for (int i = 0; i < tokens.size(); i++) {

‎src/com/annimon/ownlang/parser/Lexer.java

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@
1212
*/
1313
public final class Lexer {
1414

15+
public static List<Token> tokenize(String input) {
16+
return new Lexer(input).tokenize();
17+
}
18+
1519
private static final String OPERATOR_CHARS = "+-*/%()[]{}=<>!&|.,^~?:";
1620

1721
private static final Map<String, TokenType> OPERATORS;
@@ -136,6 +140,12 @@ else if (OPERATOR_CHARS.indexOf(current) != -1) {
136140
private void tokenizeNumber() {
137141
clearBuffer();
138142
char current = peek(0);
143+
if (current == '0' && (peek(1) == 'x' || (peek(1) == 'X'))) {
144+
next();
145+
next();
146+
tokenizeHexNumber();
147+
return;
148+
}
139149
while (true) {
140150
if (current == '.') {
141151
if (buffer.indexOf(".") != -1) throw error("Invalid float number");
@@ -151,11 +161,16 @@ private void tokenizeNumber() {
151161
private void tokenizeHexNumber() {
152162
clearBuffer();
153163
char current = peek(0);
154-
while (isHexNumber(current)) {
155-
buffer.append(current);
164+
while (isHexNumber(current) || (current == '_')) {
165+
if (current != '_') {
166+
// allow _ symbol
167+
buffer.append(current);
168+
}
156169
current = next();
157170
}
158-
addToken(TokenType.HEX_NUMBER, buffer.toString());
171+
if (buffer.length() > 0) {
172+
addToken(TokenType.HEX_NUMBER, buffer.toString());
173+
}
159174
}
160175

161176
private static boolean isHexNumber(char current) {

‎src/com/annimon/ownlang/parser/Parser.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@
1616
*/
1717
public final class Parser {
1818

19+
public static Statement parse(List<Token> tokens) {
20+
final Parser parser = new Parser(tokens);
21+
final Statement program = parser.parse();
22+
if (parser.getParseErrors().hasErrors()) {
23+
throw new ParseException();
24+
}
25+
return program;
26+
}
27+
1928
private static final Token EOF = new Token(TokenType.EOF, "", -1, -1);
2029

2130
private static final Map<TokenType, BinaryExpression.Operator> assignOperator;

‎src/com/annimon/ownlang/parser/ast/IncludeStatement.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public IncludeStatement(Expression expression) {
2323
public void execute() {
2424
try {
2525
final String input = SourceLoader.readSource(expression.eval().asString());
26-
final List<Token> tokens = new Lexer(input).tokenize();
26+
final List<Token> tokens = Lexer.tokenize(input);
2727
final Parser parser = new Parser(tokens);
2828
final Statement program = parser.parse();
2929
if (!parser.getParseErrors().hasErrors()) {

‎test/com/annimon/ownlang/parser/LexerTest.java

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
package com.annimon.ownlang.parser;
2+
3+
import com.annimon.ownlang.exceptions.LexerException;
4+
import static com.annimon.ownlang.parser.TokenType.*;
5+
import java.util.ArrayList;
6+
import java.util.List;
7+
import org.junit.Test;
8+
import static org.junit.Assert.*;
9+
10+
/**
11+
*
12+
* @author aNNiMON
13+
*/
14+
public class LexerTest {
15+
16+
@Test
17+
public void testNumbers() {
18+
String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF #";
19+
List<Token> expList = list(NUMBER, NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
20+
List<Token> result = Lexer.tokenize(input);
21+
assertTokens(expList, result);
22+
assertEquals("0", result.get(0).getText());
23+
assertEquals("3.1415", result.get(1).getText());
24+
assertEquals("CAFEBABE", result.get(2).getText());
25+
assertEquals("f7d6c5", result.get(3).getText());
26+
}
27+
28+
@Test(expected = LexerException.class)
29+
public void testNumbersError() {
30+
String input = "3.14.15 0Xf7_p6_s5";
31+
Lexer.tokenize(input);
32+
}
33+
34+
@Test
35+
public void testArithmetic() {
36+
String input = "x = -1 + 2 * 3 % 4 / 5";
37+
List<Token> expList = list(WORD, EQ, MINUS, NUMBER, PLUS, NUMBER, STAR, NUMBER, PERCENT, NUMBER, SLASH, NUMBER);
38+
List<Token> result = Lexer.tokenize(input);
39+
assertTokens(expList, result);
40+
assertEquals("x", result.get(0).getText());
41+
}
42+
43+
@Test
44+
public void testKeywords() {
45+
String input = "if else while for include";
46+
List<Token> expList = list(IF, ELSE, WHILE, FOR, INCLUDE);
47+
List<Token> result = Lexer.tokenize(input);
48+
assertTokens(expList, result);
49+
}
50+
51+
@Test
52+
public void testWord() {
53+
String input = "if bool include \"text\n\ntext\"";
54+
List<Token> expList = list(IF, WORD, INCLUDE, TEXT);
55+
List<Token> result = Lexer.tokenize(input);
56+
assertTokens(expList, result);
57+
}
58+
59+
@Test
60+
public void testString() {
61+
String input = "\"1\\\"2\"";
62+
List<Token> expList = list(TEXT);
63+
List<Token> result = Lexer.tokenize(input);
64+
assertTokens(expList, result);
65+
assertEquals("1\"2", result.get(0).getText());
66+
}
67+
68+
@Test
69+
public void testEmptyString() {
70+
String input = "\"\"";
71+
List<Token> expList = list(TEXT);
72+
List<Token> result = Lexer.tokenize(input);
73+
assertTokens(expList, result);
74+
assertEquals("", result.get(0).getText());
75+
}
76+
77+
@Test(expected = LexerException.class)
78+
public void testStringError() {
79+
String input = "\"1\"\"";
80+
List<Token> expList = list(TEXT);
81+
List<Token> result = Lexer.tokenize(input);
82+
assertTokens(expList, result);
83+
assertEquals("1", result.get(0).getText());
84+
}
85+
86+
@Test
87+
public void testOperators() {
88+
String input = "=+-*/%<>!&|";
89+
List<Token> expList = list(EQ, PLUS, MINUS, STAR, SLASH, PERCENT, LT, GT, EXCL, AMP, BAR);
90+
List<Token> result = Lexer.tokenize(input);
91+
assertTokens(expList, result);
92+
}
93+
94+
@Test
95+
public void testOperators2Char() {
96+
String input = "== != <= >= && || ==+ >=- ->";
97+
List<Token> expList = list(EQEQ, EXCLEQ, LTEQ, GTEQ, AMPAMP, BARBAR,
98+
EQEQ, PLUS, GTEQ, MINUS, MINUS, GT);
99+
List<Token> result = Lexer.tokenize(input);
100+
assertTokens(expList, result);
101+
}
102+
103+
@Test
104+
public void testComments() {
105+
String input = "// 1234 \n /* */ 123 /* \n 12345 \n\n\n */";
106+
List<Token> expList = list(NUMBER);
107+
List<Token> result = Lexer.tokenize(input);
108+
assertTokens(expList, result);
109+
assertEquals("123", result.get(0).getText());
110+
}
111+
112+
@Test
113+
public void testComments2() {
114+
String input = "// /* 1234 \n */";
115+
List<Token> expList = list(STAR, SLASH);
116+
List<Token> result = Lexer.tokenize(input);
117+
assertTokens(expList, result);
118+
}
119+
120+
@Test(expected = LexerException.class)
121+
public void testCommentsError() {
122+
String input = "/* 1234 \n";
123+
Lexer.tokenize(input);
124+
}
125+
126+
private static void assertTokens(List<Token> expList, List<Token> result) {
127+
final int length = expList.size();
128+
assertEquals(length, result.size());
129+
for (int i = 0; i < length; i++) {
130+
assertEquals(expList.get(i).getType(), result.get(i).getType());
131+
}
132+
}
133+
134+
private static List<Token> list(TokenType... types) {
135+
final List<Token> list = new ArrayList<Token>();
136+
for (TokenType t : types) {
137+
list.add(token(t));
138+
}
139+
return list;
140+
}
141+
142+
private static Token token(TokenType type) {
143+
return token(type, "", 0, 0);
144+
}
145+
146+
private static Token token(TokenType type, String text, int row, int col) {
147+
return new Token(type, text, row, col);
148+
}
149+
150+
}

‎test/com/annimon/ownlang/parser/ParserTest.java

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package com.annimon.ownlang.parser;
2+
3+
import com.annimon.ownlang.lib.Value;
4+
import com.annimon.ownlang.lib.Variables;
5+
import com.annimon.ownlang.parser.ast.*;
6+
import static com.annimon.ownlang.parser.ast.ASTHelper.*;
7+
import org.junit.Test;
8+
import static org.junit.Assert.*;
9+
10+
/**
11+
* @author aNNiMON
12+
*/
13+
public class ParserTest {
14+
15+
@Test
16+
public void testParsePrimary() {
17+
assertEval(number(2), "2", value(2));
18+
assertEval(string("test"), "\"test\"", value("test"));
19+
}
20+
21+
@Test
22+
public void testParseAdditive() {
23+
assertEval( number(5), "2 + 3", operator(BinaryExpression.Operator.ADD, value(2), value(3)) );
24+
assertEval( number(-1), "2 - 3", operator(BinaryExpression.Operator.SUBTRACT, value(2), value(3)) );
25+
}
26+
27+
@Test
28+
public void testParseMultiplicative() {
29+
assertEval( number(6), "2 * 3", operator(BinaryExpression.Operator.MULTIPLY, value(2), value(3)) );
30+
assertEval( number(4), "12 / 3", operator(BinaryExpression.Operator.DIVIDE, value(12), value(3)) );
31+
assertEval( number(2), "12 % 5", operator(BinaryExpression.Operator.REMAINDER, value(12), value(5)) );
32+
}
33+
34+
private static void assertEval(Value expectedValue, String input, Expression expected) {
35+
BlockStatement program = assertExpression(input, expected);
36+
program.execute();
37+
final Value actual = Variables.get("a");
38+
assertEquals(expectedValue.asNumber(), actual.asNumber(), 0.001);
39+
assertEquals(expectedValue.asString(), actual.asString());
40+
}
41+
42+
private static BlockStatement assertExpression(String input, Expression expected) {
43+
return assertProgram("a = " + input, block(assign("a", expected)));
44+
}
45+
46+
private static BlockStatement assertProgram(String input, BlockStatement actual) {
47+
BlockStatement result = (BlockStatement) parse(input);
48+
assertStatements(result, actual);
49+
return result;
50+
}
51+
52+
private static void assertStatements(BlockStatement expected, BlockStatement actual) {
53+
for (int i = 0; i < expected.statements.size(); i++) {
54+
assertEquals(expected.statements.get(i).getClass(), actual.statements.get(i).getClass());
55+
}
56+
}
57+
58+
private static Statement parse(String input) {
59+
return Parser.parse(Lexer.tokenize(input));
60+
}
61+
}

‎test/com/annimon/ownlang/parser/ast/ASTHelper.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package com.annimon.ownlang.parser.ast;
2+
3+
import com.annimon.ownlang.lib.NumberValue;
4+
import com.annimon.ownlang.lib.StringValue;
5+
import com.annimon.ownlang.lib.Types;
6+
import com.annimon.ownlang.lib.Value;
7+
import static org.junit.Assert.*;
8+
9+
/**
10+
* Helper for build and test AST nodes.
11+
* @author aNNiMON
12+
*/
13+
public final class ASTHelper {
14+
15+
public static void assertValue(NumberValue expected, Value actual) {
16+
assertEquals(Types.NUMBER, actual.type());
17+
if (expected.raw() instanceof Double) {
18+
assertEquals(expected.asNumber(), actual.asNumber(), 0.001);
19+
}
20+
assertEquals(expected.asInt(), actual.asInt());
21+
}
22+
23+
public static void assertValue(StringValue expected, Value actual) {
24+
assertEquals(Types.STRING, actual.type());
25+
assertEquals(expected.asString(), actual.asString());
26+
}
27+
28+
29+
public static BlockStatement block(Statement... statements) {
30+
final BlockStatement result = new BlockStatement();
31+
for (Statement statement : statements) {
32+
result.add(statement);
33+
}
34+
return result;
35+
}
36+
37+
public static AssignmentExpression assign(String variable, Expression expr) {
38+
return assign(var(variable), expr);
39+
}
40+
41+
public static AssignmentExpression assign(Accessible accessible, Expression expr) {
42+
return assign(null, accessible, expr);
43+
}
44+
45+
public static AssignmentExpression assign(BinaryExpression.Operator op, Accessible accessible, Expression expr) {
46+
return new AssignmentExpression(op, accessible, expr);
47+
}
48+
49+
public static BinaryExpression operator(BinaryExpression.Operator op, Expression left, Expression right) {
50+
return new BinaryExpression(op, left, right);
51+
}
52+
53+
public static ValueExpression value(Number value) {
54+
return new ValueExpression(value);
55+
}
56+
57+
public static ValueExpression value(String value) {
58+
return new ValueExpression(value);
59+
}
60+
61+
public static VariableExpression var(String value) {
62+
return new VariableExpression(value);
63+
}
64+
65+
66+
public static NumberValue number(Number value) {
67+
return new NumberValue(value);
68+
}
69+
70+
public static StringValue string(String value) {
71+
return new StringValue(value);
72+
}
73+
}

0 commit comments

Comments
 (0)