Add test cases from sqlite/test/main.test

[fractaledmind]

I have extracted test cases from the SQLite test suite for the lexer. I originally wrote them using a basic helper method while I was building the lever. Now, they need to be converted into the MiniTest suite. Add a new describe block to the test/feather/test_lexer.rb file and file the code style already present. Here are the test cases:

# adapted from sqlite/test/main.test
assert_tokenized_as("'xyz;", [[:TK_ILLEGAL, "'xyz;"]])
assert_tokenized_as("'xyz';", [[:TK_STRING, "'xyz'"], [:TK_SEMI, ";"]])
assert_tokenized_as('"xyz;', [[:TK_ILLEGAL, "\"xyz;"]])
assert_tokenized_as(%q["xyz';], [[:TK_ILLEGAL, "\"xyz';"]])
assert_tokenized_as('"xyz";', [[:TK_ID, "\"xyz\""], [:TK_SEMI, ";"]])
assert_tokenized_as("xyz; hi", [[:TK_ID, "xyz"], [:TK_SEMI, ";"], [:TK_SPACE, " "], [:TK_ID, "hi"]])
assert_tokenized_as("xyz; ", [[:TK_ID, "xyz"], [:TK_SEMI, ";"], [:TK_SPACE, " "]] )
assert_tokenized_as("xyz; -- hi ", [[:TK_ID, "xyz"], [:TK_SEMI, ";"], [:TK_SPACE, " "], [:TK_SPACE, "-- hi "]])
assert_tokenized_as("xyz; -- hi\n", [[:TK_ID, "xyz"], [:TK_SEMI, ";"], [:TK_SPACE, " "], [:TK_SPACE, "-- hi"], [:TK_SPACE, "\n"]])
assert_tokenized_as("SELECT a-b FROM t1", [[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_MINUS, "-"], [:TK_ID, "b"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"]])
assert_tokenized_as("SELECT a/e FROM t1", [[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_SLASH, "/"], [:TK_ID, "e"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"]])
assert_tokenized_as("CREATE TABLE abc(x,y)", [[:TK_CREATE, "CREATE"], [:TK_SPACE, " "], [:TK_TABLE, "TABLE"], [:TK_SPACE, " "], [:TK_ID, "abc"], [:TK_LP, "("], [:TK_ID, "x"], [:TK_COMMA, ","], [:TK_ID, "y"], [:TK_RP, ")"]])
assert_tokenized_as("SELECT * from T1 where x!!5", [[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "from"], [:TK_SPACE, " "], [:TK_ID, "T1"], [:TK_SPACE, " "], [:TK_WHERE, "where"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_ILLEGAL, "!"], [:TK_ILLEGAL, "!"], [:TK_INTEGER, "5"]])
assert_tokenized_as("SELECT * from T1 where ^x", [[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "from"], [:TK_SPACE, " "], [:TK_ID, "T1"], [:TK_SPACE, " "], [:TK_WHERE, "where"], [:TK_SPACE, " "], [:TK_ILLEGAL, "^"], [:TK_ID, "x"]])
assert_tokenized_as("'abc", [[:TK_ILLEGAL, "'abc"]])
assert_tokenized_as('"abc', [[:TK_ILLEGAL, '"abc']])
assert_tokenized_as("[abc", [[:TK_ILLEGAL, "[abc"]])
assert_tokenized_as("x'4869", [[:TK_ILLEGAL, "x'4869"]])
assert_tokenized_as("select x'4869'", [[:TK_SELECT, "select"], [:TK_SPACE, " "], [:TK_BLOB, "x'4869'"]])
assert_tokenized_as("select x'48695'", [[:TK_SELECT, "select"], [:TK_SPACE, " "], [:TK_ILLEGAL, "x'48695'"]])
assert_tokenized_as("x'486x'", [[:TK_ILLEGAL, "x'486x'"]])
assert_tokenized_as("$abc(", [[:TK_ILLEGAL, "$abc("]])
assert_tokenized_as("$abc(x", [[:TK_ILLEGAL, "$abc(x"]])
assert_tokenized_as("$::xyz", [[:TK_VARIABLE, "$::xyz"]])
assert_tokenized_as("$testnamespace::xyz", [[:TK_VARIABLE, "$testnamespace::xyz"]])
assert_tokenized_as("$(abc)", [[:TK_ILLEGAL, "$"], [:TK_LP, "("], [:TK_ID, "abc"], [:TK_RP, ")"]])
assert_tokenized_as("\$hi\u1234x", [[:TK_VARIABLE, "$hi\xE1\x88\xB4x"]])
assert_tokenized_as("456\u1234", [[:TK_ILLEGAL, "456\xE1\x88\xB4"]])
# --- If executed, these values are all equal to 31400
assert_tokenized_as("3.14e+4", [[:TK_FLOAT, "3.14e+4"]])
assert_tokenized_as("3.14e+04", [[:TK_FLOAT, "3.14e+04"]])
assert_tokenized_as("3.14e+004", [[:TK_FLOAT, "3.14e+004"]])
assert_tokenized_as("3.14e4", [[:TK_FLOAT, "3.14e4"]])
assert_tokenized_as("3.14e04", [[:TK_FLOAT, "3.14e04"]])
assert_tokenized_as("3.14e004", [[:TK_FLOAT, "3.14e004"]])
assert_tokenized_as("3.14E+4", [[:TK_FLOAT, "3.14E+4"]])
assert_tokenized_as("3.14E+04", [[:TK_FLOAT, "3.14E+04"]])
assert_tokenized_as("3.14E+004", [[:TK_FLOAT, "3.14E+004"]])
assert_tokenized_as("3.14E4", [[:TK_FLOAT, "3.14E4"]])
assert_tokenized_as("3.14E04", [[:TK_FLOAT, "3.14E04"]])
assert_tokenized_as("3.14E004", [[:TK_FLOAT, "3.14E004"]])
assert_tokenized_as("3.14e-4 * 1e8", [[:TK_FLOAT, "3.14e-4"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FLOAT, "1e8"]])
assert_tokenized_as("3.14E-04 * 1E08", [[:TK_FLOAT, "3.14E-04"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FLOAT, "1E08"]])
assert_tokenized_as("3.14e-004 * 01.0E+8", [[:TK_FLOAT, "3.14e-004"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FLOAT, "01.0E+8"]])
# ---
assert_tokenized_as("123--5", [[:TK_INTEGER, "123"], [:TK_SPACE, "--5"]])