Open
Description
I have extracted test cases from the SQLite test suite for the lexer. I originally wrote them using a basic helper method while I was building the lever. Now, they need to be converted into the MiniTest suite. Add a new describe block to the test/feather/test_lexer.rb file and file the code style already present. Here are the test cases:
# adapted from sqlite/test/normalize.test
assert_tokenized_as(
"SELECT * FROM t1 WHERE a IN (1) AND b=51.42",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_AND, "AND"], [:TK_SPACE, " "], [:TK_ID, "b"], [:TK_EQ, "="], [:TK_FLOAT, "51.42"]]
)
assert_tokenized_as(
"SELECT a, b+15, c FROM t1 WHERE d NOT IN (SELECT x FROM t2)",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "b"], [:TK_PLUS, "+"], [:TK_INTEGER, "15"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "c"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "d"], [:TK_SPACE, " "], [:TK_NOT, "NOT"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t2"], [:TK_RP, ")"]]
)
assert_tokenized_as(
<<~SQL,
SELECT NULL, b FROM t1 -- comment text
WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
SELECT a FROM t)
OR e='hello';
SQL
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_NULL, "NULL"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "b"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_SPACE, "-- comment text"], [:TK_SPACE, "\n "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "d"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_WITH, "WITH"], [:TK_SPACE, " "], [:TK_ID, "t"], [:TK_LP, "("], [:TK_ID, "a"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_AS, "AS"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_VALUES, "VALUES"], [:TK_LP, "("], [:TK_INTEGER, "5"], [:TK_RP, ")"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_SPACE, "/* CTE */"], [:TK_SPACE, "\n "], [:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t"], [:TK_RP, ")"], [:TK_SPACE, "\n "], [:TK_OR, "OR"], [:TK_SPACE, " "], [:TK_ID, "e"], [:TK_EQ, "="], [:TK_STRING, "'hello'"], [:TK_SEMI, ";"], [:TK_SPACE, "\n"]]
)
assert_tokenized_as(
<<~SQL,
/*Initial comment*/
-- another comment line
SELECT NULL /* comment */ , b FROM t1 -- comment text
WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
SELECT a FROM t)
OR e='hello';
SQL
[[:TK_SPACE, "/*Initial comment*/"], [:TK_SPACE, "\n"], [:TK_SPACE, "-- another comment line"], [:TK_SPACE, "\n"], [:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_NULL, "NULL"], [:TK_SPACE, " "], [:TK_SPACE, "/* comment */"], [:TK_SPACE, " "], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "b"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_SPACE, "-- comment text"], [:TK_SPACE, "\n "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "d"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_WITH, "WITH"], [:TK_SPACE, " "], [:TK_ID, "t"], [:TK_LP, "("], [:TK_ID, "a"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_AS, "AS"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_VALUES, "VALUES"], [:TK_LP, "("], [:TK_INTEGER, "5"], [:TK_RP, ")"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_SPACE, "/* CTE */"], [:TK_SPACE, "\n "], [:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t"], [:TK_RP, ")"], [:TK_SPACE, "\n "], [:TK_OR, "OR"], [:TK_SPACE, " "], [:TK_ID, "e"], [:TK_EQ, "="], [:TK_STRING, "'hello'"], [:TK_SEMI, ";"], [:TK_SPACE, "\n"]]
)
assert_tokenized_as(
"SELECT x,$::abc(15),y,@abc,z,?99,w FROM t1",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_COMMA, ","], [:TK_VARIABLE, "$::abc(15)"], [:TK_COMMA, ","], [:TK_ID, "y"], [:TK_COMMA, ","], [:TK_VARIABLE, "@abc"], [:TK_COMMA, ","], [:TK_ID, "z"], [:TK_COMMA, ","], [:TK_VARIABLE, "?99"], [:TK_COMMA, ","], [:TK_ID, "w"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"]]
)
assert_tokenized_as(
"SELECT 15 IN (1,2,3,(SELECT * FROM t1),'xyz',x'abcd',22*(x+5),null)",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_INTEGER, "15"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_INTEGER, "2"], [:TK_COMMA, ","], [:TK_INTEGER, "3"], [:TK_COMMA, ","], [:TK_LP, "("], [:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_RP, ")"], [:TK_COMMA, ","], [:TK_STRING, "'xyz'"], [:TK_COMMA, ","], [:TK_BLOB, "x'abcd'"], [:TK_COMMA, ","], [:TK_INTEGER, "22"], [:TK_STAR, "*"], [:TK_LP, "("], [:TK_ID, "x"], [:TK_PLUS, "+"], [:TK_INTEGER, "5"], [:TK_RP, ")"], [:TK_COMMA, ","], [:TK_NULL, "null"], [:TK_RP, ")"]]
)
assert_tokenized_as(
"SELECT x'abc'; -- illegal token",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ILLEGAL, "x'abc'"], [:TK_SEMI, ";"], [:TK_SPACE, " "], [:TK_SPACE, "-- illegal token"]]
)
assert_tokenized_as(
"SELECT a,NULL,b FROM t1 WHERE c IS NOT NULL or D is null or e=5",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_NULL, "NULL"], [:TK_COMMA, ","], [:TK_ID, "b"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "c"], [:TK_SPACE, " "], [:TK_IS, "IS"], [:TK_SPACE, " "], [:TK_NOT, "NOT"], [:TK_SPACE, " "], [:TK_NULL, "NULL"], [:TK_SPACE, " "], [:TK_OR, "or"], [:TK_SPACE, " "], [:TK_ID, "D"], [:TK_SPACE, " "], [:TK_IS, "is"], [:TK_SPACE, " "], [:TK_NULL, "null"], [:TK_SPACE, " "], [:TK_OR, "or"], [:TK_SPACE, " "], [:TK_ID, "e"], [:TK_EQ, "="], [:TK_INTEGER, "5"]]
)
assert_tokenized_as(
"SELECT * FROM t1 WHERE x IN (1,2,3)",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_INTEGER, "2"], [:TK_COMMA, ","], [:TK_INTEGER, "3"], [:TK_RP, ")"]]
)
assert_tokenized_as(
"CREATE TABLE t1(a,b)",
[[:TK_CREATE, "CREATE"], [:TK_SPACE, " "], [:TK_TABLE, "TABLE"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_LP, "("], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_ID, "b"], [:TK_RP, ")"]]
)
assert_tokenized_as(
"SELECT a, b FROM t1 WHERE b = ? ORDER BY a",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "b"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "b"], [:TK_SPACE, " "], [:TK_EQ, "="], [:TK_SPACE, " "], [:TK_VARIABLE, "?"], [:TK_SPACE, " "], [:TK_ORDER, "ORDER"], [:TK_SPACE, " "], [:TK_BY, "BY"], [:TK_SPACE, " "], [:TK_ID, "a"]]
)
assert_tokenized_as(
'CREATE TABLE t1(a,b,c,d,e,"col f",w,x,y,z)',
[[:TK_CREATE, "CREATE"], [:TK_SPACE, " "], [:TK_TABLE, "TABLE"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_LP, "("], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_ID, "b"], [:TK_COMMA, ","], [:TK_ID, "c"], [:TK_COMMA, ","], [:TK_ID, "d"], [:TK_COMMA, ","], [:TK_ID, "e"], [:TK_COMMA, ","], [:TK_ID, "\"col f\""], [:TK_COMMA, ","], [:TK_ID, "w"], [:TK_COMMA, ","], [:TK_ID, "x"], [:TK_COMMA, ","], [:TK_ID, "y"], [:TK_COMMA, ","], [:TK_ID, "z"], [:TK_RP, ")"]]
)
assert_tokenized_as(
'CREATE TABLE t2(x,"col y")',
[[:TK_CREATE, "CREATE"], [:TK_SPACE, " "], [:TK_TABLE, "TABLE"], [:TK_SPACE, " "], [:TK_ID, "t2"], [:TK_LP, "("], [:TK_ID, "x"], [:TK_COMMA, ","], [:TK_ID, "\"col y\""], [:TK_RP, ")"]]
)
assert_tokenized_as(
"SELECT a, b+15, c FROM t1 WHERE d NOT IN (SELECT x FROM t2)",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "b"], [:TK_PLUS, "+"], [:TK_INTEGER, "15"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "c"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "d"], [:TK_SPACE, " "], [:TK_NOT, "NOT"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t2"], [:TK_RP, ")"]]
)
assert_tokenized_as(
"SELECT a FROM t1 WHERE x IN (1,2,3) AND sqlite_version()",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_INTEGER, "2"], [:TK_COMMA, ","], [:TK_INTEGER, "3"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_AND, "AND"], [:TK_SPACE, " "], [:TK_ID, "sqlite_version"], [:TK_LP, "("], [:TK_RP, ")"]]
)
assert_tokenized_as(
"SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8()",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_INTEGER, "2"], [:TK_COMMA, ","], [:TK_INTEGER, "3"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_AND, "AND"], [:TK_SPACE, " "], [:TK_ID, "hex8"], [:TK_LP, "("], [:TK_RP, ")"]]
)
assert_tokenized_as(
"SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc')",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_INTEGER, "2"], [:TK_COMMA, ","], [:TK_INTEGER, "3"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_AND, "AND"], [:TK_SPACE, " "], [:TK_ID, "hex8"], [:TK_LP, "("], [:TK_STRING, "'abc'"], [:TK_RP, ")"]]
)
assert_tokenized_as(
%q[SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3')],
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "\"a\""], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "\"x\""], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_ID, "\"1\""], [:TK_COMMA, ","], [:TK_ID, "\"2\""], [:TK_COMMA, ","], [:TK_STRING, "'3'"], [:TK_RP, ")"]]
)
assert_tokenized_as(
"SELECT 'a' FROM t1 WHERE 'x'",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STRING, "'a'"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_STRING, "'x'"]]
)
assert_tokenized_as(
"SELECT [a] FROM t1 WHERE [x]",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "[a]"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "[x]"]]
)
assert_tokenized_as(
"SELECT * FROM t1 WHERE x IN (x)",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_ID, "x"], [:TK_RP, ")"]]
)
assert_tokenized_as(
'SELECT * FROM t1 WHERE x IN ([x],"a")',
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_ID, "[x]"], [:TK_COMMA, ","], [:TK_ID, "\"a\""], [:TK_RP, ")"]]
)
assert_tokenized_as(
%q[SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version())],
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_STAR, "*"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_ID, "[x]"], [:TK_COMMA, ","], [:TK_ID, "\"a\""], [:TK_COMMA, ","], [:TK_STRING, "'b'"], [:TK_COMMA, ","], [:TK_ID, "sqlite_version"], [:TK_LP, "("], [:TK_RP, ")"], [:TK_RP, ")"]]
)
assert_tokenized_as(
%q[SELECT a, a+1, a||'b', a+"b" FROM t1],
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_PLUS, "+"], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_CONCAT, "||"], [:TK_STRING, "'b'"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_PLUS, "+"], [:TK_ID, "\"b\""], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"]]
)
assert_tokenized_as(
"SELECT x FROM t1 WHERE x IN (1) UNION ALL SELECT x FROM t1 WHERE x IN (1)",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_UNION, "UNION"], [:TK_SPACE, " "], [:TK_ALL, "ALL"], [:TK_SPACE, " "], [:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_RP, ")"]]
)
assert_tokenized_as(
'SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y]',
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "a"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "\"col f\""], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_JOIN_KW, "LEFT"], [:TK_SPACE, " "], [:TK_JOIN_KW, "OUTER"], [:TK_SPACE, " "], [:TK_JOIN, "JOIN"], [:TK_SPACE, " "], [:TK_ID, "t2"], [:TK_SPACE, " "], [:TK_ON, "ON"], [:TK_SPACE, " "], [:TK_ID, "[t1]"], [:TK_DOT, "."], [:TK_ID, "[col f]"], [:TK_SPACE, " "], [:TK_EQ, "=="], [:TK_SPACE, " "], [:TK_ID, "[t2]"], [:TK_DOT, "."], [:TK_ID, "[col y]"]]
)
assert_tokenized_as(
"SELECT rowid, oid, _rowid_ FROM t1",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "rowid"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "oid"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "_rowid_"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"]]
)
assert_tokenized_as(
"SELECT x FROM t1 WHERE x IS NULL",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IS, "IS"], [:TK_SPACE, " "], [:TK_NULL, "NULL"]]
)
assert_tokenized_as(
"SELECT x FROM t1 WHERE x IS NOT NULL",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IS, "IS"], [:TK_SPACE, " "], [:TK_NOT, "NOT"], [:TK_SPACE, " "], [:TK_NULL, "NULL"]]
)
assert_tokenized_as(
"SELECT x FROM t1 WHERE x = NULL",
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_EQ, "="], [:TK_SPACE, " "], [:TK_NULL, "NULL"]]
)
assert_tokenized_as(
%q{SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00')},
[[:TK_SELECT, "SELECT"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_ID, "[x]"], [:TK_SPACE, " "], [:TK_IS, "IS"], [:TK_SPACE, " "], [:TK_NOT, "NOT"], [:TK_SPACE, " "], [:TK_NULL, "NULL"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_NULL, "NULL"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_STRING, "'a'"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "\"b\""], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_BLOB, "x'00'"], [:TK_RP, ")"]]
)
assert_tokenized_as(
'ATTACH "normalize800.db" AS somefile',
[[:TK_ATTACH, "ATTACH"], [:TK_SPACE, " "], [:TK_ID, "\"normalize800.db\""], [:TK_SPACE, " "], [:TK_AS, "AS"], [:TK_SPACE, " "], [:TK_ID, "somefile"]]
)
assert_tokenized_as(
%q[INSERT INTO t1 (x) VALUES("sl1"), (1), ("sl2"), ('i')],
[[:TK_INSERT, "INSERT"], [:TK_SPACE, " "], [:TK_INTO, "INTO"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_ID, "x"], [:TK_RP, ")"], [:TK_SPACE, " "], [:TK_VALUES, "VALUES"], [:TK_LP, "("], [:TK_ID, "\"sl1\""], [:TK_RP, ")"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_RP, ")"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_ID, "\"sl2\""], [:TK_RP, ")"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_STRING, "'i'"], [:TK_RP, ")"]]
)
assert_tokenized_as(
%q[UPDATE t1 SET x = "sl1" WHERE x IN (1, "sl2", 'i')],
[[:TK_UPDATE, "UPDATE"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_SET, "SET"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_EQ, "="], [:TK_SPACE, " "], [:TK_ID, "\"sl1\""], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "\"sl2\""], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_STRING, "'i'"], [:TK_RP, ")"]]
)
assert_tokenized_as(
%q[UPDATE t1 SET x = "y" WHERE x IN (1, "sl1", 'i')],
[[:TK_UPDATE, "UPDATE"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_SET, "SET"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_EQ, "="], [:TK_SPACE, " "], [:TK_ID, "\"y\""], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "\"sl1\""], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_STRING, "'i'"], [:TK_RP, ")"]]
)
assert_tokenized_as(
%q[DELETE FROM t1 WHERE x IN (1, "sl1", 'i')],
[[:TK_DELETE, "DELETE"], [:TK_SPACE, " "], [:TK_FROM, "FROM"], [:TK_SPACE, " "], [:TK_ID, "t1"], [:TK_SPACE, " "], [:TK_WHERE, "WHERE"], [:TK_SPACE, " "], [:TK_ID, "x"], [:TK_SPACE, " "], [:TK_IN, "IN"], [:TK_SPACE, " "], [:TK_LP, "("], [:TK_INTEGER, "1"], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_ID, "\"sl1\""], [:TK_COMMA, ","], [:TK_SPACE, " "], [:TK_STRING, "'i'"], [:TK_RP, ")"]]
)