Add test cases from sqlite/test/literal.test

[fractaledmind]

I have extracted test cases from the SQLite test suite for the lexer. I originally wrote them using a basic helper method while I was building the lever. Now, they need to be converted into the MiniTest suite. Add a new describe block to the test/feather/test_lexer.rb file and file the code style already present. Here are the test cases:

# adapted from sqlite/test/literal.test
assert_tokenized_as("45", [[:TK_INTEGER, "45"]])
assert_tokenized_as("0xFF", [[:TK_INTEGER, "0xFF"]])
assert_tokenized_as("0xFFFFFFFF", [[:TK_INTEGER, "0xFFFFFFFF"]])
assert_tokenized_as("0x123FFFFFFFF", [[:TK_INTEGER, "0x123FFFFFFFF"]])
assert_tokenized_as("-0x123FFFFFFFF", [[:TK_MINUS, "-"], [:TK_INTEGER, "0x123FFFFFFFF"]])
assert_tokenized_as("0xFFFFFFFFFFFFFFFF", [[:TK_INTEGER, "0xFFFFFFFFFFFFFFFF"]])
assert_tokenized_as("0x7FFFFFFFFFFFFFFF", [[:TK_INTEGER, "0x7FFFFFFFFFFFFFFF"]])
assert_tokenized_as("-0x7FFFFFFFFFFFFFFF", [[:TK_MINUS, "-"], [:TK_INTEGER, "0x7FFFFFFFFFFFFFFF"]])
assert_tokenized_as("+0x7FFFFFFFFFFFFFFF", [[:TK_PLUS, "+"], [:TK_INTEGER, "0x7FFFFFFFFFFFFFFF"]])
assert_tokenized_as("-45", [[:TK_MINUS, "-"], [:TK_INTEGER, "45"]])
assert_tokenized_as("'0xFF'", [[:TK_STRING, "'0xFF'"]])
assert_tokenized_as("'-0xFF'", [[:TK_STRING, "'-0xFF'"]])
assert_tokenized_as("-'0xFF'", [[:TK_MINUS, "-"], [:TK_STRING, "'0xFF'"]])
assert_tokenized_as("-9223372036854775808", [[:TK_MINUS, "-"], [:TK_INTEGER, "9223372036854775808"]])
assert_tokenized_as("1e123", [[:TK_FLOAT, "1e123"]])
assert_tokenized_as("1.0e123", [[:TK_FLOAT, "1.0e123"]])
assert_tokenized_as("1e1000", [[:TK_FLOAT, "1e1000"]])
assert_tokenized_as("-1e1000", [[:TK_MINUS, "-"], [:TK_FLOAT, "1e1000"]])
assert_tokenized_as("123a456", [[:TK_ILLEGAL, "123a456"]])
assert_tokenized_as("1_", [[:TK_ILLEGAL, "1_"]])
assert_tokenized_as("1_.4", [[:TK_ILLEGAL, "1_"], [:TK_INTEGER, ".4"]])
assert_tokenized_as("1e_4", [[:TK_ILLEGAL, "1e_4"]])
assert_tokenized_as("1.4_e4", [[:TK_ILLEGAL, "1.4_e4"]])
assert_tokenized_as("1.4e+_4", [[:TK_ILLEGAL, "1.4e"], [:TK_PLUS, "+"], [:TK_ID, "_4"]])
assert_tokenized_as("1.4e-_4", [[:TK_ILLEGAL, "1.4e"], [:TK_MINUS, "-"], [:TK_ID, "_4"]])
assert_tokenized_as("1.4e4_", [[:TK_ILLEGAL, "1.4e4_"]])
assert_tokenized_as("1.4e_4", [[:TK_ILLEGAL, "1.4e_4"]])
assert_tokenized_as("12__34", [[:TK_ILLEGAL, "12__34"]])
assert_tokenized_as("1234_", [[:TK_ILLEGAL, "1234_"]])
assert_tokenized_as("12._34", [[:TK_ILLEGAL, "12._34"]])
assert_tokenized_as("12.34_", [[:TK_ILLEGAL, "12.34_"]])
assert_tokenized_as("1.0e1_______2", [[:TK_ILLEGAL, "1.0e1_______2"]])