Merge pull request #27 from Pip-Install-Party/jacobs999-develop

Jacobs999 develop

Author: officialblake

tokenizer.cpp

@@ -103,6 +103,7 @@ void Tokenizer::state0(std::istringstream &inputStream, int &lineCount, std::ost
     } else if (ch == '\'') {  // Handle single quotes (character literals).
         buffer << "\nToken type: SINGLE_QUOTE\n";
         buffer << "Token: " << ch << "\n";
+        //state1(inputStream, lineCount, buffer);
         state5(inputStream, lineCount, buffer);  // Transition to state5 for single quotes.
         return state0(inputStream, lineCount, buffer);
     } else if (ch == '_') {  // Handle underscores.
@@ -136,13 +137,9 @@ void Tokenizer::state1(std::istringstream &inputStream, int &lineCount, std::ost
         std::cerr << "Error: Unterminated string\n";
         exit(1);
     } else if (ch == '"') {  // End of the string literal.
-        buffer << "\n";
-        buffer << "\nToken type: DOUBLE_QUOTE\n";
+        buffer << "\n\nToken type: DOUBLE_QUOTE\n";
         buffer << "Token: " << ch << "\n";
         return;
-    } else if (ch == '\\') {  // Handle escape characters inside the string.
-        buffer << ch;  // Add the backslash to the buffer.
-        state9(inputStream, lineCount, buffer);  // Transition to state9 to handle the escape sequence.
     } else if (ch == '\n') {  // Handle newlines within the string (error).
         lineCount++;
         std::cerr << "Error: Unterminated string on line " << lineCount << "\n";
@@ -209,7 +206,7 @@ void Tokenizer::state4(std::istringstream &inputStream, int &lineCount, std::ost
     return state4(inputStream, lineCount, buffer);  // Continue processing the identifier.
 }
 
-// Handle character literals enclosed by single quotes.
+// Handle characters enclosed by single quotes.
 void Tokenizer::state5(std::istringstream &inputStream, int &lineCount, std::ostringstream& buffer) {
     char ch;
     inputStream.get(ch);  // Get the next character.
@@ -222,9 +219,11 @@ void Tokenizer::state5(std::istringstream &inputStream, int &lineCount, std::ost
         buffer << "Token: ''\n";  // Empty literal.
         return;
     } else if (ch == '\\') {  // Escape character inside the literal.
-        state8(inputStream, lineCount, buffer);  // Handle escape sequences.
-    } else {  // Handle regular character literals.
-        buffer << "\nToken type: STRING\n";
+        buffer << "\nToken type: CHARACTER\n";
+        inputStream.get(ch); // Found backslash so need to get escaped character as well
+        buffer << "Token: \\" << ch << "\n";  // Add escape sequence to buffer.
+    } else {  // Handle regular single length character literals.
+        buffer << "\nToken type: CHARACTER\n";
         buffer << "Token: " << ch << "\n";
     }
     state11(inputStream, lineCount, buffer);  // Check for the closing single quote.
@@ -263,7 +262,7 @@ void Tokenizer::state7(std::istringstream &inputStream, int &lineCount, std::ost
     return;
 }
 
-// Handle escape sequences inside string literals (e.g., '\n', '\t').
+// Handle escape sequences inside string literals (e.g., '\n', '\t') and character literals 
 void Tokenizer::state8(std::istringstream &inputStream, int &lineCount, std::ostringstream& buffer) {
     char ch;
     inputStream.get(ch);  // Get the character after the backslash.
@@ -281,7 +280,7 @@ void Tokenizer::state8(std::istringstream &inputStream, int &lineCount, std::ost
 }
 
 // Handle character escape sequences (like '\n').
-void Tokenizer::state9(std::istringstream &inputStream, int &lineCount, std::ostringstream& buffer) {
+/*void Tokenizer::state9(std::istringstream &inputStream, int &lineCount, std::ostringstream& buffer) {
     char ch;
     inputStream.get(ch);  // Get the character after the backslash.
 
@@ -290,7 +289,7 @@ void Tokenizer::state9(std::istringstream &inputStream, int &lineCount, std::ost
     } else {
         buffer << ch;  // Handle other escape characters.
     }
-}
+}*/
 
 // Handle boolean AND (&&) and bitwise AND (&).
 void Tokenizer::state10(std::istringstream &inputStream, int &lineCount, std::ostringstream& buffer) {