Merge pull request #7 from Pip-Install-Party/Ryax3-develop

setup state 0 for tokenization implementation

Author: Ryax3

tokenizer.h

@@ -7,9 +7,12 @@
 class Tokenizer
 {
 private:
+    void Tokenizer::state0(std::ifstream& file, int& lineCount, std::ostringstream& buffer);
+
 public:
     Tokenizer() {}
     ~Tokenizer() { delete this; };
+    void begin(std::ifstream& file, int lineCount, std::ostringstream& buffer) { state0(file, lineCount, buffer); }
 };
 
 #endif /* TOKENIZER */