Skip to content

Accurate column error info with tabs. Avoid copy of pattern string. #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open

Accurate column error info with tabs. Avoid copy of pattern string. #178

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
51a6d74
Add code from dev branch
shewitt-au Jul 10, 2025
f9e9304
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
95e0ccf
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
f52b404
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
4ecaf94
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
09bdf7a
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
35a70d9
Fix build error.
shewitt-au Jul 11, 2025
8c37d1d
Make whole longest line visible
shewitt-au Jul 11, 2025
dd10d49
Fix grammar in comment
shewitt-au Jul 11, 2025
c021818
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
155dcbe
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
0c71626
Merge branch 'accurate-column' of https://github.com/shewitt-au/Steve…
shewitt-au Jul 12, 2025
b1872f6
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
b4fab84
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
6dc69fa
Fixed potential buffer overrun
shewitt-au Jul 12, 2025
fefa424
Fixed bug while investigating feedback from @paxcut
shewitt-au Jul 12, 2025
c4373e7
Longest line length
shewitt-au Jul 13, 2025
f77e420
Merge branch 'master' into accurate-column
shewitt-au Aug 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 27 additions & 5 deletions lib/include/pl/core/lexer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ namespace pl::core {
Lexer() = default;

hlp::CompileResult<std::vector<Token>> lex(const api::Source *source);
size_t getLongestLineLength() const { return m_longestLineLength; }

size_t getLongestLineLength() const {
// 'getLongestLineLength' is used by the pattern editor to control
// the x-scrolling range. Adding two makes the whole longest line visible.
// Not sure why we have to do this.
return m_longestLineLength+2;
}

private:
[[nodiscard]] char peek(size_t p = 1) const;
Expand Down Expand Up @@ -51,20 +55,38 @@ namespace pl::core {
Token makeToken(const Token& token, size_t length = 1);
static Token makeTokenAt(const Token& token, Location& location, size_t length = 1);
void addToken(const Token& token);
bool hasTheLineEnded(const char &ch) {
if(ch == '\n') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin);

bool skipLineEnding() {
char ch = m_sourceCode[m_cursor];
if (ch == '\n') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation);
m_tabCompensation = 0;
m_line++;
m_lineBegin = ++m_cursor;
return true;
}
else if (ch == '\r') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation);
m_tabCompensation = 0;
m_line++;
ch = m_sourceCode[++m_cursor];
if (ch == '\n')
++m_cursor;
m_lineBegin = m_cursor;
return true;
}

return false;
}

static constexpr int tabsize = 4;

std::string m_sourceCode;
const api::Source* m_source = nullptr;
std::vector<Token> m_tokens;
size_t m_cursor = 0;
u32 m_line = 0;
u32 m_tabCompensation = 0;
u32 m_lineBegin = 0;
size_t m_longestLineLength = 0;
u32 m_errorLength = 0;
Expand Down
7 changes: 2 additions & 5 deletions lib/include/pl/pattern_language.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ namespace pl {
*/
[[nodiscard]] std::optional<std::vector<pl::core::Token>> preprocessString(const std::string &code, const std::string &source);


/**
* @brief Parses a pattern language code string and returns the generated AST
* To get parsing errors, check PatternLanguage#getCompileErrors() after calling this method
Expand All @@ -93,7 +92,7 @@ namespace pl {
* @param checkResult Whether to check the result of the execution
* @return True if the execution was successful, false otherwise. Call PatternLanguage#getCompileErrors() AND PatternLanguage#getEvalError() to get the compilation or runtime errors if false is returned
*/
[[nodiscard]] bool executeString(std::string code, const std::string& source = api::Source::DefaultSource, const std::map<std::string, core::Token::Literal> &envVars = {}, const std::map<std::string, core::Token::Literal> &inVariables = {}, bool checkResult = true);
[[nodiscard]] bool executeString(const std::string &code, const std::string &source = api::Source::DefaultSource, const std::map<std::string, core::Token::Literal> &envVars = {}, const std::map<std::string, core::Token::Literal> &inVariables = {}, bool checkResult = true);

/**
* @brief Executes a pattern language file
Expand Down Expand Up @@ -122,7 +121,7 @@ namespace pl {

/**
* @brief Aborts the currently running execution asynchronously
*/
*/
void abort();

/**
Expand Down Expand Up @@ -159,7 +158,6 @@ namespace pl {
void setStartAddress(u64 address);
u64 getStartAddress() const;


/**
* @brief Adds a new pragma preprocessor instruction
* @param name Name of the pragma
Expand Down Expand Up @@ -271,7 +269,6 @@ namespace pl {
*/
[[nodiscard]] std::vector<ptrn::Pattern *> getPatternsAtAddress(u64 address, u64 section = 0x00) const;


/**
* @brief Get the colors of all patterns that overlap with the given address
* @param address Address to check
Expand Down
63 changes: 29 additions & 34 deletions lib/source/pl/core/lexer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ namespace pl::core {
return intLiteral.size();
}


std::optional<char> Lexer::parseCharacter() {
const char& c = m_sourceCode[m_cursor++];
if (c == '\\') {
Expand Down Expand Up @@ -134,8 +133,7 @@ namespace pl::core {
result += character.value();
}

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeString(result), location, result.size());
}
Expand All @@ -156,8 +154,7 @@ namespace pl::core {
result += character.value();
}

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeString(result), location, result.size());
}
Expand Down Expand Up @@ -317,8 +314,7 @@ namespace pl::core {
}
auto len = m_cursor - begin;

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeComment(true, result), location, len);
}
Expand All @@ -335,8 +331,7 @@ namespace pl::core {
}
auto len = m_cursor - begin;

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeDocComment(false, true, result), location, len);
}
Expand All @@ -348,8 +343,8 @@ namespace pl::core {
std::string result;

m_cursor += 3;
while(true) {
hasTheLineEnded(peek(0));
while (true) {
skipLineEnding();

if(peek(1) == '\x00') {
m_errorLength = 1;
Expand All @@ -375,7 +370,7 @@ namespace pl::core {

m_cursor += 2;
while(true) {
hasTheLineEnded(peek(0));
skipLineEnding();

if(peek(1) == '\x00') {
m_errorLength = 2;
Expand Down Expand Up @@ -419,7 +414,7 @@ namespace pl::core {
separatorToken != Token::Separators().end()) {
m_cursor++;
return makeTokenAt(separatorToken->second, location, m_cursor - begin);
}
}

return std::nullopt;
}
Expand Down Expand Up @@ -477,6 +472,7 @@ namespace pl::core {
this->m_line = 1;
this->m_lineBegin = 0;
this->m_longestLineLength = 0;
this->m_tabCompensation = 0;

const size_t end = this->m_sourceCode.size();

Expand All @@ -486,13 +482,19 @@ namespace pl::core {
const char& c = this->m_sourceCode[this->m_cursor];

if (c == '\x00') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin);
m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation);
break; // end of string
}

if (std::isblank(c) || std::isspace(c)) {
hasTheLineEnded(c);
m_cursor++;
if (c == '\t') {
u32 column = m_tabCompensation + (m_cursor - m_lineBegin + 1);
u32 tabbedColumn = (((column - 1) / tabsize + 1) * tabsize) + 1;
m_tabCompensation += tabbedColumn - column - 1;
++m_cursor;
}
else if (!skipLineEnding())
++m_cursor;
continue;
}

Expand Down Expand Up @@ -554,7 +556,7 @@ namespace pl::core {
continue;
}
if(category == '*') {
if (type != '!' && (type != '*' || peek(3) == '/' )) {
if(type != '!' && (type != '*' || peek(3) == '/')) {
const auto token = parseMultiLineComment();
if(token.has_value())
addToken(token.value());
Expand All @@ -580,7 +582,7 @@ namespace pl::core {
continue;
}

if (c == '#' && (m_tokens.empty() || m_tokens.back().location.line < m_line)) {
if (c == '#' && (m_tokens.empty() || m_tokens.back().location.line < m_line)) {
size_t length = 1;
u32 line = m_line;
while (isIdentifierCharacter(peek(length)))
Expand All @@ -590,22 +592,18 @@ namespace pl::core {
if (processToken(&Lexer::parseDirectiveName, directiveName)) {
Token::Directive directive = get<Token::Directive>(m_tokens.back().value);
if (m_line != line || directive == Token::Directive::Define || directive == Token::Directive::Undef ||
peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef ||
directive == Token::Directive::EndIf)
peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef ||
directive == Token::Directive::EndIf)
continue;
if (hasTheLineEnded(peek(0))) {
m_cursor++;
if (skipLineEnding())
continue;
}
auto directiveValue = parseDirectiveValue();
if (directiveValue.has_value()) {
addToken(directiveValue.value());
if (m_line != line || peek(0) == 0)
continue;
if (hasTheLineEnded(peek(0))) {
m_cursor++;
if (skipLineEnding())
continue;
}
directiveValue = parseDirectiveArgument();
if (directiveValue.has_value()) {
addToken(directiveValue.value());
Expand All @@ -623,7 +621,7 @@ namespace pl::core {
addToken(string.value());
continue;
}
} else if(c == '\'') {
} else if (c == '\'') {
auto location = this->location();
const auto begin = m_cursor;
m_cursor++; // skip opening '
Expand Down Expand Up @@ -671,12 +669,9 @@ namespace pl::core {
return false;
}

Location Lexer::location() {
u32 column = m_cursor - m_lineBegin;
// There is no newline before the first line so add 1 to the column
if(m_line==1) {
column += 1;
}
return Location { m_source, m_line, column, m_errorLength };
Location Lexer::location()
{
u32 column = m_tabCompensation + m_cursor - m_lineBegin + 1;
return Location{m_source, m_line, column, m_errorLength};
}
}
5 changes: 1 addition & 4 deletions lib/source/pl/pattern_language.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,16 +187,13 @@ namespace pl {
return m_currAST;
}

bool PatternLanguage::executeString(std::string code, const std::string& source, const std::map<std::string, core::Token::Literal> &envVars, const std::map<std::string, core::Token::Literal> &inVariables, bool checkResult) {
bool PatternLanguage::executeString(const std::string &code, const std::string& source, const std::map<std::string, core::Token::Literal> &envVars, const std::map<std::string, core::Token::Literal> &inVariables, bool checkResult) {
const auto startTime = std::chrono::high_resolution_clock::now();
ON_SCOPE_EXIT {
const auto endTime = std::chrono::high_resolution_clock::now();
this->m_runningTime = std::chrono::duration_cast<std::chrono::duration<double>>(endTime - startTime).count();
};

code = wolv::util::replaceStrings(code, "\r\n", "\n");
code = wolv::util::replaceStrings(code, "\t", " ");

const auto &evaluator = this->m_internals.evaluator;

evaluator->getConsole().setLogCallback(this->m_logCallback);
Expand Down
Loading