admin/pl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: lexer out of bounds

Browse Source
This commit is contained in:
Amir Adal
2026-01-03 15:03:45 +01:00
parent 629b65e151
commit 1c04a058d7
3 changed files with 74 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
include/parser/lexer.hpp
View File

@@ -92,72 +92,98 @@ public:
public:
bool NextToken()
{
if (m_pos > m_code.len())
auto len = m_code.len();
auto peek = [&]() -> char {
return (m_pos < len) ? m_code.data[m_pos] : '\0';
};
auto advance = [&]() -> char {
return (m_pos < len) ? m_code.data[m_pos++] : '\0';
};
// IMPORTANT: >= not >
if (m_pos >= len)
{
m_token = Token(TokenType::Eof);
return false;
}
char c = m_code.data[m_pos++];
char c = advance();
while(std::isspace(c)) {
// skip whitespace safely
while (c != '\0' && std::isspace((unsigned char)c))
{
if (c == '\n')
{
m_line++;
m_last_newline = m_pos;
}
c = m_code.data[m_pos++];
if (m_pos >= len) // reached real EOF while skipping whitespace
{
m_token = Token(TokenType::Eof);
return false;
}
c = advance();
}
if (m_pos-1 > m_code.len())
if (c == '\0' || m_pos > len) // paranoia guard
{
m_token = Token(TokenType::Eof);
return false;
}
if (std::isalpha(c) != 0 || c == '_')
// identifier
if (std::isalpha((unsigned char)c) || c == '_')
{
StringBuilder s;
long offset_start = m_pos - m_last_newline;
long offset_start = m_pos - m_last_newline - 1; // -1 because we already consumed c
s.Push(c);
// id
while (std::isalpha(m_code.data[m_pos]) != 0 || m_code.data[m_pos] == '_')
// NOTE: usually identifiers allow digits after first char; add isdigit if you want
while (true)
{
s.Push(m_code.data[m_pos++]);
char p = peek();
if (!(std::isalpha((unsigned char)p) || p == '_'))
break;
s.Push(advance());
}
s.Push('\0');
m_token = Token(TokenType::Id, m_line, offset_start, offset_start);
m_token = Token(TokenType::Id, m_line, offset_start, m_pos - m_last_newline);
m_token.string = s.view();
if (strcmp("extern", m_token.string.c_str()) == 0)
{
m_token.token = TokenType::Extern;
}
if (strcmp("fn", m_token.string.c_str()) == 0)
{
m_token.token = TokenType::Fn;
}
if (strcmp("local", m_token.string.c_str()) == 0)
{
m_token.token = TokenType::Local;
}
if (strcmp("extern", m_token.string.c_str()) == 0) m_token.token = TokenType::Extern;
else if (strcmp("fn", m_token.string.c_str()) == 0) m_token.token = TokenType::Fn;
else if (strcmp("local", m_token.string.c_str()) == 0) m_token.token = TokenType::Local;
return true;
}
if (std::isdigit(c) != 0)
// integer (hex supported)
if (std::isdigit((unsigned char)c))
{
StringBuilder s;
long offset_start = m_pos - m_last_newline;
bool hex = c == '0' && m_code.data[m_pos] == 'x';
long offset_start = m_pos - m_last_newline - 1;
bool hex = (c == '0' && peek() == 'x');
s.Push(c);
// integer (could be hex)
while (std::isdigit(m_code.data[m_pos]) != 0 || (hex && std::isalpha(m_code.data[m_pos]) != 0))
if (hex) s.Push(advance()); // consume 'x'
while (true)
{
s.Push(m_code.data[m_pos++]);
char p = peek();
if (std::isdigit((unsigned char)p) ||
(hex && std::isxdigit((unsigned char)p)))
{
s.Push(advance());
}
else break;
}
s.Push('\0');
m_token = Token(TokenType::IntLiteral, m_line, offset_start, m_pos - m_last_newline);
m_token.int_number = std::strtol(s.data, nullptr, hex ? 16 : 10);
@@ -165,7 +191,8 @@ public:
return true;
}
m_token = Token((TokenType)c, m_line, m_pos - m_last_newline, m_pos - m_last_newline + 1);
// single-char token fallback
m_token = Token((TokenType)c, m_line, m_pos - m_last_newline - 1, m_pos - m_last_newline);
return true;
}