teascade/c-compiler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tokenize whitespace as well, skip it in next

Browse Source
This commit is contained in:
Sofia 2026-04-12 00:49:59 +03:00
parent cdff4cc9db
commit 85644457d0
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/tokens.cpp
View File

@ -99,6 +99,9 @@ namespace token {
Token TokenStream::next() {
token::Token got = this->peek(0);
m_position++;
while (m_position < static_cast<int>(m_tokens.size()) && this->peek().type == Type::Whitespace) {
m_position++;
}
return got;
}
@ -187,8 +190,7 @@ namespace token {
if ((i + 1) >= text_length) break;
c = text[++i];
} while (iswhitespace(c));
// tokens.push_back(token::Token{ token::Type::Whitespace, content });
// Just skip whitespace tokens
tokens.push_back(token::Token{ token::Type::Whitespace, content, meta + content.size() });
}
else {
tokens.push_back(token::Token{ token::Type::Symbol, std::string{c}, meta });