teascade/c-compiler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix tokenization of long words

Browse Source
This commit is contained in:
Sofia 2026-04-11 22:06:15 +03:00
parent bd76e8676f
commit 96e4d6232f
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/tokens.cpp
View File

@ -128,7 +128,9 @@ namespace token {
uint32_t line = 0; uint32_t line = 0;
uint32_t line_start = 0; uint32_t line_start = 0;
for (int i = 0; i < static_cast<int>(text.length());) { int text_length = static_cast<int>(text.length());
for (int i = 0; i < text_length;) {
Position position{ line, i - line_start }; Position position{ line, i - line_start };
Metadata meta{ position, position, filename }; Metadata meta{ position, position, filename };
@ -138,6 +140,7 @@ namespace token {
std::string content{}; std::string content{};
do { do {
content += c; content += c;
if ((i + 1) >= text_length) break;
c = text[++i]; c = text[++i];
} while (std::isdigit(c)); } while (std::isdigit(c));
tokens.push_back(token::Token{ token::Type::LiteralInt, content, meta + content.size() }); tokens.push_back(token::Token{ token::Type::LiteralInt, content, meta + content.size() });
@ -147,6 +150,7 @@ namespace token {
c = text[++i]; // Skip initial " c = text[++i]; // Skip initial "
do { do {
content += c; content += c;
if ((i + 1) >= text_length) break;
c = text[++i]; c = text[++i];
} while (c != '\"'); } while (c != '\"');
i++; // Skip second " i++; // Skip second "
@ -156,6 +160,7 @@ namespace token {
std::string content{}; std::string content{};
do { do {
content += c; content += c;
if ((i + 1) >= text_length) break;
c = text[++i]; c = text[++i];
} while (std::isalnum(c)); } while (std::isalnum(c));
@ -179,6 +184,7 @@ namespace token {
line_start = i + 1; line_start = i + 1;
} }
content += c; content += c;
if ((i + 1) >= text_length) break;
c = text[++i]; c = text[++i];
} while (iswhitespace(c)); } while (iswhitespace(c));
// tokens.push_back(token::Token{ token::Type::Whitespace, content }); // tokens.push_back(token::Token{ token::Type::Whitespace, content });