teascade/reid-llvm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix FullToken positions

Browse Source
This commit is contained in:
Sofia 2025-07-09 18:54:51 +03:00
parent d5daaa0e87
commit 974647b401
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
reid/src/lexer.rs
View File

@ -134,6 +134,9 @@ pub fn tokenize<T: Into<String>>(to_tokenize: T) -> Result<Vec<FullToken>, Error
let mut tokens = Vec::new(); let mut tokens = Vec::new();
while let Some(character) = &cursor.next() { while let Some(character) = &cursor.next() {
// Save "current" token first character position
let position = (cursor.position.0 - 1, cursor.position.1);
let variant = match character { let variant = match character {
// Whitespace // Whitespace
w if w.is_whitespace() => continue, w if w.is_whitespace() => continue,
@ -204,7 +207,7 @@ pub fn tokenize<T: Into<String>>(to_tokenize: T) -> Result<Vec<FullToken>, Error
tokens.push(FullToken { tokens.push(FullToken {
token: variant, token: variant,
position: cursor.position, position,
}); });
} }