Fix FullToken positions
This commit is contained in:
parent
d5daaa0e87
commit
974647b401
@ -134,6 +134,9 @@ pub fn tokenize<T: Into<String>>(to_tokenize: T) -> Result<Vec<FullToken>, Error
|
||||
let mut tokens = Vec::new();
|
||||
|
||||
while let Some(character) = &cursor.next() {
|
||||
// Save "current" token first character position
|
||||
let position = (cursor.position.0 - 1, cursor.position.1);
|
||||
|
||||
let variant = match character {
|
||||
// Whitespace
|
||||
w if w.is_whitespace() => continue,
|
||||
@ -204,7 +207,7 @@ pub fn tokenize<T: Into<String>>(to_tokenize: T) -> Result<Vec<FullToken>, Error
|
||||
|
||||
tokens.push(FullToken {
|
||||
token: variant,
|
||||
position: cursor.position,
|
||||
position,
|
||||
});
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user