From 974647b401491a2d5b19887512dc1234d7410b16 Mon Sep 17 00:00:00 2001
From: sofia <sofia@teascade.net>
Date: Wed, 9 Jul 2025 18:54:51 +0300
Subject: [PATCH] Fix FullToken positions

---
 reid/src/lexer.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/reid/src/lexer.rs b/reid/src/lexer.rs
index 192077c..c1dd2be 100644
--- a/reid/src/lexer.rs
+++ b/reid/src/lexer.rs
@@ -134,6 +134,9 @@ pub fn tokenize<T: Into<String>>(to_tokenize: T) -> Result<Vec<FullToken>, Error
     let mut tokens = Vec::new();
 
     while let Some(character) = &cursor.next() {
+        // Save "current" token first character position
+        let position = (cursor.position.0 - 1, cursor.position.1);
+
         let variant = match character {
             // Whitespace
             w if w.is_whitespace() => continue,
@@ -204,7 +207,7 @@ pub fn tokenize<T: Into<String>>(to_tokenize: T) -> Result<Vec<FullToken>, Error
 
         tokens.push(FullToken {
             token: variant,
-            position: cursor.position,
+            position,
         });
     }