Fix parse error returning the token after the error

reid/src/ast/parse.rs

@@ -128,7 +128,7 @@ impl Parse for PrimaryExpression {
                     stream.expect(Token::BracketClose)?;
                     Expression(Kind::Array(expressions), stream.get_range().unwrap())
                 }
-                _ => Err(stream.expected_err("identifier, constant, parentheses or brackets")?)?,
+                _ => Err(stream.expected_err("expression")?)?,
             }
         } else {
             Err(stream.expected_err("expression")?)?
@@ -391,7 +391,8 @@ impl Parse for Block {
                 // Special list of expressions that are simply not warned about,
                 // if semicolon is missing.
                 if !matches!(e, Expression(ExpressionKind::IfExpr(_), _)) {
-                    println!("Oh no, does this statement lack ;");
+                    // In theory could ignore the missing semicolon..
+                    return Err(stream.expected_err("expected semicolon to complete statement")?);
                 }
 
                 statements.push(BlockLevelStatement::Expression(e));
@@ -445,7 +446,7 @@ impl<T: Parse + std::fmt::Debug> Parse for NamedFieldList<T> {
                     stream.next();
                 } // Consume comma
                 Some(Token::BraceClose) => break,
-                Some(_) | None => Err(stream.expected_err("another field or closing brace")?)?,
+                Some(_) | None => Err(stream.expecting_err("another field or closing brace")?)?,
             }
         }
         Ok(NamedFieldList(fields))
@@ -478,7 +479,7 @@ impl Parse for ValueIndex {
         match stream.peek() {
             Some(Token::BracketOpen) => Ok(ValueIndex::Array(stream.parse()?)),
             Some(Token::Dot) => Ok(ValueIndex::Struct(stream.parse()?)),
-            _ => Err(stream.expected_err("value or struct index")?),
+            _ => Err(stream.expecting_err("value or struct index")?),
         }
     }
 }
@@ -534,7 +535,7 @@ impl Parse for BlockLevelStatement {
                             Stmt::Return(ReturnType::Soft, e)
                         }
                     } else {
-                        Err(stream.expected_err("expression")?)?
+                        Err(stream.expecting_err("expression")?)?
                     }
                 }
             }
@@ -605,7 +606,7 @@ impl Parse for TopLevelStatement {
                     range,
                 })
             }
-            _ => Err(stream.expected_err("import or fn")?)?,
+            _ => Err(stream.expecting_err("import or fn")?)?,
         })
     }
 }

reid/src/error_raporting.rs

@@ -156,8 +156,8 @@ impl ReidError {
             let pass_err = pass::Error {
                 metadata: Metadata {
                     source_module_id: module,
-                    range: Default::default(),
+                    range: *error.get_range().unwrap_or(&Default::default()),
-                    position: error.get_position().copied(),
+                    position: None,
                 },
                 kind: error,
             };
@@ -185,24 +185,16 @@ impl std::fmt::Display for ReidError {
         for error in sorted_errors {
             let meta = error.get_meta();
             let module = self.map.get_module(&meta.source_module_id).unwrap();
-            let (position_fmt, line_fmt) = if let Some(tokens) = &module.tokens {
+            let position = if let Some(tokens) = &module.tokens {
                 let range_tokens = meta.range.into_tokens(&tokens);
-                let highlight_position = get_position(&range_tokens).unwrap();
+
-                let full_lines = get_full_lines(&tokens, highlight_position);
+                dbg!(&error);
-                let lines_position = get_position(&full_lines).unwrap();
+                dbg!(&meta.range, &tokens[meta.range.start]);
-                (
+                get_position(&range_tokens).or(meta.position.map(|p| (p, p)))
-                    fmt_positions(highlight_position),
-                    Some(fmt_lines(
-                        module.source.as_ref().unwrap(),
-                        lines_position,
-                        highlight_position,
-                        8,
-                    )?),
-                )
             } else if let Some(position) = meta.position {
-                (fmt_positions((position, position)), None)
+                Some((position, position))
             } else {
-                ("unknown".to_owned(), None)
+                None
             };
 
             if curr_module != Some(meta.source_module_id) {
@@ -223,9 +215,16 @@ impl std::fmt::Display for ReidError {
             writeln!(f)?;
             write!(f, "  Error: ")?;
             writeln!(f, "{}", color_err(format!("{}", error))?)?;
-            write!(f, "{:>20}{}", color_warn("At: ")?, position_fmt)?;
+            write!(
-            if let Some(line_fmt) = line_fmt {
+                f,
-                writeln!(f, "{}", line_fmt)?;
+                "{:>20}{}",
+                color_warn("At: ")?,
+                position
+                    .map(|p| fmt_positions(p))
+                    .unwrap_or(String::from("{unknown}")),
+            )?;
+            if let (Some(position), Some(source)) = (position, &module.source) {
+                writeln!(f, "{}", fmt_lines(source, position, 6)?)?;
             }
         }
         Ok(())
@@ -238,7 +237,7 @@ impl TokenRange {
             .iter()
             .skip(self.start)
             .by_ref()
-            .take(self.end - self.start)
+            .take(self.end + 1 - self.start)
             .collect::<Vec<_>>()
     }
 }
@@ -252,29 +251,16 @@ fn get_position(tokens: &Vec<&FullToken>) -> Option<(Position, Position)> {
     }
 }
 
-fn get_full_lines<'v>(
+fn into_full_lines<'v>((start, end): (Position, Position)) -> (Position, Position) {
-    tokens: &'v Vec<FullToken>,
+    (Position(0, start.1), Position(u32::MAX, end.1))
-    (start, end): (Position, Position),
-) -> Vec<&'v FullToken> {
-    let (first_token_pos, _) = tokens
-        .iter()
-        .enumerate()
-        .find(|(_, token)| token.position.1 == start.1)
-        .unwrap();
-    tokens
-        .iter()
-        .skip(first_token_pos)
-        .by_ref()
-        .take_while(|token| token.position.1 <= end.1)
-        .collect::<Vec<_>>()
 }
 
 fn fmt_lines(
     source: &String,
-    (line_start, line_end): (Position, Position),
     (highlight_start, highlight_end): (Position, Position),
     ident: usize,
 ) -> Result<String, std::fmt::Error> {
+    let (line_start, line_end) = into_full_lines((highlight_start, highlight_end));
     let mut cursor = Cursor {
         position: Position(0, 1),
         char_stream: source.chars(),

reid/src/token_stream.rs

@@ -24,11 +24,31 @@ impl<'a, 'b> TokenStream<'a, 'b> {
         }
     }
 
+    /// Returns expected-error for the next token in-line. Useful in conjunction
+    /// with [`TokenStream::peek`]
     pub fn expected_err<T: Into<String>>(&mut self, expected: T) -> Result<Error, Error> {
+        let next_token = self.previous().unwrap_or(Token::Eof);
         Ok(Error::Expected(
             expected.into(),
-            self.peek().unwrap_or(Token::Eof),
+            next_token,
-            self.get_next_position()?,
+            TokenRange {
+                start: self.position - 1,
+                end: self.position - 1,
+            },
+        ))
+    }
+
+    /// Returns expected-error for the previous token that was already consumed.
+    /// Useful in conjunction with [`TokenStream::next`]
+    pub fn expecting_err<T: Into<String>>(&mut self, expected: T) -> Result<Error, Error> {
+        let next_token = self.peek().unwrap_or(Token::Eof);
+        Ok(Error::Expected(
+            expected.into(),
+            next_token,
+            TokenRange {
+                start: self.position,
+                end: self.position,
+            },
         ))
     }
 
@@ -38,10 +58,10 @@ impl<'a, 'b> TokenStream<'a, 'b> {
                 self.position += 1;
                 Ok(())
             } else {
-                Err(self.expected_err(token)?)
+                Err(self.expecting_err(token)?)
             }
         } else {
-            Err(self.expected_err(token)?)
+            Err(self.expecting_err(token)?)
         }
     }
 
@@ -55,6 +75,14 @@ impl<'a, 'b> TokenStream<'a, 'b> {
         value
     }
 
+    pub fn previous(&mut self) -> Option<Token> {
+        if (self.position as i32 - 1) < 0 {
+            None
+        } else {
+            Some(self.tokens[self.position - 1].token.clone())
+        }
+    }
+
     pub fn peek(&mut self) -> Option<Token> {
         if self.tokens.len() < self.position {
             None
@@ -147,11 +175,11 @@ impl<'a, 'b> TokenStream<'a, 'b> {
         }
     }
 
-    fn get_next_position(&self) -> Result<Position, Error> {
+    fn get_position(&self, offset: usize) -> Result<Position, Error> {
         if self.tokens.is_empty() {
             Err(Error::FileEmpty)
         } else {
-            let token_idx = self.position.min(self.tokens.len() - 1);
+            let token_idx = (self.position - 1).min(self.tokens.len() - 1);
             Ok(self.tokens[token_idx].position)
         }
     }
@@ -210,7 +238,7 @@ impl std::iter::Sum for TokenRange {
 #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub enum Error {
     #[error("Expected {} got {:?}", .0, .1)]
-    Expected(String, Token, Position),
+    Expected(String, Token, TokenRange),
     #[error("Source file contains no tokens")]
     FileEmpty,
     /// Only use this error in situations where the error never ends up for the end-user!
@@ -222,7 +250,7 @@ pub enum Error {
 }
 
 impl Error {
-    pub fn get_position(&self) -> Option<&Position> {
+    pub fn get_range(&self) -> Option<&TokenRange> {
         match self {
             Error::Expected(_, _, pos) => Some(pos),
             Error::FileEmpty => None,

reid_src/array_structs.reid

@@ -5,7 +5,7 @@ struct Test {
     second: [u32; 4]
 }
 
-fn main() -> u32 {
+fn main() -> v u32 {
     let mut value = [Test {
         field: 5,
         second: [6, 3, "hello", 8],