From 0f782dcb964ed453201e34e6a5ab7e9d22e1428c Mon Sep 17 00:00:00 2001 From: sofia Date: Thu, 17 Jul 2025 20:16:37 +0300 Subject: [PATCH] Fix parse error returning the token after the error --- reid/src/ast/parse.rs | 13 ++++---- reid/src/error_raporting.rs | 60 ++++++++++++++----------------------- reid/src/token_stream.rs | 44 ++++++++++++++++++++++----- reid_src/array_structs.reid | 2 +- 4 files changed, 67 insertions(+), 52 deletions(-) diff --git a/reid/src/ast/parse.rs b/reid/src/ast/parse.rs index 0a1b823..1004dea 100644 --- a/reid/src/ast/parse.rs +++ b/reid/src/ast/parse.rs @@ -128,7 +128,7 @@ impl Parse for PrimaryExpression { stream.expect(Token::BracketClose)?; Expression(Kind::Array(expressions), stream.get_range().unwrap()) } - _ => Err(stream.expected_err("identifier, constant, parentheses or brackets")?)?, + _ => Err(stream.expected_err("expression")?)?, } } else { Err(stream.expected_err("expression")?)? @@ -391,7 +391,8 @@ impl Parse for Block { // Special list of expressions that are simply not warned about, // if semicolon is missing. if !matches!(e, Expression(ExpressionKind::IfExpr(_), _)) { - println!("Oh no, does this statement lack ;"); + // In theory could ignore the missing semicolon.. + return Err(stream.expected_err("expected semicolon to complete statement")?); } statements.push(BlockLevelStatement::Expression(e)); @@ -445,7 +446,7 @@ impl Parse for NamedFieldList { stream.next(); } // Consume comma Some(Token::BraceClose) => break, - Some(_) | None => Err(stream.expected_err("another field or closing brace")?)?, + Some(_) | None => Err(stream.expecting_err("another field or closing brace")?)?, } } Ok(NamedFieldList(fields)) @@ -478,7 +479,7 @@ impl Parse for ValueIndex { match stream.peek() { Some(Token::BracketOpen) => Ok(ValueIndex::Array(stream.parse()?)), Some(Token::Dot) => Ok(ValueIndex::Struct(stream.parse()?)), - _ => Err(stream.expected_err("value or struct index")?), + _ => Err(stream.expecting_err("value or struct index")?), } } } @@ -534,7 +535,7 @@ impl Parse for BlockLevelStatement { Stmt::Return(ReturnType::Soft, e) } } else { - Err(stream.expected_err("expression")?)? + Err(stream.expecting_err("expression")?)? } } } @@ -605,7 +606,7 @@ impl Parse for TopLevelStatement { range, }) } - _ => Err(stream.expected_err("import or fn")?)?, + _ => Err(stream.expecting_err("import or fn")?)?, }) } } diff --git a/reid/src/error_raporting.rs b/reid/src/error_raporting.rs index 7556d34..2b915a8 100644 --- a/reid/src/error_raporting.rs +++ b/reid/src/error_raporting.rs @@ -156,8 +156,8 @@ impl ReidError { let pass_err = pass::Error { metadata: Metadata { source_module_id: module, - range: Default::default(), - position: error.get_position().copied(), + range: *error.get_range().unwrap_or(&Default::default()), + position: None, }, kind: error, }; @@ -185,24 +185,16 @@ impl std::fmt::Display for ReidError { for error in sorted_errors { let meta = error.get_meta(); let module = self.map.get_module(&meta.source_module_id).unwrap(); - let (position_fmt, line_fmt) = if let Some(tokens) = &module.tokens { + let position = if let Some(tokens) = &module.tokens { let range_tokens = meta.range.into_tokens(&tokens); - let highlight_position = get_position(&range_tokens).unwrap(); - let full_lines = get_full_lines(&tokens, highlight_position); - let lines_position = get_position(&full_lines).unwrap(); - ( - fmt_positions(highlight_position), - Some(fmt_lines( - module.source.as_ref().unwrap(), - lines_position, - highlight_position, - 8, - )?), - ) + + dbg!(&error); + dbg!(&meta.range, &tokens[meta.range.start]); + get_position(&range_tokens).or(meta.position.map(|p| (p, p))) } else if let Some(position) = meta.position { - (fmt_positions((position, position)), None) + Some((position, position)) } else { - ("unknown".to_owned(), None) + None }; if curr_module != Some(meta.source_module_id) { @@ -223,9 +215,16 @@ impl std::fmt::Display for ReidError { writeln!(f)?; write!(f, " Error: ")?; writeln!(f, "{}", color_err(format!("{}", error))?)?; - write!(f, "{:>20}{}", color_warn("At: ")?, position_fmt)?; - if let Some(line_fmt) = line_fmt { - writeln!(f, "{}", line_fmt)?; + write!( + f, + "{:>20}{}", + color_warn("At: ")?, + position + .map(|p| fmt_positions(p)) + .unwrap_or(String::from("{unknown}")), + )?; + if let (Some(position), Some(source)) = (position, &module.source) { + writeln!(f, "{}", fmt_lines(source, position, 6)?)?; } } Ok(()) @@ -238,7 +237,7 @@ impl TokenRange { .iter() .skip(self.start) .by_ref() - .take(self.end - self.start) + .take(self.end + 1 - self.start) .collect::>() } } @@ -252,29 +251,16 @@ fn get_position(tokens: &Vec<&FullToken>) -> Option<(Position, Position)> { } } -fn get_full_lines<'v>( - tokens: &'v Vec, - (start, end): (Position, Position), -) -> Vec<&'v FullToken> { - let (first_token_pos, _) = tokens - .iter() - .enumerate() - .find(|(_, token)| token.position.1 == start.1) - .unwrap(); - tokens - .iter() - .skip(first_token_pos) - .by_ref() - .take_while(|token| token.position.1 <= end.1) - .collect::>() +fn into_full_lines<'v>((start, end): (Position, Position)) -> (Position, Position) { + (Position(0, start.1), Position(u32::MAX, end.1)) } fn fmt_lines( source: &String, - (line_start, line_end): (Position, Position), (highlight_start, highlight_end): (Position, Position), ident: usize, ) -> Result { + let (line_start, line_end) = into_full_lines((highlight_start, highlight_end)); let mut cursor = Cursor { position: Position(0, 1), char_stream: source.chars(), diff --git a/reid/src/token_stream.rs b/reid/src/token_stream.rs index 56c8802..57158dc 100644 --- a/reid/src/token_stream.rs +++ b/reid/src/token_stream.rs @@ -24,11 +24,31 @@ impl<'a, 'b> TokenStream<'a, 'b> { } } + /// Returns expected-error for the next token in-line. Useful in conjunction + /// with [`TokenStream::peek`] pub fn expected_err>(&mut self, expected: T) -> Result { + let next_token = self.previous().unwrap_or(Token::Eof); Ok(Error::Expected( expected.into(), - self.peek().unwrap_or(Token::Eof), - self.get_next_position()?, + next_token, + TokenRange { + start: self.position - 1, + end: self.position - 1, + }, + )) + } + + /// Returns expected-error for the previous token that was already consumed. + /// Useful in conjunction with [`TokenStream::next`] + pub fn expecting_err>(&mut self, expected: T) -> Result { + let next_token = self.peek().unwrap_or(Token::Eof); + Ok(Error::Expected( + expected.into(), + next_token, + TokenRange { + start: self.position, + end: self.position, + }, )) } @@ -38,10 +58,10 @@ impl<'a, 'b> TokenStream<'a, 'b> { self.position += 1; Ok(()) } else { - Err(self.expected_err(token)?) + Err(self.expecting_err(token)?) } } else { - Err(self.expected_err(token)?) + Err(self.expecting_err(token)?) } } @@ -55,6 +75,14 @@ impl<'a, 'b> TokenStream<'a, 'b> { value } + pub fn previous(&mut self) -> Option { + if (self.position as i32 - 1) < 0 { + None + } else { + Some(self.tokens[self.position - 1].token.clone()) + } + } + pub fn peek(&mut self) -> Option { if self.tokens.len() < self.position { None @@ -147,11 +175,11 @@ impl<'a, 'b> TokenStream<'a, 'b> { } } - fn get_next_position(&self) -> Result { + fn get_position(&self, offset: usize) -> Result { if self.tokens.is_empty() { Err(Error::FileEmpty) } else { - let token_idx = self.position.min(self.tokens.len() - 1); + let token_idx = (self.position - 1).min(self.tokens.len() - 1); Ok(self.tokens[token_idx].position) } } @@ -210,7 +238,7 @@ impl std::iter::Sum for TokenRange { #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum Error { #[error("Expected {} got {:?}", .0, .1)] - Expected(String, Token, Position), + Expected(String, Token, TokenRange), #[error("Source file contains no tokens")] FileEmpty, /// Only use this error in situations where the error never ends up for the end-user! @@ -222,7 +250,7 @@ pub enum Error { } impl Error { - pub fn get_position(&self) -> Option<&Position> { + pub fn get_range(&self) -> Option<&TokenRange> { match self { Error::Expected(_, _, pos) => Some(pos), Error::FileEmpty => None, diff --git a/reid_src/array_structs.reid b/reid_src/array_structs.reid index ec2d412..a67d139 100644 --- a/reid_src/array_structs.reid +++ b/reid_src/array_structs.reid @@ -5,7 +5,7 @@ struct Test { second: [u32; 4] } -fn main() -> u32 { +fn main() -> v u32 { let mut value = [Test { field: 5, second: [6, 3, "hello", 8],