//! Contains relevant code for parsing tokens received from //! Lexing/Tokenizing-stage. use crate::{ ast::parse::Parse, lexer::{FullToken, Token}, }; /// Utility struct that is able to parse [`FullToken`]s while being /// failure-resistance in that it can backtrack easily, and is able to keep /// track of parsed Token-ranges easily. pub struct TokenStream<'a, 'b> { ref_position: Option<&'b mut usize>, tokens: &'a [FullToken], pub position: usize, } impl<'a, 'b> TokenStream<'a, 'b> { pub fn from(tokens: &'a [FullToken]) -> Self { TokenStream { ref_position: None, tokens, position: 0, } } /// Returns expected-error for the next token in-line. Useful in conjunction /// with [`TokenStream::peek`] pub fn expected_err>(&mut self, expected: T) -> Result { let next_token = self.previous().unwrap_or(Token::Eof); Ok(Error::Expected( expected.into(), next_token, TokenRange { start: self.position - 1, end: self.position - 1, }, )) } /// Returns expected-error for the previous token that was already consumed. /// Useful in conjunction with [`TokenStream::next`] pub fn expecting_err>(&mut self, expected: T) -> Result { let next_token = self.peek().unwrap_or(Token::Eof); Ok(Error::Expected( expected.into(), next_token, TokenRange { start: self.position, end: self.position, }, )) } pub fn expect(&mut self, token: Token) -> Result<(), Error> { if let Some(peeked) = self.peek() { if token == peeked { self.position += 1; Ok(()) } else { Err(self.expecting_err(token)?) } } else { Err(self.expecting_err(token)?) } } pub fn next(&mut self) -> Option { let value = if self.tokens.len() < self.position { None } else { Some(self.tokens[self.position].token.clone()) }; self.position += 1; value } pub fn previous(&mut self) -> Option { if (self.position as i32 - 1) < 0 { None } else { Some(self.tokens[self.position - 1].token.clone()) } } pub fn peek(&mut self) -> Option { if self.tokens.len() < self.position { None } else { Some(self.tokens[self.position].token.clone()) } } pub fn peek2(&mut self) -> Option { if self.tokens.len() < (self.position + 1) { None } else { Some(self.tokens[self.position + 1].token.clone()) } } /// Parse the next value of trait Parse. If the parse succeeded, the related /// tokens are consumed, otherwise token stream does not advance. /// /// Parsetime-error is returned on failure. pub fn parse(&mut self) -> Result { let (res, pos) = self.parse_meta()?; self.position = pos; Ok(res) } /// Parse the next item with Parse-trait (Same as [`TokenStream::parse`]) /// without consuming the related tokens, essentially only peeking. pub fn parse_peek(&mut self) -> Result { self.parse_meta().map(|(res, _)| res) } /// Parse the next item with Parse-trait, also mapping it with the given /// function. The token-stream is only consumed, if the inner function /// retuns an Ok. #[allow(dead_code)] pub fn parse_map(&mut self, inner: F) -> Result where F: Fn(T) -> Result, { let (res, pos) = self.parse_meta::()?; match inner(res) { Ok(mapped) => { self.position = pos; Ok(mapped) } Err(e) => Err(e), } } /// Parses the item with Parse if the condition specified by the /// lambda-function is passed. Errors returned from this should not be /// passed to the end-user. pub fn parse_if(&mut self, inner: F) -> Result where F: Fn(&T) -> bool, { let (res, pos) = self.parse_meta::()?; if inner(&res) { self.position = pos; Ok(res) } else { Err(Error::IfFailed) } } /// Parse the next item with Parse-trait. If successful, returning the /// parsed item and the new position of the TokenStream. Failing, returning /// parse-error. /// /// Used for [`TokenStream::parse`] and [`TokenStream::parse_peek`] fn parse_meta(&mut self) -> Result<(T, usize), Error> { let mut ref_pos = self.position; let position = self.position; let clone = TokenStream { ref_position: Some(&mut ref_pos), tokens: self.tokens, position, }; match T::parse(clone) { Ok(res) => { let new_pos = ref_pos.max(self.position); Ok((res, new_pos)) } Err(e) => Err(e), } } pub fn get_range(&self) -> Option { self.ref_position.as_ref().map(|ref_pos| TokenRange { start: **ref_pos, end: self.position, }) } /// Gets range from the previous position to the current. Useful when using /// with [`TokenStream::next`] pub fn get_range_prev(&self) -> Option { self.ref_position.as_ref().map(|ref_pos| TokenRange { start: **ref_pos, end: self.position - 1, }) } } impl Drop for TokenStream<'_, '_> { fn drop(&mut self) { if let Some(ref_pos) = &mut self.ref_position { **ref_pos = self.position; } } } /// Index-range that can be used with the original array of [`FullToken`]s to /// retrieve the precise location of a failure. #[derive(Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct TokenRange { pub start: usize, pub end: usize, } impl std::fmt::Debug for TokenRange { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Tokens[{} - {}]", self.start, self.end) } } impl std::ops::Add for TokenRange { type Output = TokenRange; fn add(self, rhs: Self) -> Self::Output { TokenRange { start: self.start.min(rhs.start), end: self.end.min(rhs.end), } } } impl std::iter::Sum for TokenRange { fn sum>(mut iter: I) -> Self { let mut start = iter.next().unwrap_or(Default::default()); for item in iter { start = start + item; } start } } #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum Error { #[error("Expected {} got \"{:?}\"", .0, .1)] Expected(String, Token, TokenRange), #[error("Source file contains no tokens")] FileEmpty, /// Only use this error in situations where the error never ends up for the end-user! #[error("Undefined error, should only be used in situations where the error is not emitted!")] Undefined, /// Condition failed for the parse-if #[error("Condition failed for parse-if. Should never be returned to end-user.")] IfFailed, } impl Error { pub fn get_range(&self) -> Option<&TokenRange> { match self { Error::Expected(_, _, pos) => Some(pos), Error::FileEmpty => None, Error::Undefined => None, Error::IfFailed => None, } } }