diff --git a/reid/src/lexer.rs b/reid/src/lexer.rs index 7fd0517..27283bc 100644 --- a/reid/src/lexer.rs +++ b/reid/src/lexer.rs @@ -4,10 +4,12 @@ static DECIMAL_NUMERICS: &[char] = &['0', '1', '2', '3', '4', '5', '6', '7', '8' #[derive(Debug, Eq, PartialEq, Clone)] pub enum Token { - // Values + /// Values Identifier(String), /// Number with at most one decimal point DecimalValue(u64), + /// Some string literal that was surrounded by "quotes". + StringLit(String), // Keywords /// `let` @@ -162,6 +164,25 @@ pub fn tokenize>(to_tokenize: T) -> Result, Error } continue; } + '\"' => { + let mut value = String::new(); + let mut ignore_next = false; + while cursor.first().is_some() && (cursor.first() != Some('\"') || ignore_next) { + if cursor.first() == Some('\\') && !ignore_next { + cursor.next(); // Consume backslash anjd always add next character + ignore_next = true; + } else { + ignore_next = false; + value += &cursor.next().unwrap().to_string(); + } + } + if cursor.first() == Some('\"') { + cursor.next(); + } else { + return Err(Error::MissingQuotation(position)); + } + Token::StringLit(value) + } // "words" c if c.is_alphabetic() => { let mut value = character.to_string(); @@ -244,4 +265,6 @@ pub fn tokenize>(to_tokenize: T) -> Result, Error pub enum Error { #[error("Invalid token '{}' at Ln {}, Col {}", .0, (.1).1, (.1).0)] InvalidToken(char, Position), + #[error("String literal that starts at Ln {}, Col {} is never finished!", (.0).1, (.0).0)] + MissingQuotation(Position), } diff --git a/reid_src/strings.reid b/reid_src/strings.reid index 18084aa..bc56375 100644 --- a/reid_src/strings.reid +++ b/reid_src/strings.reid @@ -1,7 +1,7 @@ fn main() -> u16 { - let hello = "beep boop"; + let hello = "beep boop grw"; return 5; }