From 8176dc98a35228a6f7b4866b8c5f703b0eded914 Mon Sep 17 00:00:00 2001 From: sofia Date: Wed, 17 Jul 2024 20:08:24 +0300 Subject: [PATCH] Add TokenStream --- src/ast.rs | 94 +++++++++++++++++++++++++++++++++------------ src/lib.rs | 17 ++++---- src/token_stream.rs | 19 +++++++++ 3 files changed, 96 insertions(+), 34 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index e74f06c..774b3f4 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -1,6 +1,6 @@ use crate::{ lexer::Token, - token_stream::{Error, TokenStream}, + token_stream::{Error, TokenRange, TokenStream}, }; pub trait Parse @@ -11,20 +11,25 @@ where } #[derive(Debug, Clone)] -pub enum Type { +pub struct Type(pub TypeKind, pub TokenRange); + +#[derive(Debug, Clone)] +pub enum TypeKind { I32, } impl Parse for Type { fn parse(mut stream: TokenStream) -> Result { - if let Some(Token::Identifier(ident)) = stream.next() { + let kind = if let Some(Token::Identifier(ident)) = stream.next() { Ok(match &*ident { - "i32" => Type::I32, + "i32" => TypeKind::I32, _ => panic!("asd"), }) } else { Err(stream.expected_err("type identifier")?) - } + }?; + + Ok(Type(kind, stream.get_range().unwrap())) } } @@ -34,7 +39,10 @@ pub enum Literal { } #[derive(Debug, Clone)] -pub enum Expression { +pub struct Expression(pub ExpressionKind, pub TokenRange); + +#[derive(Debug, Clone)] +pub enum ExpressionKind { VariableName(String), Literal(Literal), Binop(BinaryOperator, Box, Box), @@ -51,16 +59,32 @@ impl Parse for Expression { } fn parse_primary_expression(stream: &mut TokenStream) -> Result { + use ExpressionKind as Kind; + if let Ok(exp) = stream.parse() { - Ok(Expression::FunctionCall(Box::new(exp))) + Ok(Expression( + Kind::FunctionCall(Box::new(exp)), + stream.get_range().unwrap(), + )) } else if let Ok(block) = stream.parse() { - Ok(Expression::BlockExpr(Box::new(block))) + Ok(Expression( + Kind::BlockExpr(Box::new(block)), + stream.get_range().unwrap(), + )) } else if let Ok(ifexpr) = stream.parse() { - Ok(Expression::IfExpr(Box::new(ifexpr))) + Ok(Expression( + Kind::IfExpr(Box::new(ifexpr)), + stream.get_range().unwrap(), + )) } else if let Some(token) = stream.next() { Ok(match &token { - Token::Identifier(v) => Expression::VariableName(v.clone()), - Token::DecimalValue(v) => Expression::Literal(Literal::I32(v.parse().unwrap())), + Token::Identifier(v) => { + Expression(Kind::VariableName(v.clone()), stream.get_range().unwrap()) + } + Token::DecimalValue(v) => Expression( + Kind::Literal(Literal::I32(v.parse().unwrap())), + stream.get_range().unwrap(), + ), Token::ParenOpen => { let exp = stream.parse()?; stream.expect(Token::ParenClose)?; @@ -109,7 +133,10 @@ fn parse_binop_rhs( } } - lhs = Expression::Binop(op, Box::new(lhs), Box::new(rhs)); + lhs = Expression( + ExpressionKind::Binop(op, Box::new(lhs), Box::new(rhs)), + stream.get_range().unwrap(), + ); } Ok(lhs) @@ -156,7 +183,7 @@ impl BinaryOperator { } #[derive(Debug, Clone)] -pub struct FunctionCallExpression(pub String, pub Vec); +pub struct FunctionCallExpression(pub String, pub Vec, pub TokenRange); impl Parse for FunctionCallExpression { fn parse(mut stream: TokenStream) -> Result { @@ -175,7 +202,11 @@ impl Parse for FunctionCallExpression { stream.expect(Token::ParenClose)?; - Ok(FunctionCallExpression(name, args)) + Ok(FunctionCallExpression( + name, + args, + stream.get_range().unwrap(), + )) } else { Err(stream.expected_err("identifier")?) } @@ -183,17 +214,21 @@ impl Parse for FunctionCallExpression { } #[derive(Debug, Clone)] -pub struct IfExpression(Expression, pub Block); +pub struct IfExpression(Expression, pub Block, pub TokenRange); impl Parse for IfExpression { fn parse(mut stream: TokenStream) -> Result { stream.expect(Token::If)?; - Ok(IfExpression(stream.parse()?, stream.parse()?)) + Ok(IfExpression( + stream.parse()?, + stream.parse()?, + stream.get_range().unwrap(), + )) } } #[derive(Debug, Clone)] -pub struct LetStatement(pub String, pub Expression); +pub struct LetStatement(pub String, pub Expression, pub TokenRange); impl Parse for LetStatement { fn parse(mut stream: TokenStream) -> Result { @@ -204,7 +239,11 @@ impl Parse for LetStatement { let expression = stream.parse()?; stream.expect(Token::Semi)?; - Ok(LetStatement(variable, expression)) + Ok(LetStatement( + variable, + expression, + stream.get_range().unwrap(), + )) } else { Err(stream.expected_err("identifier")?) } @@ -212,7 +251,7 @@ impl Parse for LetStatement { } #[derive(Debug, Clone)] -pub struct ImportStatement(Vec); +pub struct ImportStatement(Vec, pub TokenRange); impl Parse for ImportStatement { fn parse(mut stream: TokenStream) -> Result { @@ -235,17 +274,21 @@ impl Parse for ImportStatement { stream.expect(Token::Semi)?; - Ok(ImportStatement(import_list)) + Ok(ImportStatement(import_list, stream.get_range().unwrap())) } } #[derive(Debug)] -pub struct FunctionDefinition(pub FunctionSignature, pub Block); +pub struct FunctionDefinition(pub FunctionSignature, pub Block, pub TokenRange); impl Parse for FunctionDefinition { fn parse(mut stream: TokenStream) -> Result { stream.expect(Token::FnKeyword)?; - Ok(FunctionDefinition(stream.parse()?, stream.parse()?)) + Ok(FunctionDefinition( + stream.parse()?, + stream.parse()?, + stream.get_range().unwrap(), + )) } } @@ -254,6 +297,7 @@ pub struct FunctionSignature { pub name: String, pub args: Vec<(String, Type)>, pub return_type: Option, + pub range: TokenRange, } impl Parse for FunctionSignature { @@ -279,6 +323,7 @@ impl Parse for FunctionSignature { name, args, return_type, + range: stream.get_range().unwrap(), }) } else { Err(stream.expected_err("identifier")?)? @@ -296,6 +341,7 @@ pub enum ReturnType { pub struct Block( pub Vec, pub Option<(ReturnType, Expression)>, + pub TokenRange, ); impl Parse for Block { @@ -308,7 +354,7 @@ impl Parse for Block { if let Some((r_type, e)) = return_stmt.take() { // Special list of expressions that are simply not warned about, // if semicolon is missing. - if !matches!(&e, &Expression::IfExpr(_)) { + if !matches!(e, Expression(ExpressionKind::IfExpr(_), _)) { dbg!(r_type, &e); println!("Oh no, does this statement lack ;"); } @@ -332,7 +378,7 @@ impl Parse for Block { statements.push(statement); } stream.expect(Token::BraceClose)?; - Ok(Block(statements, return_stmt)) + Ok(Block(statements, return_stmt, stream.get_range().unwrap())) } } diff --git a/src/lib.rs b/src/lib.rs index afbc452..79147d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,7 @@ -use crate::{ - ast::TopLevelStatement, codegen::codegen_from_statements, lexer::Token, - token_stream::TokenStream, -}; +use crate::{ast::TopLevelStatement, lexer::Token, token_stream::TokenStream}; mod ast; -mod codegen; +// mod codegen; mod lexer; mod llvm_ir; mod token_stream; @@ -21,8 +18,8 @@ pub enum ReidError { LexerError(#[from] lexer::Error), #[error(transparent)] ParserError(#[from] token_stream::Error), - #[error(transparent)] - CodegenError(#[from] codegen::Error), + // #[error(transparent)] + // CodegenError(#[from] codegen::Error), } pub fn compile(source: &str) -> Result { @@ -40,7 +37,7 @@ pub fn compile(source: &str) -> Result { statements.push(statement); } - let mut module = codegen_from_statements(statements)?; - let text = module.print_to_string().unwrap(); - Ok(text.to_owned()) + // let mut module = codegen_from_statements(statements)?; + // let text = module.print_to_string().unwrap(); + Ok("text".to_owned()) } diff --git a/src/token_stream.rs b/src/token_stream.rs index 0cd31fc..12daa54 100644 --- a/src/token_stream.rs +++ b/src/token_stream.rs @@ -139,6 +139,13 @@ impl<'a, 'b> TokenStream<'a, 'b> { Ok(self.tokens[token_idx].position) } } + + pub fn get_range(&self) -> Option { + self.ref_position.as_ref().map(|ref_pos| TokenRange { + start: **ref_pos, + end: self.position, + }) + } } impl Drop for TokenStream<'_, '_> { @@ -149,6 +156,18 @@ impl Drop for TokenStream<'_, '_> { } } +#[derive(Clone)] +pub struct TokenRange { + pub start: usize, + pub end: usize, +} + +impl std::fmt::Debug for TokenRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Tokens[{} - {}]", self.start, self.end) + } +} + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Expected {} at Ln {}, Col {}, got {:?}", .0, (.2).1, (.2).0, .1)]