Add TokenStream

src/main.cpp

@@ -50,6 +50,9 @@ int main() {
         std::cout << token << std::endl;
     }
 
+    auto stream = token::TokenStream{ tokens };
+    stream.expect(token::Type::Eof);
+
     // LLVM Hello World
     // llvm_hello_world();
 

src/tokens.cpp

@@ -14,7 +14,8 @@ static bool iswhitespace(char& character) {
 }
 
 namespace token {
-    std::string Token::name() {
+
+    std::string type_name(Type& type) {
         switch (type) {
         case token::Type::Ident:
             return "Ident";
@@ -29,16 +30,51 @@ namespace token {
         case token::Type::Whitespace:
             return "Whitespace";
 
+        case token::Type::Eof:
+            return "EOF";
+
         default:
             return "Unknown";
         }
     }
 
+    std::string Token::name() {
+        return type_name(this->type);
+    }
+
     std::ostream& operator<<(std::ostream& stream, Token& token) {
         stream << token.name() << "(" << token.content << ")";
         return stream;
     }
 
+    TokenStream::TokenStream(std::vector<Token>& tokens)
+        : m_tokens{ tokens }, m_position{ 0 } {
+    };
+
+    Token TokenStream::peek(int length) {
+        int new_pos = m_position + length;
+        if (new_pos < 0 || new_pos > m_tokens.size()) {
+            return Token{ Type::Eof, {} };
+        }
+        return m_tokens[new_pos];
+    }
+
+    Token TokenStream::peek() {
+        return this->peek(0);
+    }
+
+    Token TokenStream::next() {
+        return this->peek(++m_position);
+    }
+
+    Token TokenStream::expect(Type type) {
+        auto next = this->next();
+        if (next.type == type) {
+            return next;
+        }
+        throw std::runtime_error("Expected " + type_name(type) + ", got " + type_name(next.type));
+    }
+
     std::vector<token::Token> tokenize(std::string_view text) {
         std::vector<token::Token> tokens{};
 
@@ -80,6 +116,8 @@ namespace token {
             }
         }
 
+        tokens.push_back(token::Token{ token::Type::Eof, {} });
+
         return tokens;
     }
 }

src/tokens.h

@@ -14,8 +14,12 @@ namespace token {
         ReturnKeyword,
 
         Whitespace,
+
+        Eof,
     };
 
+    std::string type_name(Type& type);
+
     struct Token {
         Type type;
         std::string content;
@@ -23,6 +27,18 @@ namespace token {
         std::string name();
     };
 
+    class TokenStream {
+    private:
+        std::vector<Token>& m_tokens;
+        int m_position;
+    public:
+        TokenStream(std::vector<Token>& tokens);
+        Token peek(int length);
+        Token peek();
+        Token next();
+        Token expect(Type type);
+    };
+
     std::ostream& operator<<(std::ostream& stream, Token& token);
 
     std::vector<Token> tokenize(std::string_view text);