48 lines
867 B
C++
48 lines
867 B
C++
#ifndef TOKENS_H
|
|
#define TOKENS_H
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
#include <iostream>
|
|
|
|
namespace token {
|
|
enum class Type {
|
|
Ident,
|
|
Symbol,
|
|
LiteralInt,
|
|
|
|
ReturnKeyword,
|
|
|
|
Whitespace,
|
|
|
|
Eof,
|
|
};
|
|
|
|
std::string type_name(Type& type);
|
|
|
|
struct Token {
|
|
Type type;
|
|
std::string content;
|
|
|
|
std::string formatted();
|
|
};
|
|
|
|
class TokenStream {
|
|
private:
|
|
std::vector<Token>& m_tokens;
|
|
public:
|
|
int m_position;
|
|
TokenStream(std::vector<Token>& tokens);
|
|
Token peek(int length);
|
|
Token peek();
|
|
Token next();
|
|
Token expect(Type type);
|
|
Token expect(Type type, std::string_view content);
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& stream, Token& token);
|
|
|
|
std::vector<Token> tokenize(std::string_view text);
|
|
}
|
|
|
|
#endif |