68 lines
1.3 KiB
C++
68 lines
1.3 KiB
C++
#ifndef TOKENS_H
|
|
#define TOKENS_H
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
#include <iostream>
|
|
#include <cstdint>
|
|
|
|
namespace token {
|
|
enum class Type {
|
|
Ident,
|
|
Symbol,
|
|
LiteralInt,
|
|
LiteralStr,
|
|
|
|
ReturnKeyword,
|
|
IfKeyword,
|
|
ElseKeyword,
|
|
ForKeyword,
|
|
|
|
Whitespace,
|
|
|
|
Eof,
|
|
};
|
|
|
|
std::string type_name(Type& type);
|
|
|
|
struct Position {
|
|
uint32_t line;
|
|
uint32_t col;
|
|
};
|
|
|
|
struct Metadata {
|
|
Position start;
|
|
Position end;
|
|
std::string filename;
|
|
};
|
|
|
|
Metadata operator+(Metadata meta, Metadata other);
|
|
|
|
struct Token {
|
|
Type type;
|
|
std::string content;
|
|
Metadata metadata;
|
|
|
|
std::string formatted();
|
|
};
|
|
|
|
class TokenStream {
|
|
private:
|
|
std::vector<Token>& m_tokens;
|
|
public:
|
|
int m_position;
|
|
TokenStream(std::vector<Token>& tokens);
|
|
Token peek(int length);
|
|
Token peek();
|
|
Token next();
|
|
Token expect(Type type);
|
|
Token expect(Type type, std::string_view content);
|
|
Metadata metadata();
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& stream, Token& token);
|
|
|
|
std::vector<Token> tokenize(std::string_view text, std::string filename);
|
|
}
|
|
|
|
#endif |