diff --git a/src/tokens.cpp b/src/tokens.cpp index 3b64209..b9535ff 100644 --- a/src/tokens.cpp +++ b/src/tokens.cpp @@ -99,6 +99,9 @@ namespace token { Token TokenStream::next() { token::Token got = this->peek(0); m_position++; + while (m_position < static_cast(m_tokens.size()) && this->peek().type == Type::Whitespace) { + m_position++; + } return got; } @@ -187,8 +190,7 @@ namespace token { if ((i + 1) >= text_length) break; c = text[++i]; } while (iswhitespace(c)); - // tokens.push_back(token::Token{ token::Type::Whitespace, content }); - // Just skip whitespace tokens + tokens.push_back(token::Token{ token::Type::Whitespace, content, meta + content.size() }); } else { tokens.push_back(token::Token{ token::Type::Symbol, std::string{c}, meta });