Parse comments correctly

src/tokens.cpp

@@ -66,6 +66,8 @@ namespace token {
 
         case token::Type::Whitespace:
             return "Whitespace";
+        case token::Type::Comment:
+            return "Comment";
 
         case token::Type::Eof:
             return "EOF";
@@ -130,7 +132,8 @@ namespace token {
     Token TokenStream::next() {
         token::Token got = this->peek(0);
         m_position++;
-        while (m_position < static_cast<int>(m_tokens.size()) && this->peek().type == Type::Whitespace) {
+        while (m_position < static_cast<int>(m_tokens.size())
+            && (this->peek().type == Type::Whitespace || this->peek().type == Type::Comment)) {
             m_position++;
         }
         return got;
@@ -179,6 +182,28 @@ namespace token {
                 } while (std::isdigit(c));
                 tokens.push_back(token::Token{ token::Type::LiteralInt, content, meta + content.size() });
             }
+            // Single-line comments
+            else if (c == '/' && text_length > (i + 2) && text[i + 1] == '/') {
+                std::string content{  };
+                i += 2;
+                while (text_length > i && text[i] != '\n') {
+                    content += text[i++];
+                }
+                tokens.push_back(token::Token{ token::Type::Comment, content, meta + content.size() });
+            }
+            // Multi-line comments
+            else if (c == '/' && text_length > (i + 2) && text[i + 1] == '*') {
+                std::string content{  };
+                i += 2;
+                while (text_length > i) {
+                    if (text[i] == '*' && text_length > (i + 1) && text[i + 1] == '/') {
+                        i += 2;
+                        break;
+                    }
+                    content += text[i++];
+                }
+                tokens.push_back(token::Token{ token::Type::Comment, content, meta + content.size() });
+            }
             else if (c == '\"') {
                 std::string content{};
                 c = text[++i]; // Skip initial "

src/tokens.h

@@ -20,6 +20,7 @@ namespace token {
         WhileKeyword,
 
         Whitespace,
+        Comment,
 
         Eof,
     };

test.c

@@ -26,9 +26,11 @@ void update_ptr(char* ptr) {
 }
 
 int main() {
+    // Test fibonacci sequence
     char text[30] = "10th fibonacci number is %d!\n";
     printf(text, fibonacci(10));
 
+    // Test arrays
     char somelist[5] = { 1, 2, 3, 4, 5 };
     char* somelist_ptr = somelist;
 
@@ -37,23 +39,28 @@ int main() {
     printf("first element: %d!\n", somelist[0]);
     printf("first element via ptr: %d!\n", somelist_ptr[0]);
 
+    // Test structs and nested fields
     struct Otus otus = { 5, { 7, 3 } };
     update(otus);
 
     printf("first field: %d!\n", otus.field);
     printf("second field's second element: %d!\n", otus.second[1]);
 
+    // Test pointers
     char hello = 10;
     update_ptr(&hello);
     printf("hello: %d!\n", hello);
 
+    /** Test nested arrays */
     char twod_array[5][5];
     twod_array[0][0] = 50;
     printf("2d array: %d!\n", twod_array[0][0]);
 
+    // Test for-loops
     for (int counter = 0; counter < 10; counter++)
         printf("for-counter: %d\n", counter);
 
+    // Test while-loops
     int counter = 0;
     while (counter < 10)
         printf("while-counter: %d\n", counter++);