Improve syntax highlighting

README.md

@@ -71,8 +71,8 @@ Currently missing big features (TODOs) are:
 Big features that I want later but are not necessary:
 - ~~User-defined binary operations~~ (DONE)
 - ~~Asymmetric binary operations (e.g. string + u32)~~ (DONE)
-- Error handling
-- Lexing & parsing of whitespace and comments as well
+- ~~Error handling~~ (Not Doing It)
+- ~~Lexing & parsing of whitespace and comments as well~~ (DONE)
 - LSP implementation
 
 Smaller features:

examples/a.reid

@@ -1,10 +0,0 @@
-struct Foo {
-    a: i32,
-    b: i32,
-}
-
-fn main() -> i32 {
-    // ISSUE: The debugger says b is 1
-    let foos = [Foo { a: 1, b: 2}];
-    return 0;
-}

examples/fibonacci.reid

@@ -1,12 +1,13 @@
-// Main
-fn main() -> bool {
-    return 144 == fibonacci(0xc);
-}
-
-// Fibonacci
-fn fibonacci(value: u16) -> u16 {
-    if value <= 2 {
-        return 1;
-    }
-    fibonacci(value - 1) + fibonacci(value - 2)
-}
+a
+(
+    b
+)
+x
+(
+    (
+        c
+        xyz
+    )
+)
+(
+a

reid-lsp/package.json

@@ -43,11 +43,18 @@
           "description": "Traces the communication between VS Code and the language server."
         }
       }
-    }
+    },
+    "grammars": [
+      {
+        "language": "reid",
+        "scopeName": "source.reid",
+        "path": "./syntaxes/grammar.json"
+      }
+    ]
   },
   "scripts": {
     "vscode:prepublish": "pnpm run package",
-    "compile": "webpack",
+    "compile": "npx js-yaml syntaxes/grammar.yaml > syntaxes/grammar.json && webpack",
     "watch": "webpack --watch",
     "package": "webpack --mode production --devtool hidden-source-map",
     "compile-tests": "tsc -p . --outDir out",
@@ -65,6 +72,7 @@
     "@vscode/test-cli": "^0.0.11",
     "@vscode/test-electron": "^2.5.2",
     "eslint": "^9.25.1",
+    "js-yaml": "^4.1.0",
     "ts-loader": "^9.5.2",
     "typescript": "^5.8.3",
     "webpack": "^5.99.7",

reid-lsp/src/main.rs

@@ -12,9 +12,9 @@ use reid::{compile_module, parse_module, perform_all_passes};
 use tower_lsp::lsp_types::{
     self, CompletionItem, CompletionOptions, CompletionParams, CompletionResponse, Diagnostic, DiagnosticSeverity,
     DidChangeTextDocumentParams, DidOpenTextDocumentParams, Hover, HoverContents, HoverParams, HoverProviderCapability,
-    InitializeParams, InitializeResult, InitializedParams, MarkedString, MessageType, OneOf, Range, ServerCapabilities,
-    TextDocumentItem, TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions,
-    WorkspaceFoldersServerCapabilities, WorkspaceServerCapabilities,
+    InitializeParams, InitializeResult, InitializedParams, MarkedString, MarkupContent, MarkupKind, MessageType, OneOf,
+    Range, ServerCapabilities, TextDocumentItem, TextDocumentSyncCapability, TextDocumentSyncKind,
+    TextDocumentSyncOptions, WorkspaceFoldersServerCapabilities, WorkspaceServerCapabilities,
 };
 use tower_lsp::{Client, LanguageServer, LspService, Server, jsonrpc};
 
@@ -91,24 +91,38 @@ impl LanguageServer for Backend {
             None
         };
 
-        let ty = if let Some(token) = token {
+        let (range, ty) = if let Some(token) = token {
             if let Some(possible_ty) = self.types.get(&file_name).unwrap().get(token) {
+                let start = token.position;
+                let end = token.position.add(token.token.len() as u32);
+                let range = Range {
+                    start: lsp_types::Position {
+                        line: (start.1 as i32 - 1).max(0) as u32,
+                        character: (start.0 as i32 - 1).max(0) as u32,
+                    },
+                    end: lsp_types::Position {
+                        line: (end.1 as i32 - 1).max(0) as u32,
+                        character: (end.0 as i32 - 1).max(0) as u32,
+                    },
+                };
                 if let Some(ty) = possible_ty.clone() {
-                    format!("{}", ty)
+                    (Some(range), format!("{}", ty))
                 } else {
-                    String::from("no type")
+                    (Some(range), String::from("no type"))
                 }
             } else {
-                String::from("no token")
+                (None, String::from("no token"))
             }
         } else {
-            String::from("no token")
+            (None, String::from("no token"))
         };
 
-        Ok(Some(Hover {
-            contents: HoverContents::Scalar(MarkedString::String(format!("{}", ty))),
-            range: None,
-        }))
+        let contents = HoverContents::Markup(MarkupContent {
+            kind: MarkupKind::Markdown,
+            value: format!("`{ty}`"),
+        });
+
+        Ok(Some(Hover { contents, range }))
     }
 
     async fn did_open(&self, params: DidOpenTextDocumentParams) {
@@ -272,6 +286,40 @@ pub fn find_type_in_context(module: &mir::Module, token_idx: usize) -> Option<Ty
         }
     }
 
+    for binop in &module.binop_defs {
+        if let Some(meta) = binop.block_meta() {
+            if !meta.contains(token_idx) {
+                continue;
+            }
+        } else {
+            continue;
+        }
+
+        return match &binop.fn_kind {
+            mir::FunctionDefinitionKind::Local(block, _) => find_type_in_block(&block, module.module_id, token_idx),
+            mir::FunctionDefinitionKind::Extern(_) => None,
+            mir::FunctionDefinitionKind::Intrinsic(_) => None,
+        };
+    }
+
+    for (_, function) in &module.associated_functions {
+        if !(function.signature() + function.block_meta()).contains(token_idx) {
+            continue;
+        }
+
+        for param in &function.parameters {
+            if param.meta.contains(token_idx) {
+                return Some(param.ty.clone());
+            }
+        }
+
+        return match &function.kind {
+            mir::FunctionDefinitionKind::Local(block, _) => find_type_in_block(&block, module.module_id, token_idx),
+            mir::FunctionDefinitionKind::Extern(_) => None,
+            mir::FunctionDefinitionKind::Intrinsic(_) => None,
+        };
+    }
+
     for function in &module.functions {
         if !(function.signature() + function.block_meta()).contains(token_idx) {
             continue;
@@ -372,7 +420,7 @@ pub fn find_type_in_expr(expr: &mir::Expression, module_id: SourceModuleId, toke
             Some(TypeKind::CustomType(mir::CustomTypeKey(name.clone(), module_id)))
         }
         mir::ExprKind::Literal(literal) => Some(literal.as_type()),
-        mir::ExprKind::BinOp(binary_operator, lhs, rhs, type_kind) => {
+        mir::ExprKind::BinOp(_, lhs, rhs, type_kind) => {
             if let Some(ty) = find_type_in_expr(lhs, module_id, token_idx) {
                 return Some(ty);
             }

reid-lsp/syntaxes/grammar.json

@@ -0,0 +1,395 @@
+{
+  "scopeName": "source.reid",
+  "patterns": [
+    {
+      "include": "#import"
+    },
+    {
+      "include": "#expression"
+    }
+  ],
+  "repository": {
+    "import": {
+      "begin": "(import)\\s*",
+      "end": ";",
+      "beginCaptures": {
+        "1": {
+          "name": "keyword"
+        }
+      },
+      "endCaptures": {
+        "0": {
+          "name": "punctuation.semi.reid"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#identifier"
+        },
+        {
+          "include": "#punctiation"
+        }
+      ]
+    },
+    "punctuation": {
+      "patterns": [
+        {
+          "match": "::",
+          "name": "keyword.operator.namespace.reid"
+        },
+        {
+          "match": ";",
+          "name": "punctuation.semi.reid"
+        },
+        {
+          "match": ".",
+          "name": "punctuation.dot.reid"
+        },
+        {
+          "match": ",",
+          "name": "punctuation.comma.reid"
+        }
+      ]
+    },
+    "expression": {
+      "patterns": [
+        {
+          "include": "#comment"
+        },
+        {
+          "include": "#fn-signature"
+        },
+        {
+          "include": "#common-type"
+        },
+        {
+          "include": "#binop-impl"
+        },
+        {
+          "include": "#type-impl"
+        },
+        {
+          "include": "#struct-definition"
+        },
+        {
+          "include": "#block"
+        },
+        {
+          "include": "#binop"
+        },
+        {
+          "include": "#namespace"
+        },
+        {
+          "include": "#cast"
+        },
+        {
+          "include": "#function-call"
+        },
+        {
+          "include": "#parenthesis"
+        },
+        {
+          "include": "#array"
+        },
+        {
+          "include": "#keywords"
+        },
+        {
+          "include": "#struct-expression"
+        },
+        {
+          "include": "#number-literal"
+        },
+        {
+          "include": "#string-literal"
+        },
+        {
+          "include": "#identifier"
+        },
+        {
+          "include": "#punctuation"
+        }
+      ]
+    },
+    "comment": {
+      "match": "\\/\\/(.|\\/)*",
+      "name": "comment.line.double-slash.reid"
+    },
+    "fn-signature": {
+      "begin": "(fn)\\s*(\\w+)\\(",
+      "beginCaptures": {
+        "1": {
+          "name": "keyword.fn.reid"
+        },
+        "2": {
+          "name": "entity.name.function.reid"
+        }
+      },
+      "end": "\\)",
+      "patterns": [
+        {
+          "include": "#annotated-identifier"
+        },
+        {
+          "include": "#keywords"
+        },
+        {
+          "include": "#binop"
+        }
+      ],
+      "endCaptures": {
+        "2": {
+          "name": "entity.name.type.reid"
+        }
+      }
+    },
+    "type-impl": {
+      "begin": "(impl)\\s* (\\w+)\\s* \\{\n",
+      "end": "\\}",
+      "captures": {
+        "1": {
+          "name": "keyword.impl.reid"
+        },
+        "2": {
+          "name": "entity.name.type"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "binop-impl": {
+      "begin": "(impl)\\s+(binop)\\s+\\(((.*)\\s*:\\s*(.*))\\)(.*)\\(((.*)\\s*:\\s*(.*))\\)\\s*->\\s*(\\w+)\\s*\\{",
+      "end": "\\}",
+      "beginCaptures": {
+        "1": {
+          "name": "keyword.impl.reid"
+        },
+        "2": {
+          "name": "keyword.impl.reid"
+        },
+        "4": {
+          "name": "variable.parameter.binop.reid"
+        },
+        "5": {
+          "name": "entity.name.type.parameter.binop.reid"
+        },
+        "6": {
+          "name": "keyword.operator.math.reid"
+        },
+        "8": {
+          "name": "variable.parameter.binop.reid"
+        },
+        "9": {
+          "name": "entity.name.type.parameter.binop.reid"
+        },
+        "10": {
+          "name": "entity.name.type.return.binop.reid"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "struct-definition": {
+      "begin": "(struct)\\s*(\\w+)\\s*\\{",
+      "end": "\\}",
+      "captures": {
+        "1": {
+          "name": "keyword.struct.reid"
+        },
+        "2": {
+          "name": "entity.name.type"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#annotated-identifier"
+        }
+      ]
+    },
+    "struct-expression": {
+      "begin": "([A-Z]\\w*)\\s*\\{",
+      "end": "\\}",
+      "captures": {
+        "1": {
+          "name": "entity.name.type.struct.reid"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "number-literal": {
+      "patterns": [
+        {
+          "match": "0x[0-9a-fA-F]+(\\.[0-9a-fA-F]+)?",
+          "name": "constant.hexadecimal"
+        },
+        {
+          "match": "0o[0-7]+(\\.[0-7]+)?",
+          "name": "constant.octal"
+        },
+        {
+          "match": "0b[01]+(\\.[01]+)?",
+          "name": "constant.binary"
+        },
+        {
+          "match": "[0-9]+(\\.[0-9]+)?",
+          "name": "constant.numeric"
+        }
+      ]
+    },
+    "string-literal": {
+      "begin": "\"",
+      "end": "\"",
+      "name": "string.quoted.double",
+      "patterns": [
+        {
+          "match": "\\.",
+          "name": "constant.character.escape"
+        }
+      ]
+    },
+    "block": {
+      "begin": "\\{",
+      "end": "\\}",
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "namespace": {
+      "match": "(\\w+)(\\:\\:)",
+      "captures": {
+        "1": {
+          "name": "entity.name.function.reid"
+        },
+        "2": {
+          "name": "keyword.operator.namespace.reid"
+        }
+      }
+    },
+    "cast": {
+      "match": "(as)\\s+(\\w+)",
+      "captures": {
+        "1": {
+          "name": "keyword.cast.reid"
+        },
+        "2": {
+          "name": "entity.name.type.reid"
+        }
+      }
+    },
+    "function-call": {
+      "begin": "(\\w+)?\\(",
+      "end": "\\)",
+      "beginCaptures": {
+        "1": {
+          "name": "entity.name.function.reid"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "parenthesis": {
+      "begin": "\\(",
+      "end": "\\)",
+      "beginCaptures": {
+        "0": {
+          "name": "keyword.operator.parenthesis.reid"
+        }
+      },
+      "endCaptures": {
+        "0": {
+          "name": "keyword.operator.parenthesis.reid"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "annotated-identifier": {
+      "begin": "(\\w+)\\:",
+      "end": ",",
+      "beginCaptures": {
+        "1": {
+          "name": "variable.language.reid"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "identifier": {
+      "patterns": [
+        {
+          "match": "[A-Z]\\w*",
+          "name": "entity.name.type.reid"
+        },
+        {
+          "match": "\\w+",
+          "name": "variable.language.reid"
+        }
+      ]
+    },
+    "keywords": {
+      "patterns": [
+        {
+          "match": "let|mut|pub|extern",
+          "name": "storage.type.reid"
+        },
+        {
+          "match": "if|return",
+          "name": "keyword.control"
+        },
+        {
+          "match": "self",
+          "name": "variable.language.self.reid"
+        }
+      ]
+    },
+    "binop": {
+      "match": "\\<\\=|\\>\\=|\\=\\=|\\<|\\>|\\*|\\+|\\-|\\^|\\&\\&|\\&",
+      "name": "keyword.operator.math.reid"
+    },
+    "array": {
+      "begin": "\\[",
+      "end": "\\]",
+      "beginCaptures": {
+        "0": {
+          "name": "entity.name.type.array.reid"
+        }
+      },
+      "endCaptures": {
+        "0": {
+          "name": "entity.name.type.array.reid"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
+    "common-type": {
+      "match": "u8|u16|u32|u64|u128|i8|i16|i32|i64|i128|bool",
+      "name": "entity.name.type.common.reid"
+    }
+  }
+}

reid-lsp/syntaxes/grammar.yaml

@@ -0,0 +1,232 @@
+scopeName: source.reid
+patterns:
+  - include: "#import"
+  - include: "#expression"
+repository:
+  # function-definition:
+  #   begin: "(fn)\\s*(\\w+)\\(((\\w+)\\s*\\:\\s*(\\w+),?)*\\)\\s*->\\s*(\\w+)\\s*\\{"
+  #   end: "\\}"
+  #   beginCaptures:
+  #     1:
+  #       name: "keyword.other"
+  #     2:
+  #       name: "entity.name.function"
+  #     4:
+  #       name: "entity.name.parameter"
+  #     5:
+  #       name: "entity.name.type"
+  #     6:
+  #       name: "entity.name.type"
+  #   patterns:
+  #     - include: "#type"
+  #     - include: "#expression"
+  import:
+    begin: "(import)\\s*"
+    end: ";"
+    beginCaptures:
+      1:
+        name: keyword
+    endCaptures:
+      0:
+        name: punctuation.semi.reid
+    patterns:
+      - include: "#identifier"
+      - include: "#punctiation"
+  punctuation:
+    patterns:
+      - match: "::"
+        name: keyword.operator.namespace.reid
+      - match: ";"
+        name: punctuation.semi.reid
+      - match: "."
+        name: punctuation.dot.reid
+      - match: ","
+        name: punctuation.comma.reid
+  expression:
+    patterns:
+      - include: "#comment"
+      - include: "#fn-signature"
+      - include: "#common-type"
+      - include: "#binop-impl"
+      - include: "#type-impl"
+      - include: "#struct-definition"
+      - include: "#block"
+      - include: "#binop"
+      - include: "#namespace"
+      - include: "#cast"
+      - include: "#function-call"
+      - include: "#parenthesis"
+      - include: "#array"
+      - include: "#keywords"
+      - include: "#struct-expression"
+      - include: "#number-literal"
+      - include: "#string-literal"
+      - include: "#identifier"
+      - include: "#punctuation"
+  comment:
+    match: "\\/\\/(.|\\/)*"
+    name: comment.line.double-slash.reid
+  fn-signature:
+    begin: "(fn)\\s*(\\w+)\\("
+    beginCaptures:
+      1:
+        name: keyword.fn.reid
+      2:
+        name: entity.name.function.reid
+    end: "\\)"
+    patterns:
+      - include: "#annotated-identifier"
+      - include: "#keywords"
+      - include: "#binop"
+    endCaptures:
+      2:
+        name: entity.name.type.reid
+  type-impl:
+    begin: >
+      (impl)\s*
+      (\w+)\s*
+      \{
+    end: "\\}"
+    captures:
+      1:
+        name: keyword.impl.reid
+      2:
+        name: entity.name.type
+    patterns:
+      - include: "#expression"
+  binop-impl:
+    begin: "(impl)\\s+(binop)\\s+\\(((.*)\\s*:\\s*(.*))\\)(.*)\\(((.*)\\s*:\\s*(.*))\\)\\s*->\\s*(\\w+)\\s*\\{"
+    end: "\\}"
+    beginCaptures:
+      1:
+        name: keyword.impl.reid
+      2:
+        name: keyword.impl.reid
+      4:
+        name: variable.parameter.binop.reid
+      5:
+        name: entity.name.type.parameter.binop.reid
+      6:
+        name: keyword.operator.math.reid
+      8:
+        name: variable.parameter.binop.reid
+      9:
+        name: entity.name.type.parameter.binop.reid
+      10:
+        name: entity.name.type.return.binop.reid
+    patterns:
+      - include: "#expression"
+  struct-definition:
+    begin: "(struct)\\s*(\\w+)\\s*\\{"
+    end: "\\}"
+    captures:
+      1:
+        name: keyword.struct.reid
+      2:
+        name: entity.name.type
+    patterns:
+      - include: "#annotated-identifier"
+  struct-expression:
+    begin: "([A-Z]\\w*)\\s*\\{"
+    end: "\\}"
+    captures:
+      1:
+        name: entity.name.type.struct.reid
+    patterns:
+      - include: "#expression"
+  number-literal:
+    patterns:
+      - match: "0x[0-9a-fA-F]+(\\.[0-9a-fA-F]+)?"
+        name: "constant.hexadecimal"
+      - match: "0o[0-7]+(\\.[0-7]+)?"
+        name: "constant.octal"
+      - match: "0b[01]+(\\.[01]+)?"
+        name: "constant.binary"
+      - match: "[0-9]+(\\.[0-9]+)?"
+        name: "constant.numeric"
+  string-literal:
+    begin: '"'
+    end: '"'
+    name: string.quoted.double
+    patterns:
+      - match: "\\."
+        name: constant.character.escape
+  block:
+    begin: "\\{"
+    end: "\\}"
+    patterns:
+      - include: "#expression"
+  namespace:
+    match: "(\\w+)(\\:\\:)"
+    captures:
+      1:
+        name: entity.name.function.reid
+      2:
+        name: keyword.operator.namespace.reid
+  cast:
+    match: "(as)\\s+(\\w+)"
+    captures:
+      1:
+        name: keyword.cast.reid
+      2:
+        name: entity.name.type.reid
+  function-call:
+    begin: "(\\w+)?\\("
+    end: "\\)"
+    beginCaptures:
+      1:
+        name: entity.name.function.reid
+    patterns:
+      - include: "#expression"
+  parenthesis:
+    begin: "\\("
+    end: "\\)"
+    beginCaptures:
+      0:
+        name: keyword.operator.parenthesis.reid
+    endCaptures:
+      0:
+        name: keyword.operator.parenthesis.reid
+    patterns:
+      - include: "#expression"
+  annotated-identifier:
+    begin: "(\\w+)\\:"
+    end: ","
+    beginCaptures:
+      1:
+        name: variable.language.reid
+    patterns:
+      - include: "#expression"
+  identifier:
+    patterns:
+      - match: "[A-Z]\\w*"
+        name: entity.name.type.reid
+      - match: "\\w+"
+        name: variable.language.reid
+  keywords:
+    patterns: 
+      - match: "let|mut|pub|extern"
+        name: "storage.type.reid"
+      - match: "if|return"
+        name: "keyword.control"
+      - match: "self"
+        name: "variable.language.self.reid"
+  binop:
+    match: "\\<\\=|\\>\\=|\\=\\=|\\<|\\>|\\*|\\+|\\-|\\^|\\&\\&|\\&"
+    name: keyword.operator.math.reid
+  array:
+    begin: "\\["
+    end: "\\]"
+    beginCaptures:
+      0:
+        name: entity.name.type.array.reid
+    endCaptures:
+      0:
+        name: entity.name.type.array.reid
+    patterns:
+      - include: "#expression"
+  common-type:
+    match: "u8|u16|u32|u64|u128|i8|i16|i32|i64|i128|bool"
+    name: entity.name.type.common.reid
+  
+    

reid/tests/e2e.rs

@@ -6,12 +6,12 @@ use reid::{
     mir::{self},
     parse_module, perform_all_passes,
 };
-use reid_lib::Context;
+use reid_lib::{compile::CompileOutput, Context};
 use util::assert_err;
 
 mod util;
 
-fn test(source: &str, name: &str, expected_exit_code: Option<i32>) {
+fn test_compile(source: &str, name: &str) -> CompileOutput {
     assert_err(assert_err(std::panic::catch_unwind(|| {
         let mut map = Default::default();
         let (id, tokens) = assert_err(parse_module(source, name, &mut map));
@@ -24,7 +24,14 @@ fn test(source: &str, name: &str, expected_exit_code: Option<i32>) {
 
         let codegen = assert_err(mir_context.codegen(&context));
 
-        let output = codegen.compile(None, Vec::new()).output();
+        Ok::<_, ()>(codegen.compile(None, Vec::new()).output())
+    })))
+}
+
+fn test(source: &str, name: &str, expected_exit_code: Option<i32>) {
+    assert_err(assert_err(std::panic::catch_unwind(|| {
+        let output = test_compile(source, name);
+
         let time = SystemTime::now();
         let in_path = PathBuf::from(format!(
             "/tmp/temp-{}.o",
@@ -157,3 +164,8 @@ fn associated_functions() {
 fn mutable_inner_functions() {
     test(include_str!("../../examples/mutable_inner.reid"), "test", Some(0));
 }
+
+#[test]
+fn cpu_raytracer_compiles() {
+    test_compile(include_str!("../../examples/cpu_raytracer.reid"), "test");
+}