feat: Add line number to token information

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
           submodules: true
       - uses: Swatinem/rust-cache@v2
       - name: setup toolchain
-        uses: hecrj/setup-rust-action@v1
+        uses: hecrj/setup-rust-action@v2
         with:
           rust-version: stable
           components: rustfmt, clippy
@@ -33,4 +33,4 @@ jobs:
       - name: rustfmt
         run: make format-check
       - name: clippy
-        run: cargo clippy --all --all-features --tests -- -D warnings
+        run: make lint

.pre-commit-config.yaml

@@ -0,0 +1,52 @@
+fail_fast: true
+
+exclude: |
+  (?x)^(
+  )$
+
+repos:
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.17
+    hooks:
+      - id: mdformat
+        additional_dependencies:
+          - mdformat-mkdocs
+          - mdformat-admon
+        exclude: |
+          (?x)^(
+            docs/formatter/black\.md
+            | docs/\w+\.md
+          )$
+
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.39.0
+    hooks:
+      - id: markdownlint-fix
+        exclude: |
+          (?x)^(
+            docs/formatter/black\.md
+            | docs/\w+\.md
+          )$
+
+  - repo: https://github.com/crate-ci/typos
+    rev: v1.20.8
+    hooks:
+      - id: typos
+
+  - repo: local
+    hooks:
+      - id: cargo-fmt
+        name: cargo fmt
+        entry: cargo fmt --
+        language: system
+        types: [rust]
+        pass_filenames: false # This makes it a lot faster
+
+  - repo: local
+    hooks:
+      - id: cargo-fmt
+        name: cargo clippy
+        entry: make lint
+        language: system
+        types: [rust]
+        pass_filenames: false # This makes it a lot faster

Makefile

@@ -23,4 +23,4 @@ format-check:
 
 .PHONY: lint
 lint:
-	@cargo clippy --all --all-features --tests -- -D warnings
+	@cargo clippy --all --all-features --tests

client/vscode/package.json

@@ -1,79 +1,80 @@
 {
-    "name": "enderpy-language-server",
-    "description": "nano rust language server",
-    "license": "MIT",
-    "version": "1.5.0",
-    "categories": [],
-    "keywords": [
-        "language-server",
-        "tower-lsp"
+  "name": "enderpy-language-server",
+  "description": "nano rust language server",
+  "license": "MIT",
+  "version": "1.5.0",
+  "categories": [],
+  "keywords": [
+    "language-server",
+    "tower-lsp"
+  ],
+  "repository": {
+    "url": "https://github.com/Glyphack/enderpy"
+  },
+  "engines": {
+    "vscode": "^1.88.0"
+  },
+  "enabledApiProposals": [],
+  "activationEvents": [
+    "onLanguage:python"
+  ],
+  "main": "./dist/extension.js",
+  "contributes": {
+    "commands": [
+      {
+        "command": "enderpy.restart",
+        "title": "Restart Enderpy Language Server"
+      }
     ],
-    "repository": {
-        "url": "https://github.com/Glyphack/enderpy"
-    },
-    "engines": {
-        "vscode": "^1.88.0"
-    },
-    "enabledApiProposals": [],
-    "activationEvents": [
-        "onLanguage:python"
+    "languages": [
+      {
+        "id": "python",
+        "extensions": [
+          ".py"
+        ]
+      }
     ],
-    "main": "./dist/extension.js",
-    "contributes": {
-        "commands": [
-            {
-                "command": "helloworld.helloWorld",
-                "title": "Say Hello"
-            }
-        ],
-        "languages": [
-            {
-                "id": "python",
-                "extensions": [
-                    ".py"
-                ]
-            }
-        ],
-        "configuration": {
-            "type": "object",
-            "title": "enderpy-language-server",
-            "properties": {
-                "enderpy-language-server.trace.server": {
-                    "type": "string",
-                    "scope": "window",
-                    "enum": [
-                        "off",
-                        "messages",
-                        "verbose"
-                    ],
-                    "enumDescriptions": [
-                        "No traces",
-                        "Error only",
-                        "Full log"
-                    ],
-                    "default": "off",
-                    "description": "Traces the communication between VS Code and the language server."
-                }
-            }
+    "configuration": {
+      "type": "object",
+      "title": "enderpy-language-server",
+      "properties": {
+        "enderpy-language-server.trace.server": {
+          "type": "string",
+          "scope": "window",
+          "enum": [
+            "off",
+            "messages",
+            "verbose"
+          ],
+          "enumDescriptions": [
+            "No traces",
+            "Error only",
+            "Full log"
+          ],
+          "default": "off",
+          "description": "Traces the communication between VS Code and the language server."
         }
-    },
-    "scripts": {
-        "vscode:prepublish": "npm run compile",
-        "compile": "tsc -b",
-        "watch": "tsc -b -w",
-        "lint": "eslint ./src --ext .ts,.tsx"
-    },
-    "devDependencies": {
-        "@types/mocha": "^10.0.6",
-        "@types/node": "^18.14.6",
-        "@typescript-eslint/eslint-plugin": "^7.1.0",
-        "@typescript-eslint/parser": "^7.1.0",
-        "eslint": "^8.57.0",
-        "mocha": "^10.3.0",
-        "typescript": "^5.3.3"
-    },
-    "dependencies": {
-        "vscode-languageclient": "^8.1.0",
-        "@types/vscode": "^1.75.1"
+      }
     }
-}
+  },
+  "scripts": {
+    "vscode:prepublish": "npm run compile",
+    "compile": "tsc -b",
+    "watch": "tsc -b -w",
+    "lint": "eslint ./src --ext .ts,.tsx"
+  },
+  "devDependencies": {
+    "@types/mocha": "^10.0.6",
+    "@types/node": "^18.14.6",
+    "@typescript-eslint/eslint-plugin": "^7.1.0",
+    "@typescript-eslint/parser": "^7.1.0",
+    "eslint": "^8.57.0",
+    "mocha": "^10.3.0",
+    "typescript": "^5.3.3"
+  },
+  "dependencies": {
+    "vscode-languageclient": "^8.1.0",
+    "@types/vscode": "^1.75.1"
+  }
+}
+

client/vscode/src/extension.ts

@@ -5,10 +5,6 @@ import { Executable, LanguageClient, LanguageClientOptions, ServerOptions } from
 let client: LanguageClient;
 
 export async function activate(context: ExtensionContext) {
-  const disposable = commands.registerCommand("helloworld.helloWorld", () => {
-    window.showInformationMessage("Hello World!");
-  });
-  context.subscriptions.push(disposable);
   const traceOutputChannel = window.createOutputChannel("Enderpy Language Server trace");
   const command = process.env.SERVER_PATH || "enderpy-lsp";
   console.log(`Running Lsp ${command}`);
@@ -31,6 +27,15 @@ export async function activate(context: ExtensionContext) {
   };
 
   client = new LanguageClient("enderpy-language-server", "enderpy language server", serverOptions, clientOptions);
+  const restartCommand = commands.registerCommand("enderpy.restart", () => {
+    if (client) {
+      console.log("Restarting language server");
+      client.stop().then(() => {
+        client.start();
+      });
+    }
+  });
+  context.subscriptions.push(restartCommand);
 
   client.start();
 }

enderpy/src/main.rs

@@ -36,6 +36,7 @@ fn symbols(path: &Path) -> Result<()> {
     let module = manager.get_state(path.to_str().expect(""));
     println!("{}", module.module_name());
     println!("{}", module.get_symbol_table());
+    println!("{:#?}", module.get_symbol_table().scope_starts);
 
     Ok(())
 }
@@ -61,7 +62,28 @@ fn tokenize(file: &PathBuf) -> Result<()> {
     let mut lexer = Lexer::new(&source);
     let tokens = enderpy_python_parser::utils::lex(&mut lexer);
     for token in tokens {
-        println!("{}", token);
+        let (start_line_num, start_line_offset) =
+            match lexer.line_starts.binary_search(&token.start) {
+                Ok(idx) => (idx, lexer.line_starts[idx]),
+                Err(idx) => (idx - 1, lexer.line_starts[idx - 1]),
+            };
+        let start_line_column = token.start - start_line_offset;
+        let (end_line_num, end_line_offset) = match lexer.line_starts.binary_search(&token.end) {
+            Ok(idx) => (idx, lexer.line_starts[idx]),
+            Err(idx) => (idx - 1, lexer.line_starts[idx - 1]),
+        };
+        let end_line_column = token.end - end_line_offset;
+        println!(
+            "{}-{}, {}-{}:   {} {} {} {}",
+            start_line_num,
+            start_line_column,
+            end_line_num,
+            end_line_column,
+            token.kind,
+            token.value,
+            token.start,
+            token.end,
+        );
     }
     Ok(())
 }

lsp/src/main.rs

@@ -149,7 +149,6 @@ impl LanguageServer for Backend {
     }
 
     async fn hover(&self, params: HoverParams) -> Result<Option<Hover>> {
-        self.client.log_message(MessageType::INFO, "hover!").await;
         let uri = params.text_document_position_params.text_document.uri;
         let Ok(path) = uri.to_file_path() else {
             return Ok(None);
@@ -171,25 +170,20 @@ impl LanguageServer for Backend {
         // For now, let's provide a sample hover message with placeholder values
         let hover_message =
             self.manager
-                .get_type_information(&path, position.line, position.character);
+                .get_hover_information(&path, position.line, position.character);
 
         let markup_content = MarkupContent {
             kind: MarkupKind::Markdown,
-            value: format!("**Hover Information**\n\n{}\n\n- Type: `<type>`\n- Documentation: `<documentation>`", hover_message),
+            value: format!(
+                "**Hover Information**\n\n- Definition: `{}`\n\n- Type: `<type>`\n",
+                hover_message
+            ),
         };
         let hover = Hover {
             contents: HoverContents::Markup(markup_content),
             range: None,
         };
 
-        // Log the hover content for debugging
-        self.client
-            .log_message(
-                MessageType::INFO,
-                format!("Hover content:\n{:?}", hover.contents),
-            )
-            .await;
-
         return Ok(Some(hover));
     }
 

parser/src/lexer/mod.rs

@@ -31,6 +31,8 @@ pub struct Lexer {
 
     // TODO: Hacky way to handle emitting multiple de indents
     next_token_is_dedent: u8,
+    /// Array of all line starts offsets
+    pub line_starts: Vec<u32>,
 }
 
 impl Lexer {
@@ -45,6 +47,7 @@ impl Lexer {
             fstring_stack: vec![],
             inside_fstring_bracket: 0,
             next_token_is_dedent: 0,
+            line_starts: vec![],
         }
     }
 
@@ -191,6 +194,7 @@ impl Lexer {
 
     fn next_kind(&mut self) -> Result<Kind, LexError> {
         if self.start_of_line {
+            self.line_starts.push(self.current);
             if let Some(indent_kind) = self.match_indentation()? {
                 self.start_of_line = false; // WHY!?
                 return Ok(indent_kind);