Merge pull request #216 from Glyphack/builtins

Use builtins from typeshed

.github/workflows/test.yml

@@ -15,24 +15,20 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: Swatinem/rust-cache@v2
       - uses: actions/checkout@v4
         with:
           submodules: true
+      - uses: Swatinem/rust-cache@v2
       - name: setup toolchain
         uses: hecrj/setup-rust-action@v1
         with:
           rust-version: stable
-      - name: setup toolchain
-        uses: hecrj/setup-rust-action@v1
-        with:
-          rust-version: nightly
           components: rustfmt, clippy
       - name: Build
-        run: cargo build --verbose
+        run: cargo build
       - name: Run tests
-        run: cargo test --verbose
+        run: cargo test
       - name: rustfmt
-        run: cargo +nightly fmt --all -- --check
+        run: make format-check
       - name: clippy
-        run: cargo +nightly clippy --all --all-features --tests -- -D warnings
+        run: cargo clippy --all --all-features --tests -- -D warnings

Makefile

@@ -15,12 +15,12 @@ check:
 
 .PHONY: format
 format:
-	@cargo +nightly fmt --all
+	@cargo fmt --all
 
 .PHONY: format-check
 format-check:
-	cargo +nightly fmt --all -- --check
+	@cargo fmt --all -- --check
 
 .PHONY: lint
 lint:
-	@cargo +nightly clippy --all --all-features --tests -- -D warnings
+	@cargo clippy --all --all-features --tests -- -D warnings

parser/src/parser/parser.rs

@@ -742,7 +742,7 @@ impl Parser {
         while self.eat(Kind::MatrixMul) {
             let name = self.parse_named_expression()?;
             decorators.push(name);
-            self.bump(Kind::NewLine);
+            self.consume_whitespace_and_comments();
         }
 
         if self.at(Kind::Def) || self.at(Kind::Async) {
@@ -2062,6 +2062,18 @@ impl Parser {
         consumed
     }
 
+    fn consume_whitespace_and_comments(&mut self) -> bool {
+        let mut consumed = false;
+        while matches!(
+            self.cur_kind(),
+            Kind::WhiteSpace | Kind::NewLine | Kind::Comment
+        ) {
+            self.bump(self.cur_kind());
+            consumed = true;
+        }
+        consumed
+    }
+
     // https://docs.python.org/3/reference/expressions.html#expression-lists
     // termination_kind is used to know when to stop parsing the list
     // for example to parse a tuple the termination_kind is Kind::RightParen

typechecker/Cargo.toml

@@ -12,6 +12,7 @@ thiserror = "1.0.48"
 log.workspace = true
 env_logger = "0.10.0"
 tempfile = "3.8.0"
+elsa = "1.10.0"
 
 [dev-dependencies]
 insta = { version = "1.28.0", features = ["yaml", "filters"] }

typechecker/src/build.rs

@@ -71,16 +71,26 @@ impl BuildManager {
 
     // Entry point to analyze the program
     pub fn build(&mut self) {
-        self.populate_modules();
-    }
-
-    // Resolves imports in all files and then populates the symbol table
-    fn populate_modules(&mut self) {
         for build_source in self.build_sources.iter() {
             let build_source: BuildSource = build_source.clone();
             let state: EnderpyFile = build_source.into();
             self.modules.insert(state.module_name(), state);
         }
+        let builtins_file = self
+            .options
+            .import_discovery
+            .typeshed_path
+            .clone()
+            .unwrap()
+            .join("stdlib/builtins.pyi");
+        let builtins = BuildSource::from_path(builtins_file, true);
+        match builtins {
+            Ok(b) => {
+                let file: EnderpyFile = b.into();
+                self.modules.insert(file.module_name(), file)
+            }
+            Err(e) => panic!("error loading builtins file: {}", e),
+        };
         let (new_files, imports) = match self.options.follow_imports {
             crate::settings::FollowImports::All => {
                 self.gather_files(self.build_sources.clone(), true)
@@ -394,6 +404,10 @@ mod tests {
 
     #[allow(dead_code)]
     fn snapshot_symbol_table(source: &str) -> String {
+        let _ = env_logger::builder()
+            .filter_level(log::LevelFilter::Debug)
+            .is_test(true)
+            .try_init();
         let mut manager = BuildManager::new(
             vec![BuildSource {
                 path: PathBuf::from("test.py"),
@@ -410,66 +424,6 @@ mod tests {
         format!("{}", module.get_symbol_table())
     }
 
-    fn snapshot_type_check(source: &str) -> String {
-        let mut manager = BuildManager::new(
-            vec![BuildSource {
-                path: PathBuf::from("test.py"),
-                module: String::from("test"),
-                source: source.to_string(),
-                followed: false,
-            }],
-            Settings::test_settings(),
-        );
-        manager.type_check();
-
-        let errors = manager.errors;
-        errors
-            .iter()
-            .map(|x| format!("{:?}", x))
-            .collect::<Vec<String>>()
-            .join("\n")
-    }
-
-    macro_rules! snap_type {
-        ($name:tt, $path:tt) => {
-            #[test]
-            fn $name() {
-                let contents = include_str!($path);
-                let result = snapshot_type_check(contents);
-                let mut settings = insta::Settings::clone_current();
-                settings.set_snapshot_path("../test_data/output/");
-                settings.set_description(contents);
-                settings.add_filter(
-                    r"module_name: .*.typechecker.test_data.inputs.symbol_table..*.py",
-                    "module_name: [REDACTED]",
-                );
-                settings.bind(|| {
-                    insta::assert_snapshot!(result);
-                });
-            }
-        };
-    }
-
-    snap_type!(test_type_check_var, "../test_data/inputs/type_check_var.py");
-    snap_type!(
-        test_type_check_call,
-        "../test_data/inputs/type_check_call.py"
-    );
-    // snap_type!(
-    //     test_type_check_list,
-    //     "../test_data/inputs/type_check_list.py"
-    // );
-
-    snap_type!(
-        test_type_check_undefined,
-        "../test_data/inputs/type_check_undefined.py"
-    );
-
-    snap_type!(
-        test_undefined_names,
-        "../test_data/inputs/test_undefined_name.py"
-    );
-
     #[test]
     fn test_symbol_table() {
         glob!("../test_data/inputs/", "symbol_table/*.py", |path| {

typechecker/src/nodes.rs

@@ -59,6 +59,12 @@ impl EnderpyFile {
     pub fn get_position(&self, pos: usize) -> Position {
         let mut line_number = 0;
         let mut line_start = 0;
+        if pos == 0 {
+            return Position {
+                line: 0,
+                character: 0,
+            };
+        }
         for (i, c) in self.build_source.source.chars().enumerate() {
             if i == pos {
                 break;

typechecker/src/settings.rs

@@ -7,6 +7,7 @@ use serde::Deserialize;
 #[allow(unused)]
 pub struct ImportDiscovery {
     pub python_executable: Option<PathBuf>,
+    // TODO make required and give error if not found
     pub typeshed_path: Option<PathBuf>,
 }
 

typechecker/src/symbol_table.rs

@@ -2,7 +2,7 @@ use std::{collections::HashMap, fmt::Display, path::PathBuf};
 
 use enderpy_python_parser::ast::{self, Node};
 
-use crate::{ruff_python_import_resolver::import_result::ImportResult, type_check::builtins};
+use crate::ruff_python_import_resolver::import_result::ImportResult;
 
 #[derive(Debug, Clone)]
 pub struct SymbolTable {
@@ -223,95 +223,16 @@ pub enum SymbolScope {
 
 impl SymbolTable {
     pub fn global(module_name: String, file_path: PathBuf) -> Self {
-        let mut builtin_scope = SymbolTableScope {
-            id: get_id(),
-            symbol_table_type: SymbolTableType::BUILTIN,
-            symbols: HashMap::new(),
-            name: String::from("builtins"),
-            parent: None,
-            start_pos: 0,
-        };
-        // TODO: This will be removed once we can import the builtins from the stdlib
-        // Hacky way of putting the builtin in symbol table so I can implement some
-        // tests
-        let list_class = Class {
-            name: builtins::LIST_TYPE.to_string(),
-            declaration_path: DeclarationPath {
-                module_name: PathBuf::from("typeshed/stdlib/builtins.pyi"),
-                node: Node { start: 0, end: 0 },
-            },
-            methods: vec![],
-            attributes: HashMap::new(),
-            special: false,
-        };
-        builtin_scope.symbols.insert(
-            builtins::LIST_TYPE.to_string(),
-            SymbolTableNode {
-                name: builtins::LIST_TYPE.to_string(),
-                declarations: vec![Declaration::Class(list_class)],
-            },
-        );
-        let tuple_class = Class {
-            name: builtins::TUPLE_TYPE.to_string(),
-            declaration_path: DeclarationPath {
-                module_name: PathBuf::from("typeshed/stdlib/builtins.pyi"),
-                node: Node { start: 0, end: 0 },
-            },
-            methods: vec![],
-            attributes: HashMap::new(),
-            special: false,
-        };
-        builtin_scope.symbols.insert(
-            builtins::TUPLE_TYPE.to_string(),
-            SymbolTableNode {
-                name: builtins::TUPLE_TYPE.to_string(),
-                declarations: vec![Declaration::Class(tuple_class)],
-            },
-        );
-        let set_class = Class {
-            name: builtins::SET_TYPE.to_string(),
-            declaration_path: DeclarationPath {
-                module_name: PathBuf::from("typeshed/stdlib/builtins.pyi"),
-                node: Node { start: 0, end: 0 },
-            },
-            methods: vec![],
-            attributes: HashMap::new(),
-            special: false,
-        };
-        builtin_scope.symbols.insert(
-            builtins::SET_TYPE.to_string(),
-            SymbolTableNode {
-                name: builtins::SET_TYPE.to_string(),
-                declarations: vec![Declaration::Class(set_class)],
-            },
-        );
-        let dict_class = Class {
-            name: builtins::DICT_TYPE.to_string(),
-            declaration_path: DeclarationPath {
-                module_name: PathBuf::from("typeshed/stdlib/builtins.pyi"),
-                node: Node { start: 0, end: 0 },
-            },
-            methods: vec![],
-            attributes: HashMap::new(),
-            special: false,
-        };
-        builtin_scope.symbols.insert(
-            builtins::DICT_TYPE.to_string(),
-            SymbolTableNode {
-                name: builtins::DICT_TYPE.to_string(),
-                declarations: vec![Declaration::Class(dict_class)],
-            },
-        );
         let global_scope = SymbolTableScope {
             id: get_id(),
             symbol_table_type: SymbolTableType::Module,
             symbols: HashMap::new(),
             name: String::from("global"),
-            parent: Some(builtin_scope.id),
+            parent: None,
             start_pos: 0,
         };
         SymbolTable {
-            scopes: vec![builtin_scope, global_scope],
+            scopes: vec![global_scope],
             all_scopes: vec![],
             _locals: HashMap::new(),
             module_name,
@@ -377,17 +298,9 @@ impl SymbolTable {
                 }
             }
         }
-        if let Some(symbol) = self.current_scope().symbols.get(&lookup_request.name) {
-            return Some(symbol);
-        }
         None
     }
 
-    pub fn lookup_in_builtin_scope(&self, name: &str) -> Option<&SymbolTableNode> {
-        let builtin_scope = self.get_builtin_scope();
-        builtin_scope.symbols.get(name)
-    }
-
     pub fn enter_scope(&mut self, new_scope: SymbolTableScope) {
         self.scopes.push(new_scope);
     }
@@ -403,10 +316,6 @@ impl SymbolTable {
     pub fn exit_scope(&mut self) {
         let finished_scope = self.scopes.pop();
         if let Some(scope) = finished_scope {
-            // Also pop the builtin scope if we are exiting the global scope
-            // if scope.symbol_table_type == SymbolTableType::Module {
-            //     self.all_scopes.push(self.scopes.pop().unwrap());
-            // }
             self.all_scopes.push(scope);
         }
     }
@@ -425,15 +334,6 @@ impl SymbolTable {
         };
     }
 
-    pub(crate) fn get_builtin_scope(&self) -> &SymbolTableScope {
-        // builtin scope always exists
-        self.scopes
-            .iter()
-            .filter(|scope| scope.symbol_table_type == SymbolTableType::BUILTIN)
-            .last()
-            .unwrap()
-    }
-
     // TODO: this can be attribute of symbol table
     pub fn is_pyi(&self) -> bool {
         self.file_path.extension().unwrap() == "pyi"