diff --git a/enderpy/src/main.rs b/enderpy/src/main.rs index 0c892c03..ae28ae3d 100644 --- a/enderpy/src/main.rs +++ b/enderpy/src/main.rs @@ -1,5 +1,5 @@ use std::{ - fs, + fs, io, path::{Path, PathBuf}, }; @@ -25,7 +25,7 @@ fn main() -> Result<()> { fn symbols(path: &Path) -> Result<()> { let dir_of_path = path.parent().unwrap(); let typeshed_path = get_typeshed_path()?; - let settings = Settings { typeshed_path }; + let settings = Settings::from_typeshed(typeshed_path); let manager = BuildManager::new(settings); let root = find_project_root(dir_of_path); @@ -40,13 +40,30 @@ fn symbols(path: &Path) -> Result<()> { } fn get_python_executable() -> Result { - let output = std::process::Command::new("python") - .arg("-c") - .arg("import sys; print(sys.executable)") - .output() - .into_diagnostic()?; - let path = String::from_utf8(output.stdout).into_diagnostic()?; - Ok(PathBuf::from(path)) + let possible_executables = ["python3", "python"]; + for executable_name in possible_executables { + let res = std::process::Command::new(executable_name) + .arg("-c") + .arg("import sys; print(sys.executable)") + .output(); + match res { + Ok(output) => { + let mut path = String::from_utf8(output.stdout).into_diagnostic()?; + // Like calling trim but I didn't want to re-allocate the str slice + while path.ends_with("\n") || path.ends_with("\r") { + path.pop(); + } + return Ok(PathBuf::from(path)); + } + Err(e) => { + if e.kind() != io::ErrorKind::NotFound { + bail!("Unknown error when looking for python executable: {e}"); + } + } + } + } + + bail!("Failed to find Python executable."); } fn get_typeshed_path() -> Result { @@ -100,9 +117,12 @@ fn check(path: &Path) -> Result<()> { bail!("Path must be a file"); } let root = find_project_root(path); - let _python_executable = Some(get_python_executable()?); + let python_executable = Some(get_python_executable()?); let typeshed_path = get_typeshed_path()?; - let settings = Settings { typeshed_path }; + let settings = Settings { + typeshed_path, + python_executable, + }; let build_manager = BuildManager::new(settings); build_manager.build(root); build_manager.build_one(root, path); @@ -124,3 +144,21 @@ fn check(path: &Path) -> Result<()> { fn watch() -> Result<()> { todo!() } + +#[cfg(test)] +mod tests { + use crate::get_python_executable; + + #[test] + fn test_get_python_successfully() { + let executable = get_python_executable().expect("No python executable found!"); + // Makes sure the python executable is working, without relying on a specific filename + assert!(executable.is_file()); + let output = std::process::Command::new(executable) + .arg("-c") + .arg("print('Working')") + .output() + .unwrap(); + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), "Working"); + } +} diff --git a/lsp/src/main.rs b/lsp/src/main.rs index 4fff805d..e36b782f 100644 --- a/lsp/src/main.rs +++ b/lsp/src/main.rs @@ -143,7 +143,7 @@ async fn main() { .parent() .unwrap() .join("typeshed"); - let settings = Settings { typeshed_path }; + let settings = Settings::from_typeshed(typeshed_path); let manager = BuildManager::new(settings); let (service, socket) = LspService::new(|client| Backend { client, manager }); Server::new(stdin, stdout, socket).serve(service).await; diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs index 2dee9189..25f0fb53 100755 --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -77,6 +77,10 @@ impl BuildManager { let module = EnderpyFile::new(path, true); self.modules.insert(module.path(), module); } + for (_, implicit_import) in imported_module.1.implicit_imports.iter() { + let module = EnderpyFile::new(&implicit_import.path, true); + self.modules.insert(module.path(), module); + } } log::debug!("Imports resolved"); for mut module in self.modules.iter_mut() { @@ -198,6 +202,17 @@ impl BuildManager { let e = EnderpyFile::new(resolved_path, true); files_to_resolve.push(e); } + + for (_, implicit_import) in resolved.implicit_imports.iter() { + let source = match std::fs::read_to_string(&implicit_import.path) { + Ok(source) => source, + Err(e) => { + panic!("cannot read implicit import"); + } + }; + let e = EnderpyFile::new(&implicit_import.path, true); + files_to_resolve.push(e); + } } } } diff --git a/typechecker/src/checker.rs b/typechecker/src/checker.rs index f93e0277..32d4ad95 100644 --- a/typechecker/src/checker.rs +++ b/typechecker/src/checker.rs @@ -1,8 +1,11 @@ +use std::path::PathBuf; + use ast::{Expression, Statement}; use enderpy_python_parser as parser; use enderpy_python_parser::ast::{self, *}; use super::{type_evaluator::TypeEvaluator, types::PythonType}; +use crate::types::ModuleRef; use crate::{ ast_visitor::TraversalVisitor, diagnostic::CharacterSpan, file::EnderpyFile, symbol_table::SymbolTable, @@ -164,6 +167,17 @@ impl TraversalVisitor for TypeChecker { for alias in _i.names.iter() { self.infer_name_type(&alias.name, alias.node.start, alias.node.end) } + + // Just to show type module when modules are hovered in imports. + let start = _i.node.start + 5; + let stop = start + _i.module.len() as u32 + 1; + self.types.insert(Interval { + start, + stop, + val: PythonType::Module(ModuleRef { + module_path: PathBuf::new(), + }), + }); } fn visit_if(&mut self, i: &parser::ast::If) { diff --git a/typechecker/src/semantic_analyzer.rs b/typechecker/src/semantic_analyzer.rs index d1b98c56..f70d8f6f 100644 --- a/typechecker/src/semantic_analyzer.rs +++ b/typechecker/src/semantic_analyzer.rs @@ -346,7 +346,8 @@ impl TraversalVisitor for SemanticAnalyzer { declaration_path, import_from_node: None, import_node: Some(i.clone()), - symbol_name: Some(alias.name()), + symbol_name: None, + module_name: Some(alias.name()), import_result, }); @@ -377,6 +378,7 @@ impl TraversalVisitor for SemanticAnalyzer { import_from_node: Some(_i.clone()), import_node: None, symbol_name: Some(alias.name()), + module_name: None, import_result: module_import_result.clone(), }); diff --git a/typechecker/src/settings.rs b/typechecker/src/settings.rs index 889f3260..68237043 100644 --- a/typechecker/src/settings.rs +++ b/typechecker/src/settings.rs @@ -7,6 +7,7 @@ use serde::Deserialize; #[allow(unused)] pub struct Settings { pub typeshed_path: PathBuf, + pub python_executable: Option, } impl Settings { @@ -20,10 +21,18 @@ impl Settings { s.try_deserialize() } + pub fn from_typeshed(typeshed_path: PathBuf) -> Self { + Settings { + typeshed_path, + python_executable: None, + } + } + pub fn test_settings() -> Self { let file_dir = env::current_dir().unwrap(); Settings { typeshed_path: file_dir.parent().unwrap().join("typeshed"), + python_executable: None, } } } diff --git a/typechecker/src/symbol_table.rs b/typechecker/src/symbol_table.rs index 037f3cd3..ed2d12b1 100644 --- a/typechecker/src/symbol_table.rs +++ b/typechecker/src/symbol_table.rs @@ -511,6 +511,9 @@ pub struct Alias { /// Name of the imported symbol in case of ImportFrom /// e.g. From bar import baz -> baz is the symbol name pub symbol_name: Option, + /// Name of imported module in case of Import + /// e.g. import os.path -> os.path is the module name + pub module_name: Option, /// The result of the import pub import_result: ImportResult, } diff --git a/typechecker/src/type_evaluator.rs b/typechecker/src/type_evaluator.rs index 511b649b..3e71dca6 100755 --- a/typechecker/src/type_evaluator.rs +++ b/typechecker/src/type_evaluator.rs @@ -16,6 +16,7 @@ use super::{ types::{CallableType, LiteralValue, PythonType}, }; use crate::{ + ruff_python_import_resolver::import_result::ImportResult, semantic_analyzer::get_member_access_info, symbol_table::{self, Class, Declaration, LookupSymbolRequest, SymbolTable, SymbolTableNode}, types::{ClassType, TypeVar}, @@ -306,6 +307,15 @@ impl TypeEvaluator { } } } + PythonType::Module(module) => { + todo!(); + let module_sym_table = + self.get_symbol_table_of(&module.module_path).unwrap(); + let typ = self + .infer_type_from_symbol_table(&a.attr, None, module_sym_table, Some(0)) + .unwrap(); + Ok(typ) + } _ => Ok(PythonType::Unknown), } } @@ -444,35 +454,12 @@ impl TypeEvaluator { name, symbol_table.file_path, ); - let result = match symbol_table.lookup_in_scope(lookup_request.clone()) { - Some(symbol) => self.get_symbol_node_type(symbol), - None => { - // Check if there's any import * and try to find the symbol in those files - let mut found = None; - for star_import in symbol_table.star_imports.iter() { - for resolved in star_import.resolved_paths.iter() { - let star_import_sym_table = self.get_symbol_table_of(resolved); - let Some(sym_table) = star_import_sym_table else { - continue; - }; - let res = sym_table.lookup_in_scope(LookupSymbolRequest { - name, - // Look in global scope of the other file - scope: Some(0), - }); - if res.is_some() { - found = res; - break; - } - } - } - match found { - Some(s) => self.get_symbol_node_type(s), - None => bail!("name {name} is not defined"), - } - } - }; - result + let result = + match self.lookup_in_table_and_star_imports(symbol_table, lookup_request.clone()) { + Some(s) => s, + None => bail!("name {name} is not defined"), + }; + self.get_symbol_node_type(result) } /// Get the type of a symbol node based on declarations @@ -561,10 +548,35 @@ impl TypeEvaluator { } } Declaration::Alias(a) => { - let resolved = self.resolve_alias(a, symbol_table); - match resolved { - Some(node) => self.get_symbol_node_type(node), - None => Ok(PythonType::Unknown), + log::debug!("evaluating alias {:?}", a); + // when symbol is an alias that is named to that symbol return Module type + // e.g. from . import path as _path + // then type of _path is Module(path) + match &a.symbol_name { + Some(name) => { + let resolved = self.resolve_alias(name, &a.import_result, symbol_table); + match resolved { + Some(node) => self.get_symbol_node_type(node), + None => Ok(PythonType::Unknown), + } + } + None => { + let mut found_module: Option = None; + for resolved_path in a.import_result.resolved_paths.iter() { + let Some(sym_table_alias_pointing_to) = + self.get_symbol_table_of(resolved_path) + else { + break; + }; + found_module = Some(PythonType::Module(crate::types::ModuleRef { + module_path: sym_table_alias_pointing_to.file_path.clone(), + })); + } + match found_module { + Some(s) => Ok(s), + None => Ok(PythonType::Unknown), + } + } } } Declaration::TypeParameter(_) => Ok(PythonType::Unknown), @@ -990,25 +1002,18 @@ impl TypeEvaluator { false } - // Follows Alias declaration and resolves it to a class declaration + // Follows Alias declaration and resolves it to another symbol node // It searches through imported symbol tables for the module alias imports - // and resolves the alias to the class declaration - // TODO: refactor all aliases and not only classes fn resolve_alias( &self, - a: &symbol_table::Alias, + // a: &symbol_table::Alias, + imported_symbol_name: &str, + import_result: &ImportResult, alias_symbol_table: &SymbolTable, ) -> Option<&symbol_table::SymbolTableNode> { - log::debug!("resolving alias: {:?}", a); - let class_name = match a.symbol_name { - Some(ref name) => name.clone(), - None => panic!("Alias {:?} has no symbol name", a.import_node), - }; - - for resolved_path in a.import_result.resolved_paths.iter() { + for resolved_path in import_result.resolved_paths.iter() { log::debug!("checking path {:?}", resolved_path); - // TODO: This is a hack to resolve Iterator alias in sys/__init__.pyi - let symbol_table_with_alias_def = if class_name == "Iterator" { + let symbol_table_with_alias_def = if imported_symbol_name == "Iterator" { self.imported_symbol_tables .iter() .find(|symbol_table| symbol_table.file_path.ends_with("stdlib/typing.pyi")) @@ -1025,12 +1030,12 @@ impl TypeEvaluator { // then it's cyclic and do not resolve if alias_symbol_table.file_path.as_path() == resolved_path { log::debug!("alias resolution skipped"); - return None; + continue; } let resolved_symbol = symbol_table_with_alias_def?.lookup_in_scope(LookupSymbolRequest { - name: &class_name, + name: imported_symbol_name, scope: None, }); @@ -1041,6 +1046,38 @@ impl TypeEvaluator { return resolved_symbol; } } + + for (_, implicit_import) in import_result.implicit_imports.iter() { + let resolved_path = &implicit_import.path; + log::debug!("checking path {:?}", resolved_path); + // TODO: This is a hack to resolve Iterator alias in sys/__init__.pyi + let symbol_table_with_alias_def = if imported_symbol_name == "Iterator" { + self.imported_symbol_tables + .iter() + .find(|symbol_table| symbol_table.file_path.ends_with("stdlib/typing.pyi")) + } else { + self.get_symbol_table_of(resolved_path) + }; + + if symbol_table_with_alias_def.is_none() { + panic!("Symbol table not found for alias: {:?}", resolved_path); + } + + let resolved_symbol = self.lookup_in_table_and_star_imports( + symbol_table_with_alias_def?, + LookupSymbolRequest { + name: imported_symbol_name, + scope: None, + }, + ); + + log::debug!("symbols {:?}", symbol_table_with_alias_def?.global_scope()); + + if resolved_symbol.is_some() { + log::debug!("alias resolved to {:?}", resolved_symbol); + return resolved_symbol; + } + } None } @@ -1080,4 +1117,38 @@ impl TypeEvaluator { ) -> PythonType { f_type.return_type.clone() } + + fn lookup_in_table_and_star_imports<'a>( + &'a self, + symbol_table: &'a SymbolTable, + lookup_request: LookupSymbolRequest, + ) -> Option<&SymbolTableNode> { + let find_in_current_symbol_table = symbol_table.lookup_in_scope(lookup_request.clone()); + if find_in_current_symbol_table.is_some() { + return find_in_current_symbol_table; + } + + log::debug!( + "did not find symbol {} in symbol table, checking star imports", + lookup_request.name + ); + // Check if there's any import * and try to find the symbol in those files + let mut found = None; + for star_import in symbol_table.star_imports.iter() { + log::debug!("checking star imports {:?}", star_import); + for resolved in star_import.resolved_paths.iter() { + log::debug!("checking path {:?}", resolved); + let star_import_sym_table = self.get_symbol_table_of(resolved); + let Some(sym_table) = star_import_sym_table else { + panic!("symbol table of star import not found at {:?}", resolved); + }; + let res = sym_table.lookup_in_scope(lookup_request.clone()); + if res.is_some() { + found = res; + break; + } + } + } + found + } } diff --git a/typechecker/src/types.rs b/typechecker/src/types.rs index e583adfc..ff63142b 100644 --- a/typechecker/src/types.rs +++ b/typechecker/src/types.rs @@ -1,5 +1,6 @@ use is_macro::Is; use std::fmt::Display; +use std::path::PathBuf; use enderpy_python_parser::ast; @@ -22,6 +23,7 @@ pub enum PythonType { /// In type inference the values are not assumed to be literals unless they /// are explicitly declared as such. KnownValue(KnownValue), + Module(ModuleRef), /// Union type MultiValue(Vec), Callable(Box), @@ -190,11 +192,17 @@ impl Display for LiteralValue { } } +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct ModuleRef { + pub module_path: PathBuf, +} + impl Display for PythonType { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { let type_str = match self { PythonType::None => "None", PythonType::Any => "Any", + PythonType::Module(_) => "Module", PythonType::Unknown => "Unknown", PythonType::Callable(callable_type) => { let fmt = format!("(function) {}", callable_type.name.as_str()); diff --git a/typechecker/test_data/inputs/import_star_test/a.py b/typechecker/test_data/inputs/import_star_test/a.py index dbae10ec..d2e0c915 100644 --- a/typechecker/test_data/inputs/import_star_test/a.py +++ b/typechecker/test_data/inputs/import_star_test/a.py @@ -1,3 +1,4 @@ from .b import * +import os print(in_b) diff --git a/typechecker/test_data/output/enderpy_python_type_checker__build__tests__symbols_import_star.snap b/typechecker/test_data/output/enderpy_python_type_checker__build__tests__symbols_import_star.snap index afbd16b2..e32cc306 100644 --- a/typechecker/test_data/output/enderpy_python_type_checker__build__tests__symbols_import_star.snap +++ b/typechecker/test_data/output/enderpy_python_type_checker__build__tests__symbols_import_star.snap @@ -1,10 +1,13 @@ --- source: typechecker/src/build.rs -description: "from .b import *\n\nprint(in_b)\n" +description: "from .b import *\nimport os\n\nprint(in_b)\n\nos.path.dirname(\"\")\n" expression: result --- [ImportResult { is_relative: true, is_import_found: true, is_partly_resolved: false, is_namespace_package: false, is_init_file_present: false, is_stub_package: false, import_type: Local, resolved_paths: ["test_data/inputs/import_star_test/b.py"], search_path: Some("test_data/inputs/import_star_test"), is_stub_file: false, is_native_lib: false, is_stdlib_typeshed_file: false, is_third_party_typeshed_file: false, is_local_typings_file: false, implicit_imports: ImplicitImports({}), filtered_implicit_imports: ImplicitImports({}), non_stub_import_result: None, py_typed_info: None, package_directory: None }] Symbols in global +os - declaration: Alias - properties: SymbolFlags(0x0) +- Declarations: +--: Alias Scopes: diff --git a/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_generics.snap b/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_generics.snap index c20ce796..41b18970 100644 --- a/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_generics.snap +++ b/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_generics.snap @@ -16,18 +16,21 @@ Tests for basic usage of generics. Line 5: from __future__ import annotations Expr types in the line --->: + __future__ => Module annotations => (class) _Feature --- Line 7: from collections.abc import Sequence Expr types in the line --->: + collections.abc => Module Sequence => Unknown --- Line 8: from typing import Any, Generic, TypeVar, assert_type Expr types in the line --->: + typing => Module Any => (class) object Generic => (class) Generic TypeVar => TypeVar[, ] @@ -281,12 +284,14 @@ Expr types in the line --->: Line 79: from logging import Logger Expr types in the line --->: + logging => Module Logger => (class) Logger --- Line 80: from collections.abc import Iterable Expr types in the line --->: + collections.abc => Module Iterable => Unknown --- @@ -486,6 +491,7 @@ Expr types in the line --->: Line 127: from collections.abc import Iterator, Mapping Expr types in the line --->: + collections.abc => Module Iterator => (class) typing.Iterator[TypeVar[_T_co, ]] Mapping => Unknown @@ -648,6 +654,7 @@ Expr types in the line --->: Line 161: from collections.abc import Sized, Container Expr types in the line --->: + collections.abc => Module Sized => Unknown Container => Unknown diff --git a/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_types.snap b/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_types.snap index 6adbc98f..ef0629e3 100644 --- a/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_types.snap +++ b/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__basic_types.snap @@ -6,6 +6,7 @@ expression: result Line 1: from typing import Dict, Set, List Expr types in the line --->: + typing => Module Dict => (class) builtins.dict[TypeVar[_KT, ], TypeVar[_VT, ]] Set => (class) set List => (class) builtins.list[TypeVar[_T, ]] diff --git a/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__import_star_lookup.snap b/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__import_star_lookup.snap index 33d2dc8e..ef3accc0 100644 --- a/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__import_star_lookup.snap +++ b/typechecker/test_data/output/enderpy_python_type_checker__checker__tests__import_star_lookup.snap @@ -1,15 +1,22 @@ --- source: typechecker/src/checker.rs -description: "1: from .b import *\n2: \n3: print(in_b)\n" +description: "1: from .b import *\n2: import os\n3: \n4: print(in_b)\n" expression: result --- Line 1: from .b import * Expr types in the line --->: + .b => Module * => Unknown --- -Line 3: print(in_b) +Line 2: import os + +Expr types in the line --->: + os => Module + +--- +Line 4: print(in_b) Expr types in the line --->: print => (function) print