From 891e054093c7d8980cc6c7fd2dce3778a893215c Mon Sep 17 00:00:00 2001 From: Glyphack Date: Mon, 13 Nov 2023 22:08:48 +0100 Subject: [PATCH] Add debug logs --- typechecker/src/build.rs | 86 +++++++++----------- typechecker/src/semantic_analyzer.rs | 1 + typechecker/src/settings.rs | 2 + typechecker/src/state.rs | 12 +++ typechecker/src/type_check/checker.rs | 25 ++---- typechecker/src/type_check/type_evaluator.rs | 41 ++++------ 6 files changed, 76 insertions(+), 91 deletions(-) diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs index 3e47b543..d6a9284b 100644 --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -39,6 +39,10 @@ impl BuildManager { builder.filter(None, log::LevelFilter::Warn); } + log::debug!("Initialized build manager"); + log::debug!("build sources: {:?}", sources); + log::debug!("options: {:?}", options); + BuildManager { errors: vec![], build_sources: sources, @@ -78,14 +82,14 @@ impl BuildManager { fn pre_analysis(&mut self) { let execution_environment = &execution_environment::ExecutionEnvironment { root: self.options.root.clone(), - python_version: ruff_python_resolver::python_version::PythonVersion::Py311, - python_platform: ruff_python_resolver::python_platform::PythonPlatform::Linux, + python_version: ruff_python_resolver::python_version::PythonVersion::Py312, + python_platform: ruff_python_resolver::python_platform::PythonPlatform::Darwin, // Adding a blank path to the extra paths is a hack to make the resolver work extra_paths: vec![PathBuf::from("")], }; let import_config = &Config { - typeshed_path: None, + typeshed_path: self.options.import_discovery.typeshed_path.clone(), stub_path: None, venv_path: Some(self.options.root.clone()), venv: None, @@ -119,8 +123,15 @@ impl BuildManager { } // Performs type checking passes over the code + // This step hapens after the binding phase pub fn type_check(&mut self) { self.build(); + // TODO: This is a hack to get all the symbol tables so we can resolve imports + let mut all_symbol_tables = Vec::new(); + for module in self.modules.values() { + all_symbol_tables.push(module.get_symbol_table()); + } + for state in self.modules.iter_mut() { if !state.1.file.errors.is_empty() { for err in state.1.file.errors.iter() { @@ -151,7 +162,7 @@ impl BuildManager { } } } - let mut checker = TypeChecker::new(state.1, &self.options); + let mut checker = TypeChecker::new(state.1, &self.options, all_symbol_tables.clone()); for stmt in &state.1.file.body { checker.type_check(stmt); } @@ -177,10 +188,25 @@ impl BuildManager { } fn gather_files(&self, current_files: Vec<&State>, add_indirect_imports: bool) -> Vec { + let execution_environment = &execution_environment::ExecutionEnvironment { + root: self.options.root.clone(), + python_version: ruff_python_resolver::python_version::PythonVersion::Py312, + python_platform: ruff_python_resolver::python_platform::PythonPlatform::Darwin, + // Adding a blank path to the extra paths is a hack to make the resolver work + extra_paths: vec![PathBuf::from("")], + }; + let import_config = &Config { + typeshed_path: self.options.import_discovery.typeshed_path.clone(), + stub_path: None, + venv_path: Some(self.options.root.clone()), + venv: None, + }; + + log::debug!("import options: {:?}", execution_environment); let mut new_imports = vec![]; let mut discovered_files = vec![]; for state in current_files { - let resolved_imports = self.resolve_file_imports(state); + let resolved_imports = self.resolve_file_imports(state, execution_environment, import_config); // check if the resolved_imports are not in the current files and add them to the new imports for (_, state) in resolved_imports { if !self.modules.contains_key(&state.file.module_name()) { @@ -194,11 +220,12 @@ impl BuildManager { } discovered_files.extend(new_imports.clone()); + while !new_imports.is_empty() { let mut next_imports = vec![]; for state in new_imports { - let resolved_imports = self.resolve_file_imports(&state); + let resolved_imports = self.resolve_file_imports(&state, execution_environment, import_config); // check if the resolved_imports are not in the current files and add them to the new imports for (_, state) in resolved_imports { if !self.modules.contains_key(&state.file.module_name()) { @@ -221,22 +248,12 @@ impl BuildManager { // Resolves imports in a file and return the resolved paths // TODO: This function is doing duplicate work because we resolve the imports in the State // module as well. We should refactor this and possibly only do it in the State module - fn resolve_file_imports(&self, state: &State) -> HashMap { - let execution_environment = &execution_environment::ExecutionEnvironment { - root: self.options.root.clone(), - python_version: ruff_python_resolver::python_version::PythonVersion::Py311, - python_platform: ruff_python_resolver::python_platform::PythonPlatform::Linux, - // Adding a blank path to the extra paths is a hack to make the resolver work - extra_paths: vec![PathBuf::from("")], - }; - log::debug!("import options: {:?}", execution_environment); - - let import_config = &Config { - typeshed_path: None, - stub_path: None, - venv_path: Some(self.options.root.clone()), - venv: None, - }; + fn resolve_file_imports( + &self, + state: &State, + execution_environment: &execution_environment::ExecutionEnvironment, + import_config: &Config + ) -> HashMap { let host = &ruff_python_resolver::host::StaticHost::new(vec![]); let mut resolved_paths = HashMap::new(); let mut resolved_imports = vec![]; @@ -278,8 +295,6 @@ impl BuildManager { } }; - log::debug!("import descriptions: {:?}", import_descriptions); - for import_desc in import_descriptions { let mut resolved = resolver::resolve_import( state.file.path().as_path(), @@ -288,10 +303,6 @@ impl BuildManager { import_config, host, ); - if !resolved.is_import_found { - let error = format!("cannot import name '{}'", import_desc.name()); - log::warn!("{}", error); - } if resolved.is_import_found { for resolved_path in resolved.resolved_paths.iter() { let source = match std::fs::read_to_string(resolved_path) { @@ -305,12 +316,6 @@ impl BuildManager { resolved_imports.push(build_source); } - // log what the import resolved to - log::debug!( - "resolved import: {} -> {:?}", - import_desc.name(), - resolved.resolved_paths - ); for (name, implicit_import) in resolved.implicit_imports.iter() { let source = std::fs::read_to_string(implicit_import.path.clone()).unwrap(); @@ -332,19 +337,6 @@ impl BuildManager { } } -fn get_line_number_of_character_position(source: &str, pos: usize) -> usize { - let mut line_number = 1; - for (i, c) in source.chars().enumerate() { - if i == pos { - break; - } - if c == '\n' { - line_number += 1; - } - } - line_number -} - #[cfg(test)] mod tests { use std::fs; diff --git a/typechecker/src/semantic_analyzer.rs b/typechecker/src/semantic_analyzer.rs index 9a4d6ad3..8cf5a97d 100644 --- a/typechecker/src/semantic_analyzer.rs +++ b/typechecker/src/semantic_analyzer.rs @@ -39,6 +39,7 @@ pub struct SemanticAnalyzer { impl SemanticAnalyzer { pub fn new(file: EnderpyFile, imports: HashMap) -> Self { let globals = SymbolTable::global(); + log::debug!("Creating semantic analyzer for {}", file.module_name()); SemanticAnalyzer { globals, file, diff --git a/typechecker/src/settings.rs b/typechecker/src/settings.rs index 2afa219c..15636fda 100644 --- a/typechecker/src/settings.rs +++ b/typechecker/src/settings.rs @@ -6,6 +6,7 @@ use std::{env, path::PathBuf}; #[allow(unused)] pub struct ImportDiscovery { pub python_executable: Option, + pub typeshed_path: Option, } #[derive(Debug, Deserialize)] @@ -45,6 +46,7 @@ impl Settings { follow_imports: FollowImports::All, import_discovery: ImportDiscovery { python_executable: None, + typeshed_path: None, }, } } diff --git a/typechecker/src/state.rs b/typechecker/src/state.rs index e052304f..928aad61 100644 --- a/typechecker/src/state.rs +++ b/typechecker/src/state.rs @@ -50,6 +50,7 @@ impl State { import_config: &ruff_python_resolver::config::Config, host: &ruff_python_resolver::host::StaticHost, ) { + log::debug!("resolving imports for file: {}", self.file.module_name()); for import in self.file.imports.iter() { let import_descriptions = match import { crate::nodes::ImportKinds::Import(i) => i @@ -63,6 +64,7 @@ impl State { vec![ruff_python_resolver::module_descriptor::ImportModuleDescriptor::from(i)] } }; + log::debug!("import descriptions: {:?}", import_descriptions); for import_desc in import_descriptions { let resolved = resolver::resolve_import( @@ -72,6 +74,16 @@ impl State { import_config, host, ); + if !resolved.is_import_found { + let error = format!("cannot import name '{}'", import_desc.name()); + log::warn!("{}", error); + continue; + } + log::debug!( + "resolved import: {} -> {:?}", + import_desc.name(), + resolved.resolved_paths + ); self.imports.insert(import_desc.name(), resolved); } } diff --git a/typechecker/src/type_check/checker.rs b/typechecker/src/type_check/checker.rs index 9b2969b4..5be77afb 100644 --- a/typechecker/src/type_check/checker.rs +++ b/typechecker/src/type_check/checker.rs @@ -1,9 +1,11 @@ +use std::collections::HashMap; + use ast::{Expression, Statement}; use enderpy_python_parser as parser; use enderpy_python_parser::ast::{self, *}; use crate::diagnostic::CharacterSpan; -use crate::symbol_table::LookupSymbolRequest; +use crate::symbol_table::{LookupSymbolRequest, self}; use crate::{ ast_visitor::TraversalVisitor, settings::Settings, state::State, symbol_table::SymbolTable, }; @@ -12,8 +14,6 @@ use super::{type_evaluator::TypeEvaluator, types::PythonType}; pub struct TypeChecker<'a> { pub errors: Vec, - // The symbol table of the module being type checked - symbol_table: SymbolTable, pub options: &'a Settings, type_evaluator: TypeEvaluator, } @@ -26,13 +26,12 @@ pub struct TypeCheckError { #[allow(unused)] impl<'a> TypeChecker<'a> { - pub fn new(module: &'a State, options: &'a Settings) -> Self { + pub fn new(module: &'a State, options: &'a Settings, symbol_tables: Vec) -> Self { let symbol_table = module.get_symbol_table(); TypeChecker { errors: vec![], - symbol_table, options, - type_evaluator: TypeEvaluator::new(module.get_symbol_table()), + type_evaluator: TypeEvaluator { symbol_table: symbol_table.clone(), imported_symbol_tables: symbol_tables.clone() }, } } @@ -489,19 +488,7 @@ impl<'a> TraversalVisitor for TypeChecker<'a> { for target in &_a.targets { match target { ast::Expression::Name(n) => { - let lookup_request = LookupSymbolRequest { - name: n.id.clone(), - position: None, - }; - let symbol = self.symbol_table.lookup_in_scope(lookup_request); - if let Some(symbol) = symbol { - let prev_target_type = self - .type_evaluator - .get_symbol_node_type(symbol, Some(n.node.start)) - .unwrap_or(PythonType::Unknown); - let value_type = self.infer_expr_type(&_a.value, true); - // TODO: Check reassignment - } + // TODO: Check reassignment } _ => {} } diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index 9744ce40..99acf22d 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -1,14 +1,12 @@ #![allow(dead_code)] #![allow(unused_variables)] -use std::collections::HashMap; - -use enderpy_python_parser as parser; -use enderpy_python_parser::ast; -use log::debug; -use miette::{bail, miette, Result}; -use parser::ast::{Expression, GetNode, Statement}; - +use super::types::LiteralValue; +use super::{ + builtins, + types::{CallableType, PythonType}, +}; +use crate::type_check::types::ClassType; use crate::{ ast_visitor::TraversalVisitor, ast_visitor_generic::TraversalVisitorImmutGeneric, @@ -16,17 +14,13 @@ use crate::{ state::State, symbol_table::{self, Declaration, LookupSymbolRequest, SymbolTable, SymbolTableNode}, }; - -use super::{ - builtins, - types::{CallableType, PythonType}, -}; - use core::panic; - -use crate::type_check::types::ClassType; - -use super::types::LiteralValue; +use enderpy_python_parser as parser; +use enderpy_python_parser::ast; +use log::debug; +use miette::{bail, miette, Result}; +use parser::ast::{Expression, GetNode, Statement}; +use std::collections::HashMap; const LITERAL_TYPE_PARAMETER_MSG: &str = "Type arguments for 'Literal' must be None, a literal value (int, bool, str, or bytes), or an enum value"; // TODO: this is not the right message there are other types like Dict that are allowed as parameters @@ -35,6 +29,7 @@ const UNION_TYPE_PARAMETER_MSG: &str = "Type arguments for 'Union' must be names pub struct TypeEvaluator { // TODO: make this a reference to the symbol table in the checker pub symbol_table: SymbolTable, + pub imported_symbol_tables: Vec, } pub struct TypeEvalError { @@ -44,10 +39,6 @@ pub struct TypeEvalError { /// Struct for evaluating the type of an expression impl TypeEvaluator { - pub fn new(symbol_table: SymbolTable) -> Self { - Self { symbol_table } - } - /// Get the type of a symbol node based on declarations pub fn get_symbol_node_type( &self, @@ -179,7 +170,7 @@ impl TypeEvaluator { todo!() } ast::Expression::Starred(s) => Ok(PythonType::Unknown), - ast::Expression::Generator(g) => { + ast::Expression::Generator(g) => {typeeval // This is not correct // let mut comp_targets: HashMap = HashMap::new(); // for gens in &g.generators { @@ -1050,7 +1041,7 @@ mod tests { module.populate_symbol_table(); let symbol_table = module.get_symbol_table(); - let type_eval = TypeEvaluator::new(symbol_table); + let type_eval = TypeEvaluator{ symbol_table, imported_symbol_tables: vec![] }; let mut type_eval_visitor = TypeEvalVisitor::new(module.file); type_eval_visitor.visit_module(); @@ -1099,7 +1090,7 @@ impl TypeEvalVisitor { let symbol_table = state.get_symbol_table(); Self { types: HashMap::new(), - type_eval: TypeEvaluator::new(symbol_table), + type_eval: TypeEvaluator { symbol_table, imported_symbol_tables: vec![] }, state, } }