From f48df1e9240c485c7b371672b8e360349034bffe Mon Sep 17 00:00:00 2001 From: Glyphack Date: Wed, 31 Jul 2024 23:53:38 +0200 Subject: [PATCH] feat: distinguish between logical and physical lines --- enderpy/src/cli.rs | 7 +- enderpy/src/main.rs | 21 ++++-- parser/src/lexer/compat.rs | 71 +++++++++--------- parser/src/lexer/mod.rs | 21 ++++-- parser/src/parser/parser.rs | 13 ++-- parser/src/token.rs | 2 + ...ts__snapshot_test_lexer@indentation-3.snap | 2 +- ...ts__snapshot_test_lexer@indentation-4.snap | 4 +- ...t_lexer@newline-in-nested-structure-0.snap | 4 +- ...er_and_errors@annotated_assignment.py.snap | 22 +++--- ...ot_test_lexer_and_errors@binary_op.py.snap | 26 +++---- ...apshot_test_lexer_and_errors@class.py.snap | 42 +++++------ ...hot_test_lexer_and_errors@comments.py.snap | 6 +- ...napshot_test_lexer_and_errors@dict.py.snap | 44 ++++++------ ...snapshot_test_lexer_and_errors@for.py.snap | 12 ++-- ...test_lexer_and_errors@function_def.py.snap | 72 +++++++++---------- ...ot_test_lexer_and_errors@functions.py.snap | 10 +-- ...r_and_errors@generator_expressions.py.snap | 16 ++--- ..._snapshot_test_lexer_and_errors@if.py.snap | 18 ++--- ...apshot_test_lexer_and_errors@lists.py.snap | 38 +++++----- ...apshot_test_lexer_and_errors@match.py.snap | 14 ++-- ...hot_test_lexer_and_errors@newlines.py.snap | 32 ++++----- ...xer_and_errors@separate_statements.py.snap | 6 +- ...napshot_test_lexer_and_errors@sets.py.snap | 46 ++++++------ ...pshot_test_lexer_and_errors@string.py.snap | 16 ++--- ...ot_test_lexer_and_errors@subscript.py.snap | 16 ++--- ...snapshot_test_lexer_and_errors@try.py.snap | 2 +- ...t_test_lexer_and_errors@type_alias.py.snap | 2 +- ...napshot_test_lexer_and_errors@with.py.snap | 16 ++--- 29 files changed, 308 insertions(+), 293 deletions(-) diff --git a/enderpy/src/cli.rs b/enderpy/src/cli.rs index e4dd8c3e..02267048 100644 --- a/enderpy/src/cli.rs +++ b/enderpy/src/cli.rs @@ -8,15 +8,14 @@ use clap::{Parser, Subcommand}; pub struct Cli { #[command(subcommand)] pub command: Commands, + #[arg(short, long)] + pub file: Option, } #[derive(Subcommand)] pub enum Commands { /// Print lexer tokens - Tokenize { - /// Path to source file - file: PathBuf, - }, + Tokenize {}, /// Print abstract syntax tree Parse { /// Path to source file diff --git a/enderpy/src/main.rs b/enderpy/src/main.rs index 91bc767e..a8cfc18a 100644 --- a/enderpy/src/main.rs +++ b/enderpy/src/main.rs @@ -1,5 +1,6 @@ use std::{ - fs, io, + fs::{self, File}, + io::{self, Read}, path::{Path, PathBuf}, }; @@ -14,7 +15,7 @@ mod cli; fn main() -> Result<()> { let cli = Cli::parse(); match &cli.command { - Commands::Tokenize { file } => tokenize(file), + Commands::Tokenize {} => tokenize(), Commands::Parse { file } => parse(file), Commands::Check { path } => check(path), Commands::Watch => watch(), @@ -70,8 +71,20 @@ fn get_typeshed_path() -> Result { Ok(path.join("typeshed")) } -fn tokenize(file: &PathBuf) -> Result<()> { - let source = fs::read_to_string(file).into_diagnostic()?; +fn tokenize() -> Result<()> { + let cli = Cli::parse(); + let mut source = String::new(); + match cli.file { + Some(path) => { + File::open(path) + .into_diagnostic()? + .read_to_string(&mut source) + .into_diagnostic()?; + } + None => { + io::stdin().read_to_string(&mut source).into_diagnostic()?; + } + } let mut lexer = Lexer::new(&source); let tokens = lexer.lex(); for token in tokens { diff --git a/parser/src/lexer/compat.rs b/parser/src/lexer/compat.rs index da5ef146..8885e03f 100644 --- a/parser/src/lexer/compat.rs +++ b/parser/src/lexer/compat.rs @@ -98,29 +98,10 @@ pub struct PythonToken { end: (u32, u32), } -fn lex_python_source(source: &str) -> Result> { - let mut process = spawn_python_script_command( - "parser/lex_python.py", - vec!["--stdin", "--output-format", "json"], - default_python_path()?, - )?; - - // Get process stdin and write the input string. - if let Some(mut stdin) = process.stdin.take() { - stdin.write_all(source.as_bytes()).into_diagnostic()?; - } else { - bail!("Failed to open stdin when running `parser/lex_python.py`"); - } - // Get process stdout and parse result. - let output = process.wait_with_output().into_diagnostic()?; - let python_tokens: Vec = - serde_json::from_str(String::from_utf8_lossy(&output.stdout).as_ref()).into_diagnostic()?; - Ok(python_tokens) -} - #[cfg(test)] mod tests { - use super::{lex_python_source, PythonKind, PythonToken}; + + use super::*; use crate::token::Kind; use crate::{lexer::Lexer, token::Token}; use tabled::{ @@ -130,6 +111,27 @@ mod tests { }; use terminal_size::{terminal_size, Width as TerminalWidth}; + fn lex_python_source(source: &str) -> Result> { + let mut process = spawn_python_script_command( + "parser/lex_python.py", + vec!["--stdin", "--output-format", "json"], + default_python_path()?, + )?; + + // Get process stdin and write the input string. + if let Some(mut stdin) = process.stdin.take() { + stdin.write_all(source.as_bytes()).into_diagnostic()?; + } else { + bail!("Failed to open stdin when running `parser/lex_python.py`"); + } + // Get process stdout and parse result. + let output = process.wait_with_output().into_diagnostic()?; + let python_tokens: Vec = + serde_json::from_str(String::from_utf8_lossy(&output.stdout).as_ref()) + .into_diagnostic()?; + Ok(python_tokens) + } + #[test] fn test_simple_compat() { let source = r#" @@ -168,8 +170,6 @@ print(a) ";", "@", "=", - // TODO lex_python: Python lexer chokes on single backslash. - // "\\", "#", "$", "?", @@ -317,18 +317,12 @@ print(a) python_tokenize_test_lexer(&["import a", "import a.b", "import a.b.c", "import a from b"]); } - // TODO lex_python: Decide whether to keep this test or not. The Python lexer + Enderpy lexer - // handle newlines in a nested context slightly differently. - // - Python increments the row counter. - // - Enderpy appends them to the original row. - // #[test] - // fn test_lex_other() { - // python_tokenize_test_lexer( - // &["(a, - // - // )"], - // ); - // } + #[test] + fn test_lex_other() { + python_tokenize_test_lexer(&["(a, + + )"]); + } #[test] fn test_lex_indentation() { @@ -672,9 +666,7 @@ def", PythonKind::FstringMiddle => enderpy_token.kind == Kind::FStringMiddle, PythonKind::FstringEnd => enderpy_token.kind == Kind::FStringEnd, PythonKind::Comment => enderpy_token.kind == Kind::Comment, - // In Python, this represents a line break within a single statement. We don't - // currently make this distinction. - PythonKind::NL => enderpy_token.kind == Kind::NewLine, + PythonKind::NL => enderpy_token.kind == Kind::NL, PythonKind::ErrorToken => { match python_token.value.as_str() { // Python 3.11 chokes on these tokens. @@ -701,7 +693,8 @@ def", || matches_python_op_token(python_token.value.as_str(), &enderpy_token.kind) || matches_python_indent_dedent_token(&python_token.kind, &enderpy_token.kind) || (python_token.kind == PythonKind::EndMarker && enderpy_token.kind == Kind::Eof) - || (python_token.value.as_str() == "\n" && enderpy_token.kind == Kind::NewLine) + || (python_token.value.as_str() == "\n" + && (matches!(enderpy_token.kind, Kind::NewLine | Kind::NL))) || python_token_value == enderpy_token_value; if !value_matches { return Some(TokenMismatch::WrongValue( diff --git a/parser/src/lexer/mod.rs b/parser/src/lexer/mod.rs index ed982c15..7ea7f2d7 100644 --- a/parser/src/lexer/mod.rs +++ b/parser/src/lexer/mod.rs @@ -61,6 +61,7 @@ pub struct Lexer<'a> { /// Array of all line starts offsets. Starts from line 0 pub line_starts: Vec, peak_mode: bool, + prev_token_newline: bool, } impl<'a> Lexer<'a> { @@ -76,6 +77,7 @@ impl<'a> Lexer<'a> { next_token_is_dedent: 0, line_starts: vec![], peak_mode: false, + prev_token_newline: false, } } @@ -125,6 +127,7 @@ impl<'a> Lexer<'a> { return self.next_token(); } + self.prev_token_newline = kind == Kind::NewLine; let value = self.parse_token_value(kind, start); let end = self.current; @@ -143,8 +146,10 @@ impl<'a> Lexer<'a> { let nesting = self.nesting; let start_of_line = self.start_of_line; let next_token_is_dedent = self.next_token_is_dedent; + let prev_token_newline = self.prev_token_newline; self.peak_mode = true; let token = self.next_token(); + self.prev_token_newline = prev_token_newline; self.peak_mode = false; self.current = current; self.current_line = current_line; @@ -255,11 +260,13 @@ impl<'a> Lexer<'a> { } fn next_kind(&mut self) -> Result { - if self.start_of_line && self.nesting == 0 { + if self.start_of_line { self.line_starts.push(self.current); - if let Some(indent_kind) = self.match_indentation()? { - self.start_of_line = false; // WHY!? - return Ok(indent_kind); + if self.nesting == 0 { + if let Some(indent_kind) = self.match_indentation()? { + self.start_of_line = false; // WHY!? + return Ok(indent_kind); + } } } @@ -510,7 +517,11 @@ impl<'a> Lexer<'a> { '\n' | '\r' => { self.current_line += 1; self.start_of_line = true; - return Ok(Kind::NewLine); + if self.nesting == 0 && !self.prev_token_newline { + return Ok(Kind::NewLine); + } else { + return Ok(Kind::NL); + } } c if match_whitespace(c) => return Ok(Kind::WhiteSpace), _ => {} diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index f40f1467..00c9c32a 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -176,12 +176,6 @@ impl<'a> Parser<'a> { } } - fn advance_to_next_line_or_semicolon(&mut self) { - while !self.eat(Kind::NewLine) && !self.eat(Kind::SemiColon) && !self.at(Kind::Eof) { - self.advance(); - } - } - /// Expect a `Kind` or return error pub fn expect(&mut self, kind: Kind) -> Result<(), ParsingError> { if !self.at(kind) { @@ -330,7 +324,10 @@ impl<'a> Parser<'a> { ) { while self.eat(Kind::WhiteSpace) || self.eat(Kind::Comment) {} - if !matches!(self.cur_kind(), Kind::NewLine | Kind::SemiColon | Kind::Eof) { + if !matches!( + self.cur_kind(), + Kind::NewLine | Kind::NL | Kind::SemiColon | Kind::Eof + ) { panic!("Statement does not end in new line or semicolon {:?}", stmt); } } @@ -1974,7 +1971,7 @@ impl<'a> Parser<'a> { fn consume_whitespace_and_newline(&mut self) -> bool { let mut consumed = false; - while matches!(self.cur_kind(), Kind::WhiteSpace | Kind::NewLine) { + while matches!(self.cur_kind(), Kind::WhiteSpace | Kind::NewLine | Kind::NL) { self.advance(); consumed = true; } diff --git a/parser/src/token.rs b/parser/src/token.rs index d5b92e05..264f1f3e 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -26,6 +26,7 @@ impl Display for Token { pub enum Kind { // Line structure NewLine, // \n + NL, // Logical newline Indent, // \t Dedent, // \t @@ -313,6 +314,7 @@ impl From for &str { Kind::Less => "<", Kind::Greater => ">", Kind::NewLine => "NewLine", + Kind::NL => "NL", Kind::Identifier => "Identifier", Kind::False => "False", Kind::None => "None", diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap index 6fe36bc9..d5280ee6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap @@ -14,7 +14,7 @@ description: "def f(x):\n y = z\n\n print(y)\n" 16,17: = (None) 18,19: Identifier (Str("z")) 19,20: NewLine (None) -20,21: NewLine (None) +20,21: NL (None) 25,30: Identifier (Str("print")) 30,31: ( (None) 31,32: Identifier (Str("y")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap index aa2a21b2..906225c1 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap @@ -6,13 +6,13 @@ description: "if a:\n\n f = c\n\n # Path: test_local.py\n" 3,4: Identifier (Str("a")) 4,5: : (None) 5,6: NewLine (None) -6,7: NewLine (None) +6,7: NL (None) 7,11: Indent (Indent(1)) 11,12: Identifier (Str("f")) 13,14: = (None) 15,16: Identifier (Str("c")) 16,17: NewLine (None) -17,18: NewLine (None) +17,18: NL (None) 22,43: Comment (Str("# Path: test_local.py")) 43,44: NewLine (None) 44,44: Dedent (Indent(1)) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap index fc90fdff..328b6a19 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap @@ -5,6 +5,6 @@ description: "(a,\n\n)" 0,1: ( (None) 1,2: Identifier (Str("a")) 2,3: , (None) -3,4: NewLine (None) -4,5: NewLine (None) +3,4: NL (None) +4,5: NL (None) 5,6: ) (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap index bd890150..d168885f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap @@ -9,7 +9,7 @@ input_file: parser/test_data/inputs/annotated_assignment.py 7,8: = (None) 9,10: Integer (Number("1")) 10,11: NewLine (None) -11,12: NewLine (None) +11,12: NL (None) 12,13: Identifier (Str("b")) 13,14: : (None) 15,20: Identifier (Str("tuple")) @@ -23,7 +23,7 @@ input_file: parser/test_data/inputs/annotated_assignment.py 34,35: , (None) 36,37: Integer (Number("2")) 37,38: NewLine (None) -38,39: NewLine (None) +38,39: NL (None) 39,40: Identifier (Str("c")) 40,41: : (None) 42,47: Identifier (Str("tuple")) @@ -38,7 +38,7 @@ input_file: parser/test_data/inputs/annotated_assignment.py 65,66: Integer (Number("2")) 66,67: , (None) 68,69: NewLine (None) -69,70: NewLine (None) +69,70: NL (None) 70,71: Identifier (Str("d")) 71,72: : (None) 73,78: Identifier (Str("tuple")) @@ -50,7 +50,7 @@ input_file: parser/test_data/inputs/annotated_assignment.py 89,90: = (None) 91,92: Identifier (Str("b")) 92,93: NewLine (None) -93,94: NewLine (None) +93,94: NL (None) 94,95: Identifier (Str("e")) 95,96: : (None) 97,102: Identifier (Str("tuple")) @@ -60,7 +60,7 @@ input_file: parser/test_data/inputs/annotated_assignment.py 108,111: Identifier (Str("int")) 111,112: ] (None) 112,113: NewLine (None) -113,114: NewLine (None) +113,114: NL (None) 114,115: Identifier (Str("f")) 115,116: : (None) 117,122: Identifier (Str("tuple")) @@ -72,14 +72,14 @@ input_file: parser/test_data/inputs/annotated_assignment.py 133,134: = (None) 135,136: Identifier (Str("e")) 136,137: NewLine (None) -137,138: NewLine (None) +137,138: NL (None) 138,139: Identifier (Str("g")) 139,140: : (None) 141,146: Identifier (Str("tuple")) 146,147: [ (None) 147,150: Identifier (Str("int")) 150,151: , (None) -151,152: NewLine (None) +151,152: NL (None) 161,164: Identifier (Str("int")) 164,165: ] (None) 166,167: = (None) @@ -87,7 +87,7 @@ input_file: parser/test_data/inputs/annotated_assignment.py 169,170: , (None) 171,172: Integer (Number("2")) 172,173: NewLine (None) -173,174: NewLine (None) +173,174: NL (None) 174,175: ( (None) 175,176: Identifier (Str("h")) 176,177: ) (None) @@ -96,11 +96,11 @@ input_file: parser/test_data/inputs/annotated_assignment.py 183,184: = (None) 185,186: Integer (Number("1")) 186,187: NewLine (None) -187,188: NewLine (None) +187,188: NL (None) 188,189: ( (None) -189,190: NewLine (None) +189,190: NL (None) 194,195: Identifier (Str("i")) -195,196: NewLine (None) +195,196: NL (None) 196,197: ) (None) 197,198: : (None) 199,202: Identifier (Str("int")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap index d731b1b7..5a3ef973 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap @@ -7,69 +7,69 @@ input_file: parser/test_data/inputs/binary_op.py 2,3: + (None) 4,5: Integer (Number("2")) 5,6: NewLine (None) -6,7: NewLine (None) +6,7: NL (None) 7,8: Integer (Number("1")) 9,10: - (None) 11,12: Integer (Number("2")) 12,13: NewLine (None) -13,14: NewLine (None) +13,14: NL (None) 14,15: Integer (Number("1")) 16,17: * (None) 18,19: Integer (Number("2")) 19,20: NewLine (None) -20,21: NewLine (None) +20,21: NL (None) 21,22: Integer (Number("1")) 23,24: / (None) 25,26: Integer (Number("2")) 26,27: NewLine (None) -27,28: NewLine (None) +27,28: NL (None) 28,29: Integer (Number("1")) 30,32: // (None) 33,34: Integer (Number("2")) 34,35: NewLine (None) -35,36: NewLine (None) +35,36: NL (None) 36,37: Integer (Number("1")) 38,39: % (None) 40,41: Integer (Number("2")) 41,42: NewLine (None) -42,43: NewLine (None) +42,43: NL (None) 43,44: Integer (Number("1")) 45,47: ** (None) 48,49: Integer (Number("2")) 49,50: NewLine (None) -50,51: NewLine (None) +50,51: NL (None) 51,52: Integer (Number("1")) 53,55: << (None) 56,57: Integer (Number("2")) 57,58: NewLine (None) -58,59: NewLine (None) +58,59: NL (None) 59,60: Integer (Number("1")) 61,63: >> (None) 64,65: Integer (Number("2")) 65,66: NewLine (None) -66,67: NewLine (None) +66,67: NL (None) 67,68: Integer (Number("1")) 69,70: & (None) 71,72: Integer (Number("2")) 72,73: NewLine (None) -73,74: NewLine (None) +73,74: NL (None) 74,75: Integer (Number("1")) 76,77: ^ (None) 78,79: Integer (Number("2")) 79,80: NewLine (None) -80,81: NewLine (None) +80,81: NL (None) 81,82: Integer (Number("1")) 83,84: | (None) 85,86: Integer (Number("2")) 86,87: NewLine (None) -87,88: NewLine (None) +87,88: NL (None) 88,89: Integer (Number("1")) 90,91: | (None) 92,93: Integer (Number("2")) 94,95: | (None) 96,97: Integer (Number("3")) 97,98: NewLine (None) -98,99: NewLine (None) +98,99: NL (None) 99,100: Integer (Number("1")) 101,102: @ (None) 103,104: Integer (Number("2")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap index 9862c37c..08b177c3 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap @@ -8,7 +8,7 @@ input_file: parser/test_data/inputs/class.py 7,8: : (None) 9,13: Pass (None) 13,14: NewLine (None) -14,15: NewLine (None) +14,15: NL (None) 15,20: Class (None) 21,22: Identifier (Str("a")) 22,23: ( (None) @@ -18,7 +18,7 @@ input_file: parser/test_data/inputs/class.py 26,30: Indent (Indent(1)) 30,34: Pass (None) 34,35: NewLine (None) -35,36: NewLine (None) +35,36: NL (None) 36,36: Dedent (Indent(1)) 36,41: Class (None) 42,43: Identifier (Str("a")) @@ -30,7 +30,7 @@ input_file: parser/test_data/inputs/class.py 49,50: : (None) 51,55: Pass (None) 55,56: NewLine (None) -56,57: NewLine (None) +56,57: NL (None) 57,62: Class (None) 63,64: Identifier (Str("a")) 64,65: ( (None) @@ -45,22 +45,22 @@ input_file: parser/test_data/inputs/class.py 76,77: : (None) 78,82: Pass (None) 82,83: NewLine (None) -83,84: NewLine (None) +83,84: NL (None) 84,89: Class (None) 90,91: Identifier (Str("a")) 91,92: ( (None) 92,93: Identifier (Str("b")) 93,94: , (None) -94,95: NewLine (None) +94,95: NL (None) 95,96: Identifier (Str("c")) 96,97: , (None) -97,98: NewLine (None) +97,98: NL (None) 98,99: Identifier (Str("d")) 99,100: ) (None) 100,101: : (None) 102,106: Pass (None) 106,107: NewLine (None) -107,108: NewLine (None) +107,108: NL (None) 108,109: @ (None) 109,114: Identifier (Str("decor")) 114,115: NewLine (None) @@ -69,7 +69,7 @@ input_file: parser/test_data/inputs/class.py 122,123: : (None) 124,128: Pass (None) 128,129: NewLine (None) -129,130: NewLine (None) +129,130: NL (None) 130,135: Class (None) 136,137: Identifier (Str("a")) 137,138: [ (None) @@ -78,7 +78,7 @@ input_file: parser/test_data/inputs/class.py 140,141: : (None) 142,146: Pass (None) 146,147: NewLine (None) -147,148: NewLine (None) +147,148: NL (None) 148,153: Class (None) 154,155: Identifier (Str("a")) 155,156: [ (None) @@ -89,7 +89,7 @@ input_file: parser/test_data/inputs/class.py 161,162: : (None) 163,167: Pass (None) 167,168: NewLine (None) -168,169: NewLine (None) +168,169: NL (None) 169,174: Class (None) 175,176: Identifier (Str("a")) 176,177: [ (None) @@ -100,7 +100,7 @@ input_file: parser/test_data/inputs/class.py 182,183: : (None) 184,188: Pass (None) 188,189: NewLine (None) -189,190: NewLine (None) +189,190: NL (None) 190,195: Class (None) 196,197: Identifier (Str("a")) 197,198: [ (None) @@ -115,7 +115,7 @@ input_file: parser/test_data/inputs/class.py 209,210: : (None) 211,215: Pass (None) 215,216: NewLine (None) -216,217: NewLine (None) +216,217: NL (None) 217,222: Class (None) 223,224: Identifier (Str("a")) 224,225: [ (None) @@ -125,7 +125,7 @@ input_file: parser/test_data/inputs/class.py 228,229: : (None) 230,234: Pass (None) 234,235: NewLine (None) -235,236: NewLine (None) +235,236: NL (None) 236,241: Class (None) 242,243: Identifier (Str("a")) 243,244: [ (None) @@ -137,7 +137,7 @@ input_file: parser/test_data/inputs/class.py 250,251: : (None) 252,256: Pass (None) 256,257: NewLine (None) -257,258: NewLine (None) +257,258: NL (None) 258,263: Class (None) 264,265: Identifier (Str("a")) 265,266: [ (None) @@ -151,7 +151,7 @@ input_file: parser/test_data/inputs/class.py 275,276: : (None) 277,281: Pass (None) 281,282: NewLine (None) -282,283: NewLine (None) +282,283: NL (None) 283,288: Class (None) 289,290: Identifier (Str("a")) 290,291: [ (None) @@ -169,7 +169,7 @@ input_file: parser/test_data/inputs/class.py 306,307: : (None) 308,312: Pass (None) 312,313: NewLine (None) -313,314: NewLine (None) +313,314: NL (None) 314,319: Class (None) 320,321: Identifier (Str("a")) 321,322: [ (None) @@ -179,7 +179,7 @@ input_file: parser/test_data/inputs/class.py 326,327: : (None) 328,332: Pass (None) 332,333: NewLine (None) -333,334: NewLine (None) +333,334: NL (None) 334,339: Class (None) 340,341: Identifier (Str("a")) 341,342: [ (None) @@ -191,7 +191,7 @@ input_file: parser/test_data/inputs/class.py 349,350: : (None) 351,355: Pass (None) 355,356: NewLine (None) -356,357: NewLine (None) +356,357: NL (None) 357,362: Class (None) 363,364: Identifier (Str("a")) 364,365: [ (None) @@ -205,7 +205,7 @@ input_file: parser/test_data/inputs/class.py 375,376: : (None) 377,381: Pass (None) 381,382: NewLine (None) -382,383: NewLine (None) +382,383: NL (None) 383,388: Class (None) 389,390: Identifier (Str("a")) 390,391: [ (None) @@ -223,7 +223,7 @@ input_file: parser/test_data/inputs/class.py 407,408: : (None) 409,413: Pass (None) 413,414: NewLine (None) -414,415: NewLine (None) +414,415: NL (None) 415,420: Class (None) 421,422: Identifier (Str("a")) 422,423: [ (None) @@ -238,7 +238,7 @@ input_file: parser/test_data/inputs/class.py 434,435: : (None) 436,440: Pass (None) 440,441: NewLine (None) -441,442: NewLine (None) +441,442: NL (None) 442,447: Class (None) 448,449: Identifier (Str("a")) 449,450: [ (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap index f64dc515..b92983e1 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap @@ -7,7 +7,7 @@ input_file: parser/test_data/inputs/comments.py 3,4: NewLine (None) 4,7: Comment (Str("# a")) 7,8: NewLine (None) -8,9: NewLine (None) +8,9: NL (None) 9,12: Comment (Str("# a")) 12,13: NewLine (None) 13,16: Comment (Str("# b")) @@ -20,7 +20,7 @@ input_file: parser/test_data/inputs/comments.py 26,29: Ellipsis (None) 30,33: Comment (Str("# a")) 33,34: NewLine (None) -34,35: NewLine (None) +34,35: NL (None) 35,38: Def (None) 39,40: Identifier (Str("b")) 40,41: ( (None) @@ -33,7 +33,7 @@ input_file: parser/test_data/inputs/comments.py 56,59: Ellipsis (None) 60,63: Comment (Str("# a")) 63,64: NewLine (None) -64,65: NewLine (None) +64,65: NL (None) 65,65: Dedent (Indent(1)) 65,94: Comment (Str("# this is a comment only line")) 94,95: NewLine (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap index fad882e2..079e315f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap @@ -13,25 +13,25 @@ input_file: parser/test_data/inputs/dict.py 10,11: Identifier (Str("d")) 11,12: } (None) 12,13: NewLine (None) -13,14: NewLine (None) +13,14: NL (None) 14,15: { (None) 15,16: Identifier (Str("a")) 16,17: : (None) 18,19: Identifier (Str("b")) 19,20: , (None) -20,21: NewLine (None) +20,21: NL (None) 21,22: Identifier (Str("c")) 22,23: : (None) 24,25: Identifier (Str("d")) 25,26: } (None) 26,27: , (None) 27,28: NewLine (None) -28,29: NewLine (None) +28,29: NL (None) 29,30: { (None) 30,31: Identifier (Str("a")) 31,32: : (None) 33,34: Identifier (Str("b")) -34,35: NewLine (None) +34,35: NL (None) 35,36: , (None) 37,38: Identifier (Str("c")) 38,39: : (None) @@ -39,35 +39,35 @@ input_file: parser/test_data/inputs/dict.py 41,42: } (None) 42,43: , (None) 43,44: NewLine (None) -44,45: NewLine (None) +44,45: NL (None) 45,46: { (None) 46,47: Identifier (Str("a")) 47,48: : (None) 49,50: Identifier (Str("b")) 50,51: , (None) -51,52: NewLine (None) +51,52: NL (None) 52,53: Identifier (Str("c")) 53,54: : (None) 55,56: Identifier (Str("d")) 56,57: , (None) -57,58: NewLine (None) +57,58: NL (None) 62,63: Identifier (Str("e")) 63,64: : (None) 65,66: Identifier (Str("f")) 66,67: } (None) 67,68: , (None) 68,69: NewLine (None) -69,70: NewLine (None) +69,70: NL (None) 70,71: { (None) 71,72: Identifier (Str("a")) 72,73: : (None) 74,75: Identifier (Str("b")) 75,76: , (None) -76,77: NewLine (None) +76,77: NL (None) 77,78: } (None) 78,79: , (None) 79,80: NewLine (None) -80,81: NewLine (None) +80,81: NL (None) 81,82: { (None) 82,83: Identifier (Str("a")) 83,84: : (None) @@ -79,7 +79,7 @@ input_file: parser/test_data/inputs/dict.py 92,93: , (None) 93,94: } (None) 94,95: NewLine (None) -95,96: NewLine (None) +95,96: NL (None) 96,97: { (None) 97,98: Integer (Number("1")) 98,99: : (None) @@ -92,7 +92,7 @@ input_file: parser/test_data/inputs/dict.py 129,130: ) (None) 130,131: } (None) 131,132: NewLine (None) -132,133: NewLine (None) +132,133: NL (None) 133,134: { (None) 134,135: Integer (Number("1")) 135,136: : (None) @@ -103,46 +103,46 @@ input_file: parser/test_data/inputs/dict.py 156,165: Identifier (Str("get_names")) 165,166: ( (None) 166,167: ) (None) -167,168: NewLine (None) +167,168: NL (None) 168,169: } (None) 169,170: NewLine (None) -170,171: NewLine (None) +170,171: NL (None) 171,191: Identifier (Str("aggregated_key_stats")) 191,192: [ (None) 192,200: Identifier (Str("idx_stat")) 200,201: ] (None) 202,203: = (None) 204,205: { (None) -205,206: NewLine (None) +205,206: NL (None) 210,212: ** (None) 212,229: Identifier (Str("filter_value_NONE")) 229,230: , (None) -230,231: NewLine (None) +230,231: NL (None) 235,237: ** (None) 237,253: Identifier (Str("filter_key_found")) 253,254: , (None) -254,255: NewLine (None) +254,255: NL (None) 259,261: ** (None) 261,273: Identifier (Str("client_types")) 273,274: , (None) -274,275: NewLine (None) +274,275: NL (None) 279,281: ** (None) 281,293: Identifier (Str("market_watch")) 293,294: , (None) -294,295: NewLine (None) +294,295: NL (None) 299,307: StringLiteral (Str("\"symbol\"")) 307,308: : (None) 309,315: Identifier (Str("symbol")) 315,316: , (None) -316,317: NewLine (None) +316,317: NL (None) 321,327: StringLiteral (Str("\"name\"")) 327,328: : (None) 329,333: Identifier (Str("name")) 333,334: , (None) -334,335: NewLine (None) +334,335: NL (None) 335,336: } (None) 336,337: NewLine (None) -337,338: NewLine (None) +337,338: NL (None) 338,340: Identifier (Str("di")) 341,342: = (None) 343,344: { (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap index 5abbf760..503df88f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap @@ -12,7 +12,7 @@ input_file: parser/test_data/inputs/for.py 12,16: Indent (Indent(1)) 16,20: Pass (None) 20,21: NewLine (None) -21,22: NewLine (None) +21,22: NL (None) 22,22: Dedent (Indent(1)) 22,25: For (None) 26,27: Identifier (Str("a")) @@ -23,7 +23,7 @@ input_file: parser/test_data/inputs/for.py 34,38: Indent (Indent(1)) 38,42: Pass (None) 42,43: NewLine (None) -43,44: NewLine (None) +43,44: NL (None) 44,44: Dedent (Indent(1)) 44,47: For (None) 48,49: Identifier (Str("a")) @@ -48,7 +48,7 @@ input_file: parser/test_data/inputs/for.py 86,87: = (None) 88,89: Integer (Number("1")) 89,90: NewLine (None) -90,91: NewLine (None) +90,91: NL (None) 91,91: Dedent (Indent(1)) 91,94: For (None) 95,96: Identifier (Str("a")) @@ -69,7 +69,7 @@ input_file: parser/test_data/inputs/for.py 128,129: = (None) 130,131: Integer (Number("1")) 131,132: NewLine (None) -132,133: NewLine (None) +132,133: NL (None) 133,133: Dedent (Indent(1)) 133,138: Async (None) 139,142: For (None) @@ -81,7 +81,7 @@ input_file: parser/test_data/inputs/for.py 151,155: Indent (Indent(1)) 155,159: Pass (None) 159,160: NewLine (None) -160,161: NewLine (None) +160,161: NL (None) 161,161: Dedent (Indent(1)) 161,166: Async (None) 167,170: For (None) @@ -93,7 +93,7 @@ input_file: parser/test_data/inputs/for.py 179,183: Indent (Indent(1)) 183,187: Pass (None) 187,188: NewLine (None) -188,189: NewLine (None) +188,189: NL (None) 189,189: Dedent (Indent(1)) 189,192: For (None) 193,194: Identifier (Str("a")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap index dd547bbb..70d6f00e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap @@ -10,7 +10,7 @@ input_file: parser/test_data/inputs/function_def.py 7,8: : (None) 9,13: Pass (None) 13,14: NewLine (None) -14,15: NewLine (None) +14,15: NL (None) 15,18: Def (None) 19,20: Identifier (Str("a")) 20,21: ( (None) @@ -20,7 +20,7 @@ input_file: parser/test_data/inputs/function_def.py 24,28: Indent (Indent(1)) 28,32: Pass (None) 32,33: NewLine (None) -33,34: NewLine (None) +33,34: NL (None) 34,34: Dedent (Indent(1)) 34,37: Def (None) 38,39: Identifier (Str("a")) @@ -34,7 +34,7 @@ input_file: parser/test_data/inputs/function_def.py 48,49: : (None) 50,54: Pass (None) 54,55: NewLine (None) -55,56: NewLine (None) +55,56: NL (None) 56,59: Def (None) 60,61: Identifier (Str("a")) 61,62: ( (None) @@ -49,22 +49,22 @@ input_file: parser/test_data/inputs/function_def.py 73,74: : (None) 75,79: Pass (None) 79,80: NewLine (None) -80,81: NewLine (None) +80,81: NL (None) 81,84: Def (None) 85,86: Identifier (Str("a")) 86,87: ( (None) 87,88: Identifier (Str("a")) 88,89: , (None) -89,90: NewLine (None) +89,90: NL (None) 90,91: Identifier (Str("b")) 91,92: , (None) -92,93: NewLine (None) +92,93: NL (None) 93,94: Identifier (Str("c")) 94,95: ) (None) 95,96: : (None) 97,101: Pass (None) 101,102: NewLine (None) -102,103: NewLine (None) +102,103: NL (None) 103,104: @ (None) 104,109: Identifier (Str("decor")) 109,110: NewLine (None) @@ -75,7 +75,7 @@ input_file: parser/test_data/inputs/function_def.py 117,118: : (None) 119,123: Pass (None) 123,124: NewLine (None) -124,125: NewLine (None) +124,125: NL (None) 125,126: @ (None) 126,131: Identifier (Str("decor")) 131,132: NewLine (None) @@ -109,7 +109,7 @@ input_file: parser/test_data/inputs/function_def.py 181,182: : (None) 183,187: Pass (None) 187,188: NewLine (None) -188,189: NewLine (None) +188,189: NL (None) 189,192: Def (None) 193,197: Identifier (Str("func")) 197,198: ( (None) @@ -119,7 +119,7 @@ input_file: parser/test_data/inputs/function_def.py 207,208: : (None) 209,213: Pass (None) 213,214: NewLine (None) -214,215: NewLine (None) +214,215: NL (None) 215,220: Async (None) 221,224: Def (None) 225,226: Identifier (Str("a")) @@ -128,7 +128,7 @@ input_file: parser/test_data/inputs/function_def.py 228,229: : (None) 230,234: Pass (None) 234,235: NewLine (None) -235,236: NewLine (None) +235,236: NL (None) 236,241: Async (None) 242,245: Def (None) 246,247: Identifier (Str("a")) @@ -139,7 +139,7 @@ input_file: parser/test_data/inputs/function_def.py 251,255: Indent (Indent(1)) 255,259: Pass (None) 259,260: NewLine (None) -260,261: NewLine (None) +260,261: NL (None) 261,261: Dedent (Indent(1)) 261,266: Async (None) 267,270: Def (None) @@ -154,7 +154,7 @@ input_file: parser/test_data/inputs/function_def.py 281,282: : (None) 283,287: Pass (None) 287,288: NewLine (None) -288,289: NewLine (None) +288,289: NL (None) 289,292: Def (None) 293,294: Identifier (Str("a")) 294,295: [ (None) @@ -165,7 +165,7 @@ input_file: parser/test_data/inputs/function_def.py 299,300: : (None) 301,305: Pass (None) 305,306: NewLine (None) -306,307: NewLine (None) +306,307: NL (None) 307,310: Def (None) 311,312: Identifier (Str("a")) 312,313: [ (None) @@ -178,7 +178,7 @@ input_file: parser/test_data/inputs/function_def.py 320,321: : (None) 322,326: Pass (None) 326,327: NewLine (None) -327,328: NewLine (None) +327,328: NL (None) 328,331: Def (None) 332,333: Identifier (Str("a")) 333,334: [ (None) @@ -191,7 +191,7 @@ input_file: parser/test_data/inputs/function_def.py 341,342: : (None) 343,347: Pass (None) 347,348: NewLine (None) -348,349: NewLine (None) +348,349: NL (None) 349,352: Def (None) 353,354: Identifier (Str("a")) 354,355: [ (None) @@ -208,7 +208,7 @@ input_file: parser/test_data/inputs/function_def.py 368,369: : (None) 370,374: Pass (None) 374,375: NewLine (None) -375,376: NewLine (None) +375,376: NL (None) 376,379: Def (None) 380,381: Identifier (Str("a")) 381,382: [ (None) @@ -220,7 +220,7 @@ input_file: parser/test_data/inputs/function_def.py 387,388: : (None) 389,393: Pass (None) 393,394: NewLine (None) -394,395: NewLine (None) +394,395: NL (None) 395,398: Def (None) 399,400: Identifier (Str("a")) 400,401: [ (None) @@ -234,7 +234,7 @@ input_file: parser/test_data/inputs/function_def.py 409,410: : (None) 411,415: Pass (None) 415,416: NewLine (None) -416,417: NewLine (None) +416,417: NL (None) 417,420: Def (None) 421,422: Identifier (Str("a")) 422,423: [ (None) @@ -250,7 +250,7 @@ input_file: parser/test_data/inputs/function_def.py 434,435: : (None) 436,440: Pass (None) 440,441: NewLine (None) -441,442: NewLine (None) +441,442: NL (None) 442,445: Def (None) 446,447: Identifier (Str("a")) 447,448: [ (None) @@ -270,7 +270,7 @@ input_file: parser/test_data/inputs/function_def.py 465,466: : (None) 467,471: Pass (None) 471,472: NewLine (None) -472,473: NewLine (None) +472,473: NL (None) 473,476: Def (None) 477,478: Identifier (Str("a")) 478,479: [ (None) @@ -282,7 +282,7 @@ input_file: parser/test_data/inputs/function_def.py 485,486: : (None) 487,491: Pass (None) 491,492: NewLine (None) -492,493: NewLine (None) +492,493: NL (None) 493,496: Def (None) 497,498: Identifier (Str("a")) 498,499: [ (None) @@ -296,7 +296,7 @@ input_file: parser/test_data/inputs/function_def.py 508,509: : (None) 510,514: Pass (None) 514,515: NewLine (None) -515,516: NewLine (None) +515,516: NL (None) 516,519: Def (None) 520,521: Identifier (Str("a")) 521,522: [ (None) @@ -312,7 +312,7 @@ input_file: parser/test_data/inputs/function_def.py 534,535: : (None) 536,540: Pass (None) 540,541: NewLine (None) -541,542: NewLine (None) +541,542: NL (None) 542,545: Def (None) 546,547: Identifier (Str("a")) 547,548: [ (None) @@ -332,7 +332,7 @@ input_file: parser/test_data/inputs/function_def.py 566,567: : (None) 568,572: Pass (None) 572,573: NewLine (None) -573,574: NewLine (None) +573,574: NL (None) 574,577: Def (None) 578,579: Identifier (Str("a")) 579,580: [ (None) @@ -349,7 +349,7 @@ input_file: parser/test_data/inputs/function_def.py 593,594: : (None) 595,599: Pass (None) 599,600: NewLine (None) -600,601: NewLine (None) +600,601: NL (None) 601,604: Def (None) 605,606: Identifier (Str("a")) 606,607: [ (None) @@ -368,35 +368,35 @@ input_file: parser/test_data/inputs/function_def.py 623,624: : (None) 625,629: Pass (None) 629,630: NewLine (None) -630,631: NewLine (None) +630,631: NL (None) 631,634: Def (None) 635,654: Identifier (Str("dataclass_transform")) 654,655: ( (None) -655,656: NewLine (None) +655,656: NL (None) 660,661: * (None) 661,662: , (None) -662,663: NewLine (None) +662,663: NL (None) 667,677: Identifier (Str("eq_default")) 677,678: : (None) 679,683: Identifier (Str("bool")) 684,685: = (None) 686,690: True (None) 690,691: , (None) -691,692: NewLine (None) +691,692: NL (None) 696,709: Identifier (Str("order_default")) 709,710: : (None) 711,715: Identifier (Str("bool")) 716,717: = (None) 718,723: False (None) 723,724: , (None) -724,725: NewLine (None) +724,725: NL (None) 729,744: Identifier (Str("kw_only_default")) 744,745: : (None) 746,750: Identifier (Str("bool")) 751,752: = (None) 753,758: False (None) 758,759: , (None) -759,760: NewLine (None) +759,760: NL (None) 764,778: Identifier (Str("frozen_default")) 778,779: : (None) 780,784: Identifier (Str("bool")) @@ -404,9 +404,9 @@ input_file: parser/test_data/inputs/function_def.py 787,792: False (None) 792,793: , (None) 795,842: Comment (Str("# on 3.11, runtime accepts it as part of kwargs")) -842,843: NewLine (None) +842,843: NL (None) 847,895: Comment (Str("# this next annotation cannot be parsed need fix")) -895,896: NewLine (None) +895,896: NL (None) 900,916: Identifier (Str("field_specifiers")) 916,917: : (None) 918,923: Identifier (Str("tuple")) @@ -429,13 +429,13 @@ input_file: parser/test_data/inputs/function_def.py 963,964: ( (None) 964,965: ) (None) 965,966: , (None) -966,967: NewLine (None) +966,967: NL (None) 971,973: ** (None) 973,979: Identifier (Str("kwargs")) 979,980: : (None) 981,984: Identifier (Str("Any")) 984,985: , (None) -985,986: NewLine (None) +985,986: NL (None) 986,987: ) (None) 988,990: -> (None) 991,1007: Identifier (Str("IdentityFunction")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap index c9fff674..ac831e16 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap @@ -20,7 +20,7 @@ input_file: parser/test_data/inputs/functions.py 83,89: Identifier (Str("symbol")) 89,90: ) (None) 90,91: NewLine (None) -91,92: NewLine (None) +91,92: NL (None) 96,98: If (None) 99,111: Identifier (Str("ticker_index")) 112,114: Is (None) @@ -61,7 +61,7 @@ input_file: parser/test_data/inputs/functions.py 320,326: Return (None) 327,339: Identifier (Str("ticker_index")) 339,340: NewLine (None) -340,341: NewLine (None) +340,341: NL (None) 341,342: NewLine (None) 342,342: Dedent (Indent(1)) 342,345: Def (None) @@ -122,7 +122,7 @@ input_file: parser/test_data/inputs/functions.py 597,603: Return (None) 604,608: Identifier (Str("data")) 608,609: NewLine (None) -609,610: NewLine (None) +609,610: NL (None) 610,610: Dedent (Indent(1)) 610,613: Def (None) 614,628: Identifier (Str("common_process")) @@ -208,7 +208,7 @@ input_file: parser/test_data/inputs/functions.py 887,888: . (None) 888,899: Identifier (Str("to_datetime")) 899,900: ( (None) -900,901: NewLine (None) +900,901: NL (None) 909,913: Identifier (Str("date")) 914,915: + (None) 916,919: StringLiteral (Str("\" \"")) @@ -226,7 +226,7 @@ input_file: parser/test_data/inputs/functions.py 950,956: Identifier (Str("format")) 956,957: = (None) 957,972: StringLiteral (Str("\"%Y%m%d %H%M%S\"")) -972,973: NewLine (None) +972,973: NL (None) 977,978: ) (None) 978,979: NewLine (None) 983,985: Identifier (Str("df")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap index 66dabf6c..1e41bd6c 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap @@ -11,7 +11,7 @@ input_file: parser/test_data/inputs/generator_expressions.py 12,13: Identifier (Str("b")) 13,14: ) (None) 14,15: NewLine (None) -15,16: NewLine (None) +15,16: NL (None) 16,17: ( (None) 17,18: Identifier (Str("a")) 19,22: For (None) @@ -22,7 +22,7 @@ input_file: parser/test_data/inputs/generator_expressions.py 33,34: Identifier (Str("c")) 34,35: ) (None) 35,36: NewLine (None) -36,37: NewLine (None) +36,37: NL (None) 37,38: ( (None) 38,39: Identifier (Str("a")) 40,43: For (None) @@ -35,7 +35,7 @@ input_file: parser/test_data/inputs/generator_expressions.py 59,60: Identifier (Str("d")) 60,61: ) (None) 61,62: NewLine (None) -62,63: NewLine (None) +62,63: NL (None) 63,64: ( (None) 64,65: Identifier (Str("a")) 66,69: For (None) @@ -48,7 +48,7 @@ input_file: parser/test_data/inputs/generator_expressions.py 86,87: Identifier (Str("d")) 87,88: ) (None) 88,89: NewLine (None) -89,90: NewLine (None) +89,90: NL (None) 90,91: ( (None) 91,94: Identifier (Str("ord")) 94,95: ( (None) @@ -64,7 +64,7 @@ input_file: parser/test_data/inputs/generator_expressions.py 124,128: Identifier (Str("line")) 128,129: ) (None) 129,130: NewLine (None) -130,131: NewLine (None) +130,131: NL (None) 131,132: ( (None) 132,133: Identifier (Str("x")) 133,134: * (None) @@ -91,7 +91,7 @@ input_file: parser/test_data/inputs/generator_expressions.py 183,185: Integer (Number("10")) 185,186: ) (None) 186,187: NewLine (None) -187,188: NewLine (None) +187,188: NL (None) 188,211: Identifier (Str("has_explicit_annotation")) 212,213: = (None) 214,224: Identifier (Str("isinstance")) @@ -105,7 +105,7 @@ input_file: parser/test_data/inputs/generator_expressions.py 250,253: And (None) 254,257: Identifier (Str("any")) 257,258: ( (None) -258,259: NewLine (None) +258,259: NL (None) 263,266: Not (None) 267,285: Identifier (Str("is_unannotated_any")) 285,286: ( (None) @@ -127,6 +127,6 @@ input_file: parser/test_data/inputs/generator_expressions.py 330,331: . (None) 331,339: Identifier (Str("ret_type")) 339,340: ] (None) -340,341: NewLine (None) +340,341: NL (None) 341,342: ) (None) 342,343: NewLine (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap index 72b2189c..e7da7fda 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap @@ -10,24 +10,24 @@ input_file: parser/test_data/inputs/if.py 6,10: Indent (Indent(1)) 10,14: Pass (None) 14,15: NewLine (None) -15,16: NewLine (None) +15,16: NL (None) 16,16: Dedent (Indent(1)) 16,18: If (None) 19,20: Identifier (Str("a")) 20,21: : (None) 21,22: NewLine (None) -22,23: NewLine (None) +22,23: NL (None) 23,27: Indent (Indent(1)) 27,31: Pass (None) 31,32: NewLine (None) -32,33: NewLine (None) +32,33: NL (None) 33,33: Dedent (Indent(1)) 33,35: If (None) 36,37: Identifier (Str("a")) 37,38: : (None) 39,43: Pass (None) 43,44: NewLine (None) -44,45: NewLine (None) +44,45: NL (None) 45,47: If (None) 48,49: Identifier (Str("a")) 49,50: : (None) @@ -37,7 +37,7 @@ input_file: parser/test_data/inputs/if.py 59,60: ; (None) 60,64: Pass (None) 64,65: NewLine (None) -65,66: NewLine (None) +65,66: NL (None) 66,66: Dedent (Indent(1)) 66,68: If (None) 69,70: Identifier (Str("a")) @@ -48,7 +48,7 @@ input_file: parser/test_data/inputs/if.py 77,89: Indent (Indent(1)) 89,93: Pass (None) 93,94: NewLine (None) -94,95: NewLine (None) +94,95: NL (None) 95,95: Dedent (Indent(1)) 95,97: If (None) 98,99: Identifier (Str("a")) @@ -69,7 +69,7 @@ input_file: parser/test_data/inputs/if.py 140,156: Indent (Indent(1)) 156,160: Pass (None) 160,161: NewLine (None) -161,162: NewLine (None) +161,162: NL (None) 162,162: Dedent (Indent(1)) 162,164: If (None) 165,166: Identifier (Str("a")) @@ -97,7 +97,7 @@ input_file: parser/test_data/inputs/if.py 234,250: Indent (Indent(1)) 250,254: Pass (None) 254,255: NewLine (None) -255,256: NewLine (None) +255,256: NL (None) 256,257: NewLine (None) 257,257: Dedent (Indent(1)) 257,259: If (None) @@ -119,7 +119,7 @@ input_file: parser/test_data/inputs/if.py 285,286: = (None) 287,288: Integer (Number("1")) 288,289: NewLine (None) -289,290: NewLine (None) +289,290: NL (None) 290,290: Dedent (Indent(1)) 290,292: If (None) 293,297: Identifier (Str("self")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap index 1862f4f6..f0a94446 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap @@ -12,21 +12,21 @@ input_file: parser/test_data/inputs/lists.py 8,9: Identifier (Str("c")) 9,10: ] (None) 10,11: NewLine (None) -11,12: NewLine (None) +11,12: NL (None) 12,13: [ (None) 13,14: Identifier (Str("a")) 14,15: , (None) -15,16: NewLine (None) +15,16: NL (None) 16,17: Identifier (Str("b")) 17,18: , (None) 19,20: Identifier (Str("c")) 20,21: ] (None) 21,22: , (None) 22,23: NewLine (None) -23,24: NewLine (None) +23,24: NL (None) 24,25: [ (None) 25,26: Identifier (Str("a")) -26,27: NewLine (None) +26,27: NL (None) 27,28: , (None) 29,30: Identifier (Str("b")) 30,31: , (None) @@ -34,27 +34,27 @@ input_file: parser/test_data/inputs/lists.py 33,34: ] (None) 34,35: , (None) 35,36: NewLine (None) -36,37: NewLine (None) +36,37: NL (None) 37,38: [ (None) 38,39: Identifier (Str("a")) 39,40: , (None) -40,41: NewLine (None) +40,41: NL (None) 41,42: Identifier (Str("b")) 42,43: , (None) -43,44: NewLine (None) +43,44: NL (None) 48,49: Identifier (Str("c")) 49,50: ] (None) 50,51: , (None) 51,52: NewLine (None) -52,53: NewLine (None) +52,53: NL (None) 53,54: [ (None) 54,55: Identifier (Str("a")) 55,56: , (None) -56,57: NewLine (None) +56,57: NL (None) 57,58: ] (None) 58,59: , (None) 59,60: NewLine (None) -60,61: NewLine (None) +60,61: NL (None) 61,62: [ (None) 62,63: Identifier (Str("a")) 63,64: , (None) @@ -64,7 +64,7 @@ input_file: parser/test_data/inputs/lists.py 69,70: , (None) 70,71: ] (None) 71,72: NewLine (None) -72,73: NewLine (None) +72,73: NL (None) 73,84: Identifier (Str("month_names")) 85,86: = (None) 87,88: [ (None) @@ -75,7 +75,7 @@ input_file: parser/test_data/inputs/lists.py 111,118: StringLiteral (Str("'Maart'")) 118,119: , (None) 125,140: Comment (Str("# These are the")) -140,141: NewLine (None) +140,141: NL (None) 144,151: StringLiteral (Str("'April'")) 151,152: , (None) 155,160: StringLiteral (Str("'Mei'")) @@ -83,7 +83,7 @@ input_file: parser/test_data/inputs/lists.py 167,173: StringLiteral (Str("'June'")) 173,174: , (None) 181,194: Comment (Str("# Dutch names")) -194,195: NewLine (None) +194,195: NL (None) 198,204: StringLiteral (Str("'July'")) 204,205: , (None) 209,219: StringLiteral (Str("'Augustus'")) @@ -91,7 +91,7 @@ input_file: parser/test_data/inputs/lists.py 221,232: StringLiteral (Str("'September'")) 232,233: , (None) 235,251: Comment (Str("# for the months")) -251,252: NewLine (None) +251,252: NL (None) 255,264: StringLiteral (Str("'October'")) 264,265: , (None) 266,276: StringLiteral (Str("'November'")) @@ -100,7 +100,7 @@ input_file: parser/test_data/inputs/lists.py 288,289: ] (None) 292,306: Comment (Str("# of the year,")) 306,307: NewLine (None) -307,308: NewLine (None) +307,308: NL (None) 308,309: [ (None) 309,310: Identifier (Str("a")) 311,314: For (None) @@ -109,7 +109,7 @@ input_file: parser/test_data/inputs/lists.py 320,321: Identifier (Str("b")) 321,322: ] (None) 322,323: NewLine (None) -323,324: NewLine (None) +323,324: NL (None) 324,325: [ (None) 325,326: Identifier (Str("a")) 327,330: For (None) @@ -120,7 +120,7 @@ input_file: parser/test_data/inputs/lists.py 341,342: Identifier (Str("c")) 342,343: ] (None) 343,344: NewLine (None) -344,345: NewLine (None) +344,345: NL (None) 345,346: [ (None) 346,347: Identifier (Str("a")) 348,351: For (None) @@ -133,7 +133,7 @@ input_file: parser/test_data/inputs/lists.py 367,368: Identifier (Str("d")) 368,369: ] (None) 369,370: NewLine (None) -370,371: NewLine (None) +370,371: NL (None) 371,372: [ (None) 372,373: Identifier (Str("a")) 374,377: For (None) @@ -146,7 +146,7 @@ input_file: parser/test_data/inputs/lists.py 394,395: Identifier (Str("d")) 395,396: ] (None) 396,397: NewLine (None) -397,398: NewLine (None) +397,398: NL (None) 398,399: [ (None) 399,400: Identifier (Str("a")) 401,404: For (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap index fc7f433b..a3f68804 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap @@ -15,7 +15,7 @@ input_file: parser/test_data/inputs/match.py 21,29: Indent (Indent(1)) 29,33: Pass (None) 33,34: NewLine (None) -34,35: NewLine (None) +34,35: NL (None) 35,35: Dedent (Indent(2)) 35,35: Dedent (None) 35,40: Identifier (Match) @@ -32,7 +32,7 @@ input_file: parser/test_data/inputs/match.py 60,68: Indent (Indent(1)) 68,72: Pass (None) 72,73: NewLine (None) -73,74: NewLine (None) +73,74: NL (None) 74,74: Dedent (Indent(2)) 74,74: Dedent (None) 74,79: Identifier (Match) @@ -49,7 +49,7 @@ input_file: parser/test_data/inputs/match.py 97,105: Indent (Indent(1)) 105,109: Pass (None) 109,110: NewLine (None) -110,111: NewLine (None) +110,111: NL (None) 111,111: Dedent (Indent(2)) 111,111: Dedent (None) 111,116: Identifier (Match) @@ -105,7 +105,7 @@ input_file: parser/test_data/inputs/match.py 270,278: Indent (Indent(1)) 278,282: Pass (None) 282,283: NewLine (None) -283,284: NewLine (None) +283,284: NL (None) 284,284: Dedent (Indent(2)) 284,284: Dedent (None) 284,289: Identifier (Match) @@ -130,7 +130,7 @@ input_file: parser/test_data/inputs/match.py 332,340: Indent (Indent(1)) 340,344: Pass (None) 344,345: NewLine (None) -345,346: NewLine (None) +345,346: NL (None) 346,346: Dedent (Indent(2)) 346,346: Dedent (None) 346,351: Identifier (Match) @@ -176,7 +176,7 @@ input_file: parser/test_data/inputs/match.py 440,448: Indent (Indent(1)) 448,452: Pass (None) 452,453: NewLine (None) -453,454: NewLine (None) +453,454: NL (None) 454,454: Dedent (Indent(2)) 454,454: Dedent (None) 454,459: Identifier (Match) @@ -217,7 +217,7 @@ input_file: parser/test_data/inputs/match.py 533,541: Indent (Indent(1)) 541,545: Pass (None) 545,546: NewLine (None) -546,547: NewLine (None) +546,547: NL (None) 547,547: Dedent (Indent(2)) 547,547: Dedent (None) 547,552: Identifier (Match) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap index a2d0a1b7..d6b617a3 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap @@ -6,64 +6,64 @@ input_file: parser/test_data/inputs/newlines.py 0,1: Identifier (Str("_")) 2,3: = (None) 4,5: ( (None) -5,6: NewLine (None) +5,6: NL (None) 10,11: Integer (Number("1")) 11,12: , (None) -12,13: NewLine (None) +12,13: NL (None) 17,18: Integer (Number("2")) 18,19: , (None) -19,20: NewLine (None) +19,20: NL (None) 20,21: ) (None) 21,22: NewLine (None) 22,23: Identifier (Str("_")) 24,25: = (None) 26,27: { (None) -27,28: NewLine (None) +27,28: NL (None) 32,33: Integer (Number("1")) 33,34: , (None) -34,35: NewLine (None) +34,35: NL (None) 39,40: Integer (Number("2")) 40,41: , (None) -41,42: NewLine (None) +41,42: NL (None) 42,43: } (None) 43,44: NewLine (None) 44,45: Identifier (Str("_")) 46,47: = (None) 48,49: [ (None) -49,50: NewLine (None) +49,50: NL (None) 54,55: Integer (Number("1")) 55,56: , (None) -56,57: NewLine (None) +56,57: NL (None) 61,62: Integer (Number("2")) 62,63: , (None) -63,64: NewLine (None) +63,64: NL (None) 64,65: ] (None) 65,66: NewLine (None) -66,67: NewLine (None) +66,67: NL (None) 67,68: NewLine (None) 68,71: Def (None) 72,77: Identifier (Str("hello")) 77,78: ( (None) -78,79: NewLine (None) +78,79: NL (None) 83,84: Identifier (Str("a")) 84,85: , (None) -85,86: NewLine (None) +85,86: NL (None) 90,91: Identifier (Str("b")) 91,92: , (None) -92,93: NewLine (None) +92,93: NL (None) 93,94: ) (None) 94,95: : (None) 96,99: Ellipsis (None) 99,100: NewLine (None) -100,101: NewLine (None) +100,101: NL (None) 101,102: NewLine (None) 102,107: Class (None) 108,109: Identifier (Str("A")) 109,110: ( (None) -110,111: NewLine (None) +110,111: NL (None) 115,121: Identifier (Str("object")) 121,122: , (None) -122,123: NewLine (None) +122,123: NL (None) 123,124: ) (None) 124,125: : (None) 125,126: NewLine (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap index eb6c3421..c1abe184 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap @@ -5,7 +5,7 @@ input_file: parser/test_data/inputs/separate_statements.py --- 0,97: Comment (Str("# Test case to check that we return correct error when two python statements are on the same line")) 97,98: NewLine (None) -98,99: NewLine (None) +98,99: NL (None) 99,102: Def (None) 103,106: Identifier (Str("foo")) 106,107: ( (None) @@ -26,7 +26,7 @@ input_file: parser/test_data/inputs/separate_statements.py 140,141: + (None) 142,143: Identifier (Str("y")) 143,144: NewLine (None) -144,145: NewLine (None) +144,145: NL (None) 145,145: Dedent (Indent(1)) 145,148: Def (None) 149,152: Identifier (Str("bar")) @@ -44,7 +44,7 @@ input_file: parser/test_data/inputs/separate_statements.py 235,236: + (None) 237,238: Identifier (Str("y")) 238,239: NewLine (None) -239,240: NewLine (None) +239,240: NL (None) 240,240: Dedent (Indent(1)) 240,283: Comment (Str("# TODO(parser): enable after error handling")) 283,284: NewLine (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap index b5c385a5..2ac637a2 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap @@ -11,45 +11,45 @@ input_file: parser/test_data/inputs/sets.py 7,8: Identifier (Str("c")) 8,9: } (None) 9,10: NewLine (None) -10,11: NewLine (None) +10,11: NL (None) 11,12: { (None) 12,13: Identifier (Str("a")) 13,14: , (None) -14,15: NewLine (None) +14,15: NL (None) 15,16: Identifier (Str("b")) 16,17: , (None) 18,19: Identifier (Str("c")) 19,20: } (None) 20,21: NewLine (None) -21,22: NewLine (None) +21,22: NL (None) 22,23: { (None) 23,24: Identifier (Str("a")) -24,25: NewLine (None) +24,25: NL (None) 25,26: , (None) 27,28: Identifier (Str("b")) 28,29: , (None) 30,31: Identifier (Str("c")) 31,32: } (None) 32,33: NewLine (None) -33,34: NewLine (None) +33,34: NL (None) 34,35: { (None) 35,36: Identifier (Str("a")) 36,37: , (None) -37,38: NewLine (None) +37,38: NL (None) 38,39: Identifier (Str("b")) 39,40: , (None) -40,41: NewLine (None) +40,41: NL (None) 45,46: Identifier (Str("c")) 46,47: } (None) 47,48: NewLine (None) -48,49: NewLine (None) +48,49: NL (None) 49,50: { (None) 50,51: Identifier (Str("a")) 51,52: , (None) -52,53: NewLine (None) +52,53: NL (None) 53,54: } (None) 54,55: NewLine (None) -55,56: NewLine (None) +55,56: NL (None) 56,57: { (None) 57,58: Identifier (Str("a")) 58,59: , (None) @@ -59,7 +59,7 @@ input_file: parser/test_data/inputs/sets.py 64,65: , (None) 65,66: } (None) 66,67: NewLine (None) -67,68: NewLine (None) +67,68: NL (None) 68,69: { (None) 69,70: Identifier (Str("a")) 71,74: For (None) @@ -71,7 +71,7 @@ input_file: parser/test_data/inputs/sets.py 88,89: ) (None) 89,90: } (None) 90,91: NewLine (None) -91,92: NewLine (None) +91,92: NL (None) 92,93: { (None) 93,94: Identifier (Str("a")) 95,98: For (None) @@ -81,10 +81,10 @@ input_file: parser/test_data/inputs/sets.py 109,110: ( (None) 110,112: Integer (Number("10")) 112,113: ) (None) -113,114: NewLine (None) +113,114: NL (None) 114,115: } (None) 115,116: NewLine (None) -116,117: NewLine (None) +116,117: NL (None) 117,118: { (None) 118,119: Identifier (Str("a")) 120,123: For (None) @@ -96,7 +96,7 @@ input_file: parser/test_data/inputs/sets.py 137,138: ) (None) 138,139: } (None) 139,140: NewLine (None) -140,141: NewLine (None) +140,141: NL (None) 141,142: { (None) 142,143: Identifier (Str("a")) 144,147: For (None) @@ -108,7 +108,7 @@ input_file: parser/test_data/inputs/sets.py 161,162: ) (None) 162,163: } (None) 163,164: NewLine (None) -164,165: NewLine (None) +164,165: NL (None) 165,166: { (None) 166,167: Identifier (Str("a")) 168,171: For (None) @@ -127,7 +127,7 @@ input_file: parser/test_data/inputs/sets.py 204,205: ) (None) 205,206: } (None) 206,207: NewLine (None) -207,208: NewLine (None) +207,208: NL (None) 208,209: { (None) 209,210: Identifier (Str("a")) 211,214: For (None) @@ -144,10 +144,10 @@ input_file: parser/test_data/inputs/sets.py 244,245: ( (None) 245,247: Integer (Number("10")) 247,248: ) (None) -248,249: NewLine (None) +248,249: NL (None) 249,250: } (None) 250,251: NewLine (None) -251,252: NewLine (None) +251,252: NL (None) 252,253: { (None) 253,254: Identifier (Str("a")) 255,258: For (None) @@ -164,12 +164,12 @@ input_file: parser/test_data/inputs/sets.py 288,289: ( (None) 289,291: Integer (Number("10")) 291,292: ) (None) -292,293: NewLine (None) +292,293: NL (None) 293,294: } (None) 294,295: NewLine (None) -295,296: NewLine (None) +295,296: NL (None) 296,297: { (None) -297,298: NewLine (None) +297,298: NL (None) 302,303: ( (None) 303,304: Identifier (Str("a")) 304,305: , (None) @@ -189,6 +189,6 @@ input_file: parser/test_data/inputs/sets.py 341,342: ( (None) 342,344: Integer (Number("10")) 344,345: ) (None) -345,346: NewLine (None) +345,346: NL (None) 346,347: } (None) 347,348: NewLine (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap index e2b1ee78..7c232b9f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap @@ -7,17 +7,17 @@ input_file: parser/test_data/inputs/string.py 2,3: = (None) 4,19: StringLiteral (Str("\"simple text\\n\"")) 19,20: NewLine (None) -20,21: NewLine (None) +20,21: NL (None) 21,24: Identifier (Str("raw")) 25,26: = (None) 27,40: StringLiteral (Str("r\"raw_text\\n\"")) 40,41: NewLine (None) -41,42: NewLine (None) +41,42: NL (None) 42,43: Identifier (Str("y")) 44,45: = (None) 46,57: StringLiteral (Str("\"\"\"multi\"\"\"")) 57,58: NewLine (None) -58,59: NewLine (None) +58,59: NL (None) 59,61: FStringStart (Str("f\"")) 61,62: { (None) 62,66: Identifier (Str("self")) @@ -38,7 +38,7 @@ input_file: parser/test_data/inputs/string.py 106,107: FstringMiddle (Str(")")) 107,108: FStringEnd (Str("\"")) 108,109: NewLine (None) -109,110: NewLine (None) +109,110: NL (None) 110,112: FStringStart (Str("f\"")) 112,113: { (None) 113,116: Identifier (Str("num")) @@ -50,7 +50,7 @@ input_file: parser/test_data/inputs/string.py 127,128: } (None) 128,129: FStringEnd (Str("\"")) 129,130: NewLine (None) -130,131: NewLine (None) +130,131: NL (None) 131,133: FStringStart (Str("f\"")) 133,148: FstringMiddle (Str("tuple argument ")) 148,149: { (None) @@ -62,7 +62,7 @@ input_file: parser/test_data/inputs/string.py 158,159: } (None) 159,160: FStringEnd (Str("\"")) 160,161: NewLine (None) -161,162: NewLine (None) +161,162: NL (None) 162,164: FStringStart (Str("f\"")) 164,175: FstringMiddle (Str("some words ")) 175,176: { (None) @@ -82,7 +82,7 @@ input_file: parser/test_data/inputs/string.py 202,214: FstringMiddle (Str(" final words")) 214,215: FStringEnd (Str("\"")) 215,216: NewLine (None) -216,217: NewLine (None) +216,217: NL (None) 217,219: FStringStart (Str("f\"")) 219,220: { (None) 220,222: FStringStart (Str("f\"")) @@ -111,7 +111,7 @@ input_file: parser/test_data/inputs/string.py 248,249: } (None) 249,250: FStringEnd (Str("\"")) 250,251: NewLine (None) -251,252: NewLine (None) +251,252: NL (None) 252,256: FStringStart (Str("f\"\"\"")) 256,257: { (None) 257,261: FStringStart (Str("f'''")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap index 2af74e4f..b735249c 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap @@ -13,7 +13,7 @@ input_file: parser/test_data/inputs/subscript.py 31,32: Integer (Number("2")) 32,33: ] (None) 33,34: NewLine (None) -34,35: NewLine (None) +34,35: NL (None) 35,41: Identifier (Str("values")) 42,43: = (None) 44,61: Identifier (Str("list_of_key_stats")) @@ -24,13 +24,13 @@ input_file: parser/test_data/inputs/subscript.py 65,66: Integer (Number("2")) 66,67: ] (None) 67,68: NewLine (None) -68,69: NewLine (None) +68,69: NL (None) 69,70: Identifier (Str("a")) 70,71: [ (None) 71,72: Identifier (Str("b")) 72,73: ] (None) 73,74: NewLine (None) -74,75: NewLine (None) +74,75: NL (None) 75,76: Identifier (Str("a")) 76,77: [ (None) 77,78: : (None) @@ -52,7 +52,7 @@ input_file: parser/test_data/inputs/subscript.py 99,100: , (None) 100,101: ] (None) 101,102: NewLine (None) -102,103: NewLine (None) +102,103: NL (None) 103,104: Identifier (Str("a")) 104,105: [ (None) 105,106: : (None) @@ -80,7 +80,7 @@ input_file: parser/test_data/inputs/subscript.py 139,143: Identifier (Str("step")) 143,144: ] (None) 144,145: NewLine (None) -145,146: NewLine (None) +145,146: NL (None) 146,147: Identifier (Str("a")) 147,148: [ (None) 148,153: Identifier (Str("lower")) @@ -118,7 +118,7 @@ input_file: parser/test_data/inputs/subscript.py 215,216: : (None) 216,217: ] (None) 217,218: NewLine (None) -218,219: NewLine (None) +218,219: NL (None) 219,220: NewLine (None) 220,221: Identifier (Str("a")) 221,222: [ (None) @@ -129,7 +129,7 @@ input_file: parser/test_data/inputs/subscript.py 228,229: Identifier (Str("d")) 229,230: ] (None) 230,231: NewLine (None) -231,232: NewLine (None) +231,232: NL (None) 232,233: Identifier (Str("a")) 233,234: [ (None) 234,235: Identifier (Str("b")) @@ -141,7 +141,7 @@ input_file: parser/test_data/inputs/subscript.py 242,243: Identifier (Str("e")) 243,244: ] (None) 244,245: NewLine (None) -245,246: NewLine (None) +245,246: NL (None) 246,247: NewLine (None) 247,248: Identifier (Str("a")) 248,249: [ (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap index 014e64ac..80236222 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap @@ -87,7 +87,7 @@ input_file: parser/test_data/inputs/try.py 373,374: = (None) 375,379: None (None) 379,380: NewLine (None) -380,381: NewLine (None) +380,381: NL (None) 381,382: NewLine (None) 382,382: Dedent (Indent(1)) 382,385: Try (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap index 231f7274..0d3ff891 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap @@ -18,7 +18,7 @@ input_file: parser/test_data/inputs/type_alias.py 48,49: = (None) 50,55: Identifier (Str("float")) 55,56: NewLine (None) -56,57: NewLine (None) +56,57: NL (None) 57,61: Identifier (Type) 62,81: Identifier (Str("AliasToAnotherAlias")) 82,83: = (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap index a2af1f1f..1f875ec6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap @@ -8,7 +8,7 @@ input_file: parser/test_data/inputs/with.py 6,7: : (None) 8,12: Pass (None) 12,13: NewLine (None) -13,14: NewLine (None) +13,14: NL (None) 14,18: With (None) 19,20: Identifier (Str("a")) 21,23: As (None) @@ -16,7 +16,7 @@ input_file: parser/test_data/inputs/with.py 25,26: : (None) 27,31: Pass (None) 31,32: NewLine (None) -32,33: NewLine (None) +32,33: NL (None) 33,37: With (None) 38,39: Identifier (Str("a")) 40,42: As (None) @@ -28,7 +28,7 @@ input_file: parser/test_data/inputs/with.py 52,53: : (None) 54,58: Pass (None) 58,59: NewLine (None) -59,60: NewLine (None) +59,60: NL (None) 60,64: With (None) 65,66: ( (None) 66,67: Identifier (Str("a")) @@ -42,7 +42,7 @@ input_file: parser/test_data/inputs/with.py 81,82: : (None) 83,87: Pass (None) 87,88: NewLine (None) -88,89: NewLine (None) +88,89: NL (None) 89,94: Async (None) 95,99: With (None) 100,101: Identifier (Str("a")) @@ -51,7 +51,7 @@ input_file: parser/test_data/inputs/with.py 106,107: : (None) 108,112: Pass (None) 112,113: NewLine (None) -113,114: NewLine (None) +113,114: NL (None) 114,119: Async (None) 120,124: With (None) 125,126: Identifier (Str("a")) @@ -66,12 +66,12 @@ input_file: parser/test_data/inputs/with.py 141,145: Indent (Indent(1)) 145,149: Pass (None) 149,150: NewLine (None) -150,151: NewLine (None) +150,151: NL (None) 151,151: Dedent (Indent(1)) 151,156: Async (None) 157,161: With (None) 162,163: ( (None) -163,164: NewLine (None) +163,164: NL (None) 172,173: Identifier (Str("a")) 174,176: As (None) 177,178: Identifier (Str("b")) @@ -79,7 +79,7 @@ input_file: parser/test_data/inputs/with.py 180,181: Identifier (Str("c")) 182,184: As (None) 185,186: Identifier (Str("d")) -186,187: NewLine (None) +186,187: NL (None) 187,188: ) (None) 188,189: : (None) 189,190: NewLine (None)