From cdf98f717c8422240710f4381473a094a70a299c Mon Sep 17 00:00:00 2001 From: ALLARD Antoine Date: Tue, 23 May 2023 14:09:44 +0200 Subject: [PATCH 1/4] Adding support for source path substitution --- ValgrindCI/__init__.py | 12 ++++++++++++ ValgrindCI/parse.py | 10 ++++++++++ ValgrindCI/render.py | 38 +++++++++++++++++++++++--------------- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/ValgrindCI/__init__.py b/ValgrindCI/__init__.py index 4786899..d05e300 100644 --- a/ValgrindCI/__init__.py +++ b/ValgrindCI/__init__.py @@ -16,6 +16,12 @@ def main(): "--source-dir", help="specifies the source directory", ) + parser.add_argument( + "--substitute-path", + action="append", + help="specifies a substitution rule `from:to` for finding source files on disk. example: --substitute-path /foo:/bar", + nargs='?' + ) parser.add_argument( "--output-dir", help="directory where the HTML report will be generated" ) @@ -55,6 +61,12 @@ def main(): data.parse(args.xml_file) data.set_source_dir(args.source_dir) + if args.substitute_path: + substitute_paths = [] + for s in args.substitute_path: + substitute_paths.append({"from": s.split(":")[0], "to": s.split(":")[1] }) + data.set_substitute_paths(substitute_paths) + errors_total = data.get_num_errors() if args.abort_on_errors and errors_total != 0: print("{} errors reported by Valgrind - Abort".format(errors_total)) diff --git a/ValgrindCI/parse.py b/ValgrindCI/parse.py index 49618a4..51f4a99 100644 --- a/ValgrindCI/parse.py +++ b/ValgrindCI/parse.py @@ -118,6 +118,7 @@ class ValgrindData: def __init__(self) -> None: self.errors: List[Error] = [] self._source_dir: Optional[str] = None + self._substitute_paths: Optional[List[dict]] = [] def parse(self, xml_file: str) -> None: root = et.parse(xml_file).getroot() @@ -130,6 +131,15 @@ def set_source_dir(self, source_dir: Optional[str]) -> None: else: self._source_dir = None + def set_substitute_paths(self, substitute_paths: Optional[List[dict]]) -> None: + if substitute_paths is not None: + self._substitute_paths = substitute_paths + + def substitute_path(self, path: str) -> str: + for s in self._substitute_paths: + path = path.replace(s.get("from"), s.get("to")) + return path + def get_num_errors(self) -> int: if self._source_dir is None: return len(self.errors) diff --git a/ValgrindCI/render.py b/ValgrindCI/render.py index 2c943e5..f532428 100644 --- a/ValgrindCI/render.py +++ b/ValgrindCI/render.py @@ -124,10 +124,14 @@ def _extract_error_data( stack["fileref"] = "{}:{}".format( frame.get_path(self._source_dir), error_line ) - with open(fullname, "r") as f: - for l, code_line in enumerate(f.readlines()): - if l >= stack["line"] and l <= error_line + lines_after - 1: - stack["code"].append(code_line) + fullname = self._data.substitute_path(fullname) + try: + with open(fullname, "r", errors="replace") as f: + for l, code_line in enumerate(f.readlines()): + if l >= stack["line"] and l <= error_line + lines_after - 1: + stack["code"].append(code_line) + except OSError as e: + print(f"Warning: cannot read stack data from missing source file: {e.filename}") issue["stack"].append(stack) return issue @@ -143,16 +147,20 @@ def _extract_data_per_source_file( else: filename = source_file - with open(filename, "r") as f: - for l, line in enumerate(f.readlines()): - klass = None - issue = None - if l + 1 in error_lines: - klass = "error" - issue = self._extract_error_data( - src_data, l + 1, lines_before, lines_after + filename = self._data.substitute_path(filename) + try: + with open(filename, "r", errors="replace") as f: + for l, line in enumerate(f.readlines()): + klass = None + issue = None + if l + 1 in error_lines: + klass = "error" + issue = self._extract_error_data( + src_data, l + 1, lines_before, lines_after + ) + lines_of_code.append( + {"line": line[:-1], "klass": klass, "issue": issue} ) - lines_of_code.append( - {"line": line[:-1], "klass": klass, "issue": issue} - ) + except OSError as e: + print(f"Warning: cannot extract data from missing source file: {e.filename}") return lines_of_code, len(error_lines) From 97b9c9e7aa814f70d636984b773f97b939286aaa Mon Sep 17 00:00:00 2001 From: ALLARD Antoine Date: Tue, 23 May 2023 14:41:35 +0200 Subject: [PATCH 2/4] Adding support for displaying relative source filenames --- ValgrindCI/__init__.py | 27 +++++++++++++++++++++++++++ ValgrindCI/parse.py | 13 +++++++++++++ ValgrindCI/render.py | 10 +++++----- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/ValgrindCI/__init__.py b/ValgrindCI/__init__.py index d05e300..d85bc85 100644 --- a/ValgrindCI/__init__.py +++ b/ValgrindCI/__init__.py @@ -22,6 +22,18 @@ def main(): help="specifies a substitution rule `from:to` for finding source files on disk. example: --substitute-path /foo:/bar", nargs='?' ) + parser.add_argument( + "--relativize", + action="append", + help="specifies a prefix to remove from displayed source filenames. example: --relativize /foo/bar", + nargs='?' + ) + parser.add_argument( + "--relativize-from-substitute-paths", + default=False, + action="store_true", + help="use the `from` values in the substitution rules as prefixes to remove from displayed source filenames", + ) parser.add_argument( "--output-dir", help="directory where the HTML report will be generated" ) @@ -67,6 +79,21 @@ def main(): substitute_paths.append({"from": s.split(":")[0], "to": s.split(":")[1] }) data.set_substitute_paths(substitute_paths) + if args.relativize: + prefixes = [] + for p in args.relativize: + prefixes.append(p) + data.set_relative_prefixes(prefixes) + + if args.relativize_from_substitute_paths: + if not args.substitute_path: + print("No substitution paths specified on the command line.") + else: + prefixes = data._relative_prefixes.copy() + for s in data._substitute_paths: + prefixes.append(s.get("from")) + data.set_relative_prefixes(prefixes) + errors_total = data.get_num_errors() if args.abort_on_errors and errors_total != 0: print("{} errors reported by Valgrind - Abort".format(errors_total)) diff --git a/ValgrindCI/parse.py b/ValgrindCI/parse.py index 51f4a99..abb586a 100644 --- a/ValgrindCI/parse.py +++ b/ValgrindCI/parse.py @@ -119,6 +119,7 @@ def __init__(self) -> None: self.errors: List[Error] = [] self._source_dir: Optional[str] = None self._substitute_paths: Optional[List[dict]] = [] + self._relative_prefixes: Optional[List[str]] = [] def parse(self, xml_file: str) -> None: root = et.parse(xml_file).getroot() @@ -135,11 +136,23 @@ def set_substitute_paths(self, substitute_paths: Optional[List[dict]]) -> None: if substitute_paths is not None: self._substitute_paths = substitute_paths + def set_relative_prefixes(self, relative_prefixes: Optional[str]) -> None: + if relative_prefixes is not None: + self._relative_prefixes = relative_prefixes + def substitute_path(self, path: str) -> str: for s in self._substitute_paths: path = path.replace(s.get("from"), s.get("to")) return path + def relativize(self, path: str) -> str: + for p in self._relative_prefixes: + if path.startswith(p): + path = path.replace(p, "") + if path.startswith("/"): + path = path[1:] + return path + def get_num_errors(self) -> int: if self._source_dir is None: return len(self.errors) diff --git a/ValgrindCI/render.py b/ValgrindCI/render.py index f532428..73069fe 100644 --- a/ValgrindCI/render.py +++ b/ValgrindCI/render.py @@ -71,14 +71,14 @@ def render(self, output_dir: str, lines_before: int, lines_after: int) -> None: f.write( self._source_tmpl.render( num_errors=num_errors, - source_file_name=source_file, + source_file_name=self._data.relativize(source_file), codelines=lines_of_code, ) ) summary.append( { - "filename": source_file, + "filename": self._data.relativize(source_file), "errors": num_errors, "link": html_filename, } @@ -121,9 +121,9 @@ def _extract_error_data( assert error_line is not None stack["line"] = error_line - lines_before - 1 stack["error_line"] = lines_before + 1 - stack["fileref"] = "{}:{}".format( - frame.get_path(self._source_dir), error_line - ) + frame_source = frame.get_path(self._source_dir) + frame_source = self._data.relativize(frame_source) + stack["fileref"] = "{}:{}".format(frame_source, error_line) fullname = self._data.substitute_path(fullname) try: with open(fullname, "r", errors="replace") as f: From ffa3c23c409b03808b46e9aa7eb27e699e9f6cca Mon Sep 17 00:00:00 2001 From: ALLARD Antoine Date: Tue, 23 May 2023 14:52:10 +0200 Subject: [PATCH 3/4] Adding support for custom title in HTML report --- ValgrindCI/__init__.py | 7 ++++++- ValgrindCI/data/index.html | 4 ++-- ValgrindCI/render.py | 6 ++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ValgrindCI/__init__.py b/ValgrindCI/__init__.py index d85bc85..1da514f 100644 --- a/ValgrindCI/__init__.py +++ b/ValgrindCI/__init__.py @@ -37,6 +37,11 @@ def main(): parser.add_argument( "--output-dir", help="directory where the HTML report will be generated" ) + parser.add_argument( + "--html-report-title", + default="ValgrindCI Report", + help="the title of the generated HTML report" + ) parser.add_argument( "--summary", default=False, @@ -102,7 +107,7 @@ def main(): if args.output_dir: renderer = HTMLRenderer(data) renderer.set_source_dir(args.source_dir) - renderer.render(args.output_dir, args.lines_before, args.lines_after) + renderer.render(args.html_report_title, args.output_dir, args.lines_before, args.lines_after) if args.number_of_errors: print("{} errors.".format(errors_total)) diff --git a/ValgrindCI/data/index.html b/ValgrindCI/data/index.html index de37adc..fb05d37 100644 --- a/ValgrindCI/data/index.html +++ b/ValgrindCI/data/index.html @@ -2,14 +2,14 @@ - Valgrind report + {{ title }}
-

ValgrindCI Report

+

{{ title }}

{{ num_errors }} errors

diff --git a/ValgrindCI/render.py b/ValgrindCI/render.py index 73069fe..e4fd87d 100644 --- a/ValgrindCI/render.py +++ b/ValgrindCI/render.py @@ -44,7 +44,7 @@ def set_source_dir(self, source_dir: Optional[str]) -> None: else: self._source_dir = None - def render(self, output_dir: str, lines_before: int, lines_after: int) -> None: + def render(self, report_title: str, output_dir: str, lines_before: int, lines_after: int) -> None: if not os.path.exists(output_dir): os.makedirs(output_dir) shutil.copy( @@ -88,7 +88,9 @@ def render(self, output_dir: str, lines_before: int, lines_after: int) -> None: with open(os.path.join(output_dir, "index.html"), "w") as f: f.write( self._index_tmpl.render( - source_list=summary, num_errors=total_num_errors + title=report_title, + source_list=summary, + num_errors=total_num_errors ) ) From bbe4e44a8f9f60d58162c8a5b30272caff39cf3f Mon Sep 17 00:00:00 2001 From: ALLARD Antoine Date: Tue, 23 May 2023 16:05:23 +0200 Subject: [PATCH 4/4] Documenting pyinstaller usage --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index baee3ae..6db864c 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,16 @@ ValgrindCI uses the `setuptools` to build its package which can then be installe > pip install ValgrindCI --no-index -f dist ``` +#### Build and package an executable with `pyinstaller` + +You can use `pyinstaller` to create a single-file executable binary: + +```bash +> pip install pyinstaller +> pyinstaller --onefile --add-data ValgrindCI:ValgrindCI valgrind-ci +> ./dist/valgrind-ci --help +``` + ## How to use ValgrindCI is a command tool designed to be executed within jobs of your favorite Continuous Integration platform. It parses the XML output of valgrind to provide its services.