From 1673301553100f567ab4140ab27f7417d84e3f08 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Wed, 24 Jul 2019 13:15:41 -0700 Subject: [PATCH] Initial commit of Code Base Investigator This is the initial commit of Code Base Investigator. Please refer to the README.md file for more information about what it is and what it can do. Co-authored-by: Doug Jacobsen Co-authored-by: Jason Sewall --- .gitignore | 6 + .pep8 | 4 + .pylintrc | 407 ++++ CONTRIBUTING.md | 31 + LICENSE | 29 + README.md | 89 + codebasin.py | 114 + codebasin/__init__.py | 2 + codebasin/config.py | 265 +++ codebasin/file_parser.py | 241 ++ codebasin/finder.py | 110 + codebasin/platform.py | 99 + codebasin/preprocessor.py | 1992 +++++++++++++++++ codebasin/report.py | 154 ++ codebasin/util.py | 79 + codebasin/walkers.py | 208 ++ docs/configuration.md | 75 + docs/example-ast.png | Bin 0 -> 15816 bytes docs/example-dendrogram.png | Bin 0 -> 16453 bytes examples/README.md | 16 + examples/disjoint-source/README.md | 26 + examples/disjoint-source/cpu/main.cpp | 49 + .../disjoint-source-dendrogram.png | Bin 0 -> 14024 bytes examples/disjoint-source/disjoint-source.yaml | 14 + examples/disjoint-source/gpu/main.cpp | 49 + examples/disjoint-source/makefile | 10 + examples/divergent-source/README.md | 27 + .../divergent-source-dendrogram.png | Bin 0 -> 14726 bytes .../divergent-source/divergent-source.yaml | 15 + examples/divergent-source/histogram.h | 8 + examples/divergent-source/main.cpp | 47 + examples/divergent-source/makefile | 10 + .../divergent-source/private_histogram.cpp | 22 + .../divergent-source/shared_histogram.cpp | 18 + examples/single-source/README.md | 25 + examples/single-source/main.cpp | 56 + examples/single-source/makefile | 7 + .../single-source-dendrogram.png | Bin 0 -> 14481 bytes examples/single-source/single-source.yaml | 14 + hooks/post-commit | 16 + hooks/pre-commit | 92 + setup.py | 26 + setup_git_hooks.sh | 35 + tests/__init__.py | 0 tests/basic_fortran/__init__.py | 2 + tests/basic_fortran/basic_fortran.yaml | 11 + tests/basic_fortran/test.f90 | 26 + tests/basic_fortran/test_basic_fortran.py | 33 + tests/commented_directive/__init__.py | 2 + .../commented_directive.yaml | 11 + tests/commented_directive/main.cpp | 14 + .../test_commented_directive.py | 45 + tests/define/__init__.py | 2 + tests/define/define.yaml | 11 + tests/define/main.cpp | 18 + tests/define/test_define.py | 31 + tests/disjoint/__init__.py | 2 + tests/disjoint/cpu.cpp | 9 + tests/disjoint/cpu_headers/header.h | 4 + tests/disjoint/disjoint.yaml | 11 + tests/disjoint/gpu.cpp | 9 + tests/disjoint/gpu_headers/header.h | 4 + tests/disjoint/test_disjoint.py | 33 + tests/failure/__init__.py | 2 + tests/failure/test_bignum.py | 29 + tests/include/__init__.py | 2 + tests/include/cpu_commands.json | 6 + tests/include/gpu_commands.json | 6 + tests/include/headers/cpu.h | 12 + tests/include/headers/gpu.h | 13 + tests/include/headers/test.h | 19 + tests/include/include-db.yaml | 9 + tests/include/include.yaml | 13 + tests/include/main.cpp | 16 + tests/include/test_include.py | 41 + tests/lexer/__init__.py | 2 + tests/lexer/test_lexer.py | 85 + tests/literals/__init__.py | 2 + tests/literals/literals.yaml | 11 + tests/literals/main.cpp | 14 + tests/literals/test_literals.py | 31 + tests/macro_expansion/__init__.py | 2 + .../defined_undefined_test.cpp | 29 + tests/macro_expansion/function_like_test.cpp | 25 + tests/macro_expansion/infinite_loop_test.cpp | 13 + .../macro_expansion-dendrogram.png | Bin 0 -> 14484 bytes tests/macro_expansion/macro_expansion.yaml | 18 + tests/macro_expansion/max_level.cpp | 216 ++ tests/macro_expansion/test_macro_expansion.py | 153 ++ tests/multi_line/__init__.py | 2 + tests/multi_line/main.cpp | 21 + tests/multi_line/multi_line.yaml | 11 + tests/multi_line/test_multi_line.py | 31 + tests/nesting/__init__.py | 2 + tests/nesting/main.cpp | 22 + tests/nesting/nesting.yaml | 11 + tests/nesting/test_nesting.py | 32 + tests/once/__init__.py | 2 + tests/once/main.cpp | 10 + tests/once/once.h | 12 + tests/once/once.yaml | 11 + tests/once/test_once.py | 31 + tests/operators/__init__.py | 2 + tests/operators/main.cpp | 37 + tests/operators/operators.yaml | 11 + tests/operators/test_operators.py | 31 + tests/parsers/__init__.py | 2 + tests/parsers/test_directive_parser.py | 150 ++ tests/safe_write/__init__.py | 2 + tests/safe_write/test_safe_write.py | 80 + tests/valid_path/__init__.py | 2 + tests/valid_path/test_valid_path.py | 33 + 112 files changed, 6082 insertions(+) create mode 100644 .gitignore create mode 100644 .pep8 create mode 100644 .pylintrc create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100755 codebasin.py create mode 100644 codebasin/__init__.py create mode 100644 codebasin/config.py create mode 100644 codebasin/file_parser.py create mode 100644 codebasin/finder.py create mode 100644 codebasin/platform.py create mode 100644 codebasin/preprocessor.py create mode 100644 codebasin/report.py create mode 100644 codebasin/util.py create mode 100644 codebasin/walkers.py create mode 100644 docs/configuration.md create mode 100755 docs/example-ast.png create mode 100755 docs/example-dendrogram.png create mode 100644 examples/README.md create mode 100644 examples/disjoint-source/README.md create mode 100644 examples/disjoint-source/cpu/main.cpp create mode 100644 examples/disjoint-source/disjoint-source-dendrogram.png create mode 100644 examples/disjoint-source/disjoint-source.yaml create mode 100644 examples/disjoint-source/gpu/main.cpp create mode 100644 examples/disjoint-source/makefile create mode 100644 examples/divergent-source/README.md create mode 100644 examples/divergent-source/divergent-source-dendrogram.png create mode 100644 examples/divergent-source/divergent-source.yaml create mode 100644 examples/divergent-source/histogram.h create mode 100644 examples/divergent-source/main.cpp create mode 100644 examples/divergent-source/makefile create mode 100644 examples/divergent-source/private_histogram.cpp create mode 100644 examples/divergent-source/shared_histogram.cpp create mode 100644 examples/single-source/README.md create mode 100644 examples/single-source/main.cpp create mode 100644 examples/single-source/makefile create mode 100644 examples/single-source/single-source-dendrogram.png create mode 100644 examples/single-source/single-source.yaml create mode 100755 hooks/post-commit create mode 100755 hooks/pre-commit create mode 100644 setup.py create mode 100755 setup_git_hooks.sh create mode 100644 tests/__init__.py create mode 100644 tests/basic_fortran/__init__.py create mode 100644 tests/basic_fortran/basic_fortran.yaml create mode 100644 tests/basic_fortran/test.f90 create mode 100644 tests/basic_fortran/test_basic_fortran.py create mode 100644 tests/commented_directive/__init__.py create mode 100644 tests/commented_directive/commented_directive.yaml create mode 100644 tests/commented_directive/main.cpp create mode 100644 tests/commented_directive/test_commented_directive.py create mode 100644 tests/define/__init__.py create mode 100644 tests/define/define.yaml create mode 100644 tests/define/main.cpp create mode 100644 tests/define/test_define.py create mode 100644 tests/disjoint/__init__.py create mode 100644 tests/disjoint/cpu.cpp create mode 100644 tests/disjoint/cpu_headers/header.h create mode 100644 tests/disjoint/disjoint.yaml create mode 100644 tests/disjoint/gpu.cpp create mode 100644 tests/disjoint/gpu_headers/header.h create mode 100644 tests/disjoint/test_disjoint.py create mode 100644 tests/failure/__init__.py create mode 100644 tests/failure/test_bignum.py create mode 100644 tests/include/__init__.py create mode 100644 tests/include/cpu_commands.json create mode 100644 tests/include/gpu_commands.json create mode 100644 tests/include/headers/cpu.h create mode 100644 tests/include/headers/gpu.h create mode 100644 tests/include/headers/test.h create mode 100644 tests/include/include-db.yaml create mode 100644 tests/include/include.yaml create mode 100644 tests/include/main.cpp create mode 100644 tests/include/test_include.py create mode 100644 tests/lexer/__init__.py create mode 100644 tests/lexer/test_lexer.py create mode 100644 tests/literals/__init__.py create mode 100644 tests/literals/literals.yaml create mode 100644 tests/literals/main.cpp create mode 100644 tests/literals/test_literals.py create mode 100644 tests/macro_expansion/__init__.py create mode 100644 tests/macro_expansion/defined_undefined_test.cpp create mode 100644 tests/macro_expansion/function_like_test.cpp create mode 100644 tests/macro_expansion/infinite_loop_test.cpp create mode 100644 tests/macro_expansion/macro_expansion-dendrogram.png create mode 100644 tests/macro_expansion/macro_expansion.yaml create mode 100644 tests/macro_expansion/max_level.cpp create mode 100644 tests/macro_expansion/test_macro_expansion.py create mode 100644 tests/multi_line/__init__.py create mode 100644 tests/multi_line/main.cpp create mode 100644 tests/multi_line/multi_line.yaml create mode 100644 tests/multi_line/test_multi_line.py create mode 100644 tests/nesting/__init__.py create mode 100644 tests/nesting/main.cpp create mode 100644 tests/nesting/nesting.yaml create mode 100644 tests/nesting/test_nesting.py create mode 100644 tests/once/__init__.py create mode 100644 tests/once/main.cpp create mode 100644 tests/once/once.h create mode 100644 tests/once/once.yaml create mode 100644 tests/once/test_once.py create mode 100644 tests/operators/__init__.py create mode 100644 tests/operators/main.cpp create mode 100644 tests/operators/operators.yaml create mode 100644 tests/operators/test_operators.py create mode 100644 tests/parsers/__init__.py create mode 100644 tests/parsers/test_directive_parser.py create mode 100644 tests/safe_write/__init__.py create mode 100644 tests/safe_write/test_safe_write.py create mode 100644 tests/valid_path/__init__.py create mode 100644 tests/valid_path/test_valid_path.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..620727e --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +*.pyo +*.pyc +build/ +dist/ +*.egg-info/ diff --git a/.pep8 b/.pep8 new file mode 100644 index 0000000..4d09de1 --- /dev/null +++ b/.pep8 @@ -0,0 +1,4 @@ +[pep8] +max_line_length = 99 + + diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..936da69 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,407 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=file-builtin,coerce-method,oct-method,buffer-builtin,map-builtin-not-iterating,input-builtin,round-builtin,cmp-method,import-star-module-level,standarderror-builtin,old-raise-syntax,zip-builtin-not-iterating,raw_input-builtin,old-ne-operator,using-cmp-argument,cmp-builtin,long-suffix,intern-builtin,unicode-builtin,reload-builtin,xrange-builtin,dict-view-method,coerce-builtin,old-octal-literal,filter-builtin-not-iterating,suppressed-message,dict-iter-method,raising-string,reduce-builtin,delslice-method,print-statement,nonzero-method,hex-method,old-division,setslice-method,parameter-unpacking,unichr-builtin,execfile-builtin,range-builtin-not-iterating,metaclass-assignment,getslice-method,no-absolute-import,backtick,indexing-exception,next-method-called,useless-suppression,unpacking-in-except,apply-builtin,long-builtin,basestring-builtin,invalid-name,too-few-public-methods,no-else-return + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=10 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=99 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..10429b5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing to Code Base Investigator +Thanks for taking the time to contribute to Code Base Investigator! + +## Bug Reports +- Open a GitHub issue containing as much information as possible, including configuration files and source code. +- If your code base is large and/or cannot be shared, we would greatly appreciate it if you could construct a minimal working example. + +## Feature Requests +- Open a GitHub issue detailing the expected behavior of the feature. +- Where appropriate, please include sample input/output of the requested feature. + +## Submitting a Patch +- Follow the style guidelines for all commits and code: + - Limit the first line of Git commit messages to 50 characters, and other lines to 72 characters. + - Follow [PEP8](https://www.python.org/dev/peps/pep-0008/) for Python code, with a line limit of 99 characters and a comment/docstring limit of 72 characters. +- Ensure that the patched code passes all existing tests: + ``` + python3 -m unittest + ``` +- Consider checking the quality of your code with [`pylint`](https://github.com/PyCQA/pylint). +- If your patch introduces a new feature, consider adding a new test. +- Open a new GitHub pull request with the patch. + +### Hooks +A number of [Git hooks](./hooks) are available to assist contributors in adhering to the guidelines above: +- pre-commit: + Optionally run [`pylint`](https://github.com/PyCQA/pylint) and/or [`autopep8`](https://github.com/hhatto/autopep8) on every commit. +- post-commit: + Run the full test suite after every commit. + +See the [setup_git_hooks.sh script](./setup_git_hooks.sh) for more information on configuring these hooks. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..22d1139 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..a378199 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# Code Base Investigator +Code Base Investigator (CBI) is a tool designed to help developers reason about the use of _specialization_ (i.e. code written specifically to provide support for or improve performance on some set of platforms) in a code base. Specialization is often necessary, but how a developer chooses to express it may impact code portability and future maintenance costs. + +The [definition of platform](https://doi.org/10.1016/j.future.2017.08.007) used by CBI is deliberately very flexible and completely user-defined; a platform can represent any execution environment for which code may be specialized. A platform could be a compiler, an operating system, a micro-architecture or some combination of these options. + +## Code Divergence +CBI measures the amount of specialization in a code base using [code divergence](http://doi.org/10.1109/P3HPC.2018.00006), which is defined as the arithmetic mean pair-wise distance between the code-paths used by each platform. + +At the two extremes, a code divergence of 0 means that all of the platforms use exactly the same code, while a code divergence of 1 means that there is no code shared between any of the platforms. The code divergence of real codes will fall somewhere in between. + +## How it Works +![Abstract Syntax Tree](./docs/example-ast.png) + +CBI tracks specialization in two forms: source files that are not compiled for all platforms; and regions of source files that are guarded by C preprocessor directives (e.g. `#ifdef`). A typical run of CBI consists of a three step process: +1) Extract source files and compilation commands from a configuration file or compilation database. +2) Build an AST representing which source lines of code (LOC) are associated with each specialization. +3) Record which specializations are used by each platform. + +## Usage +``` +usage: codebasin.py [-h] [-c FILE] [-v] [-q] [-r DIR] [-R REPORT [REPORT ...]] + +optional arguments: + -h, --help show this help message and exit + -c FILE, --config FILE configuration file + -v, --verbose increase verbosity level + -q, --quiet decrease verbosity level + -r DIR, --rootdir DIR working root directory + -R REPORT [REPORT ...] desired outout reports +``` +The `codebasin.py` script analyzes a code base described in a YAML configuration file and produces one or more output reports. Example configuration files can be found in the [examples](./examples) directory, and see the [configuration file documentation](docs/configuration.md) for a detailed description of the configuration file format. + +### Summary Report +The summary report (`-R summary`) gives a high-level summary of a code base, as shown below: +``` +--------------------------------------------- + Platform Set LOC % LOC +--------------------------------------------- + {} 2 4.88 + {GPU 1} 1 2.44 + {GPU 2} 1 2.44 + {CPU 2} 1 2.44 + {CPU 1} 1 2.44 + {FPGA} 14 34.15 + {GPU 2, GPU 1} 6 14.63 + {CPU 1, CPU 2} 6 14.63 +{FPGA, CPU 1, GPU 2, GPU 1, CPU 2} 9 21.95 +--------------------------------------------- +Code Divergence: 0.55 +Unused Code (%): 4.88 +Total SLOC: 41 +``` +Each row in the table shows the amount of code that is unique to a given set of platforms. Listed below the table are the computed code divergence, the amount of code in the code base that was not compiled for any platform, and the total size of the code base. + +### Clustering Report +The clustering report (`-R clustering`) consists of a pair-wise distance matrix, showing the ratio of platform-specific code to code used by both platforms. These distances are the same as those used to compute code divergence. +``` +Distance Matrix +----------------------------------- + FPGA CPU 1 GPU 2 GPU 1 CPU 2 +----------------------------------- + FPGA 0.00 0.70 0.70 0.70 0.70 +CPU 1 0.70 0.00 0.61 0.61 0.12 +GPU 2 0.70 0.61 0.00 0.12 0.61 +GPU 1 0.70 0.61 0.12 0.00 0.61 +CPU 2 0.70 0.12 0.61 0.61 0.00 +----------------------------------- +``` + +The distances can also be used to produce a dendrogram, showing the result of hierarchical clustering by platform similarity: +![Dendrogram](./docs/example-dendrogram.png) + +## Dependencies +- NumPy +- Matplotlib +- Python 3 +- PyYAML +- SciPy + +CBI and its dependencies can be installed using `setup.py`: +``` +python3 setup.py install +``` + +## License +[BSD 3-Clause](./LICENSE) + +## Contributing +See the [contribution guidelines](./CONTRIBUTING.md) for details. diff --git a/codebasin.py b/codebasin.py new file mode 100755 index 0000000..17e6bd9 --- /dev/null +++ b/codebasin.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +This script is the main executable of Code Base Investigator. + +usage: codebasin.py [-h] [-c FILE] [-v] [-q] [-r DIR] [-R REPORT [REPORT ...]] + +optional arguments: + -h, --help show this help message and exit + -c FILE, --config FILE + configuration file (default: config.yaml) + -v, --verbose verbosity level + -q, --quiet quiet level + -r DIR, --rootdir DIR + Set working root directory (default .) + -R REPORT [REPORT ...], --report REPORT [REPORT ...] + desired output reports (default: all) +""" + +import argparse +import os +import sys +import logging + +from codebasin import config, finder, report, util, walkers + +version = 1.0 + + +def report_enabled(name): + """ + Return true if the report with the specified name is enabled. + """ + if "all" in args.reports: + return True + return name in args.reports + + +def guess_project_name(config_path): + """ + Guess a useful name from the given path so that we can pick + meaningful filenames for output. + """ + fullpath = os.path.realpath(config_path) + (thedir, thename) = os.path.split(fullpath) + if config_path == 'config.yaml': + (base, end) = os.path.split(thedir) + res = end.strip() + else: + (base, end) = os.path.splitext(thename) + res = base.strip() + if not res: + logging.getLogger("codebasin").warning("Can't guess meaningful output name from input") + res = "unknown" + return res + + +if __name__ == '__main__': + + # Read command-line arguments + parser = argparse.ArgumentParser(description="Code Base Investigator v" + str(version)) + parser.add_argument('-c', '--config', dest='config_file', metavar='FILE', action='store', + default='config.yaml', help='configuration file (default: config.yaml)') + parser.add_argument('-v', '--verbose', dest='verbose', + action='count', default=0, help='increase verbosity level') + parser.add_argument('-q', '--quiet', dest='quiet', + action='count', default=0, help='decrease verbosity level') + parser.add_argument('-r', '--rootdir', dest="rootdir", metavar='DIR', + default=os.getcwd(), type=str, + help="Set working root directory (default .)") + parser.add_argument('-R', '--report', dest='reports', metavar='REPORT', default=['all'], + choices=['all', 'summary', 'clustering'], nargs='+', + help='desired output reports (default: all)') + args = parser.parse_args() + + stdout_log = logging.StreamHandler(sys.stdout) + stdout_log.setFormatter(logging.Formatter('[%(levelname)-8s] %(message)s')) + logging.getLogger("codebasin").addHandler(stdout_log) + logging.getLogger("codebasin").setLevel( + max(1, logging.WARNING - 10 * (args.verbose - args.quiet))) + rootdir = os.path.realpath(args.rootdir) + + # Load the configuration file into a dict + if not util.ensure_yaml(args.config_file): + logging.getLogger("codebasin").error( + "Configuration file does not have YAML file extension.") + sys.exit(1) + codebase, configuration = config.load(args.config_file, rootdir) + + # Parse the source tree, and determine source line associations. + # The trees and associations are housed in state. + state = finder.find(rootdir, codebase, configuration) + + # Count lines for platforms + platform_mapper = walkers.PlatformMapper(codebase) + setmap = platform_mapper.walk(state) + + output_prefix = os.path.realpath(guess_project_name(args.config_file)) + + # Print summary report + if report_enabled("summary"): + summary = report.summary(setmap) + if summary is not None: + print(summary) + + # Print clustering report + if report_enabled("clustering"): + clustering_output_name = output_prefix + "-dendrogram.png" + clustering = report.clustering(clustering_output_name, setmap) + if clustering is not None: + print(clustering) + + sys.exit(0) diff --git a/codebasin/__init__.py b/codebasin/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/codebasin/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/codebasin/config.py b/codebasin/config.py new file mode 100644 index 0000000..4cb49aa --- /dev/null +++ b/codebasin/config.py @@ -0,0 +1,265 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +Contains functions to build up a configuration dictionary, +defining a specific code base configuration. +""" + +import os +import collections +import glob +import itertools as it +import logging +import sys + +import yaml +from . import util + +log = logging.getLogger("codebasin") + + +def extract_defines(args): + """ + Extract definitions from command-line arguments. + Recognizes two argument "-D MACRO" and one argument "-DMACRO". + """ + defines = [] + prefix = "" + for a in args: + if a == "-D": + prefix = "-D" + elif prefix: + defines += [a] + prefix = "" + elif a[0:2] == "-D": + defines += [a[2:]] + prefix = "" + return defines + + +def extract_include_paths(args): + """ + Extract include paths from command-line arguments. + Recognizes two argument "-I path" and one argument "-Ipath". + """ + prefixes = ["-I", "-isystem"] + + include_paths = [] + prefix = "" + for a in args: + if a in prefixes: + prefix = a + elif prefix in prefixes: + include_paths += [a] + prefix = "" + elif a[0:2] == "-I": + include_paths += [a[2:]] + return include_paths + + +def extract_include_files(args): + """ + Extract include files from command-line arguments. + Recognizes two argument "-include file". + """ + includes = [] + prefix = "" + for a in args: + if a == "-include": + prefix = "-include" + elif prefix: + includes += [a] + prefix = "" + return includes + + +def expand_path(pattern): + """ + Return all valid and existing paths matching a specified pattern. + """ + if sys.version_info >= (3, 5): + paths = glob.glob(pattern, recursive=True) + else: + if "**" in pattern: + log.warning("Recursive path expansion with '**' requires Python >= 3.5") + paths = glob.glob(pattern) + if paths == []: + log.warning("Couldn't find files matching '%s' -- ignoring it.", pattern) + return [os.path.realpath(path) for path in filter(util.valid_path, paths)] + + +def flatten(nested_list): + """ + Flatten an arbitrarily nested list. + Nesting may occur when anchors are used inside a YAML list. + """ + flattened = [] + for l in nested_list: + if isinstance(l, list): + flattened.extend(flatten(l)) + else: + flattened.append(l) + return flattened + + +def load_codebase(config, rootdir): + """ + Load the code base definition into a Python object. + Return a dict of files and platform names. + """ + # Ensure expected values are present, or provide defaults + cfg_codebase = config["codebase"] + if not cfg_codebase: + raise RuntimeError("Empty 'codebase' section found in config file!") + if "files" not in cfg_codebase or cfg_codebase["files"] == []: + raise RuntimeError("Empty 'files' section found in codebase definition!") + if "platforms" not in cfg_codebase or cfg_codebase["platforms"] == []: + raise RuntimeError("Empty 'platforms' section found in codebase definition!") + + codebase = {"files": list(it.chain(*(expand_path(os.path.join(rootdir, f)) + for f in cfg_codebase["files"]))), + "platforms": cfg_codebase["platforms"]} + + # Ensure that the codebase actually specifies valid files + if not codebase["files"]: + raise RuntimeError("Specified codebase contains no valid files -- " + + "regular expressions and/or working directory may be incorrect.") + + return codebase + + +def load_database(dbpath, rootdir): + """ + Load a compilation database. + Return a list of compilation commands, where each command is + represented as a compilation database entry. + """ + with open(dbpath, 'r') as fi: + db = yaml.safe_load(fi) + + configuration = [] + for e in db: + # Database may not have tokenized arguments + if "command" in e: + args = e["command"].split() + elif "arguments" in e: + args = e["arguments"] + + # Extract defines, include paths and include files + # from command-line arguments + defines = extract_defines(args) + include_paths = extract_include_paths(args) + include_files = extract_include_files(args) + + # Include paths may be specified relative to root + include_paths = [os.path.realpath(os.path.join(rootdir, f)) for f in include_paths] + + # Files may be specified: + # - relative to root + # - relative to a directory + # - as an absolute path + filedir = rootdir + if "directory" in e: + if os.path.isabs(e["directory"]): + filedir = e["directory"] + else: + filedir = os.path.realpath(rootdir, os.path.join(e["directory"])) + + if os.path.isabs(e["file"]): + path = os.path.realpath(e["file"]) + else: + path = os.path.realpath(os.path.join(filedir, e["file"])) + + # Compilation database may contain files that don't + # exist without running make + if os.path.exists(path): + configuration += [{"file": path, + "defines": defines, + "include_paths": include_paths, + "include_files": include_files}] + else: + log.warning("Couldn't find file %s -- ignoring it.", path) + + return configuration + + +def load_platform(config, rootdir, platform_name): + """ + Load the platform specified by platform_name into a Python object. + Return a list of compilation commands, where each command is + represented as a compilation database entry. + """ + # Ensure expected values are present, or provide defaults + cfg_platform = config[platform_name] + if not cfg_platform: + raise RuntimeError("Could not find definition for platform " + + "'{}' in config file!".format(platform_name)) + if "files" not in cfg_platform and "commands" not in cfg_platform: + raise RuntimeError("Need 'files' or 'commands' section in " + + "definition of platform {}!".format(platform_name)) + if "files" not in cfg_platform: + if "defines" in cfg_platform or "include_paths" in cfg_platform: + log.warning("Extra 'defines' or 'include_paths' in definition of platform %s!", + platform_name) + else: + if "defines" not in cfg_platform: + cfg_platform["defines"] = [] + if "include_paths" not in cfg_platform: + cfg_platform["include_paths"] = [] + + # Combine manually specified files, defines and includes + # into configuration entries + configuration = [] + if "files" in cfg_platform: + include_paths = [os.path.realpath(os.path.join(rootdir, f)) + for f in flatten(cfg_platform["include_paths"])] + + # Strip optional -D prefix from defines + defines = [d[2:] if d.startswith("-D") else d + for d in flatten(cfg_platform["defines"])] + + for f in flatten(cfg_platform["files"]): + for path in expand_path(os.path.join(rootdir, f)): + configuration += [{"file": path, + "defines": defines, + "include_paths": include_paths, + "include_files": []}] + + # Add configuration entries from a compilation database + if "commands" in cfg_platform: + dbpath = os.path.realpath(os.path.join(rootdir, cfg_platform["commands"])) + configuration += load_database(dbpath, rootdir) + + # Ensure that the platform actually specifies valid files + if not configuration: + raise RuntimeError("Platform '{!s}' contains no valid files -- ".format(platform_name) + + "regular expressions and/or working directory may be incorrect.") + + return configuration + + +def load(config_file, rootdir): + """ + Load the configuration file into Python objects. + Return a (codebase, platform configuration) tuple of dicts. + """ + if os.path.isfile(config_file): + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + else: + raise RuntimeError("Could not open {!s}.".format(config_file)) + + # Read codebase definition + if "codebase" in config: + codebase = load_codebase(config, rootdir) + else: + raise RuntimeError("Missing 'codebase' section in config file!") + + log.info("Platforms: %s", ", ".join(codebase["platforms"])) + + # Read each platform definition and populate platform configuration + configuration = collections.defaultdict(list) + for platform_name in codebase["platforms"]: + configuration[platform_name] = load_platform(config, rootdir, platform_name) + + return codebase, configuration diff --git a/codebasin/file_parser.py b/codebasin/file_parser.py new file mode 100644 index 0000000..a22ddc2 --- /dev/null +++ b/codebasin/file_parser.py @@ -0,0 +1,241 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +Contains classes and functions related to parsing a file, +and building a tree of nodes from it. +""" + +from os.path import splitext + +from . import preprocessor # pylint : disable=no-name-in-module + + +class LineGroup: + """ + Represents a grouping of lines. It contains the extent, and the + number of countable lines for the group. + """ + + def __init__(self): + self.line_count = 0 + self.start_line = -1 + self.end_line = -1 + + def empty(self): + """ + Return a boolean that tells if this group contains any line + information or not. + """ + if (self.line_count != 0) or (self.start_line != -1) or (self.end_line != -1): + return False + return True + + def add_line(self, line_num, is_countable=False): + """ + Add a line to this line group. Update the extent appropriately, + and if it's a countable line, add it to the line count. + """ + + if self.start_line == -1: + self.start_line = line_num + + self.end_line = line_num + + if self.start_line == -1 or line_num < self.start_line: + self.start_line = line_num + + if line_num > self.end_line: + self.end_line = line_num + + if is_countable: + self.line_count += 1 + + def reset(self): + """ + Reset the countable group + """ + self.line_count = 0 + self.start_line = -1 + self.end_line = -1 + + def merge(self, line_group, count=False): + """ + Merge another line group into this line group, and reset the + other group. + """ + if count: + self.line_count += line_group.line_count + + if self.start_line == -1: + self.start_line = line_group.start_line + elif line_group.start_line == -1: + line_group.start_line = self.start_line + self.start_line = min(self.start_line, line_group.start_line) + + self.end_line = max(self.end_line, line_group.end_line) + line_group.reset() + + +class FileParser: + """ + Contains methods for parsing an entire source file and returning a + source tree object, along with utility methods for determining + information about source lines and helping to build the source tree. + """ + + def __init__(self, _filename): + self._filename = _filename + self.full_line = '' + + split = splitext(_filename) + if len(split) == 2: + self._file_extension = split[1].lower() + else: + self._file_extension = None + + @staticmethod + def line_info(line): + """ + Determine if the input line is a directive by checking if the + first by looking for a '#' as the first non-whitespace + character. Also determine if the last character before a new + line is a line continuation character '\'. + + Return a (directive, line_continue) tuple. + """ + + directive = False + line_continue = False + + for c in line: + if c == '#': + directive = True + break + elif c not in [' ', '\t']: + break + + if line.rstrip("\n\r")[-1:] == '\\': + line_continue = True + + return (directive, line_continue) + + def handle_directive(self, out_tree, line_num, comment_cleaner, groups): + """ + Handle inserting code and directive nodes, where appropriate. + Update the file group, and reset the code and directive groups. + """ + # We will actually use this directive, if it is not empty + self.full_line = comment_cleaner.strip_comments(self.full_line) + if self.full_line.strip(): + # We need to finalize the previously started + # CodeNode (if there was one) before processing + # this DirectiveNode + if not groups['code'].empty(): + groups['code'].add_line(line_num - 1) + self.insert_code_node(out_tree, groups['code']) + + groups['file'].merge(groups['code']) + + self.insert_directive_node(out_tree, groups['directive']) + + groups['file'].merge(groups['directive']) + else: + groups['code'].merge(groups['directive']) + + @staticmethod + def insert_code_node(tree, line_group): + """ + Build a code node, and insert it into the source tree + """ + new_node = preprocessor.CodeNode( + line_group.start_line, line_group.end_line, line_group.line_count) + tree.insert(new_node) + + def insert_directive_node(self, tree, line_group): + """ + Build a directive node by parsing a directive line, and insert a + new directive node into the tree. + """ + new_node = preprocessor.DirectiveParser(preprocessor.Lexer( + self.full_line, line_group.start_line).tokenize()).parse() + new_node.start_line = line_group.start_line + new_node.end_line = line_group.end_line + new_node.num_lines = line_group.line_count + tree.insert(new_node) + + def parse_file(self): + """ + Parse the file that this parser points at, build a SourceTree + representing this file, and return it. + """ + + file_comment_cleaner = preprocessor.CommentCleaner(self._file_extension) + if file_comment_cleaner.filetype == 'c': + cpp_comment_cleaner = file_comment_cleaner + else: + cpp_comment_cleaner = preprocessor.CommentCleaner('.c') + + out_tree = preprocessor.SourceTree(self._filename) + with open(self._filename, mode='r', errors='replace') as source_file: + previous_continue = False + + groups = {'code': LineGroup(), + 'directive': LineGroup(), + 'file': LineGroup() + } + + groups['file'].start_line = 1 + + lines = source_file.readlines() + for (line_num, line) in enumerate(lines, 1): + # Determine if this line starts with a # (directive) + # and/or ends with a \ (line continuation) + (in_directive, continue_line) = self.line_info(line) + + # Only follow continuation for directives + if previous_continue or in_directive: + + # Add this into the directive lines, even if it + # might not be a directive we count + groups['directive'].add_line(line_num, True) + + # If this line starts a new directive, flush the + # line buffer + if in_directive and not previous_continue: + self.full_line = '' + + previous_continue = continue_line + + # If this line also contains a continuation + # character + if continue_line: + self.full_line += line.rstrip("\\\n\r") + # If this line ends a previously continued line + else: + self.full_line += line.rstrip("\n\r") + + self.handle_directive(out_tree, line_num, cpp_comment_cleaner, + groups) + + # FallBack is that this line is a simple code line. + else: + previous_continue = False + + # If the line isn't empty after stripping comments, + # count it as code + if file_comment_cleaner.strip_comments(line[0:-1]).strip(): + groups['code'].add_line(line_num, True) + else: + groups['code'].add_line(line_num) + + # Insert any code lines left at the end of the file + if not groups['code'].empty(): + groups['code'].add_line(len(lines)) + self.insert_code_node(out_tree, groups['code']) + + groups['file'].merge(groups['code']) + + groups['file'].add_line(len(lines)) + out_tree.root.num_lines = groups['file'].end_line + out_tree.root.total_sloc = groups['file'].line_count + return out_tree diff --git a/codebasin/finder.py b/codebasin/finder.py new file mode 100644 index 0000000..499bbf6 --- /dev/null +++ b/codebasin/finder.py @@ -0,0 +1,110 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +Contains functions and classes related to finding +and parsing source files as part of a code base. +""" + +import logging + +from . import file_parser +from . import platform +from . import preprocessor +from . import walkers + +log = logging.getLogger("codebasin") + + +class ParserState(): + """ + Keeps track of the overall state of the parser. + Contains all of the SourceTree objects created from parsing the + source files, along with association maps, that associate nodes to + platforms. + """ + + def __init__(self): + self.trees = {} + self.maps = {} + + def insert_file(self, fn): + """ + Build a new tree for a source file, and create an association + map for it. + """ + if fn not in self.trees: + parser = file_parser.FileParser(fn) + self.trees[fn] = parser.parse_file() + self.maps[fn] = walkers.NodeAssociationMap() + + def get_filenames(self): + """ + Return all of the filenames for files parsed so far. + """ + return self.trees.keys() + + def get_tree(self, fn): + """ + Return the SourceTree associated with a filename + """ + if fn not in self.trees: + return None + return self.trees[fn] + + def get_map(self, fn): + """ + Return the NodeAssociationMap associated with a filename + """ + if fn not in self.maps: + return None + return self.maps[fn] + + +def find(rootdir, codebase, configuration): + """ + Find codepaths in the files provided and return a mapping of source + lines to platforms. + """ + + # Build a tree for each unique file for all platforms. + state = ParserState() + for f in codebase["files"]: + state.insert_file(f) + for p in configuration: + for e in configuration[p]: + if e['file'] not in codebase["files"]: + log.warning( + "%s found in definition of platform %s but missing from codebase", + e['file'], p) + state.insert_file(e['file']) + + # Process each tree, by associating nodes with platforms + for p in configuration: + for e in configuration[p]: + file_platform = platform.Platform(p, rootdir) + + for path in e['include_paths']: + file_platform.add_include_path(path) + + for definition in e['defines']: + macro = preprocessor.Macro.from_definition_string(definition) + file_platform.define(macro.name, macro) + + # Process include files. + # These modify the file_platform instance, but we throw away + # the active nodes after processing is complete. + for include in e['include_files']: + include_file = file_platform.find_include_file(include) + if include_file: + state.insert_file(include_file) + + associator = walkers.TreeAssociator(state.get_tree( + include_file), state.get_map(include_file)) + associator.walk(file_platform, state) + + # Process the file, to build a list of associate nodes + associator = walkers.TreeAssociator(state.get_tree(e['file']), + state.get_map(e['file'])) + associator.walk(file_platform, state) + + return state diff --git a/codebasin/platform.py b/codebasin/platform.py new file mode 100644 index 0000000..12c1992 --- /dev/null +++ b/codebasin/platform.py @@ -0,0 +1,99 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +Contains the Platform class used to specify definitions and include +options for a specific platform. +""" + +import os + + +class Platform(): + """ + Represents a platform, and everything associated with a platform. + Contains a list of definitions, and include paths. + """ + + def __init__(self, name, _root_dir): + self._definitions = {} + self._skip_includes = [] + self._include_paths = [] + self._root_dir = _root_dir + self.name = name + + def add_include_path(self, path): + """ + Insert a new path into the list of include paths for this + platform. + """ + self._include_paths.append(path) + + def undefine(self, identifier): + """ + Undefine a macro for this platform, if it's defined. + """ + if identifier in self._definitions: + del self._definitions[identifier] + + def define(self, identifier, macro): + """ + Define a new macro for this platform, only if it's not already + defined. + """ + if identifier not in self._definitions: + self._definitions[identifier] = macro + + def add_include_to_skip(self, fn): + """ + Define a new macro for this platform, only if it's not already + defined. + """ + if fn not in self._skip_includes: + self._skip_includes.append(fn) + + def process_include(self, fn): + """ + Return a boolean stating if this include file should be + processed or skipped. + """ + return fn not in self._skip_includes + + def is_defined(self, identifier): + """ + Return a boolean stating if the macro named by 'identifier' is + defined. + """ + if identifier in self._definitions: + return "1" + return "0" + + def get_macro(self, identifier): + """ + Return either a macro definition (if it's defined), or None. + """ + if identifier in self._definitions: + return self._definitions[identifier] + return None + + def find_include_file(self, filename, is_system_include=False): + """ + Determine and return the full path to an include file, named + 'filename' using the include paths for this platform. + + System includes do not include the rootdir, while local includes + do. + """ + include_file = None + + local_paths = [] + if not is_system_include: + local_paths += [self._root_dir] + + # Determine the path to the include file, if it exists + for path in local_paths + self._include_paths: + test_path = os.path.realpath(os.path.join(path, filename)) + if os.path.exists(test_path): + include_file = test_path + break + + return include_file diff --git a/codebasin/preprocessor.py b/codebasin/preprocessor.py new file mode 100644 index 0000000..517ffac --- /dev/null +++ b/codebasin/preprocessor.py @@ -0,0 +1,1992 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +# pylint: disable=too-many-lines +""" +Dontains classes that define: +- Nodes from the tree +- Tokens from lexing a line of code +- Operators to handle tokens +""" + +import logging +import collections +import hashlib +import numpy as np +from . import util +from . import walkers + +log = logging.getLogger('codebasin') + + +class TokenError(ValueError): + """ + Represents an error encountered during tokenization. + """ + + +class Token(): + """ + Represents a token constructed by the parser. + """ + + def __init__(self, line, col, prev_white, token): + self.line = line + self.col = col + self.prev_white = prev_white + self.token = token + + def __repr__(self): + return "Token(line={!r},col={!r},prev_white={!r},token={!r})".format( + self.line, self.col, self.prev_white, self.token) + + def __str__(self): + return str(self.token) + + +class CharacterConstant(Token): + """ + Represents a character constant. + """ + + def __repr__(self): + return "CharacterConstant(line={!r},col={!r},prev_white={!r},token={!r})".format( + self.line, self.col, self.prev_white, self.token) + + +class NumericalConstant(Token): + """ + Represents a 'preprocessing number'. + These cannot necessarily be evaluated by the preprocessor (and may + not be valid syntax). + """ + + def __repr__(self): + return "NumericalConstant(line={!r},col={!r},prev_white={!r},value={!r})".format( + self.line, self.col, self.prev_white, self.token) + + +class StringConstant(Token): + """ + Represents a string constant. + """ + + def __repr__(self): + return "StringConstant(line={!r},col={!r},prev_white={!r},token={!r})".format( + self.line, self.col, self.prev_white, self.token) + + def __str__(self): + return "\"{!s}\"".format(self.token) + + +class Identifier(Token): + """ + Represents a C identifier. + """ + + def as_str(self): + return str(self.token) + + def __repr__(self): + return "Identifier(line={!r},col={!r},prev_white={!r},token={!r})".format( + self.line, self.col, self.prev_white, self.token) + + +class Operator(Token): + """ + Represents a C operator. + """ + + def __repr__(self): + return "Operator(line={!r},col={!r},prev_white={!r},token={!r})".format( + self.line, self.col, self.prev_white, self.token) + + def __str__(self): + return str(self.token) + + +class Punctuator(Token): + """ + Represents a punctuator (e.g. parentheses) + """ + + def __repr__(self): + return "Punctuator(line={!r},col={!r},prev_white={!r},token={!r})".format( + self.line, self.col, self.prev_white, self.token) + + +class Lexer: + """ + A lexer for the C preprocessor grammar. + """ + + def __init__(self, string, line="Unknown"): + self.string = string + self.line = line + self.pos = 0 + self.prev_white = False + + def read(self, n=1): + """ + Return the next n characters in the string. + """ + return self.string[self.pos:self.pos + n] + + def eos(self): + """ + Return True when the end of the string is reached. + """ + return self.pos == len(self.string) + + def whitespace(self): + """ + Consume whitespace and advance position. + """ + while not self.eos() and self.read() in [" ", "\t", "\n", "\r"]: + self.pos += 1 + self.prev_white = True + + def match(self, literal): + """ + Match a character/string literal exactly and advance position. + """ + if self.read(len(literal)) == literal: + self.pos += len(literal) + else: + raise TokenError("Expected {}.".format(literal)) + + def match_any(self, literals): + """ + Match one from a list of character/string literals exactly. + Return the matched index and advance position. + """ + for (index, literal) in enumerate(literals): + if self.read(len(literal)) == literal: + self.pos += len(literal) + return index + + raise TokenError("Expected one of: {}.".format(", ".join(literals))) + + def number(self): + """ + Construct a NumericalConstant by parsing a string. + Return a NumericalConstant and advance position. + + := ['e'|'E'|'p'|'P']['+'|'-'] + := .?[||'_'|'.'|]* + """ + col = self.pos + try: + chars = [] + + # Match optional period + if self.read() == ".": + chars.append(self.read()) + self.pos += 1 + + # Match required decimal digit + if self.read().isdigit(): + chars.append(self.read()) + self.pos += 1 + else: + raise TokenError("Expected digit.") + + # Match any sequence of letters, digits, underscores, + # periods and exponents + exponents = ["e+", "e-", "E+", "E-", "p+", "p-", "P+", "P-"] + while not self.eos(): + if self.read(2) in exponents: + chars.append(self.read(2)) + self.pos += 2 + elif self.read().isalpha() or self.read().isdigit() or self.read() in ["_", "."]: + chars.append(self.read()) + self.pos += 1 + else: + break + + value = "".join(chars) + except TokenError: + self.pos = col + raise TokenError("Invalid preprocessing number.") + + constant = NumericalConstant(self.line, col, self.prev_white, value) + return constant + + def character_constant(self): + """ + Construct a CharacterConstant by parsing a string. + Return a CharacterConstant and advance position. + + := '''''' + """ + col = self.pos + try: + self.match('\'') + + # A character constant may be an escaped sequence + # We assume a single alpha-numerical character or space + if self.read() == '\\' and self.read(2).isprintable(): + value = self.read(2) + self.pos += 2 + elif self.read().isprintable(): + value = self.read() + self.pos += 1 + else: + raise TokenError("Expected character.") + + self.match('\'') + except TokenError: + self.pos = col + raise TokenError("Invalid character constant.") + + constant = CharacterConstant(self.line, col, self.prev_white, value) + return constant + + def string_constant(self): + """ + Construct a StringConstant by parsing a string. + Return a StringConstant and advance position. + + := '"'.*'"' + """ + col = self.pos + try: + self.match('\"') + + chars = [] + while not self.eos() and self.read() != '\"': + + # An escaped " should not close the string + if self.read(2) == "\\\"": + chars.append(self.read(2)) + self.pos += 2 + else: + chars.append(self.read()) + self.pos += 1 + + self.match('\"') + except TokenError: + self.pos = col + raise TokenError("Invalid string constant.") + + constant = StringConstant(self.line, col, self.prev_white, "".join(chars)) + return constant + + def identifier(self): + """ + Construct an Identifier by parsing a string. + Return an Identifier and advance position. + + := [|'_'][||'_']* + """ + col = self.pos + + # Match a string of characters + characters = [] + while not self.eos() and (self.read().isalnum() or self.read() == "_"): + + # First character of an identifier cannot be a digit + if self.pos == col and self.read().isdigit(): + self.pos = col + raise TokenError("Identifiers cannot start with a digit.") + + characters += self.read() + self.pos += 1 + + if not characters: + self.pos = col + raise TokenError("Invalid identifier.") + + identifier = Identifier(self.line, col, self.prev_white, "".join(characters)) + return identifier + + def operator(self): + """ + Construct an Operator by parsing a string. + Return an Operator and advance position. + + := ['-' | '+' | '!' | '#' | '~' | '*' | '/' | '|' | '&' | + '^' | '||' | '&&' | '>>' | '<<' | '!=' | '>=' | '<=' | + '==' | '##' | '?' | ':' | '<' | '>' | '%'] + """ + col = self.pos + operators = ["||", "&&", ">>", "<<", "!=", ">=", "<=", "==", "##"] + \ + ["-", "+", "!", "*", "/", "|", "&", "^", "<", ">", "?", ":", "~", "#", "=", "%"] + try: + index = self.match_any(operators) + + op = Operator(self.line, col, self.prev_white, operators[index]) + return op + except TokenError: + self.pos = col + raise TokenError("Invalid operator.") + + def punctuator(self): + """ + Construct a Punctuator by parsing a string. + Return a Punctuator and advance position. + + := ['('|')'|'{'|'}'|'['|']'|','|'.'|';'|'''|'"'|'\'] + """ + col = self.pos + punctuators = ["(", ")", "{", "}", "[", "]", ",", ".", ";", "'", "\"", "\\"] + try: + index = self.match_any(punctuators) + + punc = Punctuator(self.line, col, self.prev_white, punctuators[index]) + return punc + except TokenError: + self.pos = col + raise TokenError("Invalid punctuator.") + + def tokenize(self): + """ + Return a list of all tokens in the string. + """ + tokens = [] + while not self.eos(): + + # Try to match a new token + token = None + candidates = [self.number, self.character_constant, self.string_constant, + self.identifier, self.operator, self.punctuator] + for f in candidates: + col = self.pos + try: + self.whitespace() + token = f() + self.prev_white = False + tokens.append(token) + break + except TokenError: + self.pos = col + + # Only continue matching if a token was found + if token is None: + break + + self.whitespace() + if not self.eos(): + raise TokenError("Encountered invalid token.") + + return tokens + + +class ParseError(ValueError): + """ + Represents an error encountered during parsing. + """ + + +class Node(): + """ + Base class for all other Node types. + Contains a single parent, and an ordered list of children. + """ + + def __init__(self): + self.children = [] + self.parent = None + + def add_child(self, child): + self.children.append(child) + child.parent = self + + @staticmethod + def is_start_node(): + """ + Used to determine if a node is a start node of a tree. + Return False by default. + """ + return False + + @staticmethod + def is_cont_node(): + """ + Used to determine if a node is a continue node of a tree. + Return False by default. + """ + return False + + @staticmethod + def is_end_node(): + """ + Used to determine if a node is a end node of a tree. + Return False by default. + """ + return False + + # pylint: disable=no-self-use,unused-argument + def evaluate_for_platform(self, **kwargs): + """ + Determine if the children of this node are active, by evaluating + the statement. + Return False by default. + """ + return False + + +class FileNode(Node): + """ + Typically the root node of a tree. Simply contains a filename after + inheriting from the Node class. + """ + + def __init__(self, _filename): + super().__init__() + self.filename = _filename + # The length of the file, counting blank lines and comments + self.num_lines = 0 + # The source lines of code, ignoring blank lines and comments + self.total_sloc = 0 + self.file_hash = self.__compute_file_hash() + + def __compute_file_hash(self): + chunk_size = 4096 + hasher = hashlib.sha512() + with open(self.filename, 'rb') as in_file: + for chunk in iter(lambda: in_file.read(chunk_size), b""): + hasher.update(chunk) + + return hasher.hexdigest() + + def __repr__(self): + return "FileNode(filename={0!r})".format(self.filename) + + def __str__(self): + return "{}; Hash: {}".format(str(self.filename), str(self.file_hash)) + + def evaluate_for_platform(self, **kwargs): + """ + Since a FileNode is always used as a root node, we are only + interested in its children. + """ + return True + + +class CodeNode(Node): + """ + Represents any line of code. Contains a start and end line, and the + number of countable lines occurring between them. + """ + + def __init__(self, start_line=-1, end_line=-1, num_lines=0): + super().__init__() + self.start_line = start_line + self.end_line = end_line + self.num_lines = num_lines + + def __repr__(self): + return "CodeNode(start={0!r},end={1!r},lines={2!r}".format( + self.start_line, self.end_line, self.num_lines) + + def __str__(self): + return "Lines {}-{}; SLOC = {};".format(self.start_line, self.end_line, self.num_lines) + + +class DirectiveNode(CodeNode): + """ + A CodeNode representing a C preprocessor directive. + We need to track all of the tokens for this directive in addition to + countable lines and extent. + """ + + def __init__(self): + super().__init__() + + +class UnrecognizedDirectiveNode(DirectiveNode): + """ + A CodeNode representing an unrecognized preprocessor directive + """ + + def __init__(self, tokens): + super().__init__() + self.kind = "unrecognized" + self.tokens = tokens + + def __repr__(self): + return "DirectiveNode(kind={!r},tokens={!r})".format(self.kind, self.tokens) + + def __str__(self): + return "{}".format(" ".join([str(t) for t in self.tokens])) + + +class PragmaNode(DirectiveNode): + """ + Represents a #pragma directive + """ + + def __init__(self, tokens): + super().__init__() + self.kind = "pragma" + self.tokens = tokens + + def __repr__(self): + return "DirectiveNode(kind={0!r},tokens={1!r})".format(self.kind, self.tokens) + + def __str__(self): + return "#pragma {0!s}".format(" ".join([str(t) for t in self.tokens])) + + def evaluate_for_platform(self, **kwargs): + if self.tokens and str(self.tokens[0]) == 'once': + kwargs['platform'].add_include_to_skip(kwargs['filename']) + + +class DefineNode(DirectiveNode): + """ + A DirectiveNode representing a #define directive. + """ + + def __init__(self, identifier, args=None, value=None): + super().__init__() + self.kind = "define" + self.identifier = identifier + self.args = args + self.value = value + + def __repr__(self): + return "DirectiveNode(kind={0!r},identifier={1!r},args={2!r},value={3!r})".format( + self.kind, self.identifier, self.args, self.value) + + def __str__(self): + value_str = "".join([str(v) for v in self.value]) + + if self.args is None: + return "#define {0!s} {1!s}".format(self.identifier, value_str) + elif self.args == []: + return "#define {0!s}() {1!s}".format(self.identifier, value_str) + else: + arg_str = ", ".join([str(arg) for arg in self.args]) + return "#define {0!s}({1!s}) {2!s}".format(self.identifier, arg_str, value_str) + + def evaluate_for_platform(self, **kwargs): + """ + Add a definition into the platform, and return false + """ + macro = Macro(self.identifier, self.args, self.value) + kwargs['platform'].define(self.identifier.as_str(), macro) + return False + + +class UndefNode(DirectiveNode): + """ + A DirectiveNode representing an #undef directive. + """ + + def __init__(self, identifier): + super().__init__() + self.kind = "undefine" + self.identifier = identifier + + def __repr__(self): + return "DirectiveNode(kind={0!r},identifier={1!r})".format(self.kind, self.identifier) + + def __str__(self): + return "#undef {0!s}".format(self.identifier) + + def evaluate_for_platform(self, **kwargs): + """ + Add a definition into the platform, and return false + """ + kwargs['platform'].undefine(self.identifier.as_str()) + return False + + +class IncludePath(): + """ + Represents an include path enclosed by "" or <> + """ + + def __init__(self, path, system): + self.path = path + self.system = system + + def __repr__(self): + return "IncludePath(path={!r},system={!r})".format(self.path, self.system) + + def __str__(self): + if self.system: + return "<{0!s}>".format(self.path) + return "\"{0!s}\"".format(self.path) + + def is_system_path(self): + return self.system + + +class IncludeNode(DirectiveNode): + """ + A DirectiveNode representing an #include directive. + Its value is an IncludePath or a list of tokens. + """ + + def __init__(self, value): + super().__init__() + self.kind = "include" + self.value = value + + def __repr__(self): + return "DirectiveNode(kind={0!r},value={1!r})".format(self.kind, self.value) + + def __str__(self): + return "#include {0!s}".format(self.value) + + def evaluate_for_platform(self, **kwargs): + """ + Extract the filename from the #include. This cannot happen when + parsing because of "computed includes" like #include FOO. After + the filename is extracted, process the include file: build a + tree for it, and walk it, updating the platform definitions. + """ + + include_path = None + is_system_include = False + if isinstance(self.value, IncludePath): + include_path = self.value.path + is_system_include = self.value.system + else: + expansion = MacroExpander(self.value).expand(kwargs['platform']) + path_obj = DirectiveParser(expansion).include_path() + include_path = path_obj.path + is_system_include = path_obj.system + + include_file = kwargs['platform'].find_include_file(include_path, is_system_include) + + if include_file and kwargs['platform'].process_include(include_file): + kwargs['state'].insert_file(include_file) + + associator = walkers.TreeAssociator(kwargs['state'].get_tree( + include_file), kwargs['state'].get_map(include_file)) + associator.walk(kwargs['platform'], kwargs['state']) + + +class IfNode(DirectiveNode): + """ + Represents an #if, #ifdef or #ifndef directive. + """ + + def __init__(self, tokens): + super().__init__() + self.kind = "if" + self.tokens = tokens + + @staticmethod + def is_start_node(): + return True + + def __repr__(self): + return "DirectiveNode(kind={0!r},tokens={1!r})".format(self.kind, self.tokens) + + def __str__(self): + return "#if {0!s}".format(" ".join([str(t) for t in self.tokens])) + + def evaluate_for_platform(self, **kwargs): + # Perform macro substitution with tokens + expanded_tokens = MacroExpander(self.tokens).expand(kwargs['platform']) + + # Evaluate the expanded tokens + return ExpressionEvaluator(expanded_tokens).evaluate() + + +class ElIfNode(IfNode): + """ + Represents an #elif directive. + """ + + def __init__(self, tokens): + super().__init__(tokens) + self.kind = "elif" + + @staticmethod + def is_start_node(): + return False + + @staticmethod + def is_cont_node(): + return True + + +class ElseNode(DirectiveNode): + """ + Represents an #else directive. + """ + + def __init__(self): + super().__init__() + self.kind = "else" + + @staticmethod + def is_cont_node(): + return True + + def __repr__(self): + return "DirectiveNode(kind={0!r})".format(self.kind) + + def __str__(self): + return "#else" + + def evaluate_for_platform(self, **kwargs): + return True + + +class EndIfNode(DirectiveNode): + """ + Represents an #endif directive. + """ + + def __init__(self): + super().__init__() + self.kind = "endif" + + @staticmethod + def is_end_node(): + return True + + def __repr__(self): + return "DirectiveNode(kind={0!r})".format(self.kind) + + def __str__(self): + return "#endif" + + +class Parser: + """ + A generic token parser for matching tokens from a list. + """ + + def __init__(self, tokens): + self.tokens = tokens + self.pos = 0 + + def cursor(self): + """ + Return the current token in the list. + """ + try: + return self.tokens[self.pos] + except IndexError: + raise ParseError("No tokens left for cursor to traverse") + + def eol(self): + """ + Return True when the end of the list is reached. + """ + return self.pos == len(self.tokens) + + def match_type(self, token_type): + """ + Match a token of the specified type and advance position. + """ + if isinstance(self.cursor(), token_type): + token = self.cursor() + self.pos += 1 + else: + raise ParseError("Expected {!s}.".format(token_type)) + return token + + def match_value(self, token_type, token_value): + """ + Match a token of the specified type and value, and advance + position. + """ + if isinstance(self.cursor(), token_type) and self.cursor().token == token_value: + token = self.cursor() + self.pos += 1 + else: + raise ParseError("Expected {!s}.".format(token_value)) + return token + + +class DirectiveParser(Parser): + """ + A specialized token parser for recognizing directives. + """ + + def __arg(self): + """ + Match an Identifier, Identifier... or ... + + := ?'...' + """ + arg = None + + # Match optional identifier + initial_pos = self.pos + try: + arg = self.match_type(Identifier) + except ParseError: + self.pos = initial_pos + + # Match optional '...' + ellipsis_pos = self.pos + try: + punc = self.match_value(Punctuator, ".") + self.match_value(Punctuator, ".") + self.match_value(Punctuator, ".") + if arg is None: + arg = Identifier(punc.line, punc.col, punc.prev_white, "...") + else: + arg.token += "..." + except ParseError: + self.pos = ellipsis_pos + + if arg is not None: + return arg + raise ParseError("Invalid argument") + + def __arg_list(self): + """ + Match a comma-separated list of arguments. + Return a tuple of the Token(s) and advances position.. + + := [[',']*]? + """ + args = [] + try: + arg = self.__arg() + args.append(arg) + if arg.token.endswith("..."): + return args + + while True: + self.match_value(Punctuator, ",") + + arg = self.__arg() + if arg.token.endswith("..."): + return args + + args.append(arg) + except ParseError: + return args + + def macro_definition(self): + """ + Match a macro definition. + Return a tuple of the Identifier and argument list (or None). + """ + identifier = self.match_type(Identifier) + + # Match function-like macro definitions + arg_pos = self.pos + try: + # Read a list of arguments between parentheses. + # whitespace is NOT permitted before the opening paren. + punctuator = self.match_value(Punctuator, "(") + if punctuator.prev_white: + raise ParseError("Not a function-like macro.") + args = self.__arg_list() + punctuator = self.match_value(Punctuator, ")") + except ParseError: + args = None + self.pos = arg_pos + + return (identifier, args) + + def define(self): + """ + Match a define directive. + Return a tuple of the Define and the new string position. + + := 'define'? + := 'define' + '('?')' + ? + := [][',']* + := [|] + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "define") + (identifier, args) = self.macro_definition() + + # Any remaining tokens are the macro expansion + if not self.eol(): + expansion = self.tokens[self.pos:] + self.pos = len(self.tokens) + else: + expansion = [] + + return DefineNode(identifier, args, expansion) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid define directive.") + + def undef(self): + """ + Match an #undef directive. + Return an UndefNode. + + := 'undef' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "undef") + identifier = self.match_type(Identifier) + return UndefNode(identifier) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid undef directive.") + + def include(self): + """ + Match an #include directive. + Return an IncludeNode. + + := 'include' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "include") + path_pos = self.pos + + # Match system or local include path + try: + path = self.include_path() + return IncludeNode(path) + except ParseError: + self.pos = path_pos + + # Match computed include with a simple macro + try: + identifier = self.match_type(Identifier) + return IncludeNode([identifier]) + except ParseError: + self.pos = path_pos + + raise ParseError("Invalid include directive.") + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid include directive.") + + def __path(self, marker_type=Punctuator, initiator_value='\"', terminator_value='\"'): + """ + Match a path enclosed between the specified initiator and + terminator values. + """ + path = [] + self.match_value(marker_type, initiator_value) + while not self.eol() and not (isinstance(self.cursor(), marker_type) + and self.cursor().token == terminator_value): + path.append(self.cursor()) + self.pos += 1 + self.match_value(marker_type, terminator_value) + return path + + def include_path(self): + """ + Match an include path. + := ['<''>'|'\"''>'] + """ + initial_pos = self.pos + + # Match system include + try: + path_tokens = self.__path(Operator, "<", ">") + path_str = "".join([str(t) for t in path_tokens]) + if util.valid_path(path_str): + return IncludePath(path_str, system=True) + except ParseError: + self.pos = initial_pos + + # Match local include + try: + path_token = self.match_type(StringConstant) + path_str = path_token.token + if util.valid_path(path_str): + return IncludePath(path_str, system=False) + except ParseError: + self.pos = initial_pos + + raise ParseError("Invalid path.") + + def pragma(self): + """ + Match a #pragma directive. + Return a PragmaNode. + + := 'pragma' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "pragma") + expr = self.tokens[self.pos:] + self.pos = len(self.tokens) + + return PragmaNode(expr) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid pragma directive.") + + def if_(self): + """ + Match an #if directive. + Return an IfNode. + + := 'if' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "if") + expr = self.tokens[self.pos:] + self.pos = len(self.tokens) + + return IfNode(expr) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid if directive.") + + def ifdef(self): + """ + Match an #ifdef directive. + Return an IfNode with defined() in the expression. + + := 'ifdef' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "ifdef") + identifier = self.match_type(Identifier) + + # Wrap expression in "defined()" call + prefix = [Identifier("Unknown", -1, True, "defined"), + Punctuator("Unknown", -1, False, "(")] + suffix = [Punctuator("Unknown", -1, False, ")")] + expr = prefix + [identifier] + suffix + + return IfNode(expr) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid ifdef directive") + + def ifndef(self): + """ + Match an #ifdef directive. + Return an IfNode with !defined() in the expression. + + := 'ifndef' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "ifndef") + identifier = self.match_type(Identifier) + + # Wrap expression in "!defined()" call + prefix = [Operator("Unknown", -1, True, "!"), Identifier("Unknown", -1, + False, "defined"), + Punctuator("Unknown", + -1, False, "(")] + suffix = [Punctuator("Unknown", -1, False, ")")] + expr = prefix + [identifier] + suffix + + return IfNode(expr) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid ifndef directive") + + def elif_(self): + """ + Match an #elif directive. + Return an ElIfNode. + + := 'elif' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "elif") + expr = self.tokens[self.pos:] + self.pos = len(self.tokens) + + return ElIfNode(expr) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid elif directive.") + + def else_(self): + """ + Match an #else directive. + Return an ElseNode. + + := 'else' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "else") + return ElseNode() + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid else directive.") + + def endif(self): + """ + Match an #endif directive. + Return an EndIfNode. + + := 'endif' + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "endif") + return EndIfNode() + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid endif directive.") + + def parse(self): + """ + Parse a preprocessor directive. + Return a DirectiveNode. + + := '#'[||||| + |||] + """ + try: + self.match_value(Operator, "#") + + # Check for a match against known directives + candidates = [self.define, self.undef, self.include, self.ifdef, + self.ifndef, self.if_, self.elif_, self.else_, self.endif, self.pragma] + for f in candidates: + try: + directive = f() + if not self.eol(): + log.warning("Additional tokens at end of preprocessor directive") + return directive + except ParseError: + pass + + # Any other line beginning with '#' is a preprocessor + # directive, we just don't handle it (yet). Suppress + # warnings for common directives that shouldn't impact + # correctness. + common_unhandled = ["line", "warning", "error"] + if len(self.tokens) > 2 and str(self.tokens[1]) not in common_unhandled: + log.warning("Unrecognized directive") + return UnrecognizedDirectiveNode(self.tokens) + except ParseError: + raise ParseError("Not a directive.") + + +class Macro(): + """ + Represents a macro definition. + """ + + def __init__(self, name, args=None, expansion=None): + self.name = str(name) + if args is not None: + self.args = [str(a) for a in args] + else: + self.args = None + self.expansion = expansion + + def __repr__(self): + return "Macro(name={0!r},args={1!r},expansion={2!r})".format( + self.name, self.args, self.expansion) + + def __str__(self): + expansion_str = " ".join([str(t) for t in self.expansion]) + if self.args is None: + return "{0!s}={1!s}".format(self.name, expansion_str) + arg_str = ",".join(self.args) + return "{0!s}({1!s})={2!s}".format(self.name, arg_str, expansion_str) + + def is_function(self): + """ + Return true if macro is function-like + """ + return self.args is not None + + def is_variadic(self): + """ + Return true if macro is variadic + """ + if self.is_function() and self.args: + return self.args[-1].endswith("...") + return False + + def variable_argument(self): + """ + Return the name of the variable argument for a variadic macro. + If the macro is not variadic, return None. + """ + if self.is_variadic(): + if self.args[-1] == '...': + # An unnamed variable argument replaces __VA_ARGS__ + return "__VA_ARGS__" + else: + # Strip '...' from argument name + return self.args[-1][:-3] + else: + return None + + @staticmethod + def from_definition_string(string): + """ + Construct a Macro by parsing a string of the form + MACRO=expansion. + """ + tokens = Lexer(string).tokenize() + parser = DirectiveParser(tokens) + + (identifier, args) = parser.macro_definition() + + # Any remaining tokens after an "=" are the macro expansion + if not parser.eol(): + parser.match_value(Operator, "=") + expansion = parser.tokens[parser.pos:] + parser.pos = len(parser.tokens) + else: + expansion = [] + + return Macro(identifier, args, expansion) + + +class MacroStack: + """ + Tracks previously expanded macros during recursive expansion. + """ + + def __init__(self, level, no_expand): + self.level = level + self.no_expand = no_expand + + # Prevent infinite recursion. CPP standard requires this be at + # least 15, but cpp has been implemented to handle 200. + self.max_level = 200 + + def __contains__(self, identifier): + return str(identifier) in self.no_expand + + def push(self, identifier): + self.level += 1 + self.no_expand.append(str(identifier)) + + def pop(self): + self.level -= 1 + self.no_expand.pop() + + def overflow(self): + return self.level >= self.max_level + + +class MacroExpander(Parser): + """ + A specialized token parser for recognizing and expanding macros. + """ + + # pylint: disable=unused-argument + def defined(self, platform, stack): + """ + Expand a call to defined(X) or defined X. + """ + initial_pos = self.pos + try: + self.match_value(Identifier, "defined") + + # Match an identifier in parens + defined_pos = self.pos + try: + self.match_value(Punctuator, "(") + identifier = self.match_type(Identifier) + self.match_value(Punctuator, ")") + + value = platform.is_defined(str(identifier)) + return NumericalConstant("EXPANSION", identifier.line, + identifier.prev_white, value) + except ParseError: + self.pos = defined_pos + + # Match an identifier without parens + try: + identifier = self.match_type(Identifier) + value = platform.is_defined(str(identifier)) + return NumericalConstant("EXPANSION", identifier.line, + identifier.prev_white, value) + except ParseError: + raise ParseError("Expected identifier after \"defined\" operator") + + except ParseError: + self.pos = initial_pos + raise ParseError("Not a defined operator.") + + def __arg(self): + """ + Match an argument to a function-like macro, where: + - An argument is a (potentially empty) list of tokens + - Arguments are separated by a "," + - "(" must be matched with a ")" within an argument + """ + arg = [] + open_parens = 0 + while not self.eol(): + t = self.cursor() + if isinstance(t, Punctuator): + if t.token == "(": + open_parens += 1 + elif t.token == ")" and open_parens > 0: + open_parens -= 1 + elif (t.token == "," or t.token == ")") and open_parens == 0: + return arg + arg.append(t) + self.pos += 1 + + if open_parens > 0: + raise ParseError("Mismatched parentheses in macro argument.") + raise ParseError("Invalid macro argument.") + + def __arg_list(self): + """ + Match a list of function-like macro arguments. + """ + arg = self.__arg() + args = [arg] + try: + while not self.eol(): + self.match_value(Punctuator, ",") + arg = self.__arg() + args.append(arg) + except ParseError: + pass + return args + + @staticmethod + def grow_token_list(token_list, extension): + """ + Copy tokens from extension into token_list. + extension can be a single token or a list of tokens. + """ + if isinstance(extension, list): + token_list.extend(extension) + else: + token_list.append(extension) + + def function_macro(self, platform, stack): + """ + Expand a function-like macro. + + Follows the rules outlined in: + https://gcc.gnu.org/onlinedocs/cpp/Macro-Arguments.html#Macro-Arguments + """ + initial_pos = self.pos + try: + identifier = self.match_type(Identifier) + if identifier in stack: + raise ParseError('Cannot expand token') + + self.match_value(Punctuator, "(") + args = self.__arg_list() + self.match_value(Punctuator, ")") + except ParseError: + self.pos = initial_pos + raise ParseError("Not a function-like macro.") + + macro = platform.get_macro(str(identifier)) + if not macro: + raise ParseError("Not a function-like macro.") + + # Argument pre-scan + # Macro arguments are macro-expanded before substitution + expanded_args = [MacroExpander(arg).expand(platform) for arg in args] + + # Combine variadic arguments into one, separated by commas + va_args = None + if macro.is_variadic(): + va_args = util.interleave(expanded_args[len(macro.args) - 1:], + Punctuator("EXPANSION", -1, False, ",")) + va_args = va_args[:-1] # Remove final comma + + # Substitute each occurrence of an argument in the expansion + substituted_tokens = [] + for token in macro.expansion: + + substitution = [] + + # If a token matches an argument, it is substituted; + # otherwise it passes through + try: + substitution = expanded_args[macro.args.index(token.token)] + except (ValueError, ParseError): + substitution = token + + # If a token matches the variable argument, substitute + # precomputed va_args + if macro.is_variadic() and token.token == macro.variable_argument() and va_args: + + # Whether the token was preceded by whitespace should be + # tracked through substitution + for t in va_args[0]: + t.prev_white = token.prev_white + substitution = va_args + + self.grow_token_list(substituted_tokens, substitution) + + # Check the expansion for macros to expand + stack.push(identifier) + expansion = MacroExpander(substituted_tokens).expand(platform, stack) + stack.pop() + return expansion + + def macro(self, platform, stack): + """ + Expand a macro. + """ + initial_pos = self.pos + try: + identifier = self.match_type(Identifier) + if identifier in stack: + raise ParseError('Cannot expand this token') + + macro = platform.get_macro(str(identifier)) + if not macro: + raise TokenError('Not a macro.') + + stack.push(identifier) + expansion = MacroExpander(macro.expansion).expand(platform, stack) + stack.pop() + return expansion + except TokenError: + self.pos = initial_pos + raise ParseError("Not a macro.") + + def expand(self, platform, stack=None): + """ + Expand a list of input tokens using the specified definitions. + Return a list of new tokens, representing the result of macro + expansion. + """ + if not stack: + stack = MacroStack(0, []) + + if stack.overflow(): + return NumericalConstant("EXPANSION", -1, False, "0") + + try: + expanded_tokens = [] + while not self.eol(): + # Match and expand special tokens + expansion = None + test_pos = self.pos + + candidates = [self.defined, self.function_macro, self.macro] + for f in candidates: + try: + expansion = f(platform, stack) + break + except ParseError: + self.pos = test_pos + + # Pass all other tokens through unmodified + if expansion is None: + expansion = self.cursor() + self.pos += 1 + + self.grow_token_list(expanded_tokens, expansion) + return expanded_tokens + except ParseError: + raise ValueError("Error in macro expansion.") + + +class ExpressionEvaluator(Parser): + """ + A specialized token parser for recognizing/evaluating expressions. + """ + + # Operator precedence, associativity and Python equivalent + # Lower numbers = higher precedence + # Based on: + # https://en.cppreference.com/w/cpp/language/operator_precedence + OpInfo = collections.namedtuple("OpInfo", ["prec", "assoc"]) + UnaryOperators = { + '-': OpInfo(12, "RIGHT"), + '+': OpInfo(12, "RIGHT"), + '!': OpInfo(12, "RIGHT"), + '~': OpInfo(12, "RIGHT") + } + BinaryOperators = { + '?': OpInfo(1, "RIGHT"), + '||': OpInfo(2, "LEFT"), + '&&': OpInfo(3, "LEFT"), + '|': OpInfo(4, "LEFT"), + '^': OpInfo(5, "LEFT"), + '&': OpInfo(6, "LEFT"), + '==': OpInfo(7, "LEFT"), + '!=': OpInfo(7, "LEFT"), + '<': OpInfo(8, "LEFT"), + '<=': OpInfo(8, "LEFT"), + '>': OpInfo(8, "LEFT"), + '>=': OpInfo(8, "LEFT"), + '<<': OpInfo(9, "LEFT"), + '>>': OpInfo(9, "LEFT"), + '+': OpInfo(10, "LEFT"), + '-': OpInfo(10, "LEFT"), + '*': OpInfo(11, "LEFT"), + '/': OpInfo(11, "LEFT"), + '%': OpInfo(11, "LEFT"), + } + + def call(self): + """ + Match a built-in call or function-like macro and return 0. + + := '('?')' + """ + initial_pos = self.pos + try: + self.match_type(Identifier) + + # Read a list of arguments + self.match_value(Punctuator, "(") + self.__expression_list() + self.match_value(Punctuator, ")") + + # Any function call that still exists after substitution + # evaluates to false + return np.int64(0) + except ParseError: + self.pos = initial_pos + raise ParseError("Invalid function call.") + + def term(self): + """ + Match a constant, function call or identifier and convert it to + Python syntax. + + := [||| + ] + """ + initial_pos = self.pos + + # Match an integer constant. + # Convert from C-style literals to Python integers. + try: + constant = self.match_type(NumericalConstant) + + # Use prefix (if present) to determine base + base = 10 + bases = {"0x": 16, "0X": 16, "0b": 2, "0B": 2} + try: + prefix = constant.token[0:2] + base = bases[prefix] + value = constant.token[2:] + except KeyError: + value = constant.token + + # Strip suffix (if present) + suffix = None + suffixes = ['ull', 'ULL', 'ul', 'UL', 'll', 'LL', 'u', 'U', 'l', 'L'] + for s in suffixes: + if value.endswith(s): + suffix = s + value = value[:-len(s)] + break + + # Convert to decimal and then to integer with correct sign + # Preprocessor always uses 64-bit arithmetic! + int_value = int(value, base) + if suffix and 'u' in suffix: + return np.uint64(int_value) + else: + return np.int64(int_value) + except ParseError: + self.pos = initial_pos + + # Match a character constant. + # Convert from character literals to integer value. + try: + constant = self.match_type(CharacterConstant) + return np.int64(ord(constant.token)) + except ParseError: + self.pos = initial_pos + + # Match a function call. + try: + return self.call() + except ParseError: + self.pos = initial_pos + + # Match an identifier. + # Any identifier that still exists after substitution evaluates + # to false + try: + self.match_type(Identifier) + return np.int64(0) + except ParseError: + self.pos = initial_pos + + raise ParseError( + "Expected an integer constant, character constant, identifier or function call.") + + def primary(self): + """ + Match a simple expression + := [|'('')'|] + """ + initial_pos = self.pos + + # Match + try: + operator = self.match_type(Operator) + if operator.token in ExpressionEvaluator.UnaryOperators: + # pylint: disable=unused-variable + (prec, assoc) = ExpressionEvaluator.UnaryOperators[operator.token] + else: + raise ParseError("Not a UnaryOperator") + expr = self.expression(prec) + return self.__apply_unary_op(operator.token, expr) + except ParseError: + self.pos = initial_pos + + # Match '('')' + try: + self.match_value(Punctuator, "(") + expr = self.expression() + self.match_value(Punctuator, ")") + return expr + except ParseError: + self.pos = initial_pos + + # Match + try: + term = self.term() + return term + except ParseError: + self.pos = initial_pos + + raise ParseError( + "Expected a unary expression, an expression in parens or an identifier/constant.") + + def expression(self, min_precedence=0): + """ + Match a preprocessor expression. + Minimum precedence used to match operators during precedence + climbing. + + := []? + """ + expr = self.primary() + + # Recursion is terminated based on operator precedence + while not self.eol() and (self.cursor().token in ExpressionEvaluator.BinaryOperators) and ( + ExpressionEvaluator.BinaryOperators[self.cursor().token].prec >= min_precedence): + + operator = self.match_type(Operator) + (prec, assoc) = ExpressionEvaluator.BinaryOperators[operator.token] + + # The ternary conditional operator is treated as a + # special-case of a binary operator: + # lhs "?"":" rhs + if operator.token == "?": + true_result = self.expression() + self.match_value(Operator, ":") + + # Minimum precedence for right-hand side depends on + # associativity + if assoc == "LEFT": + rhs = self.expression(prec + 1) + elif assoc == "RIGHT": + rhs = self.expression(prec) + else: + raise ValueError("Encountered a BinaryOperator with no associativity.") + + # Converting C ternary to Python requires us to swap + # expression order: + # - C: (condition) ? true_result : false_result + # - Python: true_result if (condition) else false_result + if operator.token == "?": + condition = expr + false_result = rhs + expr = true_result if condition else false_result + else: + expr = self.__apply_binary_op(operator.token, expr, rhs) + + return expr + + def __expression_list(self): + """ + Match a comma-separated list of expressions. + Return an empty list or the expressions. + + := [][',']* + """ + exprs = [] + try: + expr = self.expression() + exprs.extend(expr) + + while True: + self.match_value(Punctuator, ",") + expr = self.expression() + exprs.extend(expr) + except ParseError: + return exprs + + @staticmethod + def __apply_unary_op(op, operand): + """ + Apply the specified unary operator: op operand + """ + if op == '-': + return -operand + elif op == '+': + return +operand + elif op == '!': + return not operand + elif op == '~': + return ~operand + else: + raise ValueError("Not a valid unary operator.") + + # pylint: disable=too-many-branches,too-many-return-statements + @staticmethod + def __apply_binary_op(op, lhs, rhs): + """ + Apply the specified binary operator: lhs op rhs + """ + if op == '||': + return lhs or rhs + elif op == '&&': + return lhs and rhs + elif op == '|': + return lhs | rhs + elif op == '^': + return lhs ^ rhs + elif op == '&': + return lhs & rhs + elif op == '==': + return lhs == rhs + elif op == '!=': + return lhs != rhs + elif op == '<': + return lhs < rhs + elif op == '<=': + return lhs <= rhs + elif op == '>': + return lhs > rhs + elif op == '>=': + return lhs >= rhs + elif op == '<<': + return lhs << rhs + elif op == '>>': + return lhs >> rhs + elif op == '+': + return lhs + rhs + elif op == '-': + return lhs - rhs + elif op == '*': + return lhs * rhs + elif op == '/': + return lhs // rhs # force integer division + elif op == '%': + return lhs % rhs + else: + raise ValueError("Not a binary operator.") + + def evaluate(self): + """ + Evaluate a preprocessor expression. + Return True/False or raises an exception if the expression is + not recognized. + """ + try: + test_val = self.expression() + return test_val != 0 + except ValueError: + raise ParseError("Could not evaluate expression.") + + +class SourceTree(): + """ + Represents a source file as a tree of directive and code nodes. + """ + + def __init__(self, filename): + self.root = FileNode(filename) + self._latest_node = self.root + + def associate_file(self, filename): + self.root.filename = filename + + def walk_to_tree_insertion_point(self): + """ + This function modifies self._latest_node to be a node that can + be a valid sibling of a tree continue node or a tree end node. + These nodes can only be inserted after an open tree start node, + or a tree continue node. + """ + + while not (self._latest_node.is_start_node() or self._latest_node.is_cont_node()): + self._latest_node = self._latest_node.parent + if self._latest_node == self.root: + log.error('Latest node == root while trying to find an insertion point.') + break + + def __insert_in_place(self, new_node, parent): + parent.add_child(new_node) + self._latest_node = new_node + + def insert(self, new_node): + """ + Handle the logic of proper node insertion. + """ + + # If there haven't been any nodes added, add this new node as a + # child of the root node. + if self._latest_node == self.root: + self.__insert_in_place(new_node, self._latest_node) + # Tree start nodes should be inserted as siblings of the + # previous node, unless it was a tree start, or tree continue + # node. In which case it's a child. + elif new_node.is_start_node(): + if self._latest_node.is_start_node() or self._latest_node.is_cont_node(): + self.__insert_in_place(new_node, self._latest_node) + else: + self.__insert_in_place(new_node, self._latest_node.parent) + + # If the node is a tree continue or a tree end node, it must be + # added as a sibling of a valid / active tree node. + elif new_node.is_cont_node() or new_node.is_end_node(): + # Need to walk back to find the previous level where an else + # or an end can be added + self.walk_to_tree_insertion_point() + + self.__insert_in_place(new_node, self._latest_node.parent) + + # Otherwise, if the previous node was a tree start or a tree + # continue, the new node is a child. If not, it's a sibling. + else: + if self._latest_node.is_start_node() or self._latest_node.is_cont_node(): + self.__insert_in_place(new_node, self._latest_node) + else: + self.__insert_in_place(new_node, self._latest_node.parent) + + +class CommentCleaner: + """ + Strip C or Fortran style comments from a source line. + """ + + def __init__(self, _file_extension): + self._in_block_comment = False + + if _file_extension in ['.f90', '.F90', '.f', '.F', '.F95', '.f95']: + self.filetype = 'fortran' + elif _file_extension in ['.cpp', '.c', '.cc', '.cu', '.cl', + '.cuh', 'clh', '.cxx', '.h', '.hpp', '.hh']: + self.filetype = 'c' + else: + log.fatal('File extension %s not recognized.', _file_extension) + + def strip_comments(self, line): + if self.filetype == 'c': + out_line = self.__remove_c_comments(line) + elif self.filetype == 'fortran': + out_line = self.__remove_fortran_comments(line) + return out_line + + def __ingest_c_block_comments(self, line, position): + """ + ingests block comments until either the line, or the comment is + exhausted. + + Return the number of ingested characters. + """ + + pos = position + while self._in_block_comment and pos < len(line): + if pos + 1 < len(line) and line[pos] == '*' and line[pos + 1] == '/': + self._in_block_comment = False + pos += 2 + pos += 1 + return pos - position + + @staticmethod + def __ingest_whitespace(line, position): + """ + ingest all contiguous whitespace characters. + + Return the number of ingested characters. + """ + pos = position + while line[pos] == ' ': + pos += 1 + return pos - position + + def __ingest_c_comment_start(self, line, pos): + """ + ingests the beginning of either a line or a block comment. + + Return the number of ingested characters for block comments, and + -1 for the start of a line comment. + """ + + if line[pos] == '/' and len(line) > pos + 1: + if line[pos + 1] == '/': + return -1 + elif line[pos + 1] == '*': + self._in_block_comment = True + return 2 + return 0 + + def __remove_c_comments(self, line): + """ + Remove C style comments. Return the stripped line. Set + self._in_block_comment if a block comment was started, but not + ended. + """ + new_chars = [] + i = 0 + while i < len(line): + blocks = self.__ingest_c_block_comments(line, i) + if blocks > 0: + i += blocks + continue + + whitespace = self.__ingest_whitespace(line, i) + if whitespace > 0: + new_chars.append(' ') + i += whitespace + continue + + comm_start = self.__ingest_c_comment_start(line, i) + if comm_start == -1: + new_chars.append(' ') + break + elif comm_start > 0: + new_chars.append(' ') + i += comm_start + + if blocks + whitespace + comm_start == 0: + new_chars.append(line[i]) + i += 1 + + new_line = ''.join(new_chars) + return new_line + + def __remove_fortran_comments(self, line): + """ + Remove Fortran style comments. Only obeys '!' line comments. + self._in_block_comment makes no sense in Fortran codes, so it + should be false both on entry and exit of this function. + """ + if self._in_block_comment: + log.error('Before Fortran line is processed, in a block comment?') + + new_chars = [] + line_len = len(line) + ignore_spaces = False + for i in range(line_len): + if line[i] == ' ': + if not ignore_spaces: + ignore_spaces = True + new_chars.append(' ') + elif line[i] == '!': + new_chars.append(' ') + break + else: + ignore_spaces = False + new_chars.append(line[i]) + + if self._in_block_comment: + log.error('Processing Fortran comment left state in a block comment?') + + new_line = ''.join(new_chars) + return new_line diff --git a/codebasin/report.py b/codebasin/report.py new file mode 100644 index 0000000..70f4e1a --- /dev/null +++ b/codebasin/report.py @@ -0,0 +1,154 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +Contains functions for generating command-line reports. +""" + +import itertools as it +import logging + +from . import util + +log = logging.getLogger("codebasin") + + +def extract_platforms(setmap): + """ + Extract a list of unique platforms from a set map + """ + unique_platforms = set(it.chain.from_iterable(setmap.keys())) + return list(unique_platforms) + + +def table(headers, rows): + """ + Convert table to ASCII string + """ + # Determine the cell widths + widths = [0] * len(headers) + for (c, h) in enumerate(headers): + widths[c] = max(widths[c], len(h)) + for r in rows: + for (c, data) in enumerate(r): + widths[c] = max(widths[c], len(data)) + hline = "-" * (sum(widths) + len(headers)) + + # Build the table as a list of strings + lines = [] + lines += [hline] + line = [h.rjust(widths[c]) for (c, h) in enumerate(headers)] + lines += [" ".join(line)] + lines += [hline] + for r in rows: + line = [data.rjust(widths[c]) for (c, data) in enumerate(r)] + lines += [" ".join(line)] + lines += [hline] + + return "\n".join(lines) + + +def distance(setmap, p1, p2): + """ + Compute distance between two platforms + """ + total = 0 + for (pset, count) in setmap.items(): + if (p1 in pset) or (p2 in pset): + total += count + d = 0 + for (pset, count) in setmap.items(): + if (p1 in pset) ^ (p2 in pset): + d += count / float(total) + return d + + +def divergence(setmap): + """ + Compute code divergence as defined by Harrell and Kitson + i.e. average of pair-wise distances between platform sets + """ + platforms = extract_platforms(setmap) + + d = 0 + npairs = 0 + for (p1, p2) in it.combinations(platforms, 2): + d += distance(setmap, p1, p2) + npairs += 1 + + if npairs == 0: + return 0 + return d / float(npairs) + + +def summary(setmap): + """ + Produce a summary report for the platform set + """ + lines = [] + + total = sum(setmap.values()) + data = [] + total_count = 0 + for pset in sorted(setmap.keys(), key=len): + name = "{%s}" % (", ".join(pset)) + count = "%d" % (setmap[pset]) + percent = "%.2f" % ((float(setmap[pset]) / float(total)) * 100) + data += [[name, count, percent]] + total_count += setmap[pset] + lines += [table(["Platform Set", "LOC", "% LOC"], data)] + + lines += ["Code Divergence: %.2f" % (divergence(setmap))] + lines += ["Unused Code (%%): %.2f" % ((setmap[frozenset()] / total_count) * 100.0)] + lines += ["Total SLOC: %d" % (total_count)] + + return "\n".join(lines) + + +def clustering(output_name, setmap): + """ + Produce a clustering report for the platform set + """ + platforms = extract_platforms(setmap) + + if len(platforms) == 1: + log.error("Error: clustering is not supported for a single platform.") + return None + + if not util.ensure_png(output_name): + log.error("Error: clustering output file name is not a png; skipping creation.") + return None + + # Import additional modules required by clustering report + # Force Agg backend to matplotlib to avoid DISPLAY errors + import matplotlib + matplotlib.use("Agg") + from matplotlib import pyplot as plt + + from scipy.cluster import hierarchy + from scipy.spatial.distance import squareform + + # Compute distance matrix between platforms + matrix = [[distance(setmap, p1, p2) for p2 in platforms] for p1 in platforms] + + # Print distance matrix as a table + lines = [] + lines += ["", "Distance Matrix"] + labelled_matrix = [[name] + [("%.2f" % column) for column in matrix[row]] + for (row, name) in enumerate(platforms)] + lines += [table([""] + platforms, labelled_matrix)] + + # Hierarchical clustering using average inter-cluster distance + clusters = hierarchy.linkage(squareform(matrix), method='average') + + # Plot dendrogram of hierarchical clustering + fig, ax = plt.subplots() + hierarchy.dendrogram(clusters, labels=platforms) + ax.set_ylim(ymin=0, ymax=1) + ax.axhline(y=divergence(setmap), linestyle='--', label="Average") + ax.legend() + plt.xlabel("Platform") + plt.ylabel("Code Divergence") + with util.safe_open_write_binary(output_name) as fp: + fig.savefig(fp) + + return "\n".join(lines) diff --git a/codebasin/util.py b/codebasin/util.py new file mode 100644 index 0000000..b65f6cc --- /dev/null +++ b/codebasin/util.py @@ -0,0 +1,79 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +Contains utility functions for common operations, including: +- Checking file extensions +- Opening files for writing +- Checking paths +""" + +from os.path import splitext +import os +import itertools as it +from collections.abc import Iterable + +import logging + +log = logging.getLogger("codebasin") + + +def ensure_ext(fname, extensions): + """Return true if the path passed in has specified extension""" + if not isinstance(extensions, Iterable): + extensions = [extensions] + + split = splitext(fname) + return len(split) == 2 and split[1].lower() in extensions + + +def ensure_png(fname): + """Return true if the path passed in specifies a png""" + return ensure_ext(fname, ".png") + + +def ensure_source(fname): + """Return true if the path passed in specifies a source file""" + extensions = [".c", ".cpp", ".cxx", ".c++", ".cc", + ".h", ".hpp", ".hxx", ".h++", ".hh", + ".inc", ".inl", ".tcc", ".icc", ".ipp"] + return ensure_ext(fname, extensions) + + +def ensure_yaml(fname): + """Return true if the path passed in specifies a YAML file""" + return ensure_ext(fname, ".yaml") + + +def safe_open_write_binary(fname): + """Open fname for (binary) writing. Truncate if not a symlink.""" + fpid = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o666) + return os.fdopen(fpid, "wb") + + +def valid_path(path): + """Return true if the path passed in is valid""" + valid = True + + # Check for null byte character(s) + if '\x00' in path: + log.critical("Null byte character in file request.") + valid = False + + # Check for carriage returns or line feed character(s) + if ('\n' in path) or ('\r' in path): + log.critical("Carriage return or line feed character in file request.") + valid = False + + return valid + + +def interleave(l1, l2): + """ + Return an interleaving of lists l1 and l2. + If l2 is not a list, broadcast its value. + """ + if isinstance(l2, list): + l2_list = l2 + else: + l2_list = [l2] * len(l1) + return list(it.chain.from_iterable(it.zip_longest(l1, l2_list))) diff --git a/codebasin/walkers.py b/codebasin/walkers.py new file mode 100644 index 0000000..3b72b64 --- /dev/null +++ b/codebasin/walkers.py @@ -0,0 +1,208 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +""" +Contains classes to help walk a tree of nodes, and create associations +between nodes and platform sets. +""" + +import logging +import collections + +log = logging.getLogger('codebasin') + + +class NodeAssociation(): + """ + Houses an association of platforms to a node. Each association + should be mapped (at a higher level) to a node, and the platforms + are contained here. + """ + + def __init__(self): + self.platforms = set() + + def add_platform(self, _platform): + """ + Add a platform to the list of associated platforms. + """ + if _platform not in self.platforms: + self.platforms.update([_platform]) + + +class NodeAssociationMap(): + """ + Contains a map of the node associations for a specific tree. + The map of trees to NodeAssociationMap happens as at higher + level, and then each node's association map is housed here. + """ + + def __init__(self): + self._node_associations = {} + + def prepare_node(self, _node): + """ + Create an empty node association map for a node. + """ + if _node not in self._node_associations: + self._node_associations[_node] = NodeAssociation() + + def add_platform(self, _node, _platform): + """ + Add a platform association to a node + """ + self.prepare_node(_node) + self._node_associations[_node].add_platform(_platform) + + def get_association(self, _node): + """ + Return the association class for a node. + """ + if _node in self._node_associations: + return self._node_associations[_node] + return None + + +class TreeWalker(): + """ + Generic tree walker class. + """ + + def __init__(self, _tree, _node_associations): + self.tree = _tree + self._node_associations = _node_associations + + +class TreePrinter(TreeWalker): + """ + Specific TreeWalker that prints the nodes for the tree + (with appropriate indentation). + """ + + def walk(self): + """ + Walk the tree, printing each node. + """ + self.__print_nodes(self.tree.root, 0) + + def __print_nodes(self, node, level): + """ + Print this specific node, then descend into it's children nodes. + """ + spacing = '' + for _ in range(level): + spacing = ' {}'.format(spacing) + + association = self._node_associations.get_association(node) + if association: + platform = ', '.join(association.platforms) + else: + platform = '' + + print('{}{} -- Platforms: {}'.format(spacing, node, platform)) + + for child in node.children: + self.__print_nodes(child, level + 1) + + +class TreeAssociator(TreeWalker): + """ + Specific TreeWalker that build associations with platforms. + """ + + def walk(self, platform, state): + """ + Walk the tree, associating nodes with platforms + """ + _ = self._associate_nodes(self.tree.root, platform, state, True) + + def _associate_nodes(self, node, platform, state, process_children): + """ + Associate this node with the platform. Evaluate the node, + and (if the evaluation say to) descend into the children nodes. + """ + self._node_associations.add_platform(node, platform.name) + + node_processed = False + eval_args = {'platform': platform, + 'filename': self.tree.root.filename, + 'state': state} + + if process_children and node.evaluate_for_platform(**eval_args): + # node_processed tells us if a child node was processed. + # This is useful for tracking which branch was taken in a + # multi-branch directive. + node_processed = True + + # process_child is used to ignore children of branch nodes + # that shouldn't be evaluated because a previous branch was + # taken + process_child = True + for child in node.children: + child_processed = self._associate_nodes(child, platform, state, process_child) + + if child_processed and (child.is_start_node() or child.is_cont_node()): + process_child = False + elif not process_child and child.is_end_node(): + process_child = True + + return node_processed + + +class TreeMapper(TreeWalker): + """ + Used to build a dictionary of associations, along with how many + lines of code each is associated with. + """ + + def __init__(self, _tree, _node_associations): + super().__init__(_tree, _node_associations) + self.line_map = collections.defaultdict(int) + + def walk(self, state): + """ + Generic tree mapping method. Returns the constructed map. + """ + if not self.line_map: + for fn in state.get_filenames(): + self._map_node(state.get_tree(fn).root, state.get_map(fn)) + return self.line_map + + def _map_node(self, node, _map): + """ + Map a specific node, and descend into the children nodes. + """ + # pass + + +class PlatformMapper(TreeMapper): + """ + Specific TreeMapper that builds a mapping of nodes to platforms. + """ + + def __init__(self, codebase, _tree=None, _node_associations=None): + super().__init__(_tree, _node_associations) + self.codebase = codebase + self._null_set = frozenset([]) + + def _map_node(self, _node, _map): + """ + Map a specific node to its platform set, and descend into the + children nodes. + """ + # Do not map files that the user does not consider to be part of + # the codebase + if type(_node).__name__ == 'FileNode' and _node.filename not in self.codebase["files"]: + return + + # This is equivalent to isinstance(CodeNode), without needing to + # import lexer. + if 'num_lines' in dir(_node) and type(_node).__name__ != 'FileNode': + association = _map.get_association(_node) + if association: + platform = frozenset(association.platforms) + self.line_map[platform] += _node.num_lines + else: + self.line_map[self._null_set] += _node.num_lines + + for child in _node.children: + self._map_node(child, _map) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..caa3598 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,75 @@ +Code Base Investigator Configuration File Format +================================================ + +Information about the type of analysis to be performed on a code base is communicated to Code Base Investigator through a configuration file expressed in YAML. This file contains information about what files should be analyzed, what platforms should be considered, and at least partial information about how the code is built on for each platform. What follows is a description of the configuration file format; users are also encouraged to consult the example configurations in the `examples/` directory. + +## YAML + +YAML is a markup language with similarities to JSON that is intended to be expressive and human readable/writable. The full details of the format are quite complex and can be found at [yaml.org](https://yaml.org/); only a subset of the full expressiveness of the format is required for Code Base Investigator. Here are the basics: + +- The format is plain text; in most cases, UTF-8 will be sufficient +- The line comment character is '#' +- YAML supports multiple 'documents' in a single stream (file), but Code Base Investigator will only load the first such document in any file. +- YAML has two main compound types: _sequences_, which express ordered relationships between values, and _mappings_, which express unordered key-value pairs. There are also numerous scalar types that YAML understands natively, including real numbers and strings. +- There are two main ways to express compound types: block-style, which rely on newlines and indentation, and 'inline', which relies on special delimiters and largely ignores whitespace. + Block-style sequences are indicated using `-` characters: + + - Item 1 + - Item 2 + - Item 3 + + Inline-style sequences are indicated using bracket delimiters `[`, `]` and comma separators: + + [ Item 1, Item 2, Item 3 ] + + Block-style mappings use the colon `:` to separate keys and values: + + key 1: item 1 + key 2: item 2 + key 3: item 3 + + Inline-style mapping use curly braces `{`, `}` and commas to enclose key-value pairs: + + { key 1: item 1, key 2: item 2, key 3: item 3 } + +## Code Base Investigator Configuration + +The configuration file is structured as a mapping at the top level. A single `codebase` key with associated mapping is required, and a key for each platform along with platform mappings should be provided. + +### Codebase Mapping + +A `codebase` key is required, which should contain a mapping containing: the key `files`, which is a sequence of file specifiers, and `platforms`, which is a sequence of user-defined names for platforms to be considered. + + codebase: + files: [ ,+ ] + platforms: [ ,+ ] + +The `` is a string that specifies a (relative) path to files in the source tree. It can be a literal path to a file, or globs are supported (for Python 3.5 and later). The files are expanded with respect to the `--rootdir` specified on the commandline at invocation, or the default, which is the working directory where Code Base Investigator was invoked. + +`` can be any string; they are referred to by later platform definitions in the configuration file. + +### Platform Mappings + +There should be a key for each platform that appears in the codebase platform sequence discussed above, and each such key should be accompanied by a mapping that describes the 'configuration': how the code base fed to a compiler for said platform. There are two supported methods: manual specification and compilation databases. + +#### Manual Platform Specification + +With manual specification, the mapping for a platform should contain a `files` key with a sequences of ``s, just as with the `codebase` mapping. The distinction is that the `files` for a platform may well be a subset of the whole set specified for the `codebase`. + +Additionally, the user may specify one each of the keys `defines` and `include_paths`, each with a sequence. `defines` is a sequence of C preprocessor symbols that should be applied when interpreting the `files` for this platform. They may be specified in the form `[-D][=]`; the optional `-D` prefix is permitted to emulate the command-line option setting of preprocessor directives found in gcc and other compilers. + +`include_paths` is used to specify (absolute) directory paths that should be included when processing `#include` directives in the source code. + +#### Compilation Database Platform Specification + +Compilation databases are simple markup files that contain information about how a compiler was invoked. They are typically named `compilation_commands.json` and are located in the build directories of projects. There are tools for generating compilation databases for most build systems: + +- CMake: add `-DCMAKE_EXPORT_COMPILE_COMMANDS` to the `cmake` invocation. +- Ninja: invoke `ninja` with `-t compdb`. +- Others: [Build EAR](https://github.com/rizsotto/Bear) + +To use a compilation database, specify the relative path in the value associated with a `commands` key under the platform mapping. Code Base Investigator will then load that compilation database and use its contents to define the configuration for that platform. + +#### Combining Manual Platform Specifications with Compilation Databases + +It is possible to combine the manual platform specification and the compilation database platform specification by specifying both the `commands` key along with some or all of the `files`/`defines`/`include_paths` used in manual specification; if either set of rules indicates that a source line belongs to the platform, the line will be associated with said platform. This is useful when the user wishes to expand the scope of what is associated with a platform beyond what a single compilation database contains. diff --git a/docs/example-ast.png b/docs/example-ast.png new file mode 100755 index 0000000000000000000000000000000000000000..ab6cdf02440f4b307bc0c969ef7e7247e602d87e GIT binary patch literal 15816 zcma*O2UJttx9%N?G^wvjmjH?)B3%I?KoC)pCZb3$A`t0F?@dLeN=F2OAiXJ}_l`jz z(n3qW*)_8FD zF$hFu41B}rX@Mi;i^xb2NGVq1uIdw?$&Cz##HXWa)zF!3f70a$*M=pN86D-V*9ee zD-pQi6LLNASaj>hi;vZNEtKC%$K?E`H<|+n_V~4#3yxPr-sI_I5r%2Xzq$*7^7p1J80ZuCKX zu1~B%Yg|mIpJ3dc|(%l#D)v&I9p%z9&89AUo9`HSGaJ&!n|o9T_G?zg?Uy4 zf86)%3SWBAdYJ4|zQYabtM#q;g09vA3bbGdy#DIsASyGhBo>q`~qM{~DSdrm2+fSA$$_F>47F zU2U|p@KZ$UVkvzb$V7Gi@ARbK16d5n_z@#jUT+`-|$wNj%&qd0Uy0KQ|%nbib6 zhnxv`mEt|{Mzm2~Gv$p=`PShW>oL+M#dIT!_l(%Kpjb&jWxAa5dR786e1W8hMZ+$_ zODV{Yes4zuM^eGiW<MLiFTBSQkkjU`LFS;n=a)yWT zIrvtO$*OSL@XurVv1ArDI+&*z z<-y)lZ_EwHXizUKGz2ZB*y)>wUlO3Zs)}aemK|!CuQy3(Y?zNfr@|k(Q$O2=_fl1S ziPFcfY%`ZDiS#cs;+Zv6*V&xh$)eCjGP>mWk&|yP^V3?F~E7&my zHK-oNfv!7ju!=hfORldcsJvP>J>i-=$HLqQNbw6$#P z(3?xc4((uC;{gVf@k`i7=Uw3dHX38|o}?^Bbl&jH29=X3nSqy0rvl6Jn&s6$%51=c z$rzfP>gal8z$Qf76&%ph+{LSG?@`<{G2{XpdU8=BrBFr?gaaqdj!gC^?1p+Pd-E-x zN5V#!GhT1VN^XsLy{*mW4mF6c2f?X$AAt?3qZ!_PGy6d+yXs?>EAt!cVe*rj!?f=S zhQ&=R| zQxW`SXv2@A7S7LTZx9^myZ6a8a%rjCbBg!$)I}l?S18;Z37$19Gz&GUVizZ3UKe;w z1*tp!6U)q253$`gwM0qDiEBcW@s4w%e@Z2w#zTfa)=8O8kDN1Kb9_O`fwjb2Wma1% zfEnnymq%V%ZtFmKiJ|Z9xrR!zeyZ^2kc&K>qi|Kx1mL+5LZg5O_#E7Zx>SfDZ?nfN zoc5Evdr!@}9Lu8-e<3FiC+7dEO83dwUKCB2vKToy=TuEd)3xEj>p>e&ul9B34Q8Iy zkh;5hZ^jFW+*9k0T1`(Sk=e&ll}R?c5?5Empc^iuRaFwg%JE&MNOoqm(QQC3oBo_~7LYc*Sl*y*g2I^wCTIg$B&}Y3oe!Ikf1vMVVYdhGy@AX@1zHJ;>kWXD=Cc`EB<3B%rv&_UA3FbF=RgnnKmy`b~f`zmLt-h^>xa z5p~)AZuWDb{w}8vUlkYEXmIE4a3>z%;dVUPkxQq@n0Wmd#W|%{FE0vS2IQPY)gP1G z6ClawfBZ#g3Rsh}uv*`Rj*o{W^)J;75MLe3+~->T*G5mJw795vYJVmsi_)y!6vmop zd13=Lo~(zjO7kO+ziuf9_ud6ogu-RGew*l1xBxr6<#18Fr38_GeU7sEL1o4%e$nF?iQ92ehDGHn-C-p{l zgf`(`^lbr-YkCM8CeVxCijU3t>VHBdUo)O9vsrUAFJ&~I?M&odAiXu$R(@hu;i^pF zRiv=FkiId-&w>qaJ1@C6nKVy)lv(kcaE#QXF@~CuVs{r{JT%6WgW%NYPaB|?^4DehSXk|AYaq*37evmJMJNSKO5qw4bTsQBuy`taw9FZtgEKaj9d1Hmez4XcIjEAz z&&>B35ceR+GgI&4J9P0Q+^W1t)6isVSO~tqcLrwvs8h2;)KisF>x1UQNv&|Ra+cmY z$n|eFoXF38mpuBgorhRw=ro1`W!XWkIhsu4@sogE!(gB|ar>93qoZTPdk)Jz;W zIuL(%s{L)*;Rhkfjl$ZuufO`%|K<{c20=Sn;3K-qG3=SmSIw1NbZi>U}_X5+-j%Bxx@uXM+%qRg1d2YmbS_u_@R;PZ1Q$0p4&B0voGEp1Qd_$baQ zXA=zsU0h=3UcY*ml>>>?Ccf3*wJ2ReCO`a$t4Fr;)=#nlk$Kt5F~~7{uCj`pE^UwssHDDcA*JaKK zDf-O6n91XB5GbB0yma+9#6JL2KAD(RW)!HfRx|X4v^9}}(^1$Oh-k+5E3P8PDJDn{ zqlTbM$C)nV)&{o@G7dZyECgtb9rcEp31shF3i6Oyq;)N`XEvHzq@wOO>Xx}|=1m(g zH<9b`QMqHVWgJw|9z#R8AujTAI2d{(3}IHgxc=b5P;gt&15BAw1MZ~1T(ILF$#>gt zXQJQ%zgrw(4U}dl<=>d;`fmg-oQG?&0B|P4 zbfvy8vs7L@H$Zfm@XEg^0B+8P_NP~gL?Bqb5}KT)t^%k}HiCxdbhWTrLvJJ6U(R;kbKb7zA;I7n%su9g;hN4DrG(;Mft0-@)8c!G%R zqot-Lm&1?E3;8X?g;gP= z{WrKEoTu`G~AV1q339f1tL;byWK?wpSRMg!y;-p&WM>`9JVI< z#Y_-j=Tqyyz2HQ`QX^hBaA)nC-rzE7NlbbsL@-NT`E>mzTT+bhAazPp4{tu@c&V)& z@7Qqz2va;I;fHwRGFW7fIz_con&|`RLis_u?xsDB*zjfrpn8dRhYbYgZg>Z3;L7LT z2)~9vPSXdDtZ@C&EX-#30{}0?FDmqMAORF=1YjrHj7jF&Ia*7SGs31wOJUoyoXT-| z>0^tcCrg;@?|(S$rEZv@cLl6^?u>Z)>~~h2USJ!7JnzOhrS%o7lqY^6Z49g<=K%^x zHhbmy7#>onFl>vfe0OhpDEdl+@43oo(HS!&>gwk>7xZ_Zm1+sY;P$IG&d~DTJG~e| zbo{_QKUyeG5F*@g#_@;^C$w4;WTKE+__gv*45o{k<7KB6w(GMcJfsm@C>3egDN+~1 zeBx8tmReHUouH#c^=*j(_I|j2>AW0E^?3nsiD~%Q2LBploquRzhfT}d`upP6*o@gv zaQ(%ey{{XMdBNBXb6{k4E+o^RTAwSeXS-dbj+QwDqBP~W*pyoxIu<ra>0a^cd$t&p)qMP;^8NYjxg(b=DVM1RYS-Wqwc#gC%Bzme? zyOlrF-(&n6cQ%6$utD9vDZih&+89mD4cj(m{^2q^4L*15#mfr<9DoJr-S=;Zd2*OK zA;FQ9cr3lcU4)j5&%J4BVrpDpheRGnfB(MgUF=|wizjz{`0(MNfIq%8H+P@hxw5p> zxw5?cdKCWWLHvlh?;14_YXavfq=85RL>7?u|Nc_pB9fi_DIG&_a;7|0Gs8|v8+ljc zy?s0L^qE1KWPBhh>-IzzFEm=$Pn6+d*KqUhx_2>pZb zp8R~fK@`)$mHce|cU#$yV^rbuPHcr%$-RxnK8h z3D{uAwjKS6YlI>;L3SqOt*1}@Z*kdRm7>ri=N>XI%AH(vZh&Tmo6ha-{JwqL2DDkYOp0RI*z#2C{ya*vX@luGv(6K)c-BZWok@t zX{EiOn1CH!$;8*JWSBJ^BRR8t+8f-LP>Gn~l|Egi(IZk7rrI1o8C&D{QY^fsaS)jq z?6W3v;y0gu`czB8sZ=%GzIl)2YEWRN^6q(Z*1L-=RM^FEmfht2{(*lL2OC2#3jmWeD$D`Ybp#LKK982GwI?@HyLi?%j;2tm^`F7{0BdqdJ?O^PUr2Q*4~d4yyam?oX4bw za!r}xue=Pc!&R3&QW}%jZp@ze0*rCrx0gl`<@kK!hwxFke!qBaa`V|wQ!KOESs+7S zWc(8ln)()uD~TlplZt2spwyZQgX<*YBqfk&`^**M?t!Y4SZ_ z9Ha5sYn{Cnc%OFx`gD!&rkw8X_Xrb1U*JZggO+6^?Cc4*Ze_ioHuj!`+!Vn3A-K#% zBjq@V9im>tRN0{{pt+qwNdRAm{-0A;uEOa)u5Yi z+nqWOX5qnI4umzsg7IQCNJiu|E2*_zV53l!{FD5h2#i=YIGgMzK5)vVyC$%P?HZ7^*IJ@2qQxFr##tu-yus zHy6llx5gTF4n2O@3CD6t!Cv**O+uo6QF8%?f>J#)vf7p>$HqUo;e7j=ow@yW^!&$p zxAX`Gk2)MFE5K*4qZ5@K?3e(Gt5av+#WrzFuEYSVS_J z&rvpg7k8G{IPs4FPT3KSkfQ_+=Xu|fx52iJCm;NN)%hv| zzyDusqNn%6*OsjVF-JTfY~ITv0?bsg{QHF1b*F6R7LRBvJRw6F2Cajm4)s;paDBMX_N2o+S*_zoEs=p=I;74aySlXPDhIE+de{C;mx}!QuR&~UU{y}e)DsdK*d4MPO zl3RZ7hsf`~^ToaOoKk$6AzL~#C#JL%-|Imld>$^}$14Y}QEzoWWd+z3mcAL`x&E7> zR3#KU1il z+)`eEQdZG9O>g!F^nSz6!Kc8+-jC*YB(;&&J^c#UEa~TZ`c`GuYdS#@x}VY(52b5w zN8CJ6p{>cBsDj}Jqz3mF7x6>I&5o_`DGnykK0X-@iNTO+pf=LD^A{l z$(ZS9s8{^PW8PN86t7n*f3&tMB9ATt4lOMGDqM>jaunKdQ<-7;D3RQ(+y1O#Niq_fm(&WPOVi3GMd*!#fZbt%qX{9rpMn?{QDvg zsH9p4r3CIBsv^#*eEqD|qN8mF2X{N5E(H_YRO=S`$AZUBXeTNtoba!kpve@Os?Lz( zTqVSkQn=J_?ygXd*qb+)E2t%iT$VEELyLT^MqL^<)H_DLyQ8c=TT{XCb5E`f!ajIM z^el^$KsOs zp%G=h&^T&IX3C{C-lamoP&8t`3qS4=L;zd+gJrOuENfW7?o@;R>!56wkaP_6u3_(4 zah|ily_jl4*!V>Xe-cGQR93W3{4kCP{c7I4-dphgV%jp*-#qj?-Qffo)+tz@Lpwk8 z0+1;vLl8CUegA0HVST(ST0(9>+@2h6NUgo2W2`Ma@0s0f75R4$_7Z@n*>iGo)})!+ zG8)e}b-LuB?6Th#1{66NCH_mGLm17v2?ME?ZOlTGJBk*;OWTq#`{%Db@ z+sjMmcv|Q0&^h}}D~~x=kGb&|mWvh8@FM{2@??Wc&A(nI8$t&ubk`nz>pEv{y@?9A z2)`ZMC2ZLp#&$T%KB)r&-Kq0Q_KETd*4heJJpVu#r!lel1X>aVK=f?4J5oA~m%zXf zS2H?Ds4zo+qH|v2>II2bEcw}jQyG0pBSWhW-VI!=^eDRO+y(4grPS>>UyyCRZfbMI zBL<@UZ?nDaFJeCx=1IDzJF{0va%dN*oy$_SoFUGz-l43g@2K{rQaDG*H~W4>{zdPn z85kIC>nj#X`ptza!Im&T*JpPO$zcR5zrh{J@;GbIz}kv398{Uk>BLaYZ#cmr1EFr? z1GgpzNO|2xuQ+G6?K)*k{&_=95+2?#17VxH)EmOccl;E{_cIjxoWY?6Yp)PVKlWQ# z0Av9E0BGUw&%&P2>b!dAa@sE?ic|R~EpujnH5hNtLn+Go{?XIUI<(wde`bc^pT}*m zsgd!OkLOo1O0z0IdYUM%Gxi);eg95U`1=-yVS^`6D2IP;+pXQ_vB9Foy<|7u)ooo> zo{_k!yuyCx^^*r^QblQD+0i{4^-&8{pD5*%54rkAuz`UAC6pvM7-(utv3hDlaQOOl z$-+kDj!&Ed82&Y8v=0ECbMPytq;q=er?>;0{r^6qJ+?g<&9#GVBa@nN)8l_KwZIv50ID^uY@> zrH8m#y8xzAO_u1WEG;e_Jqc%n`-{UPBR^1f2g=N-!)}xVcG9O$Dh}|oNF@$r($Z2n z1MT+!lT#++`9xg`J56d(myRf@YL3spxz8_Vco^)roEDZW0R$4IFkXh9atr^`nird< zTZt2uO*X<=>n@Cawjk=dySIj72;XQnB8a$jkI&Y*Hu{uPHrm*ACs@|`6YN`NO17$J zg8Ux`ZkpCf%d(bP094~L9Z?a{LnJ8S zD_}|VBf_(9#j_${SBP0cf(f}v(94we9PEzFr#NFfNOEV9pz_>W`BguVv7W)$BHw;vLkS)ev{_6u|>&2nC;MI0Sg zJ1Jk|TF-SW^R=e6fr@e76NgjXw24>tY(L`y;HXA6t$Bj*%g-|2lStVm_vlgPtzV@r z6Muf?O!Fs!ZNKd!Wn34ouqPtI-isV`xxF$ z(hYu__EleQ4R#xrPU-Ecp`|ent!`Ky)wN1bPD)N(AGm8<=V{&^kV>3axCv|Pg!5gJ(c0;#p;g*!HD1R-@ z)QCCwb!#RaJ99vEGBxGXX0*_EVRL8yGrsPPG{+;)2(lZn>cEpXb+yf&=}Gc*;9qu{}cJX=*)aYdhs=omyC?%x-x7?$}*zoC7G73FiY`i%|Y$5&c;h& zj3G~CL|fPOcKXgB?2+2LD_99pP%vI)cTj!&F0im@CcQejat<8gIjSQ@37bPy`u=Mi z&w><=v=&b7jN|bAgR;f+`w(v(|Fu*d%Bc`~g5WlQ^LDE50F$8~zImqMwlhT?dL!%J zpD)eT2NFC^)K0Na_4IxXekTtC*#U*_W*n}`kNh}C8?o`jm@;A2f%_#fgW9!W%+oz7 z_||N$GH?xj;8?Z4=1B|!(+0E~gNveyomK2nZd(qnsQO~r8Mw-mfW; z=%i~8L)DjJy~s69lWU}uf4Qin1Xq0Ou>o7q2O~Q8{067f+ z`~Lm;7^mt#!>6dkF;H=T^F)2!cQ{}CH`1i*6o)WdQ|VcQzRj#b8>_N`zcPck6#c?l zQing1)iO$5;QD!lE|#pD`lr#3BdiZpR8~YNt*sqDYHu5Etl(q3I2GI(O1X4bkaiTH z8)q;Svsr)W))G_~#D?65%x(aYak_HDwncfr!eFP^Zdrr9e#C(Vz(mKC1*^$%^ygny z(?>RTCHqHrPKUUm>c1OiQ<}+Qt~pN%qq>C(F%zsVp-QQrytf5#%cPBJU(?Ond5u_J zMX3d7LHQ{F-Uf-i?FlDVF6KrW$5L6OE8knP%BG>cocUEz$tLa<%ggiKx;p^2<@WCf zbbk~kNq%)~wBIw|dEv|YxmT_PqDQbUVtTUegTP2;lYBuE4n2Y-%}Ng78*(-9C$o1&KNqcZY2xT9!&<&)y78Z8WKN2#RlZ z4}H*#%-UzhS08N%-L*z6NMQ8&QTb(w{AaE&=|yO%GXAM!VcY!_LB+~%KBd-83McUc zq|nme_Ua4Ffv{4H8_5dj9S(d~OYTsMq_dy)3xSEUQzHTY?R_X&!yne?Z~0HuFhEV^ zSM3plpPH|&a)AdenP4EzqR{)?ZkdmskVA49>04KliMkPxjgDVUwAO{UZWvD`GQBl^0Anfn6QMa5hK z*}hVXfxb5DMf&fv%h1aOPj0mv0+vi^3gg_b`O!1^bA>Qoc6Vf#ENvk-BqQmEg=K<{ zsSGc`01kwXoZLX?^_2a8X^OGXkgIbj^l`<8b5?w?>-CL;l&+8*luPU{;Y78jAF&vA z_NNpJe4j^OEF+(!<{aca#MMql?{Ju6iUhEr{wADTQdOc{3E{brt(xN~r}fbR{aowW zP6-1Y=w9A-ayL=|#p|u=fRo#>T602FRD}C8TU~H)Pwl}UaLUj>bk8O*Z-U=n|62Z;{3*@bA%}js7|YAhf|#a%hU=ah{50Arrs<-gY=)E zG^=V7q)lPA)W&(JL@N3aOGyY{&DpImj#Rb-Pz)_=qnq4K*m~OccSgIJGEXqwABI9U2WLw&}iEr5%6|;f{pl5 zc`@@93oFL}K-3T#W5n5;H{NoZI?)OxZ+>Dl#?iHLL?wLZ>VR2Wy-uQ3sI!B9-2TkF zAF^{HZ9ZYM{Zdg-8wtb|6A_plDqHEpZlJO@zuZ;VcF^F-D1IrehB>Y`y-{8`HQ%J! zx1){J5NVKhtf-pKfbOv({+N8Rq@U~`Xn10*63ywFcYH;f!~0sRy9i4>JiiV@xlyiDCxL1x+3>i>&Gw$q_G|Mf#V0@$75zrL$7nZ zQj!u&IZmy{WJxFAmt7`%cH};1AhEys!ZAtl`4FdtgqYZo=e)bQ;N1sr&XlrFbw=z^ zhK6?K`cT(#<%MPbCdn$r*UF#puS@_gAS()mvJkm`y*}wH{haC=6C%l@Tu5VRcC-mF z4#S@f0l%2dHHd(X$ZU=U9L1YHU)!<8%afCO=X;`$h;!kz7)4kCg!RXN)X?H&i8MK>Bs!qbH&y3bN$eiTz#R=L1KE40lS=Cd;?Hna zzthdU;(O!q(>4Rd#wyWX?*!L0XerkanSFTnlRo34w92ORE76Jt-DFK&e4eB%!~zC|&45`0V1;_#sXcpJbx zLa?k-hWp9(h&~R;Ci$DV)LjAK>Oo&#fIzzQVVxvtbIpCwX`^96KhMXz+#LIfvR&^P zG;_~P7n*^u<2VJZM$4#-h#niwl&jb*VL4~5hS-!WzoKBQUcyyh))F&O^jYNN`#?$n z`l^?0E9B$lCdFrE`&50cgI=dgUeWt^Yp90n9M?-TD&%Q?>S`1pp2*3{gzy{wno^xt znZn(xZr0+utkps`jol&rg%P`S+$@FM^nRh+*P`XXxP=LkE)Fl`KXhs#d!0}W5|frX z7dYWSKrb6kRk^;H5Qy#$EDUXnN$G|$gJ%b8oc4x8K;_{G@G>mSJ?Oiq!#TjZEDCru zcc>mZhU#fQpi9V4;-@ukR1j4;I~zjZcR@ZxbhBO_ZER78&ZS)G>MDgO;Y7&^#Ti0A z(<2(E*vN9mNnb788~=g*<0|a6d+NYxax8e`&PNYRH3b~^BUja)%)aEjt125&hkIl^ zRVMydJU_&f3m$5BN_i^x%$*CUSf|DK3+k5k`sGo`-5-7b5?sPsT$vj)#8wtTLm3hF z`5eLODTLFWnz!W~lTeLR;2%ugetjCJ{_rCVA0_>ZPA^(PUuz46^o=+>VJzcRR5-8! z=S{5)Gc*mIEE**40|&s#k{iT~{H$i={4)W6+ZF=B)JZ)$MK~D7Jira_zf0ZN=0)-8 zw}U+vrmw`X$q91tL9S_PTeMPQr-CEjZq;90AsbWRGs2jT#} z#EC3K4ff1)nHS-udO|t2c($N7K6fa6qf?)b3g4Gx+>B~h-soP1$}F1* z?{3_dM}O+stR9RpI-(Ek>@ppm2)g&@ZAA3JFJM`O!Wi#{3Wld)KruApyvH{LzMaH} zvc52)S!puUH4G6^S=G`t8ugmf_<6zQLmaiQ{#&ulGpdYSX|mvFdykDeFX`@9A4_>l z(Z?Ao_-kXK66W7Kvog=y($qv&BwDCVpB!R zslv_Z&V+nr)a57+5o?_-7c-|~B$`I43&e0g=*{91wYi@(Xs!q0NMub7TblZE9w!#u zVSx4Z0X1n^SDN1U%{uf@ktDctrh}wb%jyJZ$4u2uoQOJOEF=$8 z7GkDMOw5>7uHPI2Mk)^^V!gXl=Q@J*f%*zJz$?R11(Mc3{`g+?ny#TnSATf36JIW@ zY<`Q!>VKrG{@Y7iv1(6KeIp}+Dd0G!Cnb%#{CI(#xPJZm%r#Nb6W@9FV4$MxHLJ)B zqEtkp{1}_daDSuOZBz|6=AT8VR@v7~D>0c$Zt(W4F*|W`+3n<(;ACfS5gLE!OXC#i z)M0l2Wbe4;<@OK1BJB+5QPr0(UwB`$iMSw~3S9hFHm5R2LJ|mufGt>BhT(}wR6zqy z=hmoY0%2eBj+s-3v*2A6FFf79YW){*%%iv*Psn7u^j~?Z)7m`-POAd z<~X-6B+_A_TW86531#fZy1lpYyGszC9+1K} zri|0&Qn|9rL)yz?QJQS2wwO`HeG=U~%^iRzzvT$~j6SWR&^3Pfg5p*13Eo=1HABxw z59EW4DwOK-Gb^AR&go-Y+xXLL*~&$p@yo=MONE2t#~6po0U5W*V>*-j2Nv@(b1Fis zS>11!81!hsCL0<b9g9S+j49G^|Q1;9G9y8_esn0R}%lf16G|Rk+W;?8;1q z-`&wzx}Iio1ZmioD9@U(u=06=1cAR2vxx$7bGa`idRF<@T+E%=keWL6y~J~cih*oQ zw_KGfX&{!!2zN8Kx}`0Z`BrqH66WLmJejk*%jPT)`SxHXAT8Ju<4hg|IOOr+X4G4w z_U|$VB&Vwxd$n1~I$1wIHTRaAm^tNm53D{Tk!Pz1H<^hdhANvh#1wxQnc{*$;5K`T zqNxRGbNrcnB+*}>Tp)E*=JH`05pWC@JUm4eg}51e$bZDgOD?>Oc?G?p3~&)dtb(PO zA0gnv*_`MiP`f;&7!cyFv^A*WP^B`<1*|4=Ciz# z@jWyizoXZzR4~08uD>EGK`GjLUoRNt5#_HL=o`1gGpnQI=!I9x*EVP#%{sDztr+8Y zk3+>DAw33aCTKUni&l_RjOAc_RS^YG;?uQ=m5+i zjv6Ko4QbeHgw0~Qj+s1hy0U_wAZGcXjO#CCgEMmvoGQ@>V#+w`efJUm^^X-UEaY)J{Lkz25$}`)9z@+K@Eb-mSSd2LQc1*;YUty;Z zLu{?S9qM~(xs%B&VnRJx9r}6I`^n+v0M#B!221DYbk%gBySOwcfOc9MEB2}org?-l zV|r~5*jEVV!w?xzNc2XIXJ|K``+nwc|JO_cjhAo@E1jXqs(r9rCJw82@2Ov5iaYVt zI*C+BkP+H{Ng+i0aB*QQwhdBZvF*obWw<^XMF$ed4Kmd*Hw(-4oJz=3Xos&Rcodqp*w#vMX8Ft=X zHYGs~RiH-|yI9IMnog~sUa!WRV(N8~A|(N340oi)W8F3?gLFNE+fjgaI)jbVv`+A_ z(-J{oXH>|tFT(G~w%905zEfgEJ~n@9X4W7)tbRVNM3LK#qTF=UBF^)`cesLluVp$j z*zZz;r>8Yt$2Pee`uF(wFFL-NA-kC-Qkt8Q?oBVFC(lMV3rkD+t^n_PN{u9L$@!cR zJE|W7yVoyAyctmT;LVFqH+V<=`2I^@BcsDXP0c=BX-P?t=2iTF!Q;pE4anSmmGfCg z>O3TX!~W;ji2ncA@@Eb7gn8HO^H$uy8d<3vNEZNww^#*Fxcs45=mrGv0M*+fZw3sT z8?Q*gm5+h)(Wd^H!))SyaLcCB3CdsED0kzyT*Z$y_~hkQ=kNe2x4Mj?-DVK~`1iXU zTgr#%boiWs;O5mdU*)J9D<1FO0Y|o%Df5jgiwj^DB8lf16TX z11%F`TuHUn3#`N8`g7BJOd6~G=FYH|fw2~d# zYE4NP`2x)`(fBP7HLx4Pa|$R@a5mA95G~(^%n(FRC*mA@CoVO)dx@83(n&GXx;U=J z^~Z$rjkmEp8!cX{p>o}~m(?2(G&;Fr#`Qn5o+or$)KEg+ct8~_m)hy*E3wfYC_7RR zsgAR%PIWU1%I9w_HUBu}@{F|?URM$DCad;&XU%}h?b9s0Ecz41CRyas7Z{_Z zsU*mpljVRXF8vukj6RDbYLVorRc&7M!V)DwX$?b=X3 zYIx07JM`p9c4)!=Aiwl4V)urUMPVNK;L%yI*kP>`2Qd8!A+oip38LtE?a4dQXW;%OMy&^Q1FEsnwYf`MI{84s+VkXAS)iw+6TVY`ZSGJUV26hmnVej<8zA6a z1c6jwe7(kRgc5@0g3o8w*a_%E&I>**T9p8%ou@rF)Q_iqwOL�Qcp479` zoP0uHJz$M=h6c3TUOByBeRPbnWkC9diC*OGlRq(mk^JJ4u?fUYpnZ?CcJe@Cz%zfW z^ClL0A7~sBF5Z4aE#}(;)cYZpRB7Ph4fg;QUh70ujbEvt_5;NbeRQh-$s+ogS*^;s z1dL}jTU2^ASs1u_s?eX!`>{C!d3V|!8LaC=Ri08?((vg1wNRji4-2&LnJ^`})62XL z%_7-VHeF|jV<}W5EAsv6iJ_M{Q~V?8K{ouAqcdhqxuyI6kTrY5BG>}UHq7Ej-oqSt zH}!pu^>}wf4T_kLv^H3?yrNp3vj~~7-t-aw)-%g_b^EvB1?szzgXXwp6N{11o4dB( z-pS{s3%h#vu}*cXYYo%*TqSb?y@2>+?UOoR(6hC{z$Mkiv6+Dy_4|lYi5m|d3xogW z8-V&UU`qv3`KQf53yJ-5pI=B5ryNN=Gx3Rcw3|#07VWbNN>;cTKZpg=9UENMh<|=_ zGw%IhgN1+659x2#F=pF(<>vh-d8ch|JkqvWGo2$_U`x?KgXAAAsnE^EEBu&|(G$TP zsiM;^mJxf1@>=DIInb>ar0s<5=?^3YT>~1NZJrW(n4_C0)5V2E?E68T=(n*#UHB6) ZtXW!>^{81C=+FjfsO#J_NS?ip2hILx2X}kB{_k9ij|8KhPswm0Pui3VSLZQ&hpFFNg zq0o9zC^TKGR^Sz(+O}@|MQ3wNUVRmQxvaYMC;q?M>ZG;}g~D*2{H94fl3rR`PI3p?$NuGyGcTI}TK;pY)Ju=BF5t(7=0?>{c!vAkx& zyNB`katdW9MgI6v_3Pn7?T+f|lS?H(nx0$I#@&o&51sS7lKPMuZ)_T*>ltDmZ%oZ* zG&4#vnye2nHVQfzn)zt*bp5*WRhd&-u=UZrZQYaqp|M>rJ`Iiy14xzlZAJ&;1XjJ_2C&vbO2pL_F=ml30SiDwffBO{|R(^Rc8L?Gep!t`)!LZp=Q zz=!RY4WFLrPJX`V-@iCF+En(CJIyK%Gw?lqtJnEQ`p4xnny=0dWmGrhI*jg~^W83@ zR33D&ermXNhxxJ<>uT<9+Lw9Zy>}hwCVUGcj{xoU7s3{8iG6*S6JA-g=a_!`aBr^$ zrcy;*vVy1R!2bRF*U;13nXRR#*DyAYUtFBCoSB{7{Aizuh*oxXwqLJ(u6~I>o7Q`8 zCZmkz)a#WLO4s;Sub6;+%3P9?dKMO+Sk`7-rQ|Wz$DghjsFu8FzZGkWjz!$|WMi_1 zK<+xVv%(b3B-Im1Nl8lP_+!nwbq8!Gl&O19J&OMP`E1EgU*C25HkCr+wte?$mn}=X zoR9Zkx@2t3wR`t5CN6cpT*i8Q#4Gbw&GKMg2Q%H5W)E9lnd^2-c64-%PfhXEux{CM z5{olB{Ka^`E{{!@1UI*$Zk9zOMv)^Tzp@;E+l_S{l{b+ay>Q_I%L1F*uk#BS+HDf{ zPwhwAl9~*hPqVpBJdpKgHCI@`NXNZ5qV4)#8OFrK6qlIDDX)6<>eI)M9}Ak+vCG%h zK9SpTUWIDbkQifB9%PD#G;7IBGH{zQ%AfqCJMigQ*#6076iVRc;5>$Nrll1X>MAPt z`T6-p>DE!C#oqL0c~;ZVQMy#Oc*O{qRB;|T(mAB6Lg(P%;CTJ|*;$stwPf$9oN{z@ z6nC0Pymswc1ABS>d`R|-uM=3gSPB6*koJ0g$;SLPPzpn;9pI=))X=`t9%rH^TP`pt}YQx!g z^D{q%u63V$rk|@R=f{%$c(AOzJoj;VNe_9Jb2Qr8+TQ;DP0!C4H{fjt&pwx~Yi?~- zPtiPW(_3XXP-s-X8B49d=*BW??52H6eV7}qp0bBbTec)tMMzv}eP!;qsC;B0GgRnG zd{&2(qL5jGm7Zhnh4*Xh`f858dGjXu?2E@1**%+w+jE;t^|uRORmm`}?Ed!cNzg%E zv;Mj`Y{!-yyObx@7_i2CXNToeFpD{&n^BY46g?Hj)R?ix1BeIB{4hqKdb+ zAb8-3|6$d9r%ClJ3;lN2>2{~N;Vh#qUTtk{f>vG92X!((CM~`7T-#XS>a3Awpii~! zjd0*Pc<{8L>*PZ!l}dKKU4PvPY#xV-43lbB*V+CAo%Bn0_v9Iwm~aaVYlewfY92dw z6Cc=+pv=}hY%gz881zgl&A_bf^$Ti1fRIrcMfq?YLBin9lqXip-g0U&i0Gpo~HN@jo$8w&%Qa{ zVJ>L=akW<_X)eLAWybkuZ*^0rlzLYdqM2^6LpHHbV-M2x;IJv>bga(Vx4jw z&7W4!zGhV7G&eOos9w@)S@;Z9Y_Q4DUE_)Dz0AwCyPK~HSB8lwA3l88tS0)#dbYz) zSF&C&I#fN>oIy5Sl3IfJ!F|~OC~=w*;&wd!4i}7kv6i~N+7GwHoemQ=?JBxKWoKYu zm>lgi#DfI0$m0HTepX3v$Yb%SX=(;>X+BN#@Sv1j>@09qx?_`~ zl{Uh-$ljmT=~jSj1R9ln^IrGz@g4Q|OHiMs@MFhPtZxwdMOYzVvW6$Lt2qm?UH zO1Mo;PxJESV>KijB}ey4XnX z<_U<>`f)@lO}+DRxR{MBrXrHmCcEL5MpK>9j{M0lci0B-`&0DMTa3)i39+&4uH&Wq zoE9dVb}WR7Sf&OBGNKZR%8q$|t3I;SkgA_6XxSlYH}v^TU2}2iQVM#=Y}YM@z_74E z(Y*O@{zt?oO*`_k4=+rposg9+E56mIx{=g6kF|5-<-C)abkn|?7?jHg^-nB23;c?U z9z1w}$`s+Xen;~Byi2>&;4?p?<8pE-=B=*=+HZRRz2ib2IZoqQta|H*f&K=>@(iWLJ#mS+6XfRl|YL;I5wBI{!11K3Ng z4bEM^{(9r4O%J1@(lFG?*55uj4ZSE~WMQdrSTxujc7BUj^NB#MkdTlyYu2dB%6g$i zR`RT*SS`o4P{97O$tl^9>g&0d`S(+O{Aji2G4nT1bdt z)W5d=){tgcsJPz!H6Oupd>RT0-g{4n#$w(*Jw5xYBt{#4Rv)pn*XFrL5aB5y`=PV< zc1V00;y>Uiud&i-%$&-DK! zSPfp|Ve`G>%%w}QE-o$&@u#<=4EN_Pm{f%)<=73DH8p8%*sx)nlvAcsgt&%6Bk=p9 zVk6pfE-rZ&E?yLJ9KXb|Z(oXOeLO)Zb#--Xh6_=L@39>*0FwLUw?je@o8Wy($zbWe zr~^G^m}iCHa=Icd1IH5#l4BlZwXdMx)`$HZ92FJS;WjsXY|irY^P|J9SxuyrY~7lK z^=pQ*%*-N$g~vInx0~|4rw8~_zP7g3Pgq1m1bgfX;HPI*eX70$2M323ZRcp=Qfg0m zaEhEiYn-Y8sD(CAX*7FmLIU@ZBZe_CF-pVuqb1wMB5-UTJA5X}$mi@%z={ z8O5qIP3svM8N06^KYl#$!h5`lqZrKSy#0n?ZeCtxH#fK0?f_+AY)iSht~a;rzP?-S zx;WFj{5!zalhoW1@ybVsifgCdtYquMZ*{bowzf6K-`>hn_Uzsrh-O+;RJ74e187;{ zt{43v@N*<%+-#Dro#qDlM|OckMcgx_8hvGCVw-c2_m_2~}Nv@6^;9%7;dd(&;-_R%(1&t87`*Q+}ym4|I(lA`}gjZ)kx6 z8ouqeA3i=l0hPvTdV1lcF{5*%TN*gw;VYe1W1QmNa#Ya>uI(sY5f-lE4#aQeaDwseOR@VMOBT%-900PqN19f5m%!^{hqSL>zY-7{7Z)c_EtnnPUO=nes|D$zEA$k-$CXb{&tRWT7Bd{V zTuS9AD=VY2W09T4P8ZG)#1sY_nNe7dxaCDUkBB|fd8uGvVBj|es69A0S}5{@A6GQt ziusw|sKx~4DB5AvTm@xik(k}&Cnjc;b?euE0uNf7Y1y290!vTm-+scvch2L(z3mC; zCct=T%p807npHeHDx>$GfAZyGIw*>KlYdwN1x^y^blLsKduBTT_fVA$ zC_aM%LKIXnO(~}d&YBJo1u&XVx4rQ$y{IG_K5p*v{uMR7*PWcW1O)2;?e*tq+Tb_5 zH3ipyToAfkQ}p+rFyA;8E_wzSGnXI_jISbf1c3oJ{bMSI8dHLUc=VG2dj|rP z4?jptOA8e=eQa4UBMbPH9}<+6Ivr^%t_`XMq!2Z|>sobq69q^q`DIG7cF z5500Kc~wyy|GNGAN`|<0M&xg>J%6NF(#?%pX>8BrQA#sL057;q0{wPj(U;r3c7;{O zgAMxVbO0<0Ui`2wE;cq1O~PriQ43{eaQ9r4sP!>rsmZ{g1KN+jmH4MPO%8;L*%-%U z#nV`w_u6IGw{zi2uINT1Yplb0^x>U5cM9gm6#*gj_v$MxaFXiP)w%gX9UbA|2?Y=0N$H^Kg0|4i1)~_!J z52ib-Lw28N)Vp^XbE^%@S7wCF>K838E-EM}5SGT{tc`|v1|o}~^VC&&>fe~ItRG9# zk8dT+eM|VxD<6EfqLU;5rSO_Yq>ky|bkX8es9Rc;ii;ngrV9^&np1~Cd}%BlPf6Ki?%qF-Dr8OR+isE=}A zOsc3jl^|yw_gCLr5-;TdK}GZQton&fIf!L(#?7rDh}SR`0?$BCu;C}0YALO5r})!h z@!%CD=sJ%-3!xAPGR6B*WaZ=(sSu#PHn=4*-_TnJ26zAdANk;U*Bl(Qd^R7n{WW$T z9ZUu#fyXv-^`^a?!or_^ND2r9`~lEcwnJ@M7nX|d?(!t{Br5p+!_d%yhRD#+WG;MGCk71X6;&T!FjUa*Qiy~1jgD8 ze(HyYrkQI$Ozv5}4x^CQ=(ooY4PB+*%72{7xM|Zj+Xo+^ zIM@RJ-`ggn04Dh<-Kbp9Za|%$?QmYV0Skp9z4a2R+(f-{<1<|ms9s!RV$If`=0Mp% z4Nhmu2RaK24V?NpKnCS2Lxt)gBoXfYw?^4OgrEfsd*-zsg@s?={LSiO@3Oj1oWZvx z*XVx04t>2*ZFJcd@L62SjV28>s|IX>z;SaF&h}gf-tjdZ>2H}v_PleYoT2`5rd?A~ zaOTzPx4qLn{C0F!O{w~*jA}QgFvAg|)^Gn_SZJOrJmhLYF}N{>p-2Ix*sG4_-ZhKj zPtcm|y=`$&xsD2lorWG$3zb61xMBl7}?m;u*SI#9jdM1$Fk7#PTJ$4k*r<| z&`EZB$@GtwVS6&eWOTp2_n8DJywdjCnt7qo3irSnjt!ums;7PT?%iay1V6R8k@nn1 z;7@F|!QLo$%@V&I<=yvU^uMsHDxH#*|2n#wlVwh}`Rkgf&fxL}nIZt}T9od!nWJsl zYTzzZb~d)CW7V55-bHbTCjS_dnr?d%BXef}z!{R(Gj@4GH3{r|^k|EG9W<+k=jUYv z%JZTjE{%VbP>XJ_G!F1j$W<$Rtf$mj_;cJSaqNa(*wyh#t{h9H$V)%xM>Fe%X9lAR z5|nz&-6Cl&9N}fCpPO~ckyt$XQ7RzK#kq81@`qJ$SZi~Eqsfr?#jN0yhL#Kt*@nkD z>u2ntUge|(=Y~nPwe#eRhPdR8SJgMwx1Ww=u5J2pB(73L$)MnyDg|5T)G6@QaX}MviJ6KJYI_QgQIujJ*Elr1~<0e_H3kve&7?-4u&5lZY zh3ShqPVJti*K+q4ZJNF?HOD>4dfqOcmA0z7x@n@nUIv79T@^7{QW;ZuD> z*WCt(CTIKoeUHr<43EZj-xU#yoOZwX#O<>~UQIzkp`YP;Oan}$3SN-+fN4$r^7@41W({xZ{?e*8$kT7x z<;&f~lfU>-eTF2y`R&cgFuKidI$5ROo^{UVfooL$vzS@oyXw*&-8+6HB(Iw-_*k$h zr>S11$&g#V#lG*hMVQ)zr@@n)hEIKARgrtsB_?BY7CT#Z&mI1p6dyTs#*k{;cim6C z;FtLW*U{|MocIX~cKKNWZmolnbMYSwT*I{L%NIfw#JA^dJEfz2sgw0yO5e*|9nnZR zKSRw+o#RuiXM{d$-Tmh1f1hVj>am&h^5t|Vj}ALOihI~)_foHd$+60hG%L_sbhau#=e}^b#V(J zhTV5ZSZI0L3s`7{m!!Uq7Mhz((~IQR)eWB**`B({dPZ|SjR5WC@m*bL9A=C2GdUwU zgB4s~(qF%2`$H=}hW*Dh9iyM?ldIN$LjxiA>l2ilHYrQSfbxcE18aN4J#u2QI(5ny zB7FmV96+&Bh(xBdKfZZJ@d0akC;}@3SjC5=;K81=qp5;QR_aONM;sw;BJa=;xuB{p zTe)5xG6;KrN?Ji3Zy}@Wkv*hJi{pAyK3-d2TtHDH^ff)bs>8(=I!>shWBOmX5Lf;B z@olorrka=$b^=eYfS#fPl%fYxhRlI-HqZ62=VUS@q1eZWDgxWWE@1eMmj!NEkQ{G@QZQ7 zhM1>M4*>f}Lzq%Uw}!ZA^7l3}M{#m=gi7cbDr)`sm1SotE7Gcfc$Jh{^+aOA$ZbLYfPW9YMjEhLlrPCtqvDz8N{jXXr0w9)z7?BSV#5`dwPrlovoCxJU z8MrDW4)=K03(SIaw?=@bv#Loei+Bru9RoH9PDn^7`1RH2=M*uP6qC4c@gV0U5*sC9nQ*Rq5R&` zb*Hw{%wq{pLQ>hh%AMK(J%>n7YngcjV+faNGEtz+LvgkJSt-IShQ}Hq7T~ z7#1xEuS~40so1zDsm4&?pp>_Dm*JN50+MK~M%uEWO>6kGO4&j->gnxOq(b5pviW}M zFLE2dqDafoRRc0}Jrr&ZSa@>Id;=-UtjKo3Y&*^I1L{B$E^TS8x|B4c>DOMDJl5@{g;%;fr8Znt$7P|0*S#dCLXn-4lhR5 z@q!aYvZT6N8CpWHr2Bs+OIAsAE-uUwv-GX*e2g`^<9ws$WKgF)aVgnoYNdl)4Ndd&+@H5>krF}}_HFa@{KKWIkS<9C% z-@j*%%#9m2-bbb#D6I;jrL9^Ff5EsS2lo zhW+_dEmw$Hg@ws2KMRX;vYfZD?|~fa{we;S9p>XTv@}z(2`S z_unRD z)-C96W%Zn$=`uRCl~DW&Y59Ssw9XTE^50ItVFXF2hhc+O_!!@33dhcGt;~8>NluY0 zew$VqY{vS)Sy#&10IY%?680yl+k~$sLV^w8=3>=)?f=Jg{IP#!78{B(GX?3l@JSPu z`r98v7z$EB&3=4+pRAXi9@0aa2D_U7T|X8vRWg4B451DVqS?0;xaRT;2u$`p@elp0 zfq^aJu4dsUk}4plizX$AeMx-3cfSi$FnANT2|SbKEmBfaJEnV3{-jG$T#$j#Mn9{2 zut$!?{0;-#r@q=Jjd6<1o49d>l|II~Im3ioKtK(~9hqRe;g?OxpqOP96%9ZvW`FPU zHov9s@bITZxRP)f3ACM(t%3Ic_ZDBcA+yrBo;1fWi=5ISRgBt!H-R!mm}E`dZXmYA zIUSI1GIpT-Au7MCp8yA?3Su{Ct#ph-=lV5k>SP$CV&O}?zy15UqW05R0T2-n+b6u^ z_3_ftCZ8_ukY@{+*7|{>b|s)wO2Xvsjfx#-@mYuZ{3qjtx$*|uw~1!Hy~!HfY5Tt zC;>BviT};ZP4A+z87PkOFJ61+>6Gr5$goqaIp>q{n8dMxT??Kla@WD{vO&1OWN## zfLH zVb&5iH5Dep?z!&o--)o@k2Rvw@%lLBl(MoGL`P8gh!z=!x+Ka9c+d5o3m>oGb*FiG;O+m| zTgn`JUHjvL1@RHCz=F`!N*(8%oSn;CTj$aT(NFjS6nzPsNJ4r-;p9oHOyV)HvnRse zI8inI)rZ%tEO5WhiwmAQudZrui$iaFRK+3t-|wIC=8fGXb~rwMKRl&Sx^9k{QMM3YV-Qnq!;4yE!C5Vc6R+5Wl%h*CwXXIHpVD!bN9Z}{4w z{x9-b_B%1MEjbUbH+z&RH_&$=)C3DigDdTK1!$wje*#V#luM;d>~w zM#roA!^d|l#%A0YHEo(vIVl;K7+x)a+j|mbrc5TIC_VwR^O`I&ssO z<>Dfpbo6a2Ccggz#{YkH_SQOyBKjY2dsXWa=(HyIJPFUi$Gd{COPu1QCc;K9UFW7$Q$eYsZL?2NTzVdA zO%yF-xC{hFkhFpOf*opG|8Q0(Pv+87?F_>H2Jh~RQ~``5;Ww!JdPvUk{1r#|T{5t- z(BMr0U;jB$kb(O8{=*0Hmp~4u6@z&WUXwzmu&lDuPO|#r$NTI?#>NIpGx23ri@m)S zr}iJ|%+qQ&JE^74)slI3t8GK2DxuKHKqb zdC%OF_w&i6+#1R3;E2#u#a|{M_=mFTS)~@I@X%*~m#Frjxb|jiVUxW}ra^+@sra~q zpQ0Y!*MDg%^}LdcMK z-oE{4-i$OymS9o31M&-V(@ikCNO+c*i_k?#b^dnDA8CfN%Hb%QYN$Y_aDl)->tS;x zK_t0?%qB^UY4f62M5}*Z$8ER_Oz_-M9_#fejudYc7yS2M!s^di5U;)gvmwSG#6Xd{ zd(R&9tz*YX>vo-LHu2>7NkUj~Fym@!RG`-N52U1NCMZQrz&&T4(JvvB%)k%~kP3Eg z_ujx#JJYiB@VzaEj=>&z03`=jX+!kUWiXKRjx0?w!b$YT}iW z%CE5#M)pgj&8m3$l{ttNWKWPZ82mtQU*Ej&V4d#|8_gdC1UwP5`A+ZvEH5?BHB5{! zin48bxe8g_?+kbjwdZOvNKM>_rbVy_(uv8~OoZu>h_+ne*8(|k?tvg(i~Y9xNRkHO zt|9O6oNa<|kfv*yuK^edx_s?{M6|w&b{i|fra-`rPqhU!5><|#KYxCpIU_;Xvcsi0 zjrn~qf7Xo`k4>UnHF2v}W^MMx(%U5M7Kn9@Sa?I?nLUV3o1*kMeZ9MHf{^=4kxttQ-YREx`Mc0Y9&%}r zDMElSs4u-dIXy>;$S5GfIqj{ejB5 zxj6x=PqsQzDx$@YRm$ww+}w67E$d9U;g$C<`Las2JZ~Ld9%R$~al=K(wVtVa_U`>~ z8S%_swE3#kD>{v)7z2c;Rh@2yBr48v8z<00Jd9bPv$7L_l^Q5rcu7*oG)+HU3l4?%`&8j9Z$bS}ckc4m zC%}m)jaFtUh@gwx5A|J}JE1L0IZ%ScKcWL86QWU=m_h6ml2K+KcmSvVaW zrhRm#nGIcta885(;%d^1v_&J#X)u0eKQBp6pv)>lRLdSRLbhFt%F?=wIU&!&2k{zt zlVy*&XR6&7pP(o}v#}4(^K~zj`%?F?WcUG<`jya|e-Jszroir@UVZ>yi}X5{e!-7gt%*FiM=?QpMu@ zPvvKN*{Vy^?n}LoI zP&}ecNl->`vSf%!pLy|%_^9;e2E_GVUTseQ*iY0^q&9kAx1Yc|U}9kzdgL}zxeKu( z(*m8BX2ias#uDKismOx*iC@32M;vz5f$RyZ-@kr#Gml4z8_m7||<4l!G~^t=ep4B5!Aq z@AP7{Mo$|Uxs zXh<)K>@`8i1#=^6W@ctL`aKuc`=kKfl7~QxTuV1D(&?gxh{B7C%2z&5B(M4)B4?KXJ`@qK|lGGWb9{v+ByqM!<$R$ko8z z^VC6m4I@`)U-VH7niC{H(*6x?cf59Cp?&ol0P6}w(a=K|y+~eva=29stG6B{Bo4}# z>->)rq`0it_4DezR)LkkJ>Fr4lRGISkA%_lgY7jb$qdi}16AV=9BSQjfWT z4iOJ0Qhy>H=#2o!_wV0h-6-Y22ZNhJrnC>)Ms>^<6zH_IJiS=`b*QAf79TZ zDo+Kb(0OVUxRSIq9C9Kza8)>L4oJb;3Xy*yUSAc)n58Fo_P6lx_$+m@#&8$~N0!lE3 zMAwsN{Pe;Qw<5(O;(azKs|00o5=AiyV#WB+pY(>?-JF?(&zEIphf5sUowa&G!Denqh$KI^`fU_tj1cHD< z{JzE{((!{RRL>XRk4#UW`FwG#>kadqYI$CI&cOmWKfAMT>HP}hv>)v-mmDivCW|OK z!d*t{i7Ia{cNXOT0GbwG=-~M-HGn$JPfpVTR27d+Ob7yk-P>-NSNjW>ctcgiSeyOFi?I#$VFd2=7KEzRFNG!pPF7gd6A<8iITPWdDlfOO3NpkV2tcos|m zo}1z&Z7gmm{V#DoAhm?m`RVbX)fA)EG=E5ETjOK|C>M?uAmXI>(yXzLxHUMFr&l)) z>Hxc-f_9idaI#=_;5?&R{Hg4=beiajAAGsc=C@$$We}jz^ZmOScrvlt04kVp7-x5R zHB2&C`Xt^sHa4bcpk!bWnU|NR5WAPcX^N9foLYJL`5JKwOyuYo=8Gf>iP=lLK(=SA zMeb?9cfz@#Q%?YgipF@tfj-&r-mqb$#tP!_`S%TVT5iFiW?CuJfCZTNg;)Y1s37NO3>%O{5;_5qwiN=5c zI!?j$0KuWHnht$_LD&y6cnReE53=i8Bf)f(rMJ7zQF^H7w?GW6!Cpbw=p$`CWZPt@ zlj4!eA^azZ4vDc{ONbnZ-Alw7H86AFSJH$N7L0Lc_Bhc;`um^(g--i3wkF~e|z{G!akmd)*R7rYyn zQPCs*Mdk(2;APcShtu{(GY-IyGGz?Q1T&`e@A zo1qh9U@IQFZN(`CIPkBD$#X(W-|itvhJ$h>;WN#^h18-5PsJgICQ`I9=giqp;lvE$ zyk;G2XMoE+B(Yh2cI1<4>_0mxAW~|OT(XT;3bO3PWQp1 lJl-B#act@luKJgMtQP-s`p +#include + +#include "histogram.h" + +int main(int argc, char* argv[]) +{ + int N = 1024; + int B = 16; + printf("Computing histogram of %d inputs and %d bins\n", N, B); + + int* input = (int*) malloc(N * sizeof(int)); + for (int i = 0; i < N; ++i) + { + input[i] = rand() % N; + } + + int* histogram = (int*) malloc(B * sizeof(int)); + for (int j = 0; j < B; ++j) + { + histogram[j] = 0; + } + + #pragma omp parallel for reduction(+:histogram[0:B]) + for (int i = 0; i < N; i += 16) + { + #pragma omp simd simdlen(16) + for (int v = 0; v < 16; ++v) + { + int b = (i + v) % B; + #pragma omp ordered simd overlap(b) + { + histogram[b]++; + } + } + } + + for (int j = 0; j < B; ++j) + { + printf("histogram[%d] = %d\n", j, histogram[j]); + } + + free(histogram); + free(input); + + return 0; +} diff --git a/examples/disjoint-source/disjoint-source-dendrogram.png b/examples/disjoint-source/disjoint-source-dendrogram.png new file mode 100644 index 0000000000000000000000000000000000000000..cc24b6ee9ffa1dfff251b7da89a393ddff6662cf GIT binary patch literal 14024 zcmeHuXH-?$y5+_ku`ERt6bz_HECVP=P!#ll31SC8&jqpA=NXP>p#{?`2FH|O-OoXnZ^Ynawh6t$jq z_Ot>;EpekLy7pBo@Dsku=5P42%u0-=vYtb24ns78MQM0)P$CE#~-(;XloYLh?OqkORQm&jdlB0vTEO|!v&&R z-8o!(G4B(t^1{4UdKG`P(Vet$mBeZL`JUxYnwRr*_?l}H@X1u@DAXWQE5FJ(c77G!) z`t?@CvuB+h7rV3`zoZ)x`xqjiC@VbfTdR+2Gg*wZpVW39uT=>+zxv`*y-SxOEv>9d z(>uGmIM~=Cs~SXUK}U?d6%-YPhu5xIV_n8qe|gf&Q8~lxtowopfAEsIfw0N_Qj+lr z-4|vS6cqMlaESU zNUx?spZDl3se1n7saIxgW6NFV_J18oE$wdW@b}G9Qt67b)N<9$84&4io3sv;it6s^ z@9y8z=DMgmFg?a?lVzQq)|a1e%R1dqQD7cen&@4X!ZF)8DDtDgEuSYt^KGJOg2N!StT~x={>g7pK+hKGoyNdE%E|2 zXI{{HGoP_~Qelnt*_p{nQ=hE|*xy%KQZ-K^cH2)3|NhH_xzAUOn2Q#@Q;HFb%a z={Ef;G()q|6nx!r%;H2je8_|idkK2roMNiTXoCi&HX5Km~6SmXFH%K z)whzOVz_lSQ3VS#CVSb~*mvzZ>Fpa76nE>^GU>oW;@;NZ-fxuM>KnOFI*8P^MykPs zuN@uZUp;yI((82;EiF^_jxVLCnk~t>8}v-C3Wly(zh3#|$&=~ybR8wiN^HwlhTCha zz2CLFF!+UQu*b0Z%?i3@RJ&Mai(m&ctxs4WHsq(wtR-N@d8Ta-dA$aSB>gUoaO@rC<6HQpzmOZ$% z-W~U~jK*Uw zR*u2($@vuS|oZ7t@TY zYm&5&=S?)47*|E!?JjAmjaTh|$84^oruMk6J~{dLQ0A$IhK9~p=~p`>(oO1R=;-MB zKEHeh@Z$+Cdk!FTm>Kx|@)Xcb#lqsn?%lhs7w0EAMMW1C<`lPGN3$su7gSf!=&Zo0 z=4+f3%bOj_F|LjC(|C0y7JrmYE?CHJHp_5wayoMP(@t)k+)S;-j5Zfd?HpU>j&~3C z(&lF;Rdsa27QTDAvL8OI2&}E=&S;LfSp3kwt|8UXOJu5jDYxdUkDFTYFiDZ&iW^UKI#4yS zxml~lszR7?Zuq?5&{DE`B_+AOCWXt4$duTz1#TT)bYi-Wnkve$}u}_|y=&w(%mpom<{&llYe8*H_y*Y51In;)+yPt+Ob4p3hG^{uCVYD1dd>m|#VEsK#3<`=Z?`JKgS z0vnAi=2kB{ZeLLNWfI@p@0$*qx=`NVaT&q3U#TR>B~NdUijYr@H!D`?(M z&dz*#g%r0|rkppak94z^#=~nfUFID+J3C`t7Zmcbe5X6|Fz&%Y3ovmFdh_ zwQQ+rj&i(;x6&7jLvbG4a8!A!uF39K&HC6~z`__HAQ0Q=EE;Yx`j%nBd@Uo7l95p? zI+L&|&LzFaDROIfnIG$0={%|J+l)Q~! zQ<7?O7&+*wy3Wdu69biv=_YJ&xT2A6?3aCQIO*vrk2t^?(=m%D6T@xEckbNb z$QbPCc(zx{pCABknd!J~fegfzONxQ8h969%;I?N{Q2`jqmxS-fH_TlyOWe;q7|P$eagYkE{99icDDb@!Qp-5 zW<@R!moJAv+44e$xa`@!n6jk&K1!}nN6kcr?vKYP3xwIhKiu4D$^)4YHZA!5a22snJ^*sUEdl8ZE4<3gGtsq zJc^$F{`>EiqFA>oQ}Sav-^iL8CE=06rG(O$Wqw&v7ACAIE-ua_XdT0LVDDb>2TXib z9|I5XrD5!=l;X{c1cTkXZ=Z2Rm~ar53MSF9`UbAn`rc$oyBpn7>#-hrfuMiM|&+y6^m+&Z^R{x-OIFVzst;VL|2pD zV%WUd;@Ia>1_s*e*M>TbTO?1OtgBo4r0Y+<%PX)Fa6;9dA*k`&I)W{QzY&uA9|7b4 z=NImG(*1V)xTcYj5z~>&zyJ90gN?R`tsVna*3jC9;=h)1gi6va&_NHX4bXLo)i3oY ze8xM?w22=FwYH=Ro0;n$MQ8E#kGl?Cfre=qtEOjX9UC3TdY*#Cy?IkYej@1mZxssPALpaBO3lI3Pqe}E>HjH z3_5u6cB|{6GjO;M*jNuL6R|iQzW5RwMp4Q7)24c61@pEoF0+Fw0r+seb)`$6u=)A= zvh(uF@&v?d&GOZ2U6dEcFSnLfd7f~Y`mz#JwDk4s*NvMqUqZHsz*x%8&#xjKbVOkI zJ4ksnAQzOH{oq0Q;G^auhKn7=UM4j$@??61?&GU%q@H zysL0KOYk>bI zd|E{Fs7z2#1(?fBPfu%q;V2 zv+pgv8o$l)uBTvRW_hv=z{xmT84~FTC&qEEiIK46cl<7-`nw1KzUGu zo~6ft8kQCwENW}h-63c#=irbHQq9@>ah)#y$YTHPkz{=@I`kG^9j<`Vy{zGf&%6O; zOpJ+%kt|}xfzEg_w*3TcW+1h~y3KhqRw{spjAv}U$*%M9B8zjQqfGnx_>?h_dnEmCLt$RO`)ZkD0To?o|q%%HO$^-Cx0Ho ziYjXBwr#P(juT3ku@4osX-0}`IUsJ1xZqYW|Hxf@cfS%CeE@<(k;jI9V0c~T)s7>U z-_HJx4y`K;0Mk%nvOE1Xt$%!@5un6y2-;NLQ~4~BYds3pgkX(2=8;Co27dwS@^E41 z$wS4o$gXy_tA&m)f2v8IhE&Cm$w^flg9IMHPvI&o#ASZwgF&kD<&1_aP<4pdd&F); zFa1hG)r~%~D?U+-*KggvZ9LeR4hH01X5dq1Q0>pHnPxkv0a41hw<_wMmzS!9#BD#K z+}be!C-&^w!-tEMg&waT71F^W#uJLcsH%~2=`I;2q5XzTJ|mA+PC$xxiP7g5Pthl8 zBE&r{2Xn3ss|4_9`-X=nV@{0M%rFb+eDGA@Zq4=(HhMeW3G>mBdDHK^+N-J*1~CDO z&6bBAS5{HkN5kr4#)MAdqFP&}Hqe2Q&NJ~}c^qc7MECa8)z2B(?*)cZ(4+d~5Y4HX3<|$vSxnT$`|K@oKj8 zPnrhn(3^-%0l+TlwU(@6u;B``TJ`a@fSB9lg+=}iZEah^miAyENl-sL4>iEMe3_`C z>uRho{X+`X12PK;?(YtFeHDiuL+w#_U>HfQ5Q!I^`?7(k;(idMsjY7To5#)74tC6&cuS6EVp$Gi1Il@+!3!2eHl0a{Gj#bWyZ zGrI6q3@U?so}ot1`aqt{{^HuI1R)t^O=ahS0%D6Wt$XCkCJk^@}2pr>Kgu zn;JPUyfD?FQQ(pXwT0+PWkGyWqVr=ue)?$Cc6>lsOf_Tg_js{ma3^e_t)QUQX;k{{ zyLWNWELHqhpedwtxo#OzM{}3=S=~~- zwtQzlAD?HShLDLeN?b5kqx+?+5fZu$@6XmslGH^NKC#u>l5rjHIx+ZiVD~Qo185*d z4JrCWvjImWdzv$xuZi|qfNzz5(my^a0}4JL(R#?pLj=+Yhd zI)>uvSh&QCODilYBgBcGTvc6dGg@{uK=T#AiKtiRdE@>#Jqhfjp~tHqS~^vJe7J`U zsJ|?*#bWD&j$Mstb(u9?%Ymf=n3olr=TA%IIgH0y+5^(l$s9)nmil#LUa&U7ezP*Q)eLVj`d?%XwsoU!w6@7`Zwui#Q zt%|d*M=!Ocedz|Fk$C6KC0p_As$Q^B`QP6$t6bTF_kESO!2jR2dfAhi68a4sT%owzPiwdd8JkVap$ss7LV(zw>F(V>&|FeM+j|1uyL!&C> zWjHE-ljv{q7KTyJpBtld3)=MgLXRTy>$O(9dnwpAhH8= z#r?8jHBjcr{RmifU4|d{_aY0{e*@`%iL)s!1NywD$(7 zvF2_0jA`1aMb5jObR!8^3cHZd+>LtOeg_SEK*dfH|yf<;OGk`=iS?rzJh> zHKyZ+E3b^l%SuY(CNe>D`)Xo+y2?X2X@8m|mFUG#LjI7fKLBbtIn*+q(>5;??mQ7l zr1;L#09D7}|0u|vO|LKVi;g+FhF)Ye`-R+^H7T!HC_axr355lHyDN)?aqXWbe%lBAJ((gkic8`GZ^S z#R(uz%!C8ht#I<>_3mybF%Il>=bNgg!euB0;3I}S-ukddkDj`@xfy#^KRLM+LI(_3 zf0K%`GUZE`F1arNLJ4{hnZn2_OUs)LLkakaRK1R)sr&e!_F#HH*huDf44tfw0|y^$ z7f?`FSI_=(o0*NK>oOgs1C>}hOem|dV`_Fb9yD80R(3MvVvl8(Z&+B;85a9Uu|<_c z4Q0DJ3k;<)A;+GgF4_0(>)I{CKo(qv@`d)rXWFPlP_}Jh;wVi4=V2TsQ%{)VkU;WF zoWAkGr5j=Z8{~73I8I!F>|+Em**g+mBcE~2SsV$a6DQozHcAp%w`|!*V`v1T0iNtbZW zz~FiQ;({Zs3QkHi%GQ%Jj%8@iCHkRFEB?Wq*6G1_T0y@(2IHYU{c9Fc7pJhisGi#`WNVut{j6Lo_Roi_1W3}__V1}UWz++;k?>KJzzSp`B4I%xnxwLm(JC-y4 zVShNeUf3&We3x5u?TI$+?uUih@q!S-j7J~gNSmFWwjU~mjp`Nz6c|8D1tEI|^6S`0 zze6zc<;CJYe@(NOFMfg;ek0`gwIod27cO438BDM5Ycxq#M&*37798&uU2OPU%jl-w z{qYAoC#O`n(~rCGix2dEeDK7_+dF{^2@$})%bv@i*rXw5v^vDs{%qW@y-yXI)F&x} z3*&N}T%7FepU9q7h|GSx@sn7E@1?T&mcixNS{d9KxN zXiAFUg>hP~sfM!9DeEqX1KYX)=UrW0Dh`VUsj!@E;CBAAi9jwfDt@>_nP;&p*nBes z!#|sd9iZCi$F~o_DkXRQg1H}_&t*d6;8Yg&W$WOFi5%rhEvl3TOI5i=1yffIl12D+#@XF;tCccf%We03Ua@vk5e%M18rSj zW}6~rDT&jkFJq)&60juJb!6(c8iUjW$Xozla|XziSLj!g!hG^5oF!QCRm7wjY)CzW z-i8lVMYl}g0I$V3A-bA)g9IH(oC5U7+w%mz>g`yv6VLp=b2)Gz{$VTpD>+`Z;o4vB z8O~I+!+76;0|$CT<_RXaLraIc&VgZCFIIYR5nNf3n9)w zHIZWRdO(KWUTElJ7?i;~(G7-3k3oA-?Jf_E*BZuvSU=WVo$fGhY=ejZza-fAixAb1 zl%#^OaU6?XI~{&3f3(@+CLv|uOzU?DDtY@tQ6NXZ>tmpIB-BS4*kzS4P#Bn*`vp{g zJ9bR%wIT0HLPEt5Fj={7>yu|^yo^hH4}kH==hzOW;o~aYcdew~{<~MjqO@IkxF`u? zWFu=(`rt4S15Z7XUy3ADly$6FPjFhW?y|B7|LyPD|G$-W;x+!?VIAl&P~HDReFbrn zC9*gO0daK^9jH|xy{hhe5S3}&t3X&app^i;lT$8KR)i{4jTRp;A2t5;a z1}@FniBms1Ittu-4&&T%5F=P)mX%>t|2GJ*jiUSaN0$u#73EWMp8!~_jbT!a>$x1y zNm<0gT2VMoVn)n zKgHDOGHANc{lVdrtV5wy1@!aMOzIM1Q{dJ?Je36j391#Zx%K<+zkrXb|`+fnrk6w(#a_IKyh~_|5 zQ2)!ht8f0(j_Y9PCY|fa%)boQ-baQ&NBGD4J;s$0?kLb_ATT7R!^CIoOF|K{1?U6N z80}rkC#0mLI49Tjq|U>I(h4!l?tLyb|B{;1w6%Rj7#5+_NMoVaV7X2@^Y-mp=Nm{a4dRV`YX!78JmS$@b8KSP0ugeLf;VivfSp>M{XkxL zLHE1)Wg#x8sHuInHxHg?R%KeU^ZrIArns0G_CYxHZ&Fgbmkl*#Bv;u$1s}>*;=x*oMpwIn8bJL)VaINPN*eXsi1vGo#tkb8dc~&vn5mODoB;loUP*r@|Cs zJi_nR$BefDoQ^n6jYv*@9~ih&7I?Tj_g(Jbt0suxqDHVfI~zScJ;yQ35wi>8VL?px z^~QI*^tMLXk9KL~+Pz${o+%O*gG1P|ueLeqKo^pL6A=-~p6QFPFlh$RkuU;G@V?f( z93m3pX`aD7)EpnqeHJF{7$+Mp640YacdQ>DQVW*Fr&ma!ezTi>PLP-w>9kPg1d`5;Iy6q+N`fK2f;$f|YKM?@Ej1rRC*y zp1k=nIAT5E+pt6wQD8RJ(npPO%3+2CSa)qJAenNE?GEKuRD}a3XufMhTBdFJ+9-Jv zNhP-k8|3{L3#~q{g+if=bFGsnZKg!lSYwfH!~ z@?kDBM%=pjT104GvUF+f(^D%Su!>^*8jo|AT~@>O=g4)8AozbZL*b^XmUbDL!8gX2 z5};nitEU=39dsCYS!{gi97PTUoz}tIFoT23_TgW|R0!MQoA z&J&2ZyK)0d4WNMc)fFmpU`hkJT!~u)wFViK;nbfDf04#I9W43zd5~e>RDd@M46~L#w9!5I5qV2 zp1_3WNMqcxrSDafnIe2uLey~ov8Yo)i*`3Q0`R1`$8kjZk*pB3{C0;VoDip^32vGy ze+6x!4|r6G$PLLupu0*VB3B2;ti*KGol!>)jF4Ou8F0tOXvhX>X-6Oi5SYKYazh;i zVA4pPCmIY%y8tJ!>|J<7wHH{d+FHo)VbGQn6ou7w?xlnmi}rbYTnxjd@Nl$jcoI6T z9INx#Bce5E+tezf5XjzP_l4)*6cz2IVXpIwo{tL9sX~4kUK{eapO6xcM$s6(|9w}O z4GESYX4yRI0?)uZ@@mIBqAhqIG5ENZ1TB$nA`vh)xPa#w-mp$u>v)ZINPN_V8*GLz}@yWzo5jo8ZYprZ~e1 z*7KO~T1N$8=#lJIlclM>y?wVfBc;XuvzE;Arws@hK!x3V_|QMuywmmFy9dnC(&_5aZXjyTfV*Q>@iig$ii2c1ed881($YaU$UVqbVYhGGh*5D<*~T&f<|5_ znSdA))n*F~+$iLDc7n^*brWSTBNW?kY7FImOJ4K}1a2Psx#)~O8 zsTt^ yEDXc4lZT>%dQL>AL>{m|D*b1M`M;LLT4YTwW?ERBZi9$KQ8aOx(@A1_H~t5mu2aGQ literal 0 HcmV?d00001 diff --git a/examples/disjoint-source/disjoint-source.yaml b/examples/disjoint-source/disjoint-source.yaml new file mode 100644 index 0000000..223ecdc --- /dev/null +++ b/examples/disjoint-source/disjoint-source.yaml @@ -0,0 +1,14 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +# Example configuration file for a disjoint codebase + +codebase: + files: [ "cpu/*.cpp", "gpu/*.cpp" ] + platforms: [ CPU, GPU ] + +CPU: + files: [ "cpu/*.cpp" ] + +GPU: + files: [ "gpu/*.cpp" ] diff --git a/examples/disjoint-source/gpu/main.cpp b/examples/disjoint-source/gpu/main.cpp new file mode 100644 index 0000000..e642d04 --- /dev/null +++ b/examples/disjoint-source/gpu/main.cpp @@ -0,0 +1,49 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause +#include +#include + +#include "histogram.h" + +int main(int argc, char* argv[]) +{ + int N = 1024; + int B = 16; + printf("Computing histogram of %d inputs and %d bins\n", N, B); + + int* input = (int*) malloc(N * sizeof(int)); + for (int i = 0; i < N; ++i) + { + input[i] = rand() % N; + } + + int* histogram = (int*) malloc(B * sizeof(int)); + for (int j = 0; j < B; ++j) + { + histogram[j] = 0; + } + + #pragma omp target enter data map(alloc:input[0:N], histogram[0:B]) + #pragma omp target update to(input[0:N], histogram[0:B]) + + #pragma omp target teams distribute parallel for simd + for (int i = 0; i < N; ++i) + { + int b = i % B; + #pragma omp atomic + histogram[b]++; + } + + #pragma omp target update from(histogram[0:B]) + #pragma omp target exit data map(delete:input[0:N], histogram[0:B]) + + for (int j = 0; j < B; ++j) + { + printf("histogram[%d] = %d\n", j, histogram[j]); + } + + free(histogram); + free(input); + + return 0; +} diff --git a/examples/disjoint-source/makefile b/examples/disjoint-source/makefile new file mode 100644 index 0000000..ee4846d --- /dev/null +++ b/examples/disjoint-source/makefile @@ -0,0 +1,10 @@ +all: histogram_cpu histogram_gpu + +histogram_cpu: makefile main.cpp + icpc -O3 -xHost -qopenmp -o histogram_cpu main.cpp private_histogram.cpp -qopenmp-offload=host -g + +histogram_gpu: makefile main.cpp + icpc -O3 -xHost -qopenmp -o histogram_gpu main.cpp shared_histogram.cpp -qopenmp-offload=host -g + +clean: + rm *.o histogram_cpu histogram_gpu diff --git a/examples/divergent-source/README.md b/examples/divergent-source/README.md new file mode 100644 index 0000000..6302722 --- /dev/null +++ b/examples/divergent-source/README.md @@ -0,0 +1,27 @@ +# Divergent Source Example + +An example codebase specializing some small regions of code for individual platforms. + +## Output +``` +----------------------- +Platform Set LOC % LOC +----------------------- + {CPU} 19 26.39 + {GPU} 11 15.28 + {GPU, CPU} 42 58.33 +----------------------- +Code Divergence: 0.42 +Unused Code (%): 0.00 +Total SLOC: 72 + +Distance Matrix +-------------- + GPU CPU +-------------- +GPU 0.00 0.42 +CPU 0.42 0.00 +-------------- +``` + +![dendrogram](./divergent-source-dendrogram.png) diff --git a/examples/divergent-source/divergent-source-dendrogram.png b/examples/divergent-source/divergent-source-dendrogram.png new file mode 100644 index 0000000000000000000000000000000000000000..74723ee7e7ff03d8c92dacd38df2aa4f2eff6794 GIT binary patch literal 14726 zcmeHu1yGgiyYC0sZe?R&0tOhQBBCN;p)ODw7LB4HAtBwcEfiFANk|(Y4N_7Pii&^& zQcKDON-q$koBR9R@&DhsXYSlHbLP&RnX_j*%36!B-sgRuU#;6p3a3^q+qjG%h!vF6 zCsYW6)r%llT9+)sEBik;x8tt`w#O)HOYooDl8d+S^HS^6TDAniexCfp5-Ss9iWd*r z$!glES{d6pp0_a~EY90mU$(NlY-+g0!N|te)XH*;sIaK;KEW*}c6QbhA|n6#fUuR# zC6VoGPAnvdEd=GnQ8lNq{uXy#HKV!tpX1gCycRrK_LSz7;^ccE)oe;=yK;H8kZR)T z^i6vE9)1kv%*?3!Xf~P}VqvV6=d5aMnP9x(`3~(Jv8K$kG{-ovi&xKG-7`F099fg* z^<>+rE*B{tKWE8D=E=4dmx^pV`|T@o-gnwbgtHQa*Ivm)VcZB2;7cqah;!>%x8TL| zOT6&X&8-AJvHs@&yz&3=NUnqu+2@NFjs9rS$uz5@eKM{Y>L~Yg47O+1$5}MJOl#^l z`|{!mr=X_w=yELji$_fhie9~v%(QILw(b15n>x?7cx89R;elFG~aw7G}DdG8eP1& z#=QK)2c>i8&c!|Rq=|XVIPvlFW=CAQbcy*q)Ux`)_ET%tuXj>s;`yxIj%#UaYs+oB zxhYZjo#^{TMfbQ4o7KguziX^d)Ov;=0(0)PSO&FNihSl?zy36_fM}JtFw~)AY5VQ) zfdka0EGzn0e-mGLWT(;eSigkZ#EYomXO2IA{!E@b?!B_kPi8K5?}hg!l@U@dv!nFU z*Nb^fYGaiHYuO1xNvnAWVcC+auB@!=utOK@cTaH5 z&CM-azWkw|pI=?H?n82W`d=$oxY?{wzjdO^?5~xA(Vi-!iP*-Wbj@Jm=l*e>7`hp9k=cGUjso0sj8H?Ck8L{Y_LX zPW5Dp5wj&POZ(;JxY}B^v%#XW7fS94X_Vtx)h6j2kd>7+W!5KY=euNzii+CJ&yE|6 z4HVmbWLrWMt@R0;&ZUqK{wqt+VifiUH+!b_S^T8yRy#AtLz;1 ziIkI?sP51!Jm5L!CNdsrZ#V(utv#xfEQQ z?>Z3;L&K-Mrr+P+ZA>%r72UVbD65H&lJaWECs4#7Ia)rb_h$)DklkFyfa^eh%*Op@ z*)Q>ran6MAcyJnbH|`&aoS!)*q@7Xq=EicP^ae4juac8fQ~aW$DyRIo>n`|7b!r=r z^)+D6RoBL;`X=p^l+?xMRmB-an2v^`CsBabwyxtp>hKiTtXsE}GBh-#GCw!FANN5~ z2^F`fP1M>)-_<%b$_xk&j`NtEqR<@rzC3Ae)>Ti?ILpGql3ap4;}c$yq?2WtI=GV} zGc$M%i>X%V;l6OuBK~dLj?wHU2G3J1(A@aNwT3?(VCY^({q zs4RWwSG~;bJM{7+U2?9A`AucVTmQU)_o-QWTIc5(Gn2#WnwqiLnmLVVmwgU-#+4CC zj~{Q58Z2Vd&bEH?{_duj8_UFhUe+%Rp)wdbqn+Pt}MDn~AWpHF`P z)zDj?l;Awv8K;q?ZF6w5+2^P4_*!^r*i>KhUCpOPdieC0FJCS&?lQQE+>O)^jF=go z7;1u*7JA0$SJ#0abSL%kA1%K5^)ZS(Sb)=H3oQDBE#T3kSbzUDLVCGsRnSjX10g3z>EG;Lq z=B0&>`^e{0KDTb&4_fy*dYkpK^#pO?rQy3}vqR+~8$~bP(#|kZ?z@6=$W>rFb>c*< zYK#K;GS=E|yu~rEo@4cDva81{MoLfNwDyCoM=csND4uh}k(tObIZn47=VJ=j)kKh#cn_wHR! zlU1R*wsr{S>An@*lA5xz(FTgQeYay7Tish6z^NZktPVSKw8fD@p>aF4 zvh;sRJ?=g|hC7Z-h(sreS5K(TG^dhBxx48=y%!6c9eOAxXkd;7_Og)vE6v@zcVnGy zoV>d+dc!{BHO$rP*T;I!Pcs?|-1E?yp2^HlR)!qB@{~6xd3mCGLLlwoLv=N^DBM7T zQF)-tR8P#>&6__u>*?bLzhxC8zVzqesgsDm2g*>pBP``Dyy4%3dtvyUy(5x}S5? z1LlLRukit4&J!Z<-p%Q|dDyN%LA>}v?nms21a!Z~)C=D4?{Ote4;0R)74vxTu>4LC zn*35ydKhnNKm%{yys1qwkeTfe@l4GX%sPq5N(64)XME@VoedZ4H=JxMqY!@$1LC*i z<1pSYWn{QE6)D&AoZ+@SrM zdk;Sut7>XKI(F=s-H+x&3FSvOV1{j&oaQ1&jqrn*n4K(a^!=8sfH-@8BX92C$Jd`% z{NMEr%e6$;$KYqTZY?3(ri6sH&dbX=Ci8gK@sx)aBrG zFP5$Xf-$L%qR5>+tB&!pcGISWj~_p(OpEAdE1|j8L{T1QxQ<(Xij*;s4;1`tTPZ~? z4PXOC6wX@em6AUe!tL6(mn`=O4<5LDzk8s4U?3%n;gMH6hn}m_RS~}5y7h=Gb~Spl zkFW1^i}G=T@M4DREGW8ql{GplYS)e(@9R1-;<3przP`$-HIlIJjY8|8woYT?0y6WD zVw$U%Eke2a1l286yPlXRxOVN@-C|-Hnyw}$v6u_0y`lg$Y8o1WmN^}qhkgMlv+k-V z>!cQj{e@PC!@|ORoN?tuzz3m|gh&bCgP47lVG(;&0VgdNOiv38*V6ZSnN7Z(SFhbI*k6)p3i5=5)nhd+Vf=SYU- zUe@mK-=ASCqDPe0by^?v_xFz?0c|e!!6mutk^qAg6|LYemRSOO_PqFlyJ>a#&PgZa zRE{0H%Iz_6o&+@q7;fpt6=8885#++q*6{G8G-jHcqq)$cFc+AabbVDdHUFp2o@IK@ zdz^i==V-p`xWmGucs7ph;w&#b=jSA@e7n9wAuM)e#4gjc=41jj#i3=r&BS1v;dYPP zw{HXQ%bht>`}x@^^=BtnkkrO=vfLoiWjXnH-v}kMWGtg&e_7AA4L6-l2C5~FOp-0& zzxO61+sL0-_rLT$E;tisJ2-GI&FnkGgh8x-mb>AbAP z+CIY+*bN>-%eYXo@@$A$X<0Qct2r>5#kx8Bip$S;>oSc$ZJeB#$ZpI7LQW~1YkX5i zej*5;z=rGv0lGbQ|NierMn+mn95}&rx)`x&@nYu0&+mMVSr$#tQ7tKFi*#X~o*^u|Ei?=oWYuB1dqYa@QW3D*Z`u1?}%bQv34+qD}u#DR5!u(otEQ;74WkpF|to;)E;MTP9fd za()OE)}Dcjikh374ObsAXzlM$3Jcr(uh)SW%}Qx}e2wcw^b5`ZD>pGm%35xP1XpH0oVBV1ALCY=1_^#=s|Q%nN&-hYH3+Whrvka#{U>U1sJn zK-GuJN=4us^?a96wG@3RvWbFP2E8gcWZSt5aI~PHz~|n*dfb3;<2+8kQQ9pVBl==i zQ(g9z3qYy8HjI@xrv2+`j-sR27L}Hl%L`~EDq%!k7H=|2F~@C_RAZCXt0&#vCZEC% zsgf!HP#q?y+=$4oT~%C;%}-D%WU~XPss{~Qvw5={pVd;_Q^=*B{a)(vYRYnQJ|W^3 zF7KGQ8V&E2Tw<^3uxyc-1z-%~N={2lQv-&os;V;V2f33| zR1~of=JA|SM!N==e*pL#y>#`CZj!6}iWF#eI{f_yloGE?2#-@SidhH{RJi^E?*yhfKUnfKPl1!W}9%}z6c$v?lny}IuC z*-aLV0=<8R&3RH$sgnWKi*z3gE|?u71vEKDsZsxv_cm+w`tK!o{o871;@Q)uVrI3w z(2>zLs1%CF`NhnIjLlH=$bM-ir7yg5CYukqT|t9HG$yZ_$u zi;6<_o11|&@--f?Y>q`|`BxW|9{X}(;~|@YBb9WD@FkZFJGgDc&MAZd)o1f549j+z2wr{PjddbiAYvqLBU#xDsmYE4e6)Q?`gaS z1`L&y)i`}oN9xF$!Q<>wQCpAS?9SSl z0kg3G>&it66Yt#ljiLhlN5WcMr5r8SaUdCLg{IaHOeLMvzKus>YT~w^x<`J3gun_4 z_jvWFUEgwU;{9JwS)Z-jPJ(=pvzWNFip4Uj^XS4vKxrD$4M>XL>*N0+CAMq1 ztQ(XU$&b1e>8_FI(4Us}8(w{;DZzdY5|e%B#|=>J%1TO3d3bod)0c4`ijoZoVE+dO zearK@R1s#{(TOK{=b|a$Hqqvt)GUcBd4R^EM60f@_6O(A&OV5#AV-s)9Z_|Mh{H)G z=SKv(8Uz?>d7#iJ8jVRqnh^>ofS077kf753sw-1s^8w2+eYc;oNm}VArKP2l#~@br zFjER&z7QoV#K@Fr&asV|8td12D#gotLQ6}FQw=xQI)@Xye*GH4qSNg$Y@9fN(0{!{ zipLP|Nd6!CX$NFqln`0c&`kfDZ-U{AjIp7AjiBOUlCPq2(Vb0Oi~jfp0W#!moC#jlNu<{Gi`qKK^VWgv{mTkq2ZiY1(zzAdNDZ*R9(!ucu1ZM6+HWBbFS84GM)qj6%+3U_pN@YS z&xhh42H!@TLTalHU$b(MvJyX!5w`jBsAVl}r$lZyY{U0OR; z=J?jxmw^rAU6r8%jyl(QOS$LL%~lFdpG-C=xEvoQw1Sr+yH21tfA<29PW#~7_XA;E)g6RW2wYd^G&u{RS7K<3PDY`2gGn&Kg zM~36;X}qD1Pb}po5oU zKx%i^g%jZ%=IuQOQ+v0~JKe8lO!u!>cJ0=7UKrqUXV-^iRX-;SxwgN#nHZ!vTJ6Vi zqrpRB>OBq<@^h0ByTI&vWxMyMWM=zu7?Wn>$**ctd*3Lx#%=eU#bLe|2n|^&jqUwb zHNv>ZQ8z$Q4tQ?FvwdYDr$ATEaLFyRe1~0+jT%aBy-2x`(IcOC`K6E4NjK^EgpJ-2 zGAC#3KS_U*xOPGV+e@3#SFjaYw8njeQR_w+Rh@q?asoUDeAo}HoM7+H%m zsaemv{5W_lrJi023Ut5oXy1#ZAD8Xt)~62TecxE{qVK}nJJZ^62GclormN`4hpRZ! z!vWLeiF+Khs86Oc0+#wX&`+t*-rMtsJ@0(E((eBwSaoc#VSE2wvEF<5HTMlMb+akB z2krrFgUV5}_Xqy?fJqLGtv67|^7GjPEm07oOUq!`-fT+inAznSz^_4N)pf^+o0^!2 zMLUD~P;-r4I+h%%ayf$Q<`!R>d0QDh$9`3{h+1f66wX>9-LKR=IWNg9Vkw<3T;z6W zp>CQ;jMt%sf_gGBUX@`Fdg~sBbx8(``+5r_(YrUJogU`4Mll*4a;ah=RB}t42Et`bdbIZ#z3aWtV@DCNp9rn zCx|l&3iXk5BRetW$&O3viyFy-(-nvC>$=kN1FYOK9@9cJUrglPwv*O>oWc zPssVcZC@AwVry^qj^}@1Bp&HV?c4D?M$5wGoaA_ua2k9Jx=G&k^p8A6cBzs3%)Cwb zpi-*f1{PSWVV0lpNm*ac9^eoGjyc{qm;f6wOTWUzSp24ZmWmxr*TNm}q14 z7jqTiXDYuSmmhA0PG$kT`B(c14Dg=t__MU1ac$~Wb!wto)p`|E;;ChN4J=qpP)}Nvm+00{PywQi~T4Jqy~Dlh{uZJVilvuXPjC(*nG_?P);=8b;- zJRU^c=k8r~W8+TQpu?z568M9o>yEv|eJ1ZD)xpG%D^AMFmKvuABv_f5ndQ(oad9QB zH5fb-Br5Lc5A_qR^c9>yaE zJ9Ct(EG^ybco!8$_mN}w0=~4FogsHxTuel@|87;ekwf4JIrQLwfN0zW%neR+c3^T6 z=j$2c?m}cdJ>qJr{Ef(kY`d|ptc3?VUHc;R0#x>sZ{NSmXUM=?oWenq8y&D0lf1cI zPUEUnA@!E$H$#YmM%wS9D|Z@>f3}9E#ELs`SRg#r!MHZ;hj%_SIw~~o5>HJLMmhcp z5L6;SK$gUaqP%>~>uZba@*J(Gp&-A!psl6@U-Qk-8g6&MZtHV!&0%82vpG@Pksec~ zB)0`M&7JHF<)bj%hVOp$n6`i}`lOi&h^q{+4LVOYL+@nU(E+YUIo;{{W~KkC;}iuFyW z#s^a1&zO9BdmFL_6{Wnk=$Re|~!xul($0KnJ<4 z`Q$5a-$NxMW>P=S@JvApq_@5FA+(Pm8ylOET2;k=2=M}IEGemmk^U=H2UNEcJ_XDy z9Zgc(oKCY_fT#MQsVVP79#ju=eQDgGnjrc{mSqdcMZvT`);WxHD+3M7k9cM*1NOe_fu8JZmdth+0TNq6VnFJQWoc{?xzwJG36T z@c98&n5p4#`u?5>EHAL=c!<>C6iNtXvx$|%Tzf`$6C&Ge~Q znfx*Rq5L5$YsGw6sDOH$p@kH|zj8Xud1B_J33pLdl@jTnEXmsj!b{UwBD#i@b>W5^B$W=U&0r5y~`Hlet zSg*5OO#}&QJ-vQ?0puVGs@<$5H`DOK1!Ntx%;qt?b_0zXcuYoWwQAir8zfTtan@l_7(+jEO#Z@?q0;ARPB4o=V!*Cr;rjBvp&+jDyhYvT6!hSf- zrvd$1Y458<;m6yctm{QYM3jB_u&X}e!GkAI0QKz}sQS*bBKae(aB%XNQr1R2z0NRmw$W?X@&946R{Fr7R936HL) zCdM4?oQ#41wCK{8En2W(SL>VpbJ!-9A1qkUK{vGj_IBynu!G7#+GO-eBSpVIcW*l~ z4MT$y7Dcu{nsfM}-vgLaFH2#9R}WXp7z9USv#RlD1uw2z_Hp~gm7Po3n%Di><_-_d zNq0`kW5E%e<5!&PztrL)!vOzXt@Z!kkL0@si#1v+8JI~)Ng+*I7=FHE^Ml3gbs46r z@b}~K0@6e3eCi-*=eS_R!Lm9r?)1? z53!(UdU+1&HXT^&fO0bLtFca*181sxa(wZp2q_&}yY;~}+}z0=+fSOnQ{)52U>E>J zX{Q?ANa$zxoVZ2?RStZ0dZyd-@t+*y+V$%-_gXEWN4=8Vy`^=gGABnRfWN+;?eVu4 zo}uu0O3Qm$`Hvxkh*?NRJpa8Bs01}2eG0tolSz)2$)OG-%bhYZ27si=T@4Klw2JxT zS6$1?r~A9YHpj(&$khf&0EyO}Xm)tG_eJ?P&i|uVK>GL3Jp@{I=Z{^{2w3LQK+A)0{Rq_Nk&eSkG9 zd-)fr#Qlw#swyh9gin8BO28aJ`(Qu@PNoRO9PyTP84S(5LASZduE?z1TRy)dCIpL$ zY{?FJdaHMwsX}UzOt&pWbATU%z=!ggGk17N}%tGsY`=xF$F3;FV*4g?q>qg&2`%IdTw*SV>8bO@n>4aSu6#bz|SYJs~yTY-?5< z`+$t|aSCZ`j&h=Q_-{0k@j}9D=!)o2y94 z1TndC2EzbxA)h;UYSH@0SXR!I6w;vY#s-$pu3Ej?*S%!=h6hycXv-8Z%iPg=os_XA zR(}7cGi+?SJ^=#McqlH2VEg#_QQ;}7!G0t)B~k?;fi3KIRaz~4DXHO)`)E<5dNSZS zpEoyAB4dUYyHUa}Ho%IlXkHT=1DgbRPJ|5ck;516B?-J>+MluQs;EO0O)bk(k1mQI z)Q=qTf=fP%m1T-}PWsjZYC``h=v-}yto`@R2d$5Q*D8^K%AsEynof@XCV0DKv<+9B z+h|5lPFl?4Ry8V#bq9B;HJG2GvTm_nL6MiY?Qfz`DDTEBC?V4Bx%!^7qlUI`9PSF5@VXy$bHygJ_74w?gbfhT z-fv!i22VXZ#E3Qf1Zlb>TseYBx__>ftFv`Qkv(>W1 zD&F-+PE2!o*kQ*C`=8_Rcf>3j#X$5--rnM%CY&LHb|LkKk*f-kN$!;cY&_U6xe?Sj z6z!C47$J_-G7WX#VwhD*C^ zljg5wcWx^Z{VGWB!5dePQQ!ihX{BXV&fR6MVEO&z(%*hlgP)%P)6zynHQ&V^Bwjbe z597i7N6|cKJyCfG>M6v@;s49EWQpjmc0%4yB?wI>fhpWtt^37Ma|doAfYL>bVM1Ly=<+$Ib;$W$skfEIYxIq&&YovGM@$Is2kD!K1TMB z--+{Kh{(qxhU$WBXTXC8%HKvaGQ_}enY8+b1gZ$>r)d|sIj4@noX^KIB;`QNpOjcj zV%-7bkLxF=rY!yv9~49kqT~?Qk}<|yfSpyW?p`QZWpOPeR{O_IC}VJR4tjt zh*uX_fe?+K)Yv1a4}f&FPi2=UX{LU2q&uQ+&qhdUkqHt-nOR;L85ziv3K!nrQTF4O z;&0<(xnJS;tq=l@cDnIhkEx!WFrVv^bf{ThT@uK>j5Mme#5`A_$g9Dt$o60PY)*(c z!8hEG5JrD%I@Ww_~HaE`$%u?6yS|6(|t87Bmz%PaJkTIVEo0s!B*H(7x;n zViqE5P(KAo!Q}PTh3{TYmIY@gu=j3E4C3TA=q@wd8ZK9-{S)-`*)m^6HE1UQAvMnk-O`j&}& zyatk@dX+I}f}i5cGMX>}Zj)j|i_`bJLn9-#D-JY2293&{g1UH`24Z)TI&Gek{~TMq zWo<81^+Zw+3)y?&=AsoF+qGXpYIo@jzNA~p&Mmo%vQf$<1H_e&0^?pyS2v7Cqt)iv zUZPar-+El`I)%s%S~!ZyFbdA9k!K16E+OR2`_;@_Gf}lF58 zmExgEwc_5zSL%9Mz2>pyto87o&qndf6M3^_h|Z*H3;E@a>Pe2*%VREz4F3GNTl^|D z5V&afp+h-ur;x*e*j_obj-8+{QD9-SmM5DQ8B}$jb|9^PXzqf0%Q#^cq0YN;QDtRi z{I+_PlEZiK*u;#=)}XN~!nH^yzhQv1ra^xe3qf7T^1GLgOMfO^4I~2NnSwx!#^uY2 zs1cpSN(gym#GaJY@I{+lwtCQ$>d3Nw=esTeAd1YU?@)};?OXIaksa{`I!{e6DQqb@ gV`2W2EN^(8NL?wS(i9XZha(UaS%nkH$Ie~-U){X56#xJL literal 0 HcmV?d00001 diff --git a/examples/divergent-source/divergent-source.yaml b/examples/divergent-source/divergent-source.yaml new file mode 100644 index 0000000..93f2304 --- /dev/null +++ b/examples/divergent-source/divergent-source.yaml @@ -0,0 +1,15 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +# Example configuration file for a divergent codebase + +codebase: + files: [ "*.cpp", "*.h" ] + platforms: [ CPU, GPU ] + +CPU: + files: [ "main.cpp", "private_histogram.cpp" ] + +GPU: + files: [ "main.cpp", "shared_histogram.cpp" ] + defines: [ -DUSE_OFFLOAD ] diff --git a/examples/divergent-source/histogram.h b/examples/divergent-source/histogram.h new file mode 100644 index 0000000..090536e --- /dev/null +++ b/examples/divergent-source/histogram.h @@ -0,0 +1,8 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause +#ifndef _HISTOGRAM_H_ +#define _HISTOGRAM_H_ + +void compute_histogram(int N, int* input, int B, int* histogram); + +#endif /* _HISTOGRAM_H_ */ diff --git a/examples/divergent-source/main.cpp b/examples/divergent-source/main.cpp new file mode 100644 index 0000000..0f10ebd --- /dev/null +++ b/examples/divergent-source/main.cpp @@ -0,0 +1,47 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause +#include +#include + +#include "histogram.h" + +int main(int argc, char* argv[]) +{ + int N = 1024; + int B = 16; + printf("Computing histogram of %d inputs and %d bins\n", N, B); + + int* input = (int*) malloc(N * sizeof(int)); + for (int i = 0; i < N; ++i) + { + input[i] = rand() % N; + } + + int* histogram = (int*) malloc(B * sizeof(int)); + for (int j = 0; j < B; ++j) + { + histogram[j] = 0; + } + +#ifdef USE_OFFLOAD + #pragma omp target enter data map(alloc:input[0:N], histogram[0:B]) + #pragma omp target update to(input[0:N], histogram[0:B]) +#endif + + compute_histogram(N, input, B, histogram); + +#ifdef USE_OFFLOAD + #pragma omp target update from(histogram[0:B]) + #pragma omp target exit data map(delete:input[0:N], histogram[0:B]) +#endif + + for (int j = 0; j < B; ++j) + { + printf("histogram[%d] = %d\n", j, histogram[j]); + } + + free(histogram); + free(input); + + return 0; +} diff --git a/examples/divergent-source/makefile b/examples/divergent-source/makefile new file mode 100644 index 0000000..ee4846d --- /dev/null +++ b/examples/divergent-source/makefile @@ -0,0 +1,10 @@ +all: histogram_cpu histogram_gpu + +histogram_cpu: makefile main.cpp + icpc -O3 -xHost -qopenmp -o histogram_cpu main.cpp private_histogram.cpp -qopenmp-offload=host -g + +histogram_gpu: makefile main.cpp + icpc -O3 -xHost -qopenmp -o histogram_gpu main.cpp shared_histogram.cpp -qopenmp-offload=host -g + +clean: + rm *.o histogram_cpu histogram_gpu diff --git a/examples/divergent-source/private_histogram.cpp b/examples/divergent-source/private_histogram.cpp new file mode 100644 index 0000000..cc948d1 --- /dev/null +++ b/examples/divergent-source/private_histogram.cpp @@ -0,0 +1,22 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause +#include + +#include "histogram.h" + +void compute_histogram(int N, int* input, int B, int* histogram) +{ + #pragma omp parallel for reduction(+:histogram[0:B]) + for (int i = 0; i < N; i += 16) + { + #pragma omp simd simdlen(16) + for (int v = 0; v < 16; ++v) + { + int b = (i + v) % B; + #pragma omp ordered simd overlap(b) + { + histogram[b]++; + } + } + } +} diff --git a/examples/divergent-source/shared_histogram.cpp b/examples/divergent-source/shared_histogram.cpp new file mode 100644 index 0000000..741d5e4 --- /dev/null +++ b/examples/divergent-source/shared_histogram.cpp @@ -0,0 +1,18 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause +#include "histogram.h" + +void compute_histogram(int N, int* input, int B, int* histogram) +{ +#ifdef USE_OFFLOAD + #pragma omp target teams distribute parallel for simd +#else + #pragma omp parallel for simd +#endif + for (int i = 0; i < N; ++i) + { + int b = i % B; + #pragma omp atomic + histogram[b]++; + } +} diff --git a/examples/single-source/README.md b/examples/single-source/README.md new file mode 100644 index 0000000..50044cd --- /dev/null +++ b/examples/single-source/README.md @@ -0,0 +1,25 @@ +# Single-Source Example + +An example codebase using exactly the same code on multiple platforms. + +## Output +``` +------------------------ +Platform Set LOC % LOC +------------------------ + {GPU, CPU} 43 100.00 +------------------------ +Code Divergence: 0.00 +Unused Code (%): 0.00 +Total SLOC: 43 + +Distance Matrix +-------------- + GPU CPU +-------------- +GPU 0.00 0.00 +CPU 0.00 0.00 +-------------- +``` + +![dendrogram](./single-source-dendrogram.png) diff --git a/examples/single-source/main.cpp b/examples/single-source/main.cpp new file mode 100644 index 0000000..ddde644 --- /dev/null +++ b/examples/single-source/main.cpp @@ -0,0 +1,56 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause +#include +#include + +int main(int argc, char* argv[]) +{ + if (argc != 2) + { + printf("Usage: histogram [use_offload]\n"); + return -1; + } + int use_offload = atoi(argv[1]); + + int N = 1024; + int B = 16; + printf("Computing histogram of %d inputs and %d bins\n", N, B); + printf("use_offload = %s\n", use_offload ? "true" : "false"); + + int* input = (int*) malloc(N * sizeof(int)); + for (int i = 0; i < N; ++i) + { + input[i] = rand() % N; + } + + int* histogram = (int*) malloc(B * sizeof(int)); + for (int j = 0; j < B; ++j) + { + histogram[j] = 0; + } + + #pragma omp target enter data map(alloc:input[0:N], histogram[0:B]) if (use_offload > 0) + #pragma omp target update to(input[0:N], histogram[0:B]) if (use_offload > 0) + + #pragma omp target teams distribute parallel for simd if (target: use_offload > 0) + for (int i = 0; i < N; ++i) + { + int b = i % B; + #pragma omp atomic + histogram[b]++; + } + + #pragma omp target update from(histogram[0:B]) if (use_offload > 0) + + #pragma omp target exit data map(delete:input[0:N], histogram[0:B]) if (use_offload > 0) + + for (int j = 0; j < B; ++j) + { + printf("histogram[%d] = %d\n", j, histogram[j]); + } + + free(histogram); + free(input); + + return 0; +} diff --git a/examples/single-source/makefile b/examples/single-source/makefile new file mode 100644 index 0000000..16c4bba --- /dev/null +++ b/examples/single-source/makefile @@ -0,0 +1,7 @@ +all: histogram + +histogram: makefile main.cpp + icpc -O3 -xHost -qopenmp -o histogram main.cpp -qopenmp-offload=host -g + +clean: + rm *.o histogram diff --git a/examples/single-source/single-source-dendrogram.png b/examples/single-source/single-source-dendrogram.png new file mode 100644 index 0000000000000000000000000000000000000000..a4564641d0eb25e31cedeed88b745976c213452f GIT binary patch literal 14481 zcmeHu2UJw)y5+$fp$&);TEzqiDj-3EiBO<`M1cZKfMg_RP_Lq*A_amH4ImU5$(a^W z5fLO6If~>QCC)zf?R)3Vn>%aLTC?V@*Qg@m8x;!!8~e*v`h?kK8%t9Q8&jhzzuW0sSsPiH|9*`37%xB1@7HZ?EJgYF{&@kf zg_R-S-pyy16U6TX|crxGex zh3PjF16t;0IejAo+ld>3+0XTxPMaO=i`KkL&-7a@r$sj!TyuT@z5T2B^Cjm*ba*di z+_;vP(A9C&d*xV2nMfCZYw*~l=~Zjd5CtZJDA>xbE`&9Bj7ky92;#^(rr+@5k!2_G zTH$X5u2^^GKi~MD*piI%1U1(w)oHSyK0It$XL|gwR!X;Gq-Lg7gjTBYzU-#>nTbA= z!NyFDybIV{ugvp=nB!=;R?794&-o4g>26blO(WC=lBEcsz08loH=H@|v}NBHa+P`&=*zP)?T?B2aw>ci>N zr~Q0=U$?j~>bSYN6}>4gDN)hY4eiS;Y%5Q?`cXAbOR&x~jk}qjATn$eB2^A31S>f> zWQd7)Y<|KXRP7d-pCEKY#M4iv|X7 zmMmE!*EoBC((&Pu%IVX$`W)@-(&BR`wFX-9^_ZBLG*a{k!qV~k_wNVy>^ZF#t(ug@ zTC#Nq(U5MD5FsD1Wt(=MOJ;q#g|w@0IC|KML&4&Et<- zG{=?(3CW8(2L<3yla@SJ+SkEcFK=)AYL;!s&gRdJdP#e;$C)*x^;B!HZ*S-6;UI_?IT@XTp zs}_?k5@b4m{vB^Hd!E^_VZ-RuRC?B~V9&6yu$FAQ*3F7WEe#6IiSvcYmkbUJZ&uG|a z{`~n3H|K-Cx3#^@ zb)J%=J?4;%U&-!pQqZ_c>Eua|ESsKJ&ZJmynQ9pJMb{%4s!BE0xLQsp+pZef zL#^y@%v9lU9rxM7OYL4`aX>Lt6|sNZyfMnpZ!>vA`=Mql;hnUvZiltYLdC0d9IYcF zB8=MK-`~z}aLz}U+!bD(tb+*09YW?+Z}m(3oly~)K_mdN$^di z?$}XXu6?^cGc-MPI>Jp-Wt8!$s47!>ExY-UliR#sf=&OmZ)fPvx)~W`Q$M~uf9*V& zDMPEub+*?xG&HG>R?BL;yTfLx$-b{MK(8jxb=GDemGQaF^7_KuPq}mF{5az@lcjeY zH&s$s_wTpM=In_G(5-kSQ4qG*H|*&BfKVI>kt3mzaYRi^TK`lfZ5K2IL}r_n+rF7IyF6 zs)5?1GwXL8&xvZeUtEjK@d#?jbIm?(TKi&Qepa5Alao`0T2T6&A8UB4FE40R@jjK! z63f6rD2GY7a#DJFdOnwys^K(LGp+O)S5auCn7eA?b+ToreZ#}t7c}Q*r!<@!0#Y^+4cJAziJjixyvlMaTq4i8T4rWAoWO-Q>f=ZEM!8t7@`uiTnQj z+M``3)Ugk}&AHjpYBBqJ`hVWOeEG5#9x=&rRzvBD^JLWPN;mMo0H# zQJHrxSwR#$I;@*>F@JtCXax&ZM1f-01kA9H5hGtfm&Dem)_8AjyFo@C3_Otk~i_@c6`F`G-e4Cg!JFv*r` zsovs~&rs8*zqWW?BVk%hP*jwamDT30Tci5>O_wiQcCm?>Xk8=fI+J2I)U5NJ-9FIKqnzfoSdYc;lK_Ix6X-ZT!**ZHAO5;2{|>iK^1l zclV!r_Tu5gb$9OEA&Z#P!q2Wa^O88S(Rf6Vee&CD8SSRK7lf@h{@Z?#K{SfJzS25m zUBY#_~{T`T6;ML0py- zXoBdYesABt-Mo1-r=p@_jCR(a-R*UGxe~?d&Pgg+-Kp+;n`M5rMbqO!@7}%JymMzP zzu{MGgIwFdKxt^c!rrfpx%c@)yc z%G|rych(T?Wd(C{E)j3ua2-EhBlrXD9T^7jnfKmE)Ta9d`lCkcELu#8O^l|mYhs+7XB*PYxNih9EnU%#W}US#)7NPus-~p$V)vQ5 z!Bf9k0;U2zQOm5Yta!_SK8b@e=1D7uuf$R_mC;BlViNMAQ&I#DQ!|w0{dvB77GE2`rLxN+?KHk6P?oTt(gHG9-R}_q|ey z@TR!`EYTCN?Nw#eW)J1Xc(OamW@cuN4j&ACBwxRD>xK<4|4dZs4|{OC_`d}Q??i3c zo;z>nCGH_1Az|ErZ<4x7mNV>MOqcYn@_{yLR}oY4ob zUcYADwCOcUmGRv4NN-(oB=;(WMaJIr)gG?1lhuH`0+-({)4z5tO25P(W$Ag}yC-;q zwXP%J*|UQ|rElK9KT9@d5{UtmOw5gyX-{my1AV@u9lZ29>(|`~llIHA1N(m!niu{5 zu`=%eV&zB@BL?eJcnp6$NN-bjPtDGm{M# zsg`Z0JAeF;qg7N^KK1jvrf-5>H!rCmgzfq-iUW$0w8O>4h1#K$W%Es_#pUW{LtHV4 zD>C#;woCx~*E(2QCO&2tH$EjpzEf5~Y(@1g^kiY%(UM~1i)LW-k9TSl^d!l)ux;Bm zz%sy4vLT_WL?EoVwaMqx#`95p_tVjfAKbgA1cDWK+${QQ$cdOUp3FdnDQ^YwE$RAy zwkwkRrJ2^fYHXx}LdBfAy+S=*b-##+2og>eY!87QYYt-J$JAg^T++LO^AMOG%9naN z^oC2z0`FyHVQ< z<~6qy*4EZujl|4O3k!?R@82($Z$MVhCJPWCgVeQKcn%(ya9?l{G^?+=Me^Uqun4xB z4>=@t-DewGQcM`UwmlV6G(Q^c6;7e&Hj6~!-xp;>^dot;9M%fQ7W5RdNcw>|#b*zS zx&m0u^yw`c|9L)r(HSh4q>5bsMi5aBwQhR*rkK8zm1zJc?_`$o88J!7t@%Z~8w2pg zbI7LND!T97+?;B*omn+qMaH(b>cX{a;ds^hkFg*T>o~k{$?;i=m~e%?=bGEh+vzw!(zCW$xji?JB5{Wb#=9J9Fou_uO-*v{yO*&IzZ**Km+T}ozz$fa6+Lb8{b>EZ%+WfM;nv|)@R!%py%>-9>!r%CwRnH zCMrkB$(=v12ExwgsEgKP+>jP$@a5TV3PJ_pP(IvNC}`JTdu1XacQWveLP#`Nb#X(( z>?{9b5r`0~Vy5|rC6+akI3H(5>+%f?OF;cE`tTv^gP54u<{dj?fT99~OM&2`@n|Ns ziI>-`UVTAT)o=g#z;LWZIlwm{ATl^Ocn=HC^uY&Yg8t{tkJ!bvxHaOd01BPHwUI=Q zi-$)QK+I?4$&=qu*!+Y``SRypk%b({Mk~vP{=u6gf#m9C;!{+5$#XHsx1i@b-uLd^ zJ2r(dvP?}J-&tVo+@EDPpmnVgxZNN*#NMJY0xMm^%v{r#kk1R4l4{hv2z0%h|KHjDE_S${&SS}ad)yWe=8e6iy6(HQkzsVr7aI|3+6%~E~0S73b zzkaPj%J5#-CEJZWUvBv7P5a4SL&9KC`5#$FuA%dZx=bg=NT33Mie=`_4xS*HmlJL$ zc;)@QjTYNbe_9oIa#=3f*rZ55-bqJMQll$NVe3d#d)%x(oL$_>&uRHk^{v@eOoY-% z;(VF|ODp22I@7xAO2^HcH_7I`d>7(5LsEX}rR&$DKp`|uT_8S?xA@09KspjpSY&AHAc2Fa{D((6n|CnnV9XC?$e)o}Aqw+6`@`{KsSR;(~CdcZ0tFYjAwfPZm= zpp+{`YXGpX|8Z-%@wZoZz;8|TgYW_Kq6%@3moHy#KXSfZuKDXOQ+_SIK29fF9mJdGSl-o-%(mU-ry!eDrf@&|&zJ zH~jf6D4>r|0|iYC{(4+u+S`4O6eXQTirAh$e}0I9V({&g&t5@; zFIy?f>E=zOTEX3b(qhT&AF{ODMl{uZ(XBe_!Wk%+pNorq!q?dHgjyq?$vUj}s+giXxF}xi{cgIIPMz3iS;D&J^kI$-5{(UFF- zha|Y!P_r&^%9k>x^&}L`z}WNHm>F(Ibe(1Y#gMb zD6~+DvXfI5V0x_SdW47>?usOg=?ZmLC<)*T@b>mL$&nA>o%s=D!PCDESGeMebaYh{vA3#u-`s z6Mz_1a0;3?UPQMhMHK`dlFX0{4}cr>G;CID8qfUxC)P;6NbDaO`rp2N14_MFefa5W z`;9AqK}NHYM=uz$ny<0%x(r?CI8y)R)<*apxf|i>kqylcys*3YKv0`TW)aBwWzxDl8`Z)0Y% zdhEr^w%T1qf3>58SHwXITEa|!a+!eJoP#TrKJ<>1tcbC84jd*cOM`_M3sYD!+Q9t$ zkh=!zHOX`ErH3`&zVA)L12xhjG?=`12q}zpl`&8%Y%0$S8O@CMWX(@DfIe90YvUsG zQk<#Ngt7R@`%PJW@!6`mPPTt7l*w|SK2@EU9S<(s$xcYlkE={{e>>CM+&lwHR;KOT zV3z(=MFV&Fad0ZeLtIP@;>^p|)=Q2Kes1d@lp1t!T|x@fCC#T@VtG;6c5sA@B0uk# z_TxwJKL;iKb(XbT3X^gxNqwLs`IH){fN|S&h1mC^u%X5!NB;tD2F2_rBP;39V$QRO z4SO)Ni$(n?4T{lBHXz}^3g(?rh$iiCTaXJIckh# zP|n!n_KuFY>j?RRIL)^!Et_*3okoifc-@ht^j1W8eL(1<7?j=miYIIg+s8BvNko{ zXDtc!M0he+z3;IxZY{X^wqamjUT8po##gfHKP#jW4xmo+M(Z^Q)LH5_|78W~%}yl` zijeasU0A_)J#PF<_QJ#BvA8gm4;@$Lj;E)|P;)LV99iDut0<9_0n$oLT{9wr1x@@( zhBj6z)HglSfnf6E#FH5TaQ&4Q>pe&fNYHuGRD7aJ1(?9*QMys3w04?lQsS@M=o=A@ zxz6bzZeEXo146_cEB->NOo<91qUf8}Nh4^WbO6yUHrR~T1)iT=8P@n>0n^$eArh`y zn9M+&EzfZr6L1{8mW$K*=hu@*3%u#~>{%4j#;Cx936dgXyr;4n1mG{A-?HMzVD?Z= zndD+DiVr|kcfrl2+r=D)rAkeplRbF&kkr+$T)C100C4QQ5Fj-~F&uv1D}UsmlFvnU zbhW7|4NJ>mDaHIte=OO@$$9Y5A&QZa5!sk9A@O_q)V8??istJJucgsW6xec2aRLwC zlLWjovjDRejGB&Bzln%=>X`Sh0q4fZL%@6JzP3o;J`(1PbbL&`{_WK&;Z5)C|w)6xorrh0%va@GknAW~@14@FDz)=D>z!(@XYa`#V(!|A(^%!6tn^Tv!?`}GI* z@9%{O6DZ-D1@(ouO$Yl%Z35WEAVm$TlA!C1P44VqHZ2@DvpX)mxq^g*r^e)DWy_)C zMgsFip_77w1}cP#(ZMe%H0+RCvcdWDp<syN!2}cL2V7fR`EAd=r%gRJf#K8=dWljnX zNd*YBXxG#AQyZ52y^FCIK+hxd-yX*LczjBYy|K=TnU~4D60PIVcF)RRAp(9APY->rp zd^^jw?@h-?Z|y40y2Q2H{(&KPF?)hbcs+7^De7L zq>Qsy=CLXxUgz*XK!V|yXD^{lK~ju{@E)y^5DMz~8n4Wcg)RQY=zG)((CdLTE6GOz zzN3gb@9l<^$K8P1X=$QS>|E+TIMsy#OkPwdVUc)BB==r0J%8Wdz(bml7w z^9!>>a&mGNh$-V!WVakFqcs#^$Pc>oGR7A*6 zfKwQuR9T6JkYFfUJH|MNKkN4GPR&MoPCff_hu^K887<+n#l$Eiy~aGf#5`TkDj#2=ZuKv< zY&A=mikU3?e&My%vAg<%RJK0jXBStknAmuVF=OK9EvOER$Y zwIU2O>|?}fM76hmk9cTD^r)HWD^K@6>wk{~Wfdf`XH~Bt~P%kxlO4 z<>X>UL*>LA=_MJ<@~dwdTidBx>@FQ-Iy#n_Q4e=Xr*N4hi#rEjiOGD9xPEdSC&h93 z9WEPcsFog|qN77_Tq&PrR6*Ml&JZ!p({bsMVhiqhKkqQJZWfgbNoXoj31w%~b$(O+ zlD~^SF*5oj!1LpB#unk7pZN~v)-+U}X%1!6)$X?BN_df2rNyVn@{@WtuCS-7hd+Q` zQ@EcaA$hSGOLogYmm11`ye{|iq-NfedU-9G;6tiLF-}3H`}4hGbCttpC1+i9We42d z2glZ~QL_xK?>!Ze_c)`feox=MwNdkxWx0`_IbtOlkEavrZ=LoWEmP6Hw(AB{s{2gl z)bJB;#vO*}PK6NOGXt3e_I7r5_ZuuWIm|m2SM&Y%@%8=6SiOAR)vH%aeE>;F0gnGu z_C+~ljA5WgQ4U(Y=;PUxf6+B^MYTX*=<)b2vrz5$N<+B?%M`kfn)Q&y)W;MR_3m^P zL%CS9E$lCrv!vUL=l48S6AgCFdSCA($kpR36OWf%KWrZmRX+W|XY-csn9kMiw&Fvx z2@^AIN(%Q=)#h_2CZ6%ecP(vceDTQHoja+9(QszRCJQTZj+sGe*ZHx8eKm3wHHPg} z$pbvK;e2j9j(qW)W)9IYCRc~QzdJl5Ft0>UjS0&*#ZXO?r-f?Iz5d~zo?lh3zgLuA zBWSqzhR-!g$C@#CD0Oo9#4c6K*aw`4em96*(lEr8dVH^lwBg%ZoD?;!#aAyBBm=a9 zexB0q$X1ys%Q#SGDc~H|M{s)b7asNuy!-Jl9h1dLLi~U_W+FY+;)J$AY zM9spCBY(&Y-Ji2q=$O2qlNnd)WbK0wqCZ{DC^W^=t8043bm*xc^UYsuqTBAHH2?9D zRkXE1*(904Q2ClZv5%t52v)i{m!)iCeO#K$`TSlAP3Eaew8(THUa8zW-k@=QALV$9 zSoQX|Z^!F7-c2-&{xA%4U_IFGsu=~7(t$9zo*yVj`at6_VKT{8@hzI)s4&7WAr(HI*ZyRhtXneetf zo7y1sIbWft@7{dn!bKUX_1JzjacEj!DQv0?0ZfAyF=c~Q;@Eq5~q zyImtU>X{xbhU20VSFMA*z5Sx?V{zM@=5ALoHf*x6c+(vBHGZ%9z^97DDSS-KR{rXQ zg}vRbq_%*oS&IX>Xtj`F`8*>`N5)jdk~(jQB-=W}aN1te*)Mv(=YGYyGXfVmw^`8eq)b z(0vRGlbqZG(!tp-Fwl_Br_4uzkja-y>R>t!5@^th+D-2}e}iWRbekJxOt%G8Z5HwAZ*c@g=OFq&WeKKmK-k282e(R#9@^ z#;0s5GYb=YhJDe3eG@cad8#f3XyzGhce&@&JW4*BnJ-;Ry|+)^KMD>X2xJeuy)|$F z%nqprrQ1n?l4PgjGkpp2G!jaEH6~gqVbcE(kY6 za$$sJ$?uk|b?|icC5O;x%HNX@Q&irZ`v$j%*7wjv8mV$dqubNMVH4>lU0v_)tet?; zGd!SVsl9W_-5a_tBT^W%8v}V49siA{3XKPh`BhcVnKNgq!zr5vnsaC6(pI-FA>phj z+}h2sm~ND8fHp!3JFFWv+-Ux3Lr#sS9~Qzabvc+6_{bxN!^QGKWtfPyG;|*gSPyxv z4GG?&_98DedEDYV|K z(ML?YnF?27M?!65f}8nI{}uGfHEY&nwdz|7?zZdkN_H#bjWEKDT0_do{u{N)Re)hUJuX@~`6`aUoa zhanIhu#7FoMWQ9@`YMkjaxnkCD=bWfujrVf3S@JPi$l{tz@=u!aPK{G z1RmmW6JN9Dv^6$JI&;p6w4E;{V#UgLOUI%>rTQy)oGG2n3H_nj^+z z+9&M#500dw1Z5i&RFU*XD=#wKz$d9SKSS~1S)RpoQYT$pn0Lv$(Dob77imRqqEqY5 zmGIwc_D?{DA~kq)J z{eGBPWToSUF@$>zhPwh_DIz-`?@7powTU#1g7EPgH+!fdGCHbaG!#jnXhV8Jpn#F` z_n?zC0JJd6o?N<;Ip>9=1iX(5ep~~&C7Lu`1LHa&3%+-BSnD_c zMo>tW<%h`#;BXA2*LaI1H$+3oJO#<>ghLC@s#ETA`}S=?7=us)qpiEXW(}k@d`H2- zIT%6A@2!ao!jBvr?=-Vo>emLd&FbxWacdSSEeJc*VZA&m^d?%9Im4Er6X`ZT)APql z4!uU@4%jI>LJw4kg{VDJJpSW48gm6W8(@isZlhoPO%(|*}W(QLLw(tbSJ zSwar&ATU?sz=RqtZ%g*_tSGgq2$W4BV{ltrT?kXqCvv zGgCte7-)S0W9>;_PF4St*;z9G{=Xrrq#wb^1NYVOcSNO=CG85(P}!aPlS+;$5*dN` zsf_AY9Ejhqd8{uK&|XGD0*v75_z;_5J`8f00LRr%!#)xVceD!X0t1=Amug02^g3uZ zq2J!a+zREtSs8vde_owv=<9G%Xd}D>^cUbSBOT3DYA}+5&q)FXdwBjcdBm9tIzOC{moeE!#f`%-3Z^@C4D^kXvJjy?kY_byW$$rm#K)pQ_m=SG+vK_xt{1wFSqAAD zG)ZXx{FGZQLXMT99KfqXy-GuQ53BGA>NQ6+Et1qjY9;#_>3aKfrc_D( zdIIwa(uD|xuo|6i94=?lsYcd)r}6HW5Fe?TdfSOP6m5?+rMM6aDEa`$jxA!Bd*41O zavq5}P<67QB2rS3mI4nHR2BI$iP>0Y4vAP;mq@b_DN2VY-yCYZXlxt}`#UeGzG8&( zjr@H80*m~KjgUB^y3LKqoJToDkl37WfH*6@Fx?T~MeZOc ztnqR;Zrq@SlVf@Erv((s1q^xjQGnKXx9n$XmAk@%ZPOoVFCsUTB*3p_SSAJ`@I1CPChV$i)(-7TYagt4r zT_O9}dGn}1U#jgH>FMcxhC!w3?avC^GOoBX6)f%{mP9F$& +#include + +#include "test.h" +#include "test.h" // repeated include to test handling of guards +#include "missing.h" // missing include to test file handling + +#include COMPUTED_INCLUDE + +void main(int argc, char* argv[]) +{ + return 0; +} diff --git a/tests/include/test_include.py b/tests/include/test_include.py new file mode 100644 index 0000000..e5a6151 --- /dev/null +++ b/tests/include/test_include.py @@ -0,0 +1,41 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import logging +from codebasin import config, finder, walkers + + +class TestExampleFile(unittest.TestCase): + """ + Simple test of ability to follow #include directives to additional + files. + """ + + def setUp(self): + self.rootdir = "./tests/include/" + logging.getLogger("codebasin").disabled = True + + self.expected_setmap = {frozenset(['CPU']): 11, + frozenset(['GPU']): 12, + frozenset(['CPU', 'GPU']): 16} + + def test_yaml(self): + """include/include.yaml""" + codebase, configuration = config.load("./tests/include/include.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + def test_db(self): + """include/include-db.yaml""" + codebase, configuration = config.load("./tests/include/include-db.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/lexer/__init__.py b/tests/lexer/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/lexer/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/lexer/test_lexer.py b/tests/lexer/test_lexer.py new file mode 100644 index 0000000..4a1e7ef --- /dev/null +++ b/tests/lexer/test_lexer.py @@ -0,0 +1,85 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +from codebasin import preprocessor + + +class TestLexer(unittest.TestCase): + """ + Test ability to tokenize strings correctly. + """ + + def test_character(self): + """characters""" + tokens = preprocessor.Lexer("'c'").tokenize() + self.assertTrue(len(tokens) == 1) + self.assertTrue(isinstance(tokens[0], preprocessor.CharacterConstant)) + + def test_numerical(self): + """numbers""" + numbers = ["123", "123ul", "123.4", "123.4e+05", ".123", "0xFF", "0b10"] + for number in numbers: + tokens = preprocessor.Lexer(number).tokenize() + self.assertTrue(len(tokens) == 1) + self.assertTrue(isinstance(tokens[0], preprocessor.NumericalConstant)) + + def test_string(self): + """strings""" + tokens = preprocessor.Lexer("\"this is a string constant\"").tokenize() + self.assertTrue(len(tokens) == 1) + self.assertTrue(isinstance(tokens[0], preprocessor.StringConstant)) + + def test_identifier(self): + """identifiers""" + tokens = preprocessor.Lexer("this is a string of words").tokenize() + self.assertTrue(len(tokens) == 6) + self.assertTrue(all([isinstance(t, preprocessor.Identifier) for t in tokens])) + + def test_operator(self): + """operators""" + operators = ["||", "&&", ">>", "<<", "!=", ">=", "<=", "==", "##"] + \ + ["-", "+", "!", "*", "/", "|", "&", "^", "<", ">", "?", ":", "~", "#", "=", "%"] + for op in operators: + tokens = preprocessor.Lexer(op).tokenize() + self.assertTrue(len(tokens) == 1) + self.assertTrue(isinstance(tokens[0], preprocessor.Operator)) + self.assertTrue(str(tokens[0].token) == op) + + def test_puncuator(self): + """punctuators""" + punctuators = ["(", ")", "{", "}", "[", "]", ",", ".", ";", "'", "\"", "\\"] + for punc in punctuators: + tokens = preprocessor.Lexer(punc).tokenize() + self.assertTrue(len(tokens) == 1) + self.assertTrue(isinstance(tokens[0], preprocessor.Punctuator)) + self.assertTrue(str(tokens[0].token) == punc) + + def test_expression(self): + """expression""" + tokens = preprocessor.Lexer("foo(a,b) * 124 + 'c'").tokenize() + self.assertTrue(len(tokens) == 10) + self.assertTrue(isinstance(tokens[0], preprocessor.Identifier)) + self.assertTrue(isinstance(tokens[1], preprocessor.Punctuator)) + self.assertTrue(isinstance(tokens[2], preprocessor.Identifier)) + self.assertTrue(isinstance(tokens[3], preprocessor.Punctuator)) + self.assertTrue(isinstance(tokens[4], preprocessor.Identifier)) + self.assertTrue(isinstance(tokens[5], preprocessor.Punctuator)) + self.assertTrue(isinstance(tokens[6], preprocessor.Operator)) + self.assertTrue(isinstance(tokens[7], preprocessor.NumericalConstant)) + self.assertTrue(isinstance(tokens[8], preprocessor.Operator)) + self.assertTrue(isinstance(tokens[9], preprocessor.CharacterConstant)) + + tokens = preprocessor.Lexer("a > b ? \"true_string\" : \"false_string\"").tokenize() + self.assertTrue(len(tokens) == 7) + self.assertTrue(isinstance(tokens[0], preprocessor.Identifier)) + self.assertTrue(isinstance(tokens[1], preprocessor.Operator)) + self.assertTrue(isinstance(tokens[2], preprocessor.Identifier)) + self.assertTrue(isinstance(tokens[3], preprocessor.Operator)) + self.assertTrue(isinstance(tokens[4], preprocessor.StringConstant)) + self.assertTrue(isinstance(tokens[5], preprocessor.Operator)) + self.assertTrue(isinstance(tokens[6], preprocessor.StringConstant)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/literals/__init__.py b/tests/literals/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/literals/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/literals/literals.yaml b/tests/literals/literals.yaml new file mode 100644 index 0000000..b49dd1f --- /dev/null +++ b/tests/literals/literals.yaml @@ -0,0 +1,11 @@ +codebase: + files: [ main.cpp ] + platforms: [ CPU, GPU ] + +CPU: + files: [ main.cpp ] + defines: [ USE_CPU ] + +GPU: + files: [ main.cpp ] + defines: [ USE_GPU ] diff --git a/tests/literals/main.cpp b/tests/literals/main.cpp new file mode 100644 index 0000000..85c083d --- /dev/null +++ b/tests/literals/main.cpp @@ -0,0 +1,14 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#if 20110325ul >= 20100325ul +void foo(); +#endif + +#if 0xFF == 255 +void bar(); +#endif + +#if 0b11 == 3 +void baz(); +#endif diff --git a/tests/literals/test_literals.py b/tests/literals/test_literals.py new file mode 100644 index 0000000..7455dd3 --- /dev/null +++ b/tests/literals/test_literals.py @@ -0,0 +1,31 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import logging +from codebasin import config, finder, walkers + + +class TestExampleFile(unittest.TestCase): + """ + Simple test of C-style literal handling. + e.g. 0x0ULL, 0b11 + """ + + def setUp(self): + self.rootdir = "./tests/literals/" + logging.getLogger("codebasin").disabled = True + + self.expected_setmap = {frozenset(['CPU', 'GPU']): 9} + + def test_yaml(self): + """literals/literals.yaml""" + codebase, configuration = config.load("./tests/literals/literals.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/macro_expansion/__init__.py b/tests/macro_expansion/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/macro_expansion/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/macro_expansion/defined_undefined_test.cpp b/tests/macro_expansion/defined_undefined_test.cpp new file mode 100644 index 0000000..b2c1481 --- /dev/null +++ b/tests/macro_expansion/defined_undefined_test.cpp @@ -0,0 +1,29 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#define CPU 1 +#define GPU 2 + +#define ARCH GPU + +#undef CPU +#undef GPU + +#if ARCH == 1 + +void my_cpu_func() { + +} + +#elif ARCH == 2 + +void my_gpu_func() { + +} + +#else + +#warning "ARCH Value is unexpected." + +#endif + diff --git a/tests/macro_expansion/function_like_test.cpp b/tests/macro_expansion/function_like_test.cpp new file mode 100644 index 0000000..2478ad8 --- /dev/null +++ b/tests/macro_expansion/function_like_test.cpp @@ -0,0 +1,25 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#define FOO 1 +#define BAR 2 + +#define MAX(a,b) (a) >= (b) ? (a) : (b) + +double a, b; + +#if MAX(FOO, BAR) == 0 +void neither_foo_nor_bar() +{ + a = b +} +#else +void both_foo_and_bar() +{ + a = 10; + b = a; + a = 15; + return; +} +#endif + diff --git a/tests/macro_expansion/infinite_loop_test.cpp b/tests/macro_expansion/infinite_loop_test.cpp new file mode 100644 index 0000000..e84bfc2 --- /dev/null +++ b/tests/macro_expansion/infinite_loop_test.cpp @@ -0,0 +1,13 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#define BAR 1 +#define FOO BAR +#undef BAR +#define BAR FOO + +#if FOO == 1 +void my_func(); +#else +void other_func(); +#endif diff --git a/tests/macro_expansion/macro_expansion-dendrogram.png b/tests/macro_expansion/macro_expansion-dendrogram.png new file mode 100644 index 0000000000000000000000000000000000000000..fe196859b2446501488af049620214bd69d2870a GIT binary patch literal 14484 zcmeHuXH-<_w(Z880JX&c2)2NMXaW@pCcpv(Bug}*l93=esEw$INRd>c0+KUG4kl2d zk_yR45{e*6GQ7Dx-F?m-@18sE`+48E4m(h_`Rd!>UTdzo=3MnaQC@~|4eJ_$AQ&lU zPAd__5?6v?XkWDguLyr>?!teTS)Qb*tim71Rafuh=hfzCE?5!-^Cj{hhG@ws1H5>| zN=m~@+3cE?%_R$c!sL>bxv`m*vBBjXxAZM64a`gr3ho!&$G_u-m6f@ukkCKh7c{fD zE|f9ozJ(xm5R}s=RBVF=TO1rKCl*UadNXP=7@PhG*64A3{W~ zbk5bu45ZK7IZ1@l+~{L=trAgXJyxQ@O9;YszgWB}Zi49cAXX8?#jQ(r;KfU;T=CMq z-w9l?_1^!!@!vR-n}I~uwF=gWfv?IP9UZA(jWzRZM?x#p4C{C^4C`cXs~JWdfPsi`mR-{ROnf8?Jcu-CoJ@xi{3s z+3nbJf=KqS=(7?{F#Tqc_SJZ1s?)13PCZGcwJZZNN4GvB)O3GJT$;7UB{CE+|AQD2GZ(Z$NKfhiDg9Co&D3J0+~lgf&&BN1hih? zDDxMfR!a2fb-COrXsBrG(mv9-QlE>TaC+gHE7G{Tyx;}dQ zk24d5qHgmp!n84+^5x5yhmRlobMtU>?-3Q%{B76SC*0eQTD)gj!4o4NEUKAf6+>=j zvw-IJ?f|`dHnY4JSy@NqW>yl!9wF0d_M%a$k570}(Q*Cj*Dcp$Z+tFl za2F3{W>`c9W!c&Bcqg9X*Crn3%_zq#%S9*Eo8Er#DezsTwK*E(d@Dn_}^X{H>fCQuO~o_l7# zry{VsqvM6s>`w|cH8pj-Xeo2Fv?uGDRjU*O1a;D_>1yipH42gm$cui@!>x+JqSo3r zt%7<5+B7|v{5yBTW-FYe05Rb=z&gToI3`pPphref-(mGEIk+R+mL5Vbf*} zPgdb-@yX`!!)=br@=?kh931^^g@sfq^|CJ8-lIn^kPEitX0kK6 zN2E!X)u&xo5H@Soj){qRe#n4|{8o+#h>z#D?yHtUL^h@y`1lLyB|dtzuEmlqCObduo&v>{wEO)p-3n`NNbGR*u%uOcuQ)@T#D_*|* zlG|XgF{3f-W;kwCKeLI4B9uQ)-7KUV&n!8o$SgkoBqA-!sm-o;dtLPTcNwk6hO)xX zaGDiNzcQ?ie!=N7cOHLaWo4-q*J^@}T6WRtaj(t=_J4kH%Kpcjl{#bPK}VHwlh4KM zEWEoYB4Hc__$v6`x(_ zv}G97lXMfx%g;{q)yPoE=;`k1i3J$NYGXN6LW`hb}}28;yQ1A50R#B;4-oi~qPN8HN_SeRIcm-_mvpYJql+1Ivu3_b$Eq#8m0_RSgxZR2;kv}(D z)?c5X z=lKcs!RBmb%bxP{03z;S@!+{dVZ+)p!v(X6C=c3E0fnp+=v> zPT-TL?PY^Fabm<_5v0L~;$q%?`;r^d5wWL$8r3L-LX;fbUj_nbVyH#S+1WW!G_c|o zkR`^lx03bHjVG)k#?dIb25kjSYgVt`Yu$oO)=a*5TS-Z26AMeEHhhm*f>ctMj`B5kCuGo9*Sniqi z`RjXUCoeBn2qRyLE)VC8J;vjYdoFBbh)-fYc$z{rtgy)MfPrW0t5hN7dk3 z`Mn1Zl5e4urK19E{8g>iFf#gatFL-r$G2zC$BALLZzLu4^{%+4tfBGr&fUA#J>}ct znojJ^%g;YLI=P#CS9^zog2Iv&yoXGefcorPM?TwstzE>rQ=kd|Q{OOUA+A?Ek3^15 z&&=@d+9l0)*i_7He!Q;+NXvzKyLac#lUy>7Yx`=V%3`z#f-uTDM`&l6zu35O%k`Iawb)KOhr0d?gt)1xdQSi6y#jkglnJj7N zOUiq>lG9P~?g~bwnpfv08jX@ZwYBL1(6IuYmZS~>C48}nkdl^;)ygylC@+R`2;-Ea zzc=qBdU|{LPzX~}QWEzwGczN{4By>j)BN@Yv$N zft3clOSw`4ZWR6G$&;O(zyJPwW6Bko2V?7iS@^t~37Vb|BInMXC5+n+bmc|4j{V71 zLw$TK?If`G<3}w#xLmhfsf$%bDM{4@bmR67zr2(pN!zY-&sF-xlwX}afAL}%on^y@ z^N+VLB}u19R83!c&I38W&`=>zE~lvu_x`%L0He8r*&$^$HGhfeE}zR|R{gaUm+3OS zLne)fXbmWpaqb-UXR@=iM}O=I@stx?vUS6T7Xt&v%a<+F?oj}9Py8b&h<*8r6|DJ- zx3Db<2?@iNoD6V7ohvA;pFVvu1b{pk3lg=C)%Z9zKJLry=l@{vRxV)UVIJ;=mo+nM zg(4Nt(F!mY1J(e#5&TLaG|^|bf+5h18Zk9xTNi&p8TaJPP4*y3d7}?shKIBK_FH&M zu8q;$)JXr=v?#t+$mcAs%+;=B(F98rzVvn(5T9gAngx#8^x`M;vy)qnehYZ|bPu5J z{rl6~MXk;N^A$lgCrDJ7*nka{x@l=Iy}^?BJ3)kfyRg?=f9j=b_D46bHTYHSWLJwD%G`)_o8E*Yhd>1G= z(6E24C^lv8f_K8gcA%+auC79BFI-SeTO zMC&UnAipW~5FsBTcEOYFuo^lD3rov7)Me|T<_icAk{9|VAbGO{0&DuLG+W-^Ub-^f zq)7z4ix-U(BkQ5$oX7(RbTZ*m)6!VhufJ%03=y-g!9`A2?eGrDh7B9g?j`3i;xM>0 z?AJ^^J~s5ZHb&XQvvBI02T81#Zzn%MQf_HG3Ypbd#o;Y%!(YPP>*Ft6IUz$HVO2Hp zG%!%Bi{W#xt%C#K?%gGK9^>`q&`5@4yRjb4Eb~h#Bg@xrjy9-ywb=>~)xN-=zxpat zW2UKX;Z=J2A(CNJeS?DHr^kBfcbO%UQ-!fWroI6&faD=bE_gfr9kZlwXlN+-p{Ubz zcfbr?Z_!ZS2p5{tV{Mr(N`0{W1Mp|t;G)M<#ZX>VMLuqJCK>M$5b64tzqS+*dHD7m zvI*qPXKrpzCHIy|U5v7fb${)75Kz2o`KByT*vNk;7gskJ^3$hVO`5Yt@+K}^xB$xW z^zGZXS2LUHk}sVC%|%H@+MnAIFmD%*Q+2Ib)NP6+yjfrbap^M$3fg$RR#!@qeuWJsb$I#LfP?OKpel z>wxxqJUSLE|JC^Q7>8%B3hwmHn>Y6J6SQu0zer0udY0!RCeKknHNJ_VAyGbg@($fe zUY_Z9@D7zD$hV||nf_G%DEof)aLgvKVY9oTDkDgRmPxh~1-Dts2Q6pRyXWKM$Zw)RN z*!}zOC&2H&G?^79kSySoJ5wEk9Lm8Cijt#aUa==zZ!D*%RrF&{$;1s@(qupOtAr|q z#N(0a6fmy@xUc{noX5Y&fLr(=HI$T<{Q?4ZQ%Wl<>+$@AZd@SC4QP49>fd%nqt-O0 z>U*Gv9m?5%6}^_IMc4f}39y$?mxXLp+G9F4E!Or}PWbZs`x`m80odB*`JI?9T3Mw_ zKW2|Xty7Irk^qLl?}N&Nerb|if??(#d1!w(&5$2<3lR)=AO+UWDFIDX%>| zb2jV#roVStH8L`C9?UAi6t`Kl3Q>dJr|eo93tQ$_V^YDGpP3*D78d__XNX+b2MaG- zvBD6G$(}ohZTeuRK=cfYZdsO1n{FWCL3G>@qSy5UNm%+&%|!}utotsyUaJi1n-SM_ zfA<1+87=F_OLi;w)@^&a|4Om;4~2KgfUv9nt;>|cr5O{nGUbto{09n*XoA+?Kc66} z*K3m|pmJ2)=$p^lr`Hh2KfGIL`lej8eED*cqNR)=8l#bER5bQl->mQ8C*Se246`f|2z*`pS%0KnuZj!G6n+wL*De zyF+YGHf5S+rjjKoTw{j42>mCZSs7jZ6E80>;P%6_lFoBkQo(+VG^D)ZNb=(ADZzH| zZ>?`ga~o`d?2oq}yY_dD+`4H~1loj6Y;AdVX@>Rj27kZPkfbN+x$U4dFbVoK zx@yf#(_qMOkx00I5)hz6+|ABnQk*q4O#}kQBFW-_8?tB5rvF(m`-E->rMWCRo3s_^ zfEPnVRYL<2kI=WJ9w7)8=2wHHixSMOmF#mLI2(^27B|r&lALCG^?hLUr`ubppX#7H|I< zH;!wjf~@KFeZww`xxmyzWS=7?Wzbh0sg{XYY4_*PUu)Z(b(2gYCQga?K+_z`!E5E8 z2Q%9Q(LV4sk;KagDNX944|a!q2Czk7su$& zpY>6p&i1X1O{TdCK{;4upGDqk_D$w~FSYAaQ|zj-muH_YguiA84@I3R?vOTkYEBvY4NPt@V#{ zh}+-5iF7xh$8bH0w@SP?Npm8m=G4iP<0K|uA(>uJGB>e#^XAgBxbZS(adB}~MMcHO zBs+Zijv9Cz!ukp(_C2{Ry5Gj3sap&o)r;8umlZ+E8CEdLTDldVTZ$zLChE1+;x}W# zZ&B^|_oIKS4qUM#<|Ydl6Oc!|kZVAYe7H#>73w(p;6pGzIBPd(DL5jj=?0n=fTmqQ zZ258n77b>``_x{O_xc*ed~vQ8q`QPy*5cpM!6>@!8DuGu6v@mX>Fsas zvWnUMkPU+ zNL3Y@cbtTOa^u_k2dGx69-9R=ZP^l4US2*{nTdxEXE+H zZGlp8LO$romx|9^B=RXxT;&pP>puY%=>^~Iw?128m5WAjd z2WZ%`wkLo7iXUJlih%i)5r?5{=&a*kShR*=zgZ3EPZ~lOj)iLatQQYG2(6V)(%RKA zh$Z#4pedcby-@(L7$}2M#k2&i-FkX$2Za6;Lb^FlK~Am)%%cwG1MTU%jwjr9c6J5; zPBN)erU!224%~8hHjDeqVnvnWbQsvrxLs5Q9t9~yaQBhr3TGGtC=IYX`Y#FxZL26d z6NoF_TZUt=S{qEi%Z0*02*BTV%in;vfjW})2Pu-6Tc;f&kYQ@ra2-N&96=~cg#+?3 zv(;V>jF7fmfDdeSvc)D8qH6f9!A3H-MD0c+P{6q<552t9E1nCJK~D-~yu7g==O8Ll zRgBpI04?Ykc=||9zdc!56bv_-=#Ys-gU?-0lk7|>nk7O1ax=YtP}f5N{0u! z7ubua5fKsJfyPvBfcRNh6Ps;RME;d#9z1xk4%q_@ywrM)2+Y^Rt+wHt`8C2=g$*^n zasbJ+QpbZxTg>{~`_=P5-Z6`UG>~)}#v8g+88SkCF-yON@DzgB(nHPyWC_SAFsxj$ zqQ6Y2PzAy7@yM(>>pbW#DMOKr!~F@vKgf;avdMKf_d^z4XFyo%-d5s@=72~Ba~KjJOTSvqx;Qc_NLLfDe34w7R`?4J zT9>m!&Z@0dA)8&0Ja|;hcOH9sW~2&Woy=RRVk?the%p!p30{t$ZJUFa=w@3)lC&Tw zDClwxA`(62K_~Q0|3v(nCtg-H#Kk8*a(P9c50|D!fXUtf z+yTc`3=LyXo;+Eao3Zw-Nf%&BY@krUe!Ue8#QG$?BpfX|piIFuM zwq2auMM~y`hWr1HUf_iO!!$~9!jks*3u&9SZQHA-r}yFg`x|JzBkL=l9h5;GQGyzv z3JVQNb4Vs~uSn;=*1+S`5|q$^Vgf+V0LnB;YSCR-f&N$w0XD)Mzv6M#^zRaMhjx-- z(xi6qdZjNnIzK5`lT$xBAfOL~R_Z~i#`82pJ^_VZ=x6wDCqjUV< z%EkeKD-i5ZzJ)gKT%=+40w4I@9^2AZYD4I#nR^hnF}k)L%LO!&kHdFOaddK$OZdA; zEM3XudnEuD2Dk^*F={hmmLvc}2nP-vc=+i0)vJ3B9U9LYD;Teqv+r=*G7b%;AC}QE zvxHwG7lmEUi0=5V%`$}3-^xU8F(e3R+R$%iQ7F;52^&;%16Q?88q@cBso^*Hd+{4Q zd;fbNPN_z(Me5T$JUn|39H_QVxQLEdHQPeJOahuikzbDpL$X<0fvEj>^sci{!@#DH zW=a@+v1!?wisr-4cJ5}8C0YS;A|lwP^}^Mos5!N833WmeP+NWZGVPecSZp67)yR(I zaP|r^>GWj?^p*d=xkt;68>9c% zSx)*?$Z}GEFlo)Z02f>dPIEdMoBwu}8>h^WqydVBX6n@kMgu}F(;k|yjbq@OC!D31 z|3le+%s#Kmz`*d~l|uuho1a$slH@c=s$k^L$+?q9RYxVm&xuS=Yikeo9dxq@>Me9H z^a|+CaBdAjyG@PAyK|C7R8S!yDIf|;UVYD&BP#9YXQ=Wa{4eU8lsOi4Or2kxTJH8( zDZprQ)HkZcjTW-k>U+7Jp;Mk%7g)=ijO9%3qibc8CVq}T$?SEbZ59nG<=exbQD0Vb zN@B}-y1A(JOu2sSMDnK+`oN+M9JDqe<%B5rbGMRGodSd+eo|j=xV}cU@m1@!v`@o3 zjgo377pe+EwzuCm=^$Z(m-pZxK}i{?fXWtR^YJT;g-Z*%L`tp~GazuC^)eX0|Zl+=Tw%nHu> zuQ>Q%%(x->#Ol?nrHWBVNg}UA^9cyRY?K&n7HCRQ3S7PD?cP$T;S{x^?$B(ANaCbP zh}Mgm>#_?L=`lL0V?$<>C3(u~{h7+wW#dgYbA-7uXWHZ!_W9L`Qk`<%H`)vH_W8&p z;w7gZ&B*NnzkfKhX6<<0!#@ghS|{geE9T9QM7?i}U1*)LKf8rCd)v&k_LWnip9Y;i z_~P7y!`ZFOPBuOT3)8fXMmz?y2X7`y?&fcf7jovG-jm2};usrebp1!yOOqKZdWVt` zP3-2~#2DiMZt=`68G)&x<1ra!GFD0nMwJVJ1>+y&E3`5*Ya6+StrXvk@KPgOpSG+q z4plQWu~&Ljd36Pa8k@c#bt2eVFhivC?(0%XD^Kp2YU^36a{=2U>p67T6KWjp@=zAe zF~w>&GvgJl>XU`TOn%(cbqeYeTABWuv-I8ytNtiWDtFt*vvuDyvTGYV`Pk@PlNme| z31Kh3h$@@N`fE-t$9F6B%Tn|1rfm{!Z&ET!J6x+=nK{8j(WM0`YRt_j8Ci-*@j94r zQK&M0%CSeM=y)Yl`g>;5b6Ki!fnBWdRn@)>$v^0sEEQD+BfN_5xjseKhcvUqcb2`> z8Rp`c&8uxxc8KkxDJ(Qd*UrtZv77SE^E~2|9Q)XLszhYxT!-K2IG-JxcK}CN`EnOy zuD3&a0!gma%nj3pYD&e0iw)9$H0P>G)XBC=Z|a)r{}_mC?#!Az5l`dLg@O-*>@zHadXs)NN<)DzgzEoe+{lmjX zaks3iOu2f@nxlFH{Be=BP5MIPj{%Wyowava7b&Td1-D4YxBA6g8ulyjwC(pV?r$3!5V_nbRARl@go`$s8ivg#N#=Ss+}bBM-|Ji8 zxpjtS&syJ&i)?yUPdrzh5xIP^LDPgi}adUUqYI&#+CZd+4s}m9u@@tqg1jC%4 zKY!lXyPA?LPsyzMDaGNocjA#CXF7-TT#Mv`;6x)LOFhr@(*StXtGUK@XXnQhq{YtdZ_qD4a~ zU}0n9qr_fFd$sYm-(U*x=vY<=KR;^uYO|nrG_K@5a-`v#0*bwPch-l!7yAaT=%8Kok@l~cO&}=;yjoIzB)R$rk1c$>yrr>T*q@?c zbI(y%{LdaXnDYOgFkFfdclh}oP49W|%I@J|a-Klgq)`Pw)u0I(j&+#ni_%Z~s-0wM zjXA8uHtO7>H57Kfn@*UJ%jvksmN;Ir-`ul*-|-6q7gA57 z^ueJC7%G@GB`1(EwnTPzr0e`O?Ya@X{#c#d*G3`uAr5CQf?T>@eI67Xxmc5_FULM4dQNJC~gMaaY%6-kPFBPCQ{;S&m9-q%QMx zyYh= z?=cvxJNt;DR%2Da8Me2G^s_;Z$LDcw}u^)fdg=uZbv^i)j%fzal9lWl6 z*lfFmqXy||N=x5^MCKgXxGZtavg2Djn{{#_(4zmn$7cE&d8jiSbuRD~`rPc+!x+pw z2u5JptMw$|!O+N@ow4l&M^yAK_> ze#mYZ$yMT$LG=9!=bj%r56gLZQsD4mUe+Z$9&JGPl$^YeZ0Q@)EosZ(WMsa#J7zO9 zkh3<6yP%+e+GStRuqbNX?;E$G$Zx~?^V(ROQ{BA-P;1v)!hZ*@S<1(mn zuc0!jeoMD(vnyzezJWU~g=~auP9pV2s0JTi9Sg4X&?JXMyMb_s>9P*i%0wlMp5 z8&MsO8#F>s3#?b4xxz4&1UvWwoKbSrg%q*A4&EY-VG<-b9Dkg^j~F_G&L@j;hv}Mb zeSQ5t3|`maC`p?Sjdl!_L{@S8bkba43SE_z_mA`1{bgfML={{kij$spFUN@4O!$4%MhgD1ibVq{kqBfmHt?+@S2RTcyY}C@y((l=` z7<|D1)mU$woS{I-SzfST;#%g(iB+zr^0LP>)J&xHCa?b7fVKZHCYf`!(U2qYz)_7>r zxy8FN=7@1NOaYP;GD}yi`;1H$zEZr7lqSh)0S&>tdx1){yO^)pdfctcO4JYL7F-bR zckawWBKZhnazXlh1jT23q=b_WA)yW<7%z#4kbHlhDHk9}8ksu7lS3_h6llf1=yJ?$t1(M%E{6%S&!mH?U$)rk@ literal 0 HcmV?d00001 diff --git a/tests/macro_expansion/macro_expansion.yaml b/tests/macro_expansion/macro_expansion.yaml new file mode 100644 index 0000000..42333df --- /dev/null +++ b/tests/macro_expansion/macro_expansion.yaml @@ -0,0 +1,18 @@ +codebase: + files: [ "*.cpp" ] + platforms: [ CPU, GPU ] + +default: &default + files: + - defined_undefined_test.cpp + - infinite_loop_test.cpp + - function_like_test.cpp + - max_level.cpp + +CPU: + <<: *default + defines: [ CPU ] + +GPU: + <<: *default + defines: [ GPU ] diff --git a/tests/macro_expansion/max_level.cpp b/tests/macro_expansion/max_level.cpp new file mode 100644 index 0000000..9424429 --- /dev/null +++ b/tests/macro_expansion/max_level.cpp @@ -0,0 +1,216 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause +// Should be 207 lines + +#define FOO 10 +#define FOO1 FOO +#define FOO2 FOO1 +#define FOO3 FOO2 +#define FOO4 FOO3 +#define FOO5 FOO4 +#define FOO6 FOO5 +#define FOO7 FOO6 +#define FOO8 FOO7 +#define FOO9 FOO8 +#define FOO10 FOO9 +#define FOO11 FOO10 +#define FOO12 FOO11 +#define FOO13 FOO12 +#define FOO14 FOO13 +#define FOO15 FOO14 +#define FOO16 FOO15 +#define FOO17 FOO16 +#define FOO18 FOO17 +#define FOO19 FOO18 +#define FOO20 FOO19 +#define FOO21 FOO20 +#define FOO22 FOO21 +#define FOO23 FOO22 +#define FOO24 FOO23 +#define FOO25 FOO24 +#define FOO26 FOO25 +#define FOO27 FOO26 +#define FOO28 FOO27 +#define FOO29 FOO28 +#define FOO30 FOO29 +#define FOO31 FOO30 +#define FOO32 FOO31 +#define FOO33 FOO32 +#define FOO34 FOO33 +#define FOO35 FOO34 +#define FOO36 FOO35 +#define FOO37 FOO36 +#define FOO38 FOO37 +#define FOO39 FOO38 +#define FOO40 FOO39 +#define FOO41 FOO40 +#define FOO42 FOO41 +#define FOO43 FOO42 +#define FOO44 FOO43 +#define FOO45 FOO44 +#define FOO46 FOO45 +#define FOO47 FOO46 +#define FOO48 FOO47 +#define FOO49 FOO48 +#define FOO50 FOO49 +#define FOO51 FOO50 +#define FOO52 FOO51 +#define FOO53 FOO52 +#define FOO54 FOO53 +#define FOO55 FOO54 +#define FOO56 FOO55 +#define FOO57 FOO56 +#define FOO58 FOO57 +#define FOO59 FOO58 +#define FOO60 FOO59 +#define FOO61 FOO60 +#define FOO62 FOO61 +#define FOO63 FOO62 +#define FOO64 FOO63 +#define FOO65 FOO64 +#define FOO66 FOO65 +#define FOO67 FOO66 +#define FOO68 FOO67 +#define FOO69 FOO68 +#define FOO70 FOO69 +#define FOO71 FOO70 +#define FOO72 FOO71 +#define FOO73 FOO72 +#define FOO74 FOO73 +#define FOO75 FOO74 +#define FOO76 FOO75 +#define FOO77 FOO76 +#define FOO78 FOO77 +#define FOO79 FOO78 +#define FOO80 FOO79 +#define FOO81 FOO80 +#define FOO82 FOO81 +#define FOO83 FOO82 +#define FOO84 FOO83 +#define FOO85 FOO84 +#define FOO86 FOO85 +#define FOO87 FOO86 +#define FOO88 FOO87 +#define FOO89 FOO88 +#define FOO90 FOO89 +#define FOO91 FOO90 +#define FOO92 FOO91 +#define FOO93 FOO92 +#define FOO94 FOO93 +#define FOO95 FOO94 +#define FOO96 FOO95 +#define FOO97 FOO96 +#define FOO98 FOO97 +#define FOO99 FOO98 +#define FOO100 FOO99 +#define FOO101 FOO100 +#define FOO102 FOO101 +#define FOO103 FOO102 +#define FOO104 FOO103 +#define FOO105 FOO104 +#define FOO106 FOO105 +#define FOO107 FOO106 +#define FOO108 FOO107 +#define FOO109 FOO108 +#define FOO110 FOO109 +#define FOO111 FOO110 +#define FOO112 FOO111 +#define FOO113 FOO112 +#define FOO114 FOO113 +#define FOO115 FOO114 +#define FOO116 FOO115 +#define FOO117 FOO116 +#define FOO118 FOO117 +#define FOO119 FOO118 +#define FOO120 FOO119 +#define FOO121 FOO120 +#define FOO122 FOO121 +#define FOO123 FOO122 +#define FOO124 FOO123 +#define FOO125 FOO124 +#define FOO126 FOO125 +#define FOO127 FOO126 +#define FOO128 FOO127 +#define FOO129 FOO128 +#define FOO130 FOO129 +#define FOO131 FOO130 +#define FOO132 FOO131 +#define FOO133 FOO132 +#define FOO134 FOO133 +#define FOO135 FOO134 +#define FOO136 FOO135 +#define FOO137 FOO136 +#define FOO138 FOO137 +#define FOO139 FOO138 +#define FOO140 FOO139 +#define FOO141 FOO140 +#define FOO142 FOO141 +#define FOO143 FOO142 +#define FOO144 FOO143 +#define FOO145 FOO144 +#define FOO146 FOO145 +#define FOO147 FOO146 +#define FOO148 FOO147 +#define FOO149 FOO148 +#define FOO150 FOO149 +#define FOO151 FOO150 +#define FOO152 FOO151 +#define FOO153 FOO152 +#define FOO154 FOO153 +#define FOO155 FOO154 +#define FOO156 FOO155 +#define FOO157 FOO156 +#define FOO158 FOO157 +#define FOO159 FOO158 +#define FOO160 FOO159 +#define FOO161 FOO160 +#define FOO162 FOO161 +#define FOO163 FOO162 +#define FOO164 FOO163 +#define FOO165 FOO164 +#define FOO166 FOO165 +#define FOO167 FOO166 +#define FOO168 FOO167 +#define FOO169 FOO168 +#define FOO170 FOO169 +#define FOO171 FOO170 +#define FOO172 FOO171 +#define FOO173 FOO172 +#define FOO174 FOO173 +#define FOO175 FOO174 +#define FOO176 FOO175 +#define FOO177 FOO176 +#define FOO178 FOO177 +#define FOO179 FOO178 +#define FOO180 FOO179 +#define FOO181 FOO180 +#define FOO182 FOO181 +#define FOO183 FOO182 +#define FOO184 FOO183 +#define FOO185 FOO184 +#define FOO186 FOO185 +#define FOO187 FOO186 +#define FOO188 FOO187 +#define FOO189 FOO188 +#define FOO190 FOO189 +#define FOO191 FOO190 +#define FOO192 FOO191 +#define FOO193 FOO192 +#define FOO194 FOO193 +#define FOO195 FOO194 +#define FOO196 FOO195 +#define FOO197 FOO196 +#define FOO198 FOO197 +#define FOO199 FOO198 +#define FOO200 FOO199 + +#if FOO200 == 10 +void reached_max() { + int bar = 0; + int baz = 1; +} +#else +void died_before_max() { + return; +} +#endif diff --git a/tests/macro_expansion/test_macro_expansion.py b/tests/macro_expansion/test_macro_expansion.py new file mode 100644 index 0000000..b7da9b2 --- /dev/null +++ b/tests/macro_expansion/test_macro_expansion.py @@ -0,0 +1,153 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import logging +from codebasin import config, finder, walkers, preprocessor, platform + + +class TestExampleFile(unittest.TestCase): + """ + Simple test to handle macro expansion + """ + + def setUp(self): + self.rootdir = "./tests/macro_expansion/" + logging.getLogger("codebasin").disabled = True + + self.expected_setmap = {frozenset([]): 13, + frozenset(['CPU', 'GPU']): 239} + + def test_yaml(self): + """macro_expansion/macro_expansion.yaml""" + codebase, configuration = config.load( + "./tests/macro_expansion/macro_expansion.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + def test_variadic(self): + """variadic macros""" + + expected_expansion = [preprocessor.Identifier("Unknown", 0, False, "fprintf"), + preprocessor.Punctuator("Unknown", 0, False, "("), + preprocessor.Identifier("Unknown", 0, False, "stderr"), + preprocessor.Punctuator("Unknown", 0, False, ","), + preprocessor.StringConstant("Unknown", 0, True, "%d, %f, %e"), + preprocessor.Punctuator("Unknown", 0, False, ","), + preprocessor.Identifier("Unknown", 0, True, "a"), + preprocessor.Punctuator("Unknown", 0, False, ","), + preprocessor.Identifier("Unknown", 0, True, "b"), + preprocessor.Punctuator("Unknown", 0, False, ","), + preprocessor.Identifier("Unknown", 0, True, "c"), + preprocessor.Punctuator("Unknown", 0, False, ")")] + + for def_string in [ + "eprintf(...)=fprintf(stderr, __VA_ARGS__)", + "eprintf(args...)=fprintf(stderr, args)"]: + macro = preprocessor.Macro.from_definition_string(def_string) + tokens = preprocessor.Lexer("eprintf(\"%d, %f, %e\", a, b, c)").tokenize() + p = platform.Platform("Test", self.rootdir) + p._definitions = {macro.name: macro} + expanded_tokens = preprocessor.MacroExpander(tokens).expand(p) + self.assertTrue(len(expanded_tokens) == len(expected_expansion)) + for i in range(len(expected_expansion)): + self.assertEqual(expanded_tokens[i].prev_white, expected_expansion[i].prev_white) + self.assertEqual(expanded_tokens[i].token, expected_expansion[i].token) + + def test_self_reference_macros_1(self): + """Self referencing macros test 1""" + + expected_expansion = [preprocessor.Punctuator('Unknown', 4, False, '('), + preprocessor.NumericalConstant('Unknown', 5, False, '4'), + preprocessor.Operator('Unknown', 7, True, '+'), + preprocessor.Identifier('Unknown', 9, True, 'FOO'), + preprocessor.Punctuator('Unknown', 12, False, ')')] + + def_string = 'FOO=(4 + FOO)' + macro = preprocessor.Macro.from_definition_string(def_string) + tokens = preprocessor.Lexer("FOO").tokenize() + p = platform.Platform("Test", self.rootdir) + p._definitions = {macro.name: macro} + expanded_tokens = preprocessor.MacroExpander(tokens).expand(p) + self.assertTrue(len(expanded_tokens) == len(expected_expansion)) + for i in range(len(expected_expansion)): + self.assertEqual(expanded_tokens[i].line, expected_expansion[i].line) + self.assertEqual(expanded_tokens[i].col, expected_expansion[i].col) + self.assertEqual(expanded_tokens[i].prev_white, expected_expansion[i].prev_white) + self.assertEqual(expanded_tokens[i].token, expected_expansion[i].token) + + def test_self_reference_macros_2(self): + """Self referencing macros test 2""" + + expected_expansion = [preprocessor.Identifier('Unknown', 4, False, 'FOO')] + + def_string = 'FOO=FOO' + macro = preprocessor.Macro.from_definition_string(def_string) + tokens = preprocessor.Lexer("FOO").tokenize() + p = platform.Platform("Test", self.rootdir) + p._definitions = {macro.name: macro} + expanded_tokens = preprocessor.MacroExpander(tokens).expand(p) + self.assertTrue(len(expanded_tokens) == len(expected_expansion)) + for i in range(len(expected_expansion)): + self.assertEqual(expanded_tokens[i].line, expected_expansion[i].line) + self.assertEqual(expanded_tokens[i].col, expected_expansion[i].col) + self.assertEqual(expanded_tokens[i].prev_white, expected_expansion[i].prev_white) + self.assertEqual(expanded_tokens[i].token, expected_expansion[i].token) + + def test_indirect_self_reference_macros(self): + """ Indirect self referencing macros test""" + + x_expected_expansion = [preprocessor.Punctuator('Unknown', 2, False, '('), + preprocessor.NumericalConstant('Unknown', 3, False, '4'), + preprocessor.Operator('Unknown', 5, True, '+'), + preprocessor.Punctuator('Unknown', 2, False, '('), + preprocessor.NumericalConstant('Unknown', 3, False, '2'), + preprocessor.Operator('Unknown', 5, True, '*'), + preprocessor.Identifier('Unknown', 7, True, 'x'), + preprocessor.Punctuator('Unknown', 8, False, ')'), + preprocessor.Punctuator('Unknown', 8, False, ')')] + + y_expected_expansion = [preprocessor.Punctuator('Unknown', 2, False, '('), + preprocessor.NumericalConstant('Unknown', 3, False, '2'), + preprocessor.Operator('Unknown', 5, True, '*'), + preprocessor.Punctuator('Unknown', 2, False, '('), + preprocessor.NumericalConstant('Unknown', 3, False, '4'), + preprocessor.Operator('Unknown', 5, True, '+'), + preprocessor.Identifier('Unknown', 7, True, 'y'), + preprocessor.Punctuator('Unknown', 8, False, ')'), + preprocessor.Punctuator('Unknown', 8, False, ')')] + + x_string = 'x=(4 + y)' + x_macro = preprocessor.Macro.from_definition_string(x_string) + y_string = 'y=(2 * x)' + y_macro = preprocessor.Macro.from_definition_string(y_string) + + x_tokens = preprocessor.Lexer("x").tokenize() + y_tokens = preprocessor.Lexer("y").tokenize() + + p = platform.Platform("Test", self.rootdir) + p._definitions = {x_macro.name: x_macro, y_macro.name: y_macro} + + x_expanded_tokens = preprocessor.MacroExpander(x_tokens).expand(p) + + y_expanded_tokens = preprocessor.MacroExpander(y_tokens).expand(p) + + self.assertTrue(len(x_expanded_tokens) == len(x_expected_expansion)) + for i in range(len(x_expected_expansion)): + self.assertEqual(x_expanded_tokens[i].line, x_expected_expansion[i].line) + self.assertEqual(x_expanded_tokens[i].col, x_expected_expansion[i].col) + self.assertEqual(x_expanded_tokens[i].prev_white, x_expected_expansion[i].prev_white) + self.assertEqual(x_expanded_tokens[i].token, x_expected_expansion[i].token) + + self.assertTrue(len(y_expanded_tokens) == len(y_expected_expansion)) + for i in range(len(y_expected_expansion)): + self.assertEqual(y_expanded_tokens[i].line, y_expected_expansion[i].line) + self.assertEqual(y_expanded_tokens[i].col, y_expected_expansion[i].col) + self.assertEqual(y_expanded_tokens[i].prev_white, y_expected_expansion[i].prev_white) + self.assertEqual(y_expanded_tokens[i].token, y_expected_expansion[i].token) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/multi_line/__init__.py b/tests/multi_line/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/multi_line/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/multi_line/main.cpp b/tests/multi_line/main.cpp new file mode 100644 index 0000000..17758a5 --- /dev/null +++ b/tests/multi_line/main.cpp @@ -0,0 +1,21 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#define FOO 0 +#define BAR 10 + +#if FOO < 1 \ + && BAR == 10 +int foo() +{ + return 0; +} +#endif + +#if FOO == 1 \ + && BAR >= 2 +int bar() +{ + return 1; +} +#endif diff --git a/tests/multi_line/multi_line.yaml b/tests/multi_line/multi_line.yaml new file mode 100644 index 0000000..4c97002 --- /dev/null +++ b/tests/multi_line/multi_line.yaml @@ -0,0 +1,11 @@ +codebase: + files: [ main.cpp ] + platforms: [ CPU, GPU ] + +CPU: + files: [ main.cpp ] + defines: [ CPU ] + +GPU: + files: [ main.cpp ] + defines: [ GPU ] diff --git a/tests/multi_line/test_multi_line.py b/tests/multi_line/test_multi_line.py new file mode 100644 index 0000000..3a3f3b7 --- /dev/null +++ b/tests/multi_line/test_multi_line.py @@ -0,0 +1,31 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import logging +from codebasin import config, finder, walkers + + +class TestExampleFile(unittest.TestCase): + """ + Simple test of ability to handle counting of multi-line directives + """ + + def setUp(self): + self.rootdir = "./tests/multi_line/" + logging.getLogger("codebasin").disabled = True + + self.expected_setmap = {frozenset([]): 4, + frozenset(['CPU', 'GPU']): 12} + + def test_yaml(self): + """multi_line/multi_line.yaml""" + codebase, configuration = config.load("./tests/multi_line/multi_line.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/nesting/__init__.py b/tests/nesting/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/nesting/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/nesting/main.cpp b/tests/nesting/main.cpp new file mode 100644 index 0000000..7eb20dd --- /dev/null +++ b/tests/nesting/main.cpp @@ -0,0 +1,22 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#define NESTING_TEST + +#ifdef CPU +int foo() +{ +#ifdef NESTING_TEST + return 0; +#endif +} +#endif + +#ifdef GPU +int bar() +{ +#ifdef NESTING_TEST + return 1; +#endif +} +#endif diff --git a/tests/nesting/nesting.yaml b/tests/nesting/nesting.yaml new file mode 100644 index 0000000..4c97002 --- /dev/null +++ b/tests/nesting/nesting.yaml @@ -0,0 +1,11 @@ +codebase: + files: [ main.cpp ] + platforms: [ CPU, GPU ] + +CPU: + files: [ main.cpp ] + defines: [ CPU ] + +GPU: + files: [ main.cpp ] + defines: [ GPU ] diff --git a/tests/nesting/test_nesting.py b/tests/nesting/test_nesting.py new file mode 100644 index 0000000..1e295df --- /dev/null +++ b/tests/nesting/test_nesting.py @@ -0,0 +1,32 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import logging +from codebasin import config, finder, walkers + + +class TestExampleFile(unittest.TestCase): + """ + Simple test of ability to handle nested definition scopes + """ + + def setUp(self): + self.rootdir = "./tests/nesting/" + logging.getLogger("codebasin").disabled = True + + self.expected_setmap = {frozenset(['CPU']): 6, + frozenset(['GPU']): 6, + frozenset(['CPU', 'GPU']): 5} + + def test_yaml(self): + """nesting/nesting.yaml""" + codebase, configuration = config.load("./tests/nesting/nesting.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/once/__init__.py b/tests/once/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/once/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/once/main.cpp b/tests/once/main.cpp new file mode 100644 index 0000000..6832889 --- /dev/null +++ b/tests/once/main.cpp @@ -0,0 +1,10 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#include "once.h" +#include "once.h" + +void main(int argc, char* argv[]) +{ + return 0; +} diff --git a/tests/once/once.h b/tests/once/once.h new file mode 100644 index 0000000..48d4250 --- /dev/null +++ b/tests/once/once.h @@ -0,0 +1,12 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#ifdef PROCESSED_ONCE // this should never be true +void foo() +{ + return; +}; +#endif +#define PROCESSED_ONCE diff --git a/tests/once/once.yaml b/tests/once/once.yaml new file mode 100644 index 0000000..131c1e1 --- /dev/null +++ b/tests/once/once.yaml @@ -0,0 +1,11 @@ +codebase: + files: [ main.cpp, once.h ] + platforms: [ CPU, GPU ] + +CPU: + files: [ main.cpp ] + defines: [ CPU ] + +GPU: + files: [ main.cpp ] + defines: [ GPU ] diff --git a/tests/once/test_once.py b/tests/once/test_once.py new file mode 100644 index 0000000..511af92 --- /dev/null +++ b/tests/once/test_once.py @@ -0,0 +1,31 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import logging +from codebasin import config, finder, walkers + + +class TestExampleFile(unittest.TestCase): + """ + Simple test of ability to obey #pragma once directives. + """ + + def setUp(self): + self.rootdir = "./tests/once/" + logging.getLogger("codebasin").disabled = True + + self.expected_setmap = {frozenset([]): 4, + frozenset(['CPU', 'GPU']): 10} + + def test_yaml(self): + """once/once.yaml""" + codebase, configuration = config.load("./tests/once/once.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/operators/__init__.py b/tests/operators/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/operators/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/operators/main.cpp b/tests/operators/main.cpp new file mode 100644 index 0000000..5912855 --- /dev/null +++ b/tests/operators/main.cpp @@ -0,0 +1,37 @@ +// Copyright (C) 2019 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +#define OR A || B +#define AND A && B +#define RSHIFT A >> B +#define LSHIFT A << B +#define NEQUAL A != B +#define GT_EQUAL A >= B +#define LT_EQUAL A <= B +#define EQUAL A == B +#define CONCAT A ## B +#define MINUS A - B +#define PLUS A + B +#define NOT ! A +#define MULT A * B +#define DIV A / B +#define BIT_OR A | B +#define BIT_AND A & B +#define BIT_XOR A ^ B +#define BIT_COMP A ~ B +#define LT A < B +#define GT A > B +#define TERN A > B ? A : B +#define ASSIGN A = B +#define MOD A % B +#define STRINGIFY #A + +#define PARENS ( A ) +#define CURLY_BRACES { B } +#define SQUARE_BRACES A[B] +#define COMMA , +#define PERIOD . +#define SEMI_COLON ; +#define SINGLE_QUOTE '' +#define DOUBLE_QUOTE "" + diff --git a/tests/operators/operators.yaml b/tests/operators/operators.yaml new file mode 100644 index 0000000..4c97002 --- /dev/null +++ b/tests/operators/operators.yaml @@ -0,0 +1,11 @@ +codebase: + files: [ main.cpp ] + platforms: [ CPU, GPU ] + +CPU: + files: [ main.cpp ] + defines: [ CPU ] + +GPU: + files: [ main.cpp ] + defines: [ GPU ] diff --git a/tests/operators/test_operators.py b/tests/operators/test_operators.py new file mode 100644 index 0000000..6965f80 --- /dev/null +++ b/tests/operators/test_operators.py @@ -0,0 +1,31 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import logging +from codebasin import config, finder, walkers + + +class TestExampleFile(unittest.TestCase): + """ + Simple test of ability to recognize different operators when used + within directives + """ + + def setUp(self): + self.rootdir = "./tests/operators/" + logging.getLogger("codebasin").disabled = True + + self.expected_setmap = {frozenset(['CPU', 'GPU']): 32} + + def test_yaml(self): + """operators/operators.yaml""" + codebase, configuration = config.load("./tests/operators/operators.yaml", self.rootdir) + state = finder.find(self.rootdir, codebase, configuration) + mapper = walkers.PlatformMapper(codebase) + setmap = mapper.walk(state) + self.assertDictEqual(setmap, self.expected_setmap, "Mismatch in setmap") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parsers/__init__.py b/tests/parsers/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/parsers/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/parsers/test_directive_parser.py b/tests/parsers/test_directive_parser.py new file mode 100644 index 0000000..c389735 --- /dev/null +++ b/tests/parsers/test_directive_parser.py @@ -0,0 +1,150 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +from codebasin import preprocessor + + +class TestDirectiveParser(unittest.TestCase): + """ + Test ability to parse directives correctly. + """ + + def test_define(self): + """define""" + tokens = preprocessor.Lexer("#define FOO").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.DefineNode)) + self.assertTrue(str(node.identifier) == "FOO") + self.assertTrue(node.args is None) + self.assertTrue(node.value == []) + + tokens = preprocessor.Lexer("#define FOO string").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.DefineNode)) + self.assertTrue(str(node.identifier) == "FOO") + self.assertTrue(node.args is None) + self.assertTrue(len(node.value) == 1) + self.assertTrue(isinstance(node.value[0], preprocessor.Identifier)) + + tokens = preprocessor.Lexer("#define FOO (a, b)").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.DefineNode)) + self.assertTrue(str(node.identifier) == "FOO") + self.assertTrue(node.args is None) + self.assertTrue(len(node.value) == 5) + self.assertTrue(isinstance(node.value[0], preprocessor.Punctuator)) + self.assertTrue(isinstance(node.value[1], preprocessor.Identifier)) + self.assertTrue(isinstance(node.value[2], preprocessor.Punctuator)) + self.assertTrue(isinstance(node.value[3], preprocessor.Identifier)) + self.assertTrue(isinstance(node.value[4], preprocessor.Punctuator)) + + tokens = preprocessor.Lexer("#define FOO(a, b)").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.DefineNode)) + self.assertTrue(str(node.identifier) == "FOO") + self.assertTrue(len(node.args) == 2) + self.assertTrue(node.value == []) + + tokens = preprocessor.Lexer("#define eprintf(...)").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.DefineNode)) + self.assertTrue(str(node.identifier) == "eprintf") + self.assertTrue(len(node.args) == 1) + self.assertTrue(node.args[0].token == "...") + + tokens = preprocessor.Lexer("#define eprintf(args...)").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.DefineNode)) + self.assertTrue(str(node.identifier) == "eprintf") + self.assertTrue(len(node.args) == 1) + self.assertTrue(node.args[0].token == "args...") + + def test_undef(self): + """undef""" + tokens = preprocessor.Lexer("#undef FOO").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.UndefNode)) + + def test_include(self): + """include""" + tokens = preprocessor.Lexer("#include ").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.IncludeNode)) + self.assertTrue(isinstance(node.value, preprocessor.IncludePath)) + self.assertTrue(node.value.system) + + tokens = preprocessor.Lexer("#include \"path/to/local/header\"").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.IncludeNode)) + self.assertTrue(isinstance(node.value, preprocessor.IncludePath)) + self.assertTrue(not node.value.system) + + tokens = preprocessor.Lexer("#include COMPUTED_INCLUDE").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.IncludeNode)) + self.assertTrue(len(node.value) == 1) + self.assertTrue(isinstance(node.value[0], preprocessor.Identifier)) + + def test_if(self): + """if""" + tokens = preprocessor.Lexer("#if FOO == BAR").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.IfNode)) + self.assertTrue(len(node.tokens) == 3) + + def test_ifdef(self): + """ifdef""" + tokens = preprocessor.Lexer("#ifdef FOO").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.IfNode)) + self.assertTrue(len(node.tokens) == 4) + self.assertTrue(isinstance(node.tokens[0], preprocessor.Identifier)) + self.assertTrue(node.tokens[0].token == "defined") + + def test_ifndef(self): + """ifndef""" + tokens = preprocessor.Lexer("#ifndef FOO").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.IfNode)) + self.assertTrue(len(node.tokens) == 5) + self.assertTrue(isinstance(node.tokens[0], preprocessor.Operator)) + self.assertTrue(node.tokens[0].token == "!") + self.assertTrue(isinstance(node.tokens[1], preprocessor.Identifier)) + self.assertTrue(node.tokens[1].token == "defined") + + def test_elif(self): + """elif""" + tokens = preprocessor.Lexer("#elif FOO == BAR").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.ElIfNode)) + self.assertTrue(len(node.tokens) == 3) + + def test_else(self): + """else""" + tokens = preprocessor.Lexer("#else").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.ElseNode)) + + def test_endif(self): + """endif""" + tokens = preprocessor.Lexer("#endif").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.EndIfNode)) + + def test_pragma(self): + """pragma""" + tokens = preprocessor.Lexer("#pragma anything").tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.PragmaNode)) + + def test_unsupported(self): + """unsupported""" + for directive in ["#line", "#warning", "#error"]: + tokens = preprocessor.Lexer(directive).tokenize() + node = preprocessor.DirectiveParser(tokens).parse() + self.assertTrue(isinstance(node, preprocessor.UnrecognizedDirectiveNode)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/safe_write/__init__.py b/tests/safe_write/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/safe_write/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/safe_write/test_safe_write.py b/tests/safe_write/test_safe_write.py new file mode 100644 index 0000000..dfb1a5b --- /dev/null +++ b/tests/safe_write/test_safe_write.py @@ -0,0 +1,80 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest +import tempfile +import shutil +import os + +from codebasin import util + + +class TestSafeWrite(unittest.TestCase): + """ + Test that safe_open_write_binary properly opens non-symlinks and + bails on symlinks. + """ + + def setUp(self): + self.testdir = tempfile.mkdtemp() + self.path_linkfail = os.path.join(self.testdir, "nowrite.bin") + self.path_link = os.path.join(self.testdir, "link.bin") + self.path_write = os.path.join(self.testdir, "write.bin") + self.path_create = os.path.join(self.testdir, "create.bin") + + self.initial = bytes("MAGIC", "utf-8") + self.updated = bytes("GOOD", "utf-8") + + with open(self.path_linkfail, "wb") as fp: + fp.write(self.initial) + + shutil.copyfile(self.path_linkfail, self.path_write) + + os.symlink(self.path_linkfail, self.path_link) + + def tearDown(self): + shutil.rmtree(self.testdir) + + def test_linkfail(self): + """Check that we fail to open a symlink for writing""" + with self.assertRaises(os.error): + with util.safe_open_write_binary(self.path_link) as fp: + fp.write(bytes("BAD", "utf-8")) + + with open(self.path_linkfail, "rb") as fp: + got = fp.read(5) + self.assertEqual(got, self.initial) + st = os.fstat(fp.fileno()) + self.assertEqual(st.st_mode & 0o111, 0) + + with open(self.path_link, "rb") as fp: + got = fp.read(5) + self.assertEqual(got, self.initial) + st = os.fstat(fp.fileno()) + self.assertEqual(st.st_mode & 0o111, 0) + + def test_write(self): + """Check that we can write to existing non-symlink files""" + with util.safe_open_write_binary(self.path_write) as fp: + fp.write(self.updated) + + with open(self.path_write, "rb") as fp: + got = fp.read(5) + self.assertEqual(got, self.updated) + st = os.fstat(fp.fileno()) + self.assertEqual(st.st_mode & 0o111, 0) + + def test_create(self): + """Check that we can write to non-existing files""" + with util.safe_open_write_binary(self.path_create) as fp: + fp.write(self.updated) + + with open(self.path_create, "rb") as fp: + got = fp.read(5) + self.assertEqual(got, self.updated) + st = os.fstat(fp.fileno()) + self.assertEqual(st.st_mode & 0o111, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/valid_path/__init__.py b/tests/valid_path/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/valid_path/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/valid_path/test_valid_path.py b/tests/valid_path/test_valid_path.py new file mode 100644 index 0000000..1a55dd1 --- /dev/null +++ b/tests/valid_path/test_valid_path.py @@ -0,0 +1,33 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import unittest + +from codebasin import util + + +class TestValidPath(unittest.TestCase): + """ + Test that valid_path correctly identifies null-byte, carriage return + and line feed characters. + """ + + def test_valid(self): + """Check that a valid path is accepted""" + self.assertTrue(util.valid_path("/valid/path/")) + + def test_null_byte(self): + """Check that a null-byte character is rejected""" + self.assertFalse(util.valid_path("/invalid/\x00/path/")) + + def test_carriage_return(self): + """Check that a carriage return character is rejected""" + self.assertFalse(util.valid_path("/invalid/\r/path/")) + + def test_line_feed(self): + """Check that a line feed character is rejected""" + self.assertFalse(util.valid_path("/invalid/\n/path/")) + + +if __name__ == '__main__': + unittest.main()