diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index a883ec40e..067479ef6 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -8,7 +8,7 @@ on: env: GTEST_OUTPUT: "xml:test_results" - BUILDCMD: "colcon build --event-handlers console_direct+ --executor sequential --cmake-args -DBUILD_HARDENING=ON -DBUILD_TESTING=ON -DBUILD_PYTHON_BINDING=ON -DPYTHON_BINDING_VERSION=${PYTHON_BINDING_VERSION} && colcon test --event-handlers console_direct+ --packages-select ad_map_access ad_physics && colcon test-result" + BUILDCMD: "colcon build --event-handlers console_direct+ --executor sequential --cmake-args -DBUILD_HARDENING=ON -DBUILD_TESTING=ON -DBUILD_PYTHON_BINDING=ON -DPYTHON_BINDING_VERSION=${PYTHON_BINDING_VERSION} -DPYTHON_PACKAGE_FOLDER_NAME=${PYTHON_PACKAGE_FOLDER_NAME} && colcon test --event-handlers console_direct+ --packages-select ad_map_access ad_physics && colcon test-result" permissions: contents: read @@ -18,35 +18,33 @@ jobs: strategy: matrix: include: - - os: ubuntu-20.04 - compiler: gcc9 - EXTRA_PACKAGES: "" - CC: "" - CXX: "" - PYTHON_BINDING_VERSION: "3.8" - os: ubuntu-20.04 compiler: clang10 EXTRA_PACKAGES: clang-10 CC: /usr/bin/clang-10 CXX: /usr/bin/clang++-10 PYTHON_BINDING_VERSION: "3.10" + PYTHON_PACKAGE_FOLDER_NAME: "site-packages" - os: ubuntu-22.04 compiler: gcc11 EXTRA_PACKAGES: "" CC: "" CXX: "" PYTHON_BINDING_VERSION: "3.10" + PYTHON_PACKAGE_FOLDER_NAME: "site-packages" - os: ubuntu-22.04 compiler: clang14 EXTRA_PACKAGES: clang-14 CC: /usr/bin/clang-14 CXX: /usr/bin/clang++-14 PYTHON_BINDING_VERSION: "3.10" + PYTHON_PACKAGE_FOLDER_NAME: "site-packages" name: ${{ matrix.os }}, ${{ matrix.compiler }}, python-${{ matrix.PYTHON_BINDING_VERSION }} runs-on: ${{ matrix.os }} env: PYTHON_BINDING_VERSION: ${{ matrix.PYTHON_BINDING_VERSION }} + PYTHON_PACKAGE_FOLDER_NAME: ${{ matrix.PYTHON_PACKAGE_FOLDER_NAME }} EXTRA_PACKAGES: ${{ matrix.EXTRA_PACKAGES }} steps: diff --git a/.github/workflows/install_dependencies.sh b/.github/workflows/install_dependencies.sh index ec2b5715e..411e0d578 100644 --- a/.github/workflows/install_dependencies.sh +++ b/.github/workflows/install_dependencies.sh @@ -48,7 +48,7 @@ curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python${PYTHON_BINDING_VERS # to handle some error on missing pip dependencies sudo pip${PYTHON_BINDING_VERSION} install testresources sudo pip${PYTHON_BINDING_VERSION} install --upgrade setuptools==59.6.0 -sudo pip${PYTHON_BINDING_VERSION} install colcon-common-extensions xmlrunner pygccxml pyplusplus +sudo pip${PYTHON_BINDING_VERSION} install colcon-common-extensions unittest-xml-reporting pygccxml pyplusplus if (( IS_UBUNTU_20_04 && IS_PYTHON_3_10 )); then echo "!!!!!!! Ubunut 20.04 and python 3.10: compile boost 1.80 !!!!!!!" diff --git a/README.md b/README.md index 36f704749..cd49eff3c 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,9 @@ Following compiler and Python combinations are [tested continously](https://gith | | Ubuntu 20.04 | Ubuntu 22.04 | |:---------------:|:------------:|:------------:| -| GCC 9 | x | | | Clang 10 | x | | | GCC 11 | | x | | Clang 14 | | x | -| Python 3.8 | x | | | Python 3.10 | x | x | Important: cmake is required to be at least version 3.5! diff --git a/ad_map_access/python/tests/interface_test.py b/ad_map_access/python/tests/interface_test.py index d5f6b7eff..369e3f407 100644 --- a/ad_map_access/python/tests/interface_test.py +++ b/ad_map_access/python/tests/interface_test.py @@ -64,8 +64,6 @@ def test_interface(self): if __name__ == '__main__': if os.environ.get('GTEST_OUTPUT') and os.environ['GTEST_OUTPUT'].startswith('xml:'): base_folder = os.environ['GTEST_OUTPUT'][4:] - result_filename = base_folder + 'ad_map_access_interface_test_python' + str(sys.version_info.major) + ".xml" - with open(result_filename, "w+") as result_file: - unittest.main(testRunner=xmlrunner.XMLTestRunner(output=result_file)) + unittest.main(testRunner=xmlrunner.XMLTestRunner(output=base_folder)) else: unittest.main() diff --git a/ad_physics/python/tests/interface_test.py b/ad_physics/python/tests/interface_test.py index c0cfb0997..8c08a597f 100644 --- a/ad_physics/python/tests/interface_test.py +++ b/ad_physics/python/tests/interface_test.py @@ -50,8 +50,6 @@ def test_interface(self): if __name__ == '__main__': if os.environ.get('GTEST_OUTPUT') and os.environ['GTEST_OUTPUT'].startswith('xml:'): base_folder = os.environ['GTEST_OUTPUT'][4:] - result_filename = base_folder + 'ad_pyhsics_interface_test_python' + str(sys.version_info.major) + ".xml" - with open(result_filename, "w+") as result_file: - unittest.main(testRunner=xmlrunner.XMLTestRunner(output=result_file)) + unittest.main(testRunner=xmlrunner.XMLTestRunner(output=base_folder)) else: unittest.main() diff --git a/cmake/python-binding.cmake b/cmake/python-binding.cmake index 5a59ccb39..a19aef892 100644 --- a/cmake/python-binding.cmake +++ b/cmake/python-binding.cmake @@ -69,8 +69,18 @@ function(find_python_binding_packages) endfunction() function(get_python_test_environment) + set(PACKAGE_FOLDER_NAME "${PYTHON_PACKAGE_FOLDER_NAME}") + if ("${PACKAGE_FOLDER_NAME}" STREQUAL "") + set(PACKAGE_FOLDER_NAME "dist-packages") + endif() + if ("${PACKAGE_FOLDER_NAME}" STREQUAL "dist-packages") + set(LIBDIR_PREFIX "local/") + else() + set(LIBDIR_PREFIX "") + endif() + set(TEST_LD_LIBRARY_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") - set(TEST_PYTHONPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${PYTHON_BINDING_FOLDER_NAME}/site-packages") + set(TEST_PYTHONPATH "${CMAKE_INSTALL_PREFIX}/${LIBDIR_PREFIX}${CMAKE_INSTALL_LIBDIR}/${PYTHON_BINDING_FOLDER_NAME}/${PACKAGE_FOLDER_NAME}") foreach(dep ${ARGN}) get_target_property(dep_configurations ${dep} IMPORTED_CONFIGURATIONS) set(TEST_${dep}_LOCATION False) @@ -83,7 +93,8 @@ function(get_python_test_environment) endforeach() if(TEST_${dep}_LOCATION) set(TEST_LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${TEST_${dep}_LOCATION}") - set(TEST_PYTHONPATH "${TEST_PYTHONPATH}:${TEST_${dep}_LOCATION}/${PYTHON_BINDING_FOLDER_NAME}/site-packages") + string(REGEX REPLACE "/${CMAKE_INSTALL_LIBDIR}$" "" TEST_${dep}_INSTALL_PREFIX ${TEST_${dep}_LOCATION}) + set(TEST_PYTHONPATH "${TEST_PYTHONPATH}:${TEST_${dep}_INSTALL_PREFIX}/${LIBDIR_PREFIX}${CMAKE_INSTALL_LIBDIR}/${PYTHON_BINDING_FOLDER_NAME}/${PACKAGE_FOLDER_NAME}") else() message(WARNING "Failed to query mandatory location of dependency ${dep}") endif() diff --git a/cmake/python/python_wrapper_helper.py.in b/cmake/python/python_wrapper_helper.py.in index ac72e798a..56a7bd78d 100644 --- a/cmake/python/python_wrapper_helper.py.in +++ b/cmake/python/python_wrapper_helper.py.in @@ -1,7 +1,7 @@ #!/usr/bin/python # ----------------- BEGIN LICENSE BLOCK --------------------------------- # -# Copyright (c) 2019-2020 Intel Corporation +# Copyright (c) 2019-2022 Intel Corporation # # ----------------- END LICENSE BLOCK ----------------------------------- @@ -33,7 +33,7 @@ def get_list_of_files(directory, ignore_files): for ignore_file in ignore_files: if full_path.find(ignore_file) != -1: skip = True - print ("Skipping file: " + full_path) + print("Skipping file: " + full_path) if not skip: if full_path.endswith(".h") or full_path.endswith(".hpp"): all_files.append(full_path) @@ -41,7 +41,7 @@ def get_list_of_files(directory, ignore_files): return all_files -def generate_python_wrapper(header_directories, include_paths, library_name, cpp_filename, declarations, main_namespace="", ignore_declarations={}, ignore_files={}): +def generate_python_wrapper(header_directories, include_paths, library_name, cpp_filename, declarations, main_namespace="", ignore_declarations={}, ignore_files={}, add_declarations={}): """ Function to generate Python-C++ binding code by calling pygccxml and py++ @@ -60,6 +60,10 @@ def generate_python_wrapper(header_directories, include_paths, library_name, cpp :type ignore_declarations: list :param ignore_files: a list of files to be ignored :type ignore_files: list + :param add_declarations: a list of declarations to be explicitly enabled searched for in + against the full declaration string resulting from '"{}".format(decl)' output + (useful e.g. for typedefs to template types which are not as such visible in the delcarations) + :type add_declarations: list :return: """ @@ -67,20 +71,19 @@ def generate_python_wrapper(header_directories, include_paths, library_name, cpp # Find out the xml generator (gccxml or castxml) generator_path, generator_name = utils.find_xml_generator() - compiler = "g++" - compiler_path = "/usr/bin/g++" + compiler = "@CXX@" # Create configuration for CastXML xml_generator_config = parser.xml_generator_configuration_t( xml_generator_path=generator_path, xml_generator=generator_name, compiler=compiler, - compiler_path=compiler_path, start_with_declarations=declarations) # Set include dirs and cflags to avoid warnings and errors xml_generator_config.append_cflags("-std=c++@CMAKE_CXX_STANDARD@") xml_generator_config.append_cflags("-Wno-error=invalid-constexpr") + xml_generator_config.append_cflags("-DSAFE_DATATYPES_EXPLICIT_CONVERSION=1") for inc_dir in include_paths: xml_generator_config.include_paths.append(inc_dir) @@ -124,32 +127,57 @@ def generate_python_wrapper(header_directories, include_paths, library_name, cpp top_namespace, main_namespace)) for decl in builder.decls(): - for ignore_declaration in ignore_declarations: - if ignore_declaration in decl.name or ignore_declaration in decl.alias: - decl.exclude() - decl.ignore = True - decl.already_exposed = True - # print("declaration to ignore found: {} (alias {})".format(decl, decl.alias)) - break + decl_full_string_and_type = "{}".format(decl) + # print("declaration {} (alias {}, full {})".format(decl.name, decl.alias, decl_full_string_and_type)) if main_namespace != "": - if isinstance(decl, decl_wrappers.class_wrapper.class_t) or isinstance(decl, decl_wrappers.class_wrapper.class_declaration_t) or isinstance(decl, decl_wrappers.typedef_wrapper.typedef_t): - decl_full_string = "{}".format(decl) + if isinstance(decl, decl_wrappers.class_wrapper.class_t) or isinstance(decl, decl_wrappers.class_wrapper.class_declaration_t) or isinstance(decl, decl_wrappers.typedef_wrapper.typedef_t) or isinstance(decl, decl_wrappers.enumeration_wrapper.enumeration_t): + decl_full_string = decl_full_string_and_type.split(" ")[0] if main_namespace in decl_full_string: # namespace present, ok - # print("typedef/class main namespace found: {}".format(decl_full_string)) - continue - if decl_full_string in main_namespace: + # print("typedef/class/enum main namespace found [ignore-{},exposed-{}]: + # {}".format(decl.ignore, decl.already_exposed, + # decl_full_string_and_type)) + pass + elif decl_full_string in main_namespace: # declaration is a parent of main namespace, ok - # print("typedef/class declaration of upper level namespace found: {}".format(decl_full_string)) - continue - if top_namespace != "" and not top_namespace in decl_full_string: + # print("typedef/class/enum declaration of upper level namespace found + # [ignore{},exposed{}]: {}".format(decl.ignore, decl.already_exposed, + # decl_full_string_and_type)) + pass + elif top_namespace != "" and not top_namespace in decl_full_string: # global or std defaults, ok - # print("typedef/class outside top namespace found: {}".format(decl_full_string)) - continue - # print("typedef/class outside of main namespace found. Ignoring: {}".format(decl_full_string)) + # print("typedef/class/enum outside top namespace found + # [ignore-{},exposed-{}]: {}".format(decl.ignore, decl.already_exposed, + # decl_full_string_and_type)) + pass + else: + # print("typedef/class/enum outside of main namespace found + # [ignore-{},exposed-{}]. Ignoring: {}".format(decl.ignore, + # decl.already_exposed, decl_full_string_and_type)) + decl.exclude() + decl.ignore = True + # try to enforce that it's not exported anymore + decl.already_exposed = True + + if decl.ignore: + if decl_full_string in declarations: + decl.ignore = False + decl.already_exposed = False + # print("Ignored typedef/class/enum explicitly listed in delclarations + # found. Reenable {}".format(decl_full_string_and_type)) + for add_declaration in add_declarations: + if (add_declaration in decl.name) or (add_declaration in decl.alias) or (add_declaration in decl_full_string_and_type): + decl.ignore = False + decl.already_exposed = False + # print("declaration to explicitly add found: {} (alias {})".format(decl, decl.alias)) + break + for ignore_declaration in ignore_declarations: + if (ignore_declaration in decl.name) or (ignore_declaration in decl.alias) or (ignore_declaration in decl_full_string_and_type): decl.exclude() decl.ignore = True decl.already_exposed = True + # print("declaration to ignore found: {} (alias {})".format(decl, decl.alias)) + break # debug declarations # builder.print_declarations() @@ -168,7 +196,7 @@ def generate_python_wrapper(header_directories, include_paths, library_name, cpp print("generate_python_wrapper(): {} written.".format(cpp_filename)) -def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filename_out, additional_replacements={}, additional_includes={}, spdx_license="MIT", fix_include_directives=True, fix_enum_class=True): +def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filename_out, additional_replacements=[], additional_includes={}, spdx_license="MIT", fix_include_directives=True, fix_enum_class=True): """ Post process generated binding code @@ -204,7 +232,7 @@ def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filenam file_output.write("/*\n" " * ----------------- BEGIN LICENSE BLOCK ---------------------------------\n" " *\n" - " * Copyright (c) 2020 Intel Corporation\n" + " * Copyright (c) 2020-2021 Intel Corporation\n" " *\n" " * SPDX-License-Identifier: " + spdx_license + "\n" " *\n" @@ -228,7 +256,7 @@ def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filenam for header_dir in header_directories: if not header_dir.endswith("/"): header_dir = header_dir + "/" - line = line.replace(header_dir, "") + line = str.replace(line, header_dir, "") # Fix C++ enum classes if fix_enum_class: @@ -236,7 +264,7 @@ def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filenam if line.find("export_values()") != -1: enum_started = False else: - line = line.replace(enum_namespace, enum_namespace_full) + line = str.replace(line, enum_namespace, enum_namespace_full) else: match = enum_declaration_start.match(line) if match: @@ -246,7 +274,7 @@ def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filenam enum_namespace_full = match.group(1) + "::" for replacement in additional_replacements: - line = line.replace(replacement[0], replacement[1]) + line = str.replace(line, replacement[0], replacement[1]) file_output.write(line) diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake index a5efd07c2..a8b88b189 100644 --- a/cmake/warnings.cmake +++ b/cmake/warnings.cmake @@ -9,3 +9,8 @@ list(APPEND TARGET_COMPILE_OPTIONS -Wall -Wextra -pedantic -Wfloat-equal -Wshado # treat warnings as errors list(APPEND TARGET_COMPILE_OPTIONS $<$>:-Werror>) + +# disable some warnings on 3rd party code +list(APPEND TARGET_COMPILE_OPTIONS $<$:-Wno-error=maybe-uninitialized>) +list(APPEND TARGET_COMPILE_OPTIONS -Wno-error=deprecated-declarations) + diff --git a/doc/BUILDING.md b/doc/BUILDING.md index 7dc602cea..6e7511ee4 100644 --- a/doc/BUILDING.md +++ b/doc/BUILDING.md @@ -30,7 +30,7 @@ The components within this repository have some dependencies: - Osmium >= 2.13: - ***all components when enabling unit tests***: - gtest aka. googletests < 1.10 : - - xmlrunner + - unittest-xml-reporting Dependencies provided by Ubunutu (>= 18.04): @@ -50,7 +50,7 @@ $> sudo apt-get install libboost-all-dev libpugixml-dev libgtest-dev libpython- Additional dependencies for the python bindings: ```bash $> sudo apt-get install castxml -$> pip install --user python-wheel pygccxml pyplusplus xmlrunner +$> pip install --user python-wheel pygccxml pyplusplus unittest-xml-reporting ``` Remaining dependencies are present as GIT submodules; also to fix the version of these: