Update python build

Remove EOL Python3.8
remove 3rd party warnings
support old fashion site-packages for CI
update python tests

.github/workflows/build_test.yml

@@ -8,7 +8,7 @@ on:
 
 env:
   GTEST_OUTPUT: "xml:test_results"
-  BUILDCMD: "colcon build --event-handlers console_direct+ --executor sequential --cmake-args -DBUILD_HARDENING=ON -DBUILD_TESTING=ON -DBUILD_PYTHON_BINDING=ON -DPYTHON_BINDING_VERSION=${PYTHON_BINDING_VERSION} && colcon test --event-handlers console_direct+ --packages-select ad_map_access ad_physics && colcon test-result"
+  BUILDCMD: "colcon build --event-handlers console_direct+ --executor sequential --cmake-args -DBUILD_HARDENING=ON -DBUILD_TESTING=ON -DBUILD_PYTHON_BINDING=ON -DPYTHON_BINDING_VERSION=${PYTHON_BINDING_VERSION} -DPYTHON_PACKAGE_FOLDER_NAME=${PYTHON_PACKAGE_FOLDER_NAME} && colcon test --event-handlers console_direct+ --packages-select ad_map_access ad_physics && colcon test-result"
 
 permissions:
   contents: read
@@ -18,35 +18,33 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-20.04
-            compiler: gcc9
-            EXTRA_PACKAGES: ""
-            CC: ""
-            CXX: ""
-            PYTHON_BINDING_VERSION: "3.8"
           - os: ubuntu-20.04
             compiler: clang10
             EXTRA_PACKAGES: clang-10
             CC: /usr/bin/clang-10
             CXX: /usr/bin/clang++-10
             PYTHON_BINDING_VERSION: "3.10"
+            PYTHON_PACKAGE_FOLDER_NAME: "site-packages"
           - os: ubuntu-22.04
             compiler: gcc11
             EXTRA_PACKAGES: ""
             CC: ""
             CXX: ""
             PYTHON_BINDING_VERSION: "3.10"
+            PYTHON_PACKAGE_FOLDER_NAME: "site-packages"
           - os: ubuntu-22.04
             compiler: clang14
             EXTRA_PACKAGES: clang-14
             CC: /usr/bin/clang-14
             CXX: /usr/bin/clang++-14
             PYTHON_BINDING_VERSION: "3.10"
+            PYTHON_PACKAGE_FOLDER_NAME: "site-packages"
 
     name: ${{ matrix.os }}, ${{ matrix.compiler }}, python-${{ matrix.PYTHON_BINDING_VERSION }}
     runs-on: ${{ matrix.os }}
     env:
       PYTHON_BINDING_VERSION: ${{ matrix.PYTHON_BINDING_VERSION }}
+      PYTHON_PACKAGE_FOLDER_NAME: ${{ matrix.PYTHON_PACKAGE_FOLDER_NAME }}
       EXTRA_PACKAGES: ${{ matrix.EXTRA_PACKAGES }}
 
     steps:

.github/workflows/install_dependencies.sh

@@ -48,7 +48,7 @@ curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python${PYTHON_BINDING_VERS
 # to handle some error on missing pip dependencies
 sudo pip${PYTHON_BINDING_VERSION} install testresources
 sudo pip${PYTHON_BINDING_VERSION} install --upgrade setuptools==59.6.0
-sudo pip${PYTHON_BINDING_VERSION} install colcon-common-extensions xmlrunner pygccxml pyplusplus
+sudo pip${PYTHON_BINDING_VERSION} install colcon-common-extensions unittest-xml-reporting pygccxml pyplusplus
 
 if (( IS_UBUNTU_20_04 && IS_PYTHON_3_10 )); then
   echo "!!!!!!! Ubunut 20.04 and python 3.10: compile boost 1.80 !!!!!!!"

README.md

@@ -72,11 +72,9 @@ Following compiler and Python combinations are [tested continously](https://gith
 
 |                 | Ubuntu 20.04 | Ubuntu 22.04 |
 |:---------------:|:------------:|:------------:|
-|   GCC 9         |       x      |              |
 |  Clang 10       |       x      |              |
 |   GCC 11        |              |       x      |
 |  Clang 14       |              |       x      |
-|  Python 3.8     |       x      |              |
 |  Python 3.10    |       x      |       x      |
 
 Important: cmake is required to be at least version 3.5!

ad_map_access/python/tests/interface_test.py

@@ -64,8 +64,6 @@ def test_interface(self):
 if __name__ == '__main__':
     if os.environ.get('GTEST_OUTPUT') and os.environ['GTEST_OUTPUT'].startswith('xml:'):
         base_folder = os.environ['GTEST_OUTPUT'][4:]
-        result_filename = base_folder + 'ad_map_access_interface_test_python' + str(sys.version_info.major) + ".xml"
-        with open(result_filename, "w+") as result_file:
-            unittest.main(testRunner=xmlrunner.XMLTestRunner(output=result_file))
+        unittest.main(testRunner=xmlrunner.XMLTestRunner(output=base_folder))
     else:
         unittest.main()

ad_physics/python/tests/interface_test.py

@@ -50,8 +50,6 @@ def test_interface(self):
 if __name__ == '__main__':
     if os.environ.get('GTEST_OUTPUT') and os.environ['GTEST_OUTPUT'].startswith('xml:'):
         base_folder = os.environ['GTEST_OUTPUT'][4:]
-        result_filename = base_folder + 'ad_pyhsics_interface_test_python' + str(sys.version_info.major) + ".xml"
-        with open(result_filename, "w+") as result_file:
-            unittest.main(testRunner=xmlrunner.XMLTestRunner(output=result_file))
+        unittest.main(testRunner=xmlrunner.XMLTestRunner(output=base_folder))
     else:
         unittest.main()

cmake/python-binding.cmake

@@ -69,8 +69,18 @@ function(find_python_binding_packages)
 endfunction()
 
 function(get_python_test_environment)
+  set(PACKAGE_FOLDER_NAME "${PYTHON_PACKAGE_FOLDER_NAME}")
+  if ("${PACKAGE_FOLDER_NAME}" STREQUAL "")
+    set(PACKAGE_FOLDER_NAME "dist-packages")
+  endif()
+  if ("${PACKAGE_FOLDER_NAME}" STREQUAL "dist-packages")
+    set(LIBDIR_PREFIX "local/")
+  else()
+    set(LIBDIR_PREFIX "")
+  endif()
+
   set(TEST_LD_LIBRARY_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
-  set(TEST_PYTHONPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${PYTHON_BINDING_FOLDER_NAME}/site-packages")
+  set(TEST_PYTHONPATH "${CMAKE_INSTALL_PREFIX}/${LIBDIR_PREFIX}${CMAKE_INSTALL_LIBDIR}/${PYTHON_BINDING_FOLDER_NAME}/${PACKAGE_FOLDER_NAME}")
   foreach(dep ${ARGN})
     get_target_property(dep_configurations ${dep} IMPORTED_CONFIGURATIONS)
     set(TEST_${dep}_LOCATION False)
@@ -83,7 +93,8 @@ function(get_python_test_environment)
     endforeach()
     if(TEST_${dep}_LOCATION)
       set(TEST_LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${TEST_${dep}_LOCATION}")
-      set(TEST_PYTHONPATH "${TEST_PYTHONPATH}:${TEST_${dep}_LOCATION}/${PYTHON_BINDING_FOLDER_NAME}/site-packages")
+      string(REGEX REPLACE "/${CMAKE_INSTALL_LIBDIR}$" "" TEST_${dep}_INSTALL_PREFIX ${TEST_${dep}_LOCATION})
+      set(TEST_PYTHONPATH "${TEST_PYTHONPATH}:${TEST_${dep}_INSTALL_PREFIX}/${LIBDIR_PREFIX}${CMAKE_INSTALL_LIBDIR}/${PYTHON_BINDING_FOLDER_NAME}/${PACKAGE_FOLDER_NAME}")
     else()
       message(WARNING "Failed to query mandatory location of dependency ${dep}")
     endif()

cmake/python/python_wrapper_helper.py.in

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # ----------------- BEGIN LICENSE BLOCK ---------------------------------
 #
-# Copyright (c) 2019-2020 Intel Corporation
+# Copyright (c) 2019-2022 Intel Corporation
 #
 # ----------------- END LICENSE BLOCK -----------------------------------
 
@@ -33,15 +33,15 @@ def get_list_of_files(directory, ignore_files):
             for ignore_file in ignore_files:
                 if full_path.find(ignore_file) != -1:
                     skip = True
-                    print ("Skipping file: " + full_path)
+                    print("Skipping file: " + full_path)
             if not skip:
                 if full_path.endswith(".h") or full_path.endswith(".hpp"):
                     all_files.append(full_path)
 
     return all_files
 
 
-def generate_python_wrapper(header_directories, include_paths, library_name, cpp_filename, declarations, main_namespace="", ignore_declarations={}, ignore_files={}):
+def generate_python_wrapper(header_directories, include_paths, library_name, cpp_filename, declarations, main_namespace="", ignore_declarations={}, ignore_files={}, add_declarations={}):
     """
     Function to generate Python-C++ binding code by calling pygccxml and py++
 
@@ -60,27 +60,30 @@ def generate_python_wrapper(header_directories, include_paths, library_name, cpp
     :type ignore_declarations: list<string>
     :param ignore_files: a list of files to be ignored
     :type ignore_files: list<string>
+    :param add_declarations: a list of declarations to be explicitly enabled searched for in
+      against the full declaration string resulting from '"{}".format(decl)' output
+     (useful e.g. for typedefs to template types which are not as such visible in the delcarations)
+    :type add_declarations: list<string>
     :return:
     """
 
     warnings.filterwarnings(action="once", category=DeprecationWarning)
 
     # Find out the xml generator (gccxml or castxml)
     generator_path, generator_name = utils.find_xml_generator()
-    compiler = "g++"
-    compiler_path = "/usr/bin/g++"
+    compiler = "@CXX@"
 
     # Create configuration for CastXML
     xml_generator_config = parser.xml_generator_configuration_t(
         xml_generator_path=generator_path,
         xml_generator=generator_name,
         compiler=compiler,
-        compiler_path=compiler_path,
         start_with_declarations=declarations)
 
     # Set include dirs and cflags to avoid warnings and errors
     xml_generator_config.append_cflags("-std=c++@CMAKE_CXX_STANDARD@")
     xml_generator_config.append_cflags("-Wno-error=invalid-constexpr")
+    xml_generator_config.append_cflags("-DSAFE_DATATYPES_EXPLICIT_CONVERSION=1")
 
     for inc_dir in include_paths:
         xml_generator_config.include_paths.append(inc_dir)
@@ -124,32 +127,57 @@ def generate_python_wrapper(header_directories, include_paths, library_name, cpp
             top_namespace, main_namespace))
 
     for decl in builder.decls():
-        for ignore_declaration in ignore_declarations:
-            if ignore_declaration in decl.name or ignore_declaration in decl.alias:
-                decl.exclude()
-                decl.ignore = True
-                decl.already_exposed = True
-                # print("declaration to ignore found: {} (alias {})".format(decl, decl.alias))
-                break
+        decl_full_string_and_type = "{}".format(decl)
+        # print("declaration {} (alias {}, full {})".format(decl.name, decl.alias, decl_full_string_and_type))
         if main_namespace != "":
-            if isinstance(decl, decl_wrappers.class_wrapper.class_t) or isinstance(decl, decl_wrappers.class_wrapper.class_declaration_t) or isinstance(decl, decl_wrappers.typedef_wrapper.typedef_t):
-                decl_full_string = "{}".format(decl)
+            if isinstance(decl, decl_wrappers.class_wrapper.class_t) or isinstance(decl, decl_wrappers.class_wrapper.class_declaration_t) or isinstance(decl, decl_wrappers.typedef_wrapper.typedef_t) or isinstance(decl, decl_wrappers.enumeration_wrapper.enumeration_t):
+                decl_full_string = decl_full_string_and_type.split(" ")[0]
                 if main_namespace in decl_full_string:
                     # namespace present, ok
-                    # print("typedef/class main namespace found: {}".format(decl_full_string))
-                    continue
-                if decl_full_string in main_namespace:
+                    # print("typedef/class/enum main namespace found [ignore-{},exposed-{}]:
+                    # {}".format(decl.ignore, decl.already_exposed,
+                    # decl_full_string_and_type))
+                    pass
+                elif decl_full_string in main_namespace:
                     # declaration is a parent of main namespace, ok
-                    # print("typedef/class declaration of upper level namespace found: {}".format(decl_full_string))
-                    continue
-                if top_namespace != "" and not top_namespace in decl_full_string:
+                    # print("typedef/class/enum declaration of upper level namespace found
+                    # [ignore{},exposed{}]: {}".format(decl.ignore, decl.already_exposed,
+                    # decl_full_string_and_type))
+                    pass
+                elif top_namespace != "" and not top_namespace in decl_full_string:
                     # global or std defaults, ok
-                    # print("typedef/class outside top namespace found: {}".format(decl_full_string))
-                    continue
-                # print("typedef/class outside of main namespace found. Ignoring: {}".format(decl_full_string))
+                    # print("typedef/class/enum outside top namespace found
+                    # [ignore-{},exposed-{}]: {}".format(decl.ignore, decl.already_exposed,
+                    # decl_full_string_and_type))
+                    pass
+                else:
+                    # print("typedef/class/enum outside of main namespace found
+                    # [ignore-{},exposed-{}]. Ignoring: {}".format(decl.ignore,
+                    # decl.already_exposed, decl_full_string_and_type))
+                    decl.exclude()
+                    decl.ignore = True
+                    # try to enforce that it's not exported anymore
+                    decl.already_exposed = True
+
+                if decl.ignore:
+                    if decl_full_string in declarations:
+                        decl.ignore = False
+                        decl.already_exposed = False
+                        # print("Ignored typedef/class/enum explicitly listed in delclarations
+                        # found. Reenable {}".format(decl_full_string_and_type))
+                    for add_declaration in add_declarations:
+                        if (add_declaration in decl.name) or (add_declaration in decl.alias) or (add_declaration in decl_full_string_and_type):
+                            decl.ignore = False
+                            decl.already_exposed = False
+                            # print("declaration to explicitly add found: {} (alias {})".format(decl, decl.alias))
+                            break
+        for ignore_declaration in ignore_declarations:
+            if (ignore_declaration in decl.name) or (ignore_declaration in decl.alias) or (ignore_declaration in decl_full_string_and_type):
                 decl.exclude()
                 decl.ignore = True
                 decl.already_exposed = True
+                # print("declaration to ignore found: {} (alias {})".format(decl, decl.alias))
+                break
 
     # debug declarations
     # builder.print_declarations()
@@ -168,7 +196,7 @@ def generate_python_wrapper(header_directories, include_paths, library_name, cpp
     print("generate_python_wrapper(): {} written.".format(cpp_filename))
 
 
-def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filename_out, additional_replacements={}, additional_includes={}, spdx_license="MIT", fix_include_directives=True, fix_enum_class=True):
+def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filename_out, additional_replacements=[], additional_includes={}, spdx_license="MIT", fix_include_directives=True, fix_enum_class=True):
     """
     Post process generated binding code
 
@@ -204,7 +232,7 @@ def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filenam
             file_output.write("/*\n"
                               " * ----------------- BEGIN LICENSE BLOCK ---------------------------------\n"
                               " *\n"
-                              " * Copyright (c) 2020 Intel Corporation\n"
+                              " * Copyright (c) 2020-2021 Intel Corporation\n"
                               " *\n"
                               " * SPDX-License-Identifier: " + spdx_license + "\n"
                               " *\n"
@@ -228,15 +256,15 @@ def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filenam
             for header_dir in header_directories:
                 if not header_dir.endswith("/"):
                     header_dir = header_dir + "/"
-                line = line.replace(header_dir, "")
+                line = str.replace(line, header_dir, "")
 
         # Fix C++ enum classes
         if fix_enum_class:
             if enum_started:
                 if line.find("export_values()") != -1:
                     enum_started = False
                 else:
-                    line = line.replace(enum_namespace, enum_namespace_full)
+                    line = str.replace(line, enum_namespace, enum_namespace_full)
             else:
                 match = enum_declaration_start.match(line)
                 if match:
@@ -246,7 +274,7 @@ def post_process_python_wrapper(header_directories, cpp_filename_in, cpp_filenam
                     enum_namespace_full = match.group(1) + "::"
 
         for replacement in additional_replacements:
-            line = line.replace(replacement[0], replacement[1])
+            line = str.replace(line, replacement[0], replacement[1])
 
         file_output.write(line)
 

cmake/warnings.cmake

@@ -9,3 +9,8 @@ list(APPEND TARGET_COMPILE_OPTIONS -Wall -Wextra -pedantic -Wfloat-equal -Wshado
 
 # treat warnings as errors
 list(APPEND TARGET_COMPILE_OPTIONS $<$<NOT:$<BOOL:${DISABLE_WARNINGS_AS_ERRORS}>>:-Werror>)
+
+# disable some warnings on 3rd party code
+list(APPEND TARGET_COMPILE_OPTIONS $<$<CXX_COMPILER_ID:CXX,gcc>:-Wno-error=maybe-uninitialized>)
+list(APPEND TARGET_COMPILE_OPTIONS -Wno-error=deprecated-declarations)
+

doc/BUILDING.md

@@ -30,7 +30,7 @@ The components within this repository have some dependencies:
    - Osmium >= 2.13: <https://osmcode.org/libosmium/>
  - ***all components when enabling unit tests***:
    - gtest aka. googletests < 1.10 : <https://github.com/google/googletest>
-   - xmlrunner
+   - unittest-xml-reporting
 
 Dependencies provided by Ubunutu (>= 18.04):
 
@@ -50,7 +50,7 @@ $>  sudo apt-get install libboost-all-dev libpugixml-dev libgtest-dev libpython-
 Additional dependencies for the python bindings:
 ```bash
 $>  sudo apt-get install castxml
-$>  pip install --user python-wheel pygccxml pyplusplus xmlrunner
+$>  pip install --user python-wheel pygccxml pyplusplus unittest-xml-reporting
 ```
 
 Remaining dependencies are present as GIT submodules; also to fix the version of these: