Merge pull request #272 from SpiNNakerManchester/use_build

runs an intergation test in linux, windows and macos

.github/workflows/python_actions.yml

@@ -21,8 +21,10 @@ jobs:
   call:
     uses: SpiNNakerManchester/SupportScripts/.github/workflows/python_checks.yml@main
     with:
-      base-package: spinn_utilities
       dependencies:
       test_directories: unittests
+      coverage-package: spinn_utilities
       flake8-packages: spinn_utilities unittests
+      pylint-packages: spinn_utilities
+      mypy-packages: spinn_utilities
     secrets: inherit

spinn_utilities/executable_finder.py

@@ -102,7 +102,8 @@ def get_executable_path(self, executable_name: str) -> str:
                 return potential_filename
 
         # No executable found
-        raise KeyError(f"Executable {executable_name} not found in path")
+        raise KeyError(f"Executable {executable_name} not found in paths "
+                       f"f{list(self._binary_search_paths)}")
 
     def get_executable_paths(self, executable_names: str) -> List[str]:
         """

spinn_utilities/make_tools/log_sqllite_database.py

@@ -57,19 +57,9 @@ def __init__(self, new_dict=False):
         """
         # To Avoid an Attribute error on close after an exception
         self._db = None
-        database_file = os.environ.get('C_LOGS_DICT', None)
-        if database_file is None:
-            script = sys.modules[self.__module__].__file__
-            directory = os.path.dirname(script)
-            database_file = os.path.join(directory, DB_FILE_NAME)
-
-        if not new_dict and not os.path.exists(database_file):
-            message = f"Unable to locate c_logs_dict at {database_file}. "
-            if 'C_LOGS_DICT' in os.environ:
-                message += (
-                    "This came from the environment variable 'C_LOGS_DICT'. ")
-            message += "Please rebuild the C code."
-            raise FileNotFoundError(message)
+        database_file = self._database_file()
+        if not new_dict:
+            self._check_database_file(database_file)
 
         try:
             self._db = sqlite3.connect(database_file)
@@ -91,6 +81,50 @@ def __init__(self, new_dict=False):
                 message += "Please rebuild the C code."
             raise FileNotFoundError(message) from ex
 
+    def _database_file(self) -> str:
+        """
+        Finds the database file path.
+
+        If environment variable C_LOGS_DICT exists that is used,
+        otherwise the default path in this directory is used.
+
+        :return: Absolute path to where the database file is or will be
+        :rtype: str
+        """
+        if 'C_LOGS_DICT' in os.environ:
+            return str(os.environ['C_LOGS_DICT'])
+
+        script = sys.modules[self.__module__].__file__
+        assert script is not None
+        directory = os.path.dirname(script)
+        return os.path.join(directory, DB_FILE_NAME)
+
+    def _extra_database_error_message(self) -> str:
+        """
+        Adds a possible extra part to the error message.
+
+        :return: A likely empty string
+        :rtype: str
+        """
+        return ""
+
+    def _check_database_file(self, database_file: str) -> None:
+        """
+        Checks the database file exists:
+
+        :param str database_file: Absolute path to the database file
+        :raises FileNotFoundErrorL If the file does not exists
+        """
+        if os.path.exists(database_file):
+            return
+        message = f"Unable to locate c_logs_dict at {database_file}. "
+        if 'C_LOGS_DICT' in os.environ:
+            message += (
+                "This came from the environment variable 'C_LOGS_DICT'. ")
+        message += self._extra_database_error_message()
+        message += "Please rebuild the C code."
+        raise FileNotFoundError(message)
+
     def __del__(self):
         self.close()
 

spinn_utilities/make_tools/replacer.py

@@ -13,13 +13,17 @@
 # limitations under the License.
 
 import logging
+import os
+import shutil
 import struct
 import sys
 from typing import Optional, Tuple
+from spinn_utilities.overrides import overrides
+from spinn_utilities.config_holder import get_config_str_or_none
 from spinn_utilities.log import FormatAdapter
 from .file_converter import FORMAT_EXP
 from .file_converter import TOKEN
-from .log_sqllite_database import LogSqlLiteDatabase
+from .log_sqllite_database import DB_FILE_NAME, LogSqlLiteDatabase
 
 logger = FormatAdapter(logging.getLogger(__name__))
 
@@ -34,6 +38,29 @@ class Replacer(LogSqlLiteDatabase):
     Performs replacements.
     """
 
+    @overrides(LogSqlLiteDatabase._database_file)
+    def _database_file(self) -> str:
+        database_file = super()._database_file()
+        if not os.path.isfile(database_file):
+            external_binaries = get_config_str_or_none(
+                "Mapping", "external_binaries")
+            if external_binaries is not None:
+                source_file = os.path.join(external_binaries, DB_FILE_NAME)
+                if os.path.exists(source_file):
+
+                    shutil.copyfile(source_file, database_file)
+        return database_file
+
+    @overrides(LogSqlLiteDatabase._extra_database_error_message)
+    def _extra_database_error_message(self) -> str:
+        extra__binaries = get_config_str_or_none(
+            "Mapping", "external_binaries")
+        if extra__binaries is None:
+            return ""
+        else:
+            return (f"The cfg {extra__binaries=} "
+                    f"also does not contain a {DB_FILE_NAME}. ")
+
     def __enter__(self):
         return self
 

spinn_utilities/spinn_utilities.cfg

@@ -8,3 +8,6 @@ I_have_a_sense_of_humour = True
 listen_port = None
 notify_port = 19999
 notify_hostname = localhost
+
+[Mapping]
+external_binaries = None

unittests/make_tools/test_replacer.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 """
-These test depend on unittests/make_tools/replacer_dict/logs.sqlite3
+These test depend on unittests/make_tools/replacer.sqlite3
 
 The replacer.sqlite3 file in this directory is created by
 unittests/make_tools/test_file_convert.py method test_convert
@@ -29,6 +29,9 @@
 import math
 import unittest
 import os
+import tempfile
+from spinn_utilities.config_setup import unittest_setup
+from spinn_utilities.config_holder import set_config
 from spinn_utilities.make_tools.replacer import Replacer
 from spinn_utilities.make_tools.file_converter import TOKEN
 
@@ -44,6 +47,7 @@ def test_replacer(self):
         assert ("[INFO] (weird,file.c: 37): this is ok" == new)
 
     def test_not_there_existing(self):
+        unittest_setup()
         # Point C_LOGS_DICT to somewhere that does not exist
         os.environ["C_LOGS_DICT"] = str(
             os.path.join(PATH, "foo", "not_there.sqlite3"))
@@ -53,15 +57,17 @@ def test_not_there_existing(self):
         except Exception as ex:
             assert ("Unable to locate c_logs_dict" in str(ex))
 
-    def test_not_there_new(self):
-        # Point C_LOGS_DICT to somewhere that does not exist
-        os.environ["C_LOGS_DICT"] = str(
-            os.path.join(PATH, "foo", "not_there", "bad.sqlite3"))
-        try:
-            Replacer(True)
-            raise NotImplementedError("Should not work!")
-        except Exception as ex:
-            assert ("Error accessing c_logs_dict" in str(ex))
+    def test_external_empty(self):
+        unittest_setup()
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            set_config("Mapping", "external_binaries", tmpdirname)
+            os.environ["C_LOGS_DICT"] = str(
+                os.path.join(tmpdirname, "missing.sqlite3"))
+            try:
+                Replacer()
+                raise ValueError("Should not get here")
+            except FileNotFoundError as ex:
+                self.assertIn(tmpdirname, str(ex))
 
     def test_tab(self):
         os.environ["C_LOGS_DICT"] = str(os.path.join(PATH, "replacer.sqlite3"))