From bc80b2f48f2d91605086fb42e99ef388297e599a Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Fri, 1 Mar 2024 15:15:40 +0000 Subject: [PATCH 01/10] split Chip processors between user and monitor --- spinn_machine/chip.py | 125 +++++++++++++++++++++++------- spinn_machine/machine.py | 3 +- unittests/test_chip.py | 19 ++++- unittests/test_json_machine.py | 6 +- unittests/test_machine.py | 5 +- unittests/test_virtual_machine.py | 15 ++-- 6 files changed, 128 insertions(+), 45 deletions(-) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index 4d0dd6eb..1c6d8215 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -19,6 +19,7 @@ from .router import Router standard_processors = {} +standard_monitor_processors = None class Chip(object): @@ -36,9 +37,9 @@ class Chip(object): _IPTAG_IDS = OrderedSet(range(1, 8)) __slots__ = ( - "_x", "_y", "_p", "_router", "_sdram", "_ip_address", + "_x", "_y", "_router", "_sdram", "_ip_address", "_tag_ids", "_nearest_ethernet_x", "_nearest_ethernet_y", - "_n_user_processors", "_parent_link", "_v_to_p_map" + "_user_processors", "_monitor_processors", "_parent_link", "_v_to_p_map" ) # pylint: disable=too-many-arguments, wrong-spelling-in-docstring @@ -83,7 +84,9 @@ def __init__(self, x: int, y: int, n_processors: int, router: Router, """ self._x = x self._y = y - self._p = self.__generate_processors(n_processors, down_cores) + self._monitor_processors = self.__generate_monitors() + self._user_processors = self.__generate_processors( + n_processors, down_cores) self._router = router self._sdram = sdram self._ip_address = ip_address @@ -98,30 +101,40 @@ def __init__(self, x: int, y: int, n_processors: int, router: Router, self._parent_link = parent_link self._v_to_p_map = v_to_p_map + def __generate_monitors(self): + """ + Generates the monitors assuming all Chips have the same monitor cores + + :return: Dict[int, Processor] + """ + global standard_monitor_processors + if standard_monitor_processors is None: + standard_monitor_processors = dict() + for i in range( + MachineDataView.get_machine_version().n_non_user_cores): + standard_monitor_processors[i] = Processor.factory(i, True) + return standard_monitor_processors + def __generate_processors( self, n_processors: int, down_cores: Optional[Collection[int]]) -> Dict[int, Processor]: + n_monitors = MachineDataView.get_machine_version().n_non_user_cores if down_cores is None: if n_processors not in standard_processors: processors = dict() - processors[0] = Processor.factory(0, True) - for i in range(1, n_processors): + for i in range(n_monitors, n_processors): processors[i] = Processor.factory(i) standard_processors[n_processors] = processors - self._n_user_processors = n_processors - 1 return standard_processors[n_processors] else: processors = dict() - if 0 in down_cores: - raise NotImplementedError( - "Declaring core 0 as down is not supported") - processors[0] = Processor.factory(0, True) - for i in range(1, n_processors): + for i in range(n_monitors): + if i in down_cores: + raise NotImplementedError( + f"Declaring monitor core {i} as down is not supported") + for i in range(n_monitors, n_processors): if i not in down_cores: processors[i] = Processor.factory(i) - self._n_user_processors = ( - n_processors - len(down_cores) - - MachineDataView.get_machine_version().n_non_user_cores) return processors def is_processor_with_id(self, processor_id: int) -> bool: @@ -133,7 +146,9 @@ def is_processor_with_id(self, processor_id: int) -> bool: :return: Whether the processor with the given ID exists :rtype: bool """ - return processor_id in self._p + if processor_id in self._user_processors: + return True + return processor_id in self._monitor_processors def get_processor_with_id(self, processor_id: int) -> Optional[Processor]: """ @@ -146,7 +161,9 @@ def get_processor_with_id(self, processor_id: int) -> Optional[Processor]: or ``None`` if no such processor :rtype: Processor or None """ - return self._p.get(processor_id) + if processor_id in self._user_processors: + return self._user_processors[processor_id] + return self._monitor_processors.get(processor_id) @property def x(self) -> int: @@ -169,11 +186,30 @@ def y(self) -> int: @property def processors(self) -> Iterator[Processor]: """ - An iterable of available processors. + An iterable of available all processors. + + Deprecated: There are many more efficient methods instead. + - all_processor_id + - n_processors + - n_user_processors + - user_processors + - n_monitor_processors + - monitor_processors :rtype: iterable(Processor) """ - return iter(self._p.values()) + yield from self._monitor_processors.values() + yield from self._user_processors.values() + + @property + def all_processor_ids(self) -> Iterator[int]: + """ + An iterable of id's of all available processors + + :rtype: iterable(int) + """ + yield from self._monitor_processors.keys() + yield from self._user_processors.keys() @property def n_processors(self) -> int: @@ -182,7 +218,16 @@ def n_processors(self) -> int: :rtype: int """ - return len(self._p) + return len(self._monitor_processors) + len(self._user_processors) + + @property + def user_processors(self) -> Iterator[Processor]: + """ + An iterable of available user processors. + + :rtype: iterable(Processor) + """ + return self._user_processors.values() @property def n_user_processors(self) -> int: @@ -191,7 +236,25 @@ def n_user_processors(self) -> int: :rtype: int """ - return self._n_user_processors + return len(self._user_processors) + + @property + def monitor_processors(self) -> Iterator[Processor]: + """ + An iterable of available monitor processors. + + :rtype: iterable(Processor) + """ + return self._monitor_processors.values() + + @property + def n_monitor_processors(self) -> int: + """ + The total number of processors that are not monitors. + + :rtype: int + """ + return len(self._monitor_processors) @property def router(self) -> Router: @@ -248,16 +311,15 @@ def tag_ids(self) -> Iterable[int]: """ return self._tag_ids - def get_first_none_monitor_processor(self) -> Optional[Processor]: + def get_first_none_monitor_processor(self) -> Processor: """ Get the first processor in the list which is not a monitor core. - :rtype: Processor or None + :rtype: Processor + ;raises StopIteration: If there is no user processor """ - for processor in self.processors: - if not processor.is_monitor: - return processor - return None + return next(iter(self._user_processors.values())) + @property def parent_link(self) -> Optional[int]: @@ -304,7 +366,8 @@ def __iter__(self) -> Iterator[Tuple[int, Processor]]: * ``processor`` is the processor with the ID :rtype: iterable(tuple(int,Processor)) """ - return iter(self._p.items()) + yield from self._monitor_processors.items() + yield from self._user_processors.items() def __len__(self) -> int: """ @@ -313,11 +376,13 @@ def __len__(self) -> int: :return: The number of items in the underlying iterator. :rtype: int """ - return len(self._p) + return len(self._monitor_processors) + len(self._user_processors) def __getitem__(self, processor_id: int) -> Processor: - if processor_id in self._p: - return self._p[processor_id] + if processor_id in self._user_processors: + return self._user_processors[processor_id] + if processor_id in self._monitor_processors: + return self._monitor_processors[processor_id] # Note difference from get_processor_with_id(); this is to conform to # standard Python semantics raise KeyError(processor_id) diff --git a/spinn_machine/machine.py b/spinn_machine/machine.py index 11ec02f3..42324835 100644 --- a/spinn_machine/machine.py +++ b/spinn_machine/machine.py @@ -1098,8 +1098,7 @@ def total_cores(self) -> int: :rtype: int """ - return sum( - 1 for chip in self.chips for _processor in chip.processors) + return sum(chip.n_processors for chip in self.chips) def unreachable_outgoing_chips(self) -> List[XY]: """ diff --git a/unittests/test_chip.py b/unittests/test_chip.py index c4ff6a3d..38c18555 100644 --- a/unittests/test_chip.py +++ b/unittests/test_chip.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest +from spinn_utilities.config_holder import set_config from spinn_utilities.ordered_set import OrderedSet from spinn_machine import Link, Router, Chip from spinn_machine.config_setup import unittest_setup @@ -22,6 +23,7 @@ class TestingChip(unittest.TestCase): def setUp(self): unittest_setup() + set_config("Machine", "version", 5) self._x = 0 self._y = 1 @@ -97,13 +99,28 @@ def test_getitem_and_contains(self): self.assertFalse(self.n_processors in new_chip) def test_0_down(self): + # Chip where 0 the monitor is down with self.assertRaises(NotImplementedError): Chip(1, 1, self.n_processors, self._router, self._sdram, 0, 0, self._ip, down_cores=[0]) def test_1_chip(self): + # Chip with just 1 processor new_chip = Chip(1, 1, 1, self._router, self._sdram, 0, 0, self._ip) - self.assertIsNone(new_chip.get_first_none_monitor_processor()) + with self.assertRaises(Exception): + new_chip.get_first_none_monitor_processor() + + def test_processors(self): + new_chip = self._create_chip(self._x, self._y, self.n_processors, + self._router, self._sdram, self._ip) + all_p = set() + for id in new_chip.all_processor_ids: + all_p.add(new_chip[id]) + self.assertEqual(len(all_p), new_chip.n_processors) + users = set(new_chip.user_processors) + self.assertEqual(len(users), new_chip.n_user_processors) + monitors = set(new_chip.monitor_processors) + self.assertEqual(users.union(monitors), all_p) if __name__ == '__main__': diff --git a/unittests/test_json_machine.py b/unittests/test_json_machine.py index 3a3a7330..b423f87c 100644 --- a/unittests/test_json_machine.py +++ b/unittests/test_json_machine.py @@ -63,7 +63,11 @@ def test_monitor_exceptions(self): MachineDataWriter.mock().set_machine(vm) chip02 = vm[0, 2] # Hack in an extra monitor - chip02._n_user_processors -= 1 + users = dict(chip02._user_processors) + monitors = dict(chip02._monitor_processors) + monitors[1] = users.pop(1) + chip02._monitor_processors = monitors + chip02._user_processors = users jpath = mktemp("json") # Should still be able to write json even with more than one monitor to_json_path(jpath) diff --git a/unittests/test_machine.py b/unittests/test_machine.py index b417a22b..5e100080 100644 --- a/unittests/test_machine.py +++ b/unittests/test_machine.py @@ -91,6 +91,7 @@ def test_create_new_machine(self): def test_summary(self): machine = virtual_machine(8, 8) + a = machine.summary_string() self.assertEqual( "Machine on 127.0.0.0 with 48 Chips, 856 cores and 120.0 links. " "Chips have sdram of 123469792 bytes, router table of size 1023, " @@ -389,7 +390,9 @@ def test_concentric_xys(self): def test_too_few_cores(self): machine = virtual_machine(8, 8) # Hack to get n_processors return a low number - machine.get_chip_at(0, 1)._p = [1, 2, 3] + chip01 = machine.get_chip_at(0, 1) + chip01._user_processors = dict( + list(chip01._user_processors.items())[:2]) with self.assertRaises(SpinnMachineException): machine.validate() diff --git a/unittests/test_virtual_machine.py b/unittests/test_virtual_machine.py index 29c479c4..194fba8c 100644 --- a/unittests/test_virtual_machine.py +++ b/unittests/test_virtual_machine.py @@ -21,7 +21,7 @@ from spinn_machine.ignores import IgnoreChip, IgnoreCore, IgnoreLink from spinn_machine.machine_factory import machine_repair from spinn_machine.version.version_5 import CHIPS_PER_BOARD -from .geometry import (to_xyz, shortest_mesh_path_length, +from geometry import (to_xyz, shortest_mesh_path_length, shortest_torus_path_length, minimise_xyz) @@ -173,15 +173,10 @@ def test_new_vm_with_monitor(self): vm = virtual_machine(2, 2, n_cpus_per_chip=n_cpus, validate=True) _chip = vm[1, 1] self.assertEqual(n_cpus, _chip.n_processors) - monitors = 0 - normal = 0 - for core in _chip.processors: - if core.is_monitor: - monitors += 1 - else: - normal += 1 - self.assertEqual(n_cpus - 1, normal) - self.assertEqual(1, monitors) + self.assertEqual(n_cpus - 1, _chip.n_user_processors) + self.assertEqual(1, _chip.n_monitor_processors) + self.assertEqual(n_cpus - 1, len(list(_chip.user_processors))) + self.assertEqual(1, len(list(_chip.monitor_processors))) def test_iter_chips(self): set_config("Machine", "version", 2) From f934b9db5cb37c0925a298ae9c4c2c97906863fd Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 06:30:00 +0000 Subject: [PATCH 02/10] use .geometry for github actions ect --- unittests/test_virtual_machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_virtual_machine.py b/unittests/test_virtual_machine.py index 194fba8c..353fd256 100644 --- a/unittests/test_virtual_machine.py +++ b/unittests/test_virtual_machine.py @@ -21,7 +21,7 @@ from spinn_machine.ignores import IgnoreChip, IgnoreCore, IgnoreLink from spinn_machine.machine_factory import machine_repair from spinn_machine.version.version_5 import CHIPS_PER_BOARD -from geometry import (to_xyz, shortest_mesh_path_length, +from .geometry import (to_xyz, shortest_mesh_path_length, shortest_torus_path_length, minimise_xyz) From be6ff70a9c003fd4fe28a1f08daf5878b59bf59c Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 06:35:50 +0000 Subject: [PATCH 03/10] flake8 --- spinn_machine/chip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index 1c6d8215..2d7c2091 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -39,7 +39,8 @@ class Chip(object): __slots__ = ( "_x", "_y", "_router", "_sdram", "_ip_address", "_tag_ids", "_nearest_ethernet_x", "_nearest_ethernet_y", - "_user_processors", "_monitor_processors", "_parent_link", "_v_to_p_map" + "_user_processors", "_monitor_processors", "_parent_link", + "_v_to_p_map" ) # pylint: disable=too-many-arguments, wrong-spelling-in-docstring @@ -320,7 +321,6 @@ def get_first_none_monitor_processor(self) -> Processor: """ return next(iter(self._user_processors.values())) - @property def parent_link(self) -> Optional[int]: """ From 71af355b0106c43536ec13d7e8cbebbdf32dcda7 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 06:44:43 +0000 Subject: [PATCH 04/10] flake8 --- unittests/test_machine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/unittests/test_machine.py b/unittests/test_machine.py index 5e100080..a58ded1d 100644 --- a/unittests/test_machine.py +++ b/unittests/test_machine.py @@ -91,7 +91,6 @@ def test_create_new_machine(self): def test_summary(self): machine = virtual_machine(8, 8) - a = machine.summary_string() self.assertEqual( "Machine on 127.0.0.0 with 48 Chips, 856 cores and 120.0 links. " "Chips have sdram of 123469792 bytes, router table of size 1023, " From 3509d6463e19198a552cef69206d40cea0925328 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 07:13:49 +0000 Subject: [PATCH 05/10] pylint: disable=global-statement --- spinn_machine/chip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index 2d7c2091..cbaa9b05 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -108,7 +108,7 @@ def __generate_monitors(self): :return: Dict[int, Processor] """ - global standard_monitor_processors + global standard_monitor_processors # pylint: disable=global-statement if standard_monitor_processors is None: standard_monitor_processors = dict() for i in range( From 36d65fb666573bd8f79ab92477458308702c2703 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 07:20:52 +0000 Subject: [PATCH 06/10] use yield from for mypy --- spinn_machine/chip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index cbaa9b05..47421d63 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -228,7 +228,7 @@ def user_processors(self) -> Iterator[Processor]: :rtype: iterable(Processor) """ - return self._user_processors.values() + yield from self._user_processors.values() @property def n_user_processors(self) -> int: From 2561bb568bbec650f38306e6ff991580f63f18ff Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 07:34:09 +0000 Subject: [PATCH 07/10] user_processors_ids --- spinn_machine/chip.py | 12 +++++++++++- unittests/test_chip.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index 47421d63..779df0c6 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -190,10 +190,11 @@ def processors(self) -> Iterator[Processor]: An iterable of available all processors. Deprecated: There are many more efficient methods instead. - - all_processor_id + - all_processor_ids - n_processors - n_user_processors - user_processors + - user_processors_ids - n_monitor_processors - monitor_processors @@ -230,6 +231,15 @@ def user_processors(self) -> Iterator[Processor]: """ yield from self._user_processors.values() + @property + def user_processors_ids(self) -> Iterator[Processor]: + """ + An iterable of available user processors. + + :rtype: iterable(Processor) + """ + yield from self._user_processors + @property def n_user_processors(self) -> int: """ diff --git a/unittests/test_chip.py b/unittests/test_chip.py index 38c18555..2aa25f96 100644 --- a/unittests/test_chip.py +++ b/unittests/test_chip.py @@ -119,6 +119,7 @@ def test_processors(self): self.assertEqual(len(all_p), new_chip.n_processors) users = set(new_chip.user_processors) self.assertEqual(len(users), new_chip.n_user_processors) + self.assertEqual(len(users), len(set(new_chip.user_processors_ids))) monitors = set(new_chip.monitor_processors) self.assertEqual(users.union(monitors), all_p) From e76a557cdb215d81a614a4365c6e1d6266924298 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 07:43:48 +0000 Subject: [PATCH 08/10] fix typing --- spinn_machine/chip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index 779df0c6..e7400705 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -232,7 +232,7 @@ def user_processors(self) -> Iterator[Processor]: yield from self._user_processors.values() @property - def user_processors_ids(self) -> Iterator[Processor]: + def user_processors_ids(self) -> Iterator[int]: """ An iterable of available user processors. From 073bfaed9b9e70eda580eb0cc5fb196d03bdab63 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 16:45:17 +0000 Subject: [PATCH 09/10] pylint: disable=invalid-name --- spinn_machine/chip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index e7400705..3f470358 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -18,8 +18,11 @@ from .processor import Processor from .router import Router +# global values so Chip objects can share processor dict objects +# One dict for each number of processors (none dead) standard_processors = {} -standard_monitor_processors = None +# One dict for the standard monitor processors +standard_monitor_processors = None # pylint: disable=invalid-name class Chip(object): From ee9d0ca54f310ef8de2b0566a94e94da6853d869 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Mar 2024 17:27:58 +0000 Subject: [PATCH 10/10] monitor_processors_ids --- spinn_machine/chip.py | 10 ++++++++++ unittests/test_chip.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/spinn_machine/chip.py b/spinn_machine/chip.py index 3f470358..ea899b8d 100644 --- a/spinn_machine/chip.py +++ b/spinn_machine/chip.py @@ -200,6 +200,7 @@ def processors(self) -> Iterator[Processor]: - user_processors_ids - n_monitor_processors - monitor_processors + - monitor_processors_ids :rtype: iterable(Processor) """ @@ -261,6 +262,15 @@ def monitor_processors(self) -> Iterator[Processor]: """ return self._monitor_processors.values() + @property + def monitor_processors_ids(self) -> Iterator[int]: + """ + An iterable of available user processors. + + :rtype: iterable(Processor) + """ + yield from self._monitor_processors + @property def n_monitor_processors(self) -> int: """ diff --git a/unittests/test_chip.py b/unittests/test_chip.py index 2aa25f96..083e606e 100644 --- a/unittests/test_chip.py +++ b/unittests/test_chip.py @@ -122,6 +122,8 @@ def test_processors(self): self.assertEqual(len(users), len(set(new_chip.user_processors_ids))) monitors = set(new_chip.monitor_processors) self.assertEqual(users.union(monitors), all_p) + self.assertEqual(len(monitors), + len(set(new_chip.monitor_processors_ids))) if __name__ == '__main__':