LIU-417: Swap order in LG and finalise tests.

daliuge-translator/dlg/dropmake/graph_config.py

@@ -31,33 +31,33 @@
 CONFIG_KEY = "graphConfigurations"
 GRAPH_NODES = "nodeDataArray"
 
+
 def apply_active_configuration(logical_graph: dict) -> dict:
     """
-    Given a JSON representation of the LogicalGraph template, apply the 
+    Given a JSON representation of the LogicalGraph Template (LGT), apply the 
     "Active Configuration" to the relevant constructs/fields.
 
+    :param: logical_graph, dict representation of JSON LGT
 
-    """
+    Currently, this does not raise any exceptions, but logs either warnings or errors. 
+    If there are missing keys or the configuration cannot be applied, it will return 
+    the original LGT. See graph_config.check_config_is_value for more details. 
 
-    if not ACTIVE_CONFIG_KEY in logical_graph:
-        LOGGER.warning("No %s available in Logical Graph.", ACTIVE_CONFIG_KEY)
+    return: dict, the updated LG
+    """
+    if is_config_invalid(logical_graph):
         return logical_graph
-
-    if CONFIG_KEY in logical_graph:
-        if not logical_graph[CONFIG_KEY]:
-            LOGGER.warning("No %s provided to the Logical Graph.", CONFIG_KEY)
-            return logical_graph
 
+    try:
         activeConfigurationID = logical_graph[ACTIVE_CONFIG_KEY]
         activeConfig = logical_graph[CONFIG_KEY][activeConfigurationID]
-
         nodeDataArray = logical_graph[GRAPH_NODES] 
 
         for node_id, fields in activeConfig["nodes"].items():
             idx = get_key_idx_from_list(node_id, nodeDataArray) 
             if idx is None:
                 LOGGER.warning(
-                    "%s present in activeConfig but no available in Logical Graph."
+                    "%s present in activeConfig but not available in Logical Graph."
                 )
                 continue 
             node_name = nodeDataArray[idx]["name"]
@@ -71,11 +71,40 @@ def apply_active_configuration(logical_graph: dict) -> dict:
                             node_name, field_name, str(prev_value), str(field["value"]))
                 nodeDataArray[idx]["fields"][fieldidx] = field
         logical_graph[GRAPH_NODES] = nodeDataArray
-    else:
-        LOGGER.error("No graph configuration available in Logical Graph.")
+
+    except KeyError:
+        LOGGER.warning(
+            "Graph config key does not exist in logical graph. Using base field values."
+        )
 
     return logical_graph
 
+def is_config_invalid(logical_graph: dict) -> bool:
+    """
+    Given a logical graph, verify that the correct keys are present prior to applying
+    the configuration. 
+
+    We want to make sure that: 
+        - "activeGraphConfigId" and "graphConfigurations" exist 
+        - Both of these store non-empty information
+
+    Note: We do not perform any validation on if the graphConfig is self-consistent; that 
+    is, if it mentions a graphConfig Key not in the LG, or there is a field/node ID that 
+    is not in the LG, we report this as an error and continue (this implies an error 
+    upstream that we cannot resolve).
+
+    :return: True if the config has correct keys and they are present. 
+    """
+
+    checkActiveId = logical_graph.get(ACTIVE_CONFIG_KEY, None)
+    if not checkActiveId:
+        LOGGER.warning("No %s data available in Logical Graph.", ACTIVE_CONFIG_KEY)
+    checkGraphConfig = logical_graph.get(CONFIG_KEY, None)    
+    if not checkGraphConfig:
+        LOGGER.warning("No %s data available in Logical Graph.", CONFIG_KEY)
+
+    return (not checkActiveId) or (not checkGraphConfig)
+
 
 def get_key_idx_from_list(key: str, dictList: list) -> int: 
     """

daliuge-translator/dlg/dropmake/lg.py

@@ -68,7 +68,7 @@ class LG:
     it is doing all the conversion inside __init__
     """
 
-    def __init__(self, f, ssid=None):
+    def __init__(self, f, ssid=None, apply_config=True):
         """
         parse JSON into LG object graph first
         """
@@ -88,7 +88,8 @@ def __init__(self, f, ssid=None):
         logger.info("Loading graph: %s", lg["modelData"]["filePath"])
         logger.info("Found LG version: %s", lgver)
 
-        lg = apply_active_configuration(lg)
+        if apply_config:
+            lg = apply_active_configuration(lg)
 
         if LG_VER_EAGLE == lgver:
             lg = convert_mkn(lg)

daliuge-translator/test/dropmake/test_graph_config.py

@@ -25,12 +25,20 @@
 from dlg.dropmake.graph_config import apply_active_configuration, get_key_idx_from_list
 import daliuge_tests.dropmake as test_graphs
 
+try:
+    from importlib.resources import files, as_file
+except (ImportError, ModuleNotFoundError):
+    from importlib_resources import files
 
-def get_graphconfig_fname(lg_name: str):
+LOG_PRFIX = "WARNING:dlg.dropmake.graph_config:"
+
+def get_lg_from_fname(lg_name: str) -> dict:
     """
-    Return the graph_config from the test repository 
+    Return the logical graph from the graph_config files in the test graph repository 
     """
-    return str(files(test_graphs) / f"graph_config/{lg_name}")
+    fname = str(files(test_graphs) / f"graph_config/{lg_name}")
+    with open(fname, "r") as fp:
+            return json.load(fp)
 
 def get_value_from_lgnode_field(node_id: str, field_id: str, logical_graph: dict) -> str:
     """
@@ -42,21 +50,65 @@ def get_value_from_lgnode_field(node_id: str, field_id: str, logical_graph: dict
     field_idx = get_key_idx_from_list(field_id, logical_graph["nodeDataArray"][idx]["fields"])
     return logical_graph["nodeDataArray"][idx]["fields"][field_idx]["value"]
 
+
 class TestGraphConfig(unittest.TestCase):
     """
     """
 
-    def test_apply_with_no_config(self):
-        pass
+    def test_apply_with_empty_config(self):
+        """
+        Exhaust the following possibilities that may exist when attempting to 
+        apply graph config:
+            - "activeGraphConfigId" is not in graph
+            - "activeGraphConfigId" is empty
+            - "graphConfigurations" is not in graph
+            - "graphConfigurations" is empty
+            - Keys from the activeGraphConfigId or graphConfigurations are not found in 
+            the logical graph. 
+        """ 
+        lg = get_lg_from_fname(f"ArrayLoopNoActiveID.graph")
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+        self.assertEqual(
+            [f"{LOG_PRFIX}No activeGraphConfigId data available in Logical Graph.",
+            f"{LOG_PRFIX}No graphConfigurations data available in Logical Graph."],
+            cm.output
+        )
+
+        lg['activeGraphConfigId'] = ""
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+        self.assertEqual(
+            [f"{LOG_PRFIX}No activeGraphConfigId data available in Logical Graph.",
+            f"{LOG_PRFIX}No graphConfigurations data available in Logical Graph."],
+            cm.output
+        )
+
+        lg['activeGraphConfigId'] = "temporaryString"
+
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+        self.assertEqual(
+            [f"{LOG_PRFIX}No graphConfigurations data available in Logical Graph."],
+            cm.output
+        )
+
+        lg['graphConfigurations'] = {"key": "value"}
 
+        with self.assertLogs('root', level="WARNING") as cm:
+            alt_lg = apply_active_configuration(lg)
+            self.assertEqual(
+               [f"{LOG_PRFIX}Graph config key does not exist in logical graph. "
+                "Using base field values."],
+               cm.output
+            )
+
     def test_apply_with_loop(self):
         """
         ArrayLoopLoop.graph has a GraphConfig with modified "num_of_iter" field in the 
         "Loop" node (construct). 
         """
-        fname = get_graphconfig_fname("ArrayLoopLoop.graph")
-        with open(fname, "r") as fp:
-            lg = json.load(fp)
+        lg = get_lg_from_fname("ArrayLoopLoop.graph")
         node_id = "732df21b-f714-4d25-9773-b4169db270a0"
         field_id = "3d25fcc9-50bb-4bbc-9b19-2eceabc238f2"
         value = get_value_from_lgnode_field(node_id, field_id, lg)
@@ -66,17 +118,15 @@ def test_apply_with_loop(self):
         self.assertEqual(5, int(value))
 
     def test_apply_with_scatter(self):
-        fname = get_graphconfig_fname("ArrayLoopScatter.graph")
-        with open(fname, "r") as fp:
-            lg = json.load(fp)
-        node_id = "732df21b-f714-4d25-9773-b4169db270a0"
-        field_id = "5560c56a-10f3-4d42-a436-404816dddf5f"
+        """
+        ArrayLoopLoop.graph has a GraphConfig with modified "num_of_copies" field in the 
+        "Scatter" node (construct). 
+        """
+        lg = get_lg_from_fname("ArrayLoopScatter.graph")
+        node_id = "5560c56a-10f3-4d42-a436-404816dddf5f"
+        field_id = "0057b318-2405-4b5b-ac59-06c0068f91b7"
         value = get_value_from_lgnode_field(node_id, field_id, lg)
         self.assertEqual(1, int(value))
         lg = apply_active_configuration(lg)
         value = get_value_from_lgnode_field(node_id, field_id, lg)
         self.assertEqual(5, int(value))
-
-class TestGraphUnrollWithconfig(unittest.TestCase):
-    """
-    """