LIU-420: Fix unittest errors by using correct session_dir

- Previous approach ended up using local graph as input to the script; this isn't how client.submit_job() works; it uses the remote path the phyiscal graph is expected
ICRAR · Nov 25, 2024 · 7de4581 · 7de4581
1 parent 24f82eb
commit 7de4581
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 12 deletions.
diff --git a/daliuge-engine/dlg/deploy/slurm_client.py b/daliuge-engine/dlg/deploy/slurm_client.py
@@ -263,6 +263,11 @@ def create_job_desc(self, physical_graph_file):
 
         return init_tpl.safe_substitute(pardict)
 
+    @property
+    def session_dir(self):
+        return "{0}/workspace/{1}".format(
+            self.dlg_root, self.get_session_dirname()
+        )
 
     def mk_session_dir(self, dlg_root: str = ""):
         """
@@ -282,9 +287,7 @@ def mk_session_dir(self, dlg_root: str = ""):
                 dlg_root = os.environ["DLG_ROOT"]
             else:
                 dlg_root = f"{os.environ['HOME']}.dlg"
-        session_dir = "{0}/workspace/{1}".format(
-            self.dlg_root, self.get_session_dirname()
-        )
+        session_dir =  self.session_dir
         if not self._remote and not os.path.exists(session_dir):
             os.makedirs(session_dir)
         if self._remote:

diff --git a/daliuge-engine/test/deploy/slurm_script.sh b/daliuge-engine/test/deploy/slurm_script.sh
@@ -3,11 +3,11 @@
 #SBATCH --nodes=6
 #SBATCH --ntasks-per-node=1
 #SBATCH --cpus-per-task=2
-#SBATCH --job-name=DALiuGE-EAGLE_TestSession
+#SBATCH --job-name=DALiuGE-SLURM_TestSession
 #SBATCH --time=00:45:00
 #SBATCH --error=err-%j.log
 
 export DLG_ROOT=/scratch/pawsey0411/test/dlg
 source /software/projects/pawsey0411/venv/bin/activate
 
-srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace//home/00087932/github/EAGLE_TestSession --physical-graph "/home/00087932/github/EAGLE_test_repo/eagle_test_graphs/daliuge_tests/engine/graphs/SLURM_HelloWorld_simplePG.graph"   --verbose-level 1  --max-threads 0 --app 0 --num_islands 1   --ssid EAGLE_TestSession
+srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession --physical-graph "/scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession/SLURM_HelloWorld_simplePG.graph"   --verbose-level 1  --max-threads 0 --app 0 --num_islands 1   --ssid SLURM_TestSession
diff --git a/daliuge-engine/test/deploy/slurm_script_from_template.sh b/daliuge-engine/test/deploy/slurm_script_from_template.sh
@@ -4,11 +4,11 @@
 #SBATCH --ntasks-per-node=2
 #SBATCH --cpus-per-task=4
 #SBATCH --mem=0
-#SBATCH --job-name=DALiuGE-EAGLE_TestSession
+#SBATCH --job-name=DALiuGE-SLURM_TestSession
 #SBATCH --time=00:45:00
 #SBATCH --error=err-%j.log
 
 export DLG_ROOT=/scratch/pawsey0411/test/dlg
 source /software/projects/pawsey0411/venv/bin/activate
 
-srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace//home/00087932/github/EAGLE_TestSession --physical-graph "/home/00087932/github/EAGLE_test_repo/eagle_test_graphs/daliuge_tests/engine/graphs/SLURM_HelloWorld_simplePG.graph"   --verbose-level 1  --max-threads 0 --app 0 --num_islands 1   --ssid EAGLE_TestSession
+srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession --physical-graph "/scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession/SLURM_HelloWorld_simplePG.graph"   --verbose-level 1  --max-threads 0 --app 0 --num_islands 1   --ssid SLURM_TestSession
diff --git a/daliuge-engine/test/deploy/test_slurm_client.py b/daliuge-engine/test/deploy/test_slurm_client.py
@@ -43,16 +43,18 @@ class TestSlurmClient(unittest.TestCase):
     def test_client_with_cli(self):
         # Use special graph that also contains file name. See 'create_dlg_job.py'
         pg = files(test_graphs) / "SLURM_HelloWorld_simplePG.graph"
-          
+
         client = SlurmClient(
             facility="setonix", 
             num_nodes=6,
             job_dur=45,
-            physical_graph_template_file=str(pg),
+            physical_graph_template_file=pg,
             suffix="TestSession",
             username="test"
         )
-        job_desc = client.create_job_desc(pg)
+        session_dir = client.session_dir
+        physical_graph_file_name = "{0}/{1}".format(session_dir, client._pip_name)
+        job_desc = client.create_job_desc(physical_graph_file_name)
         curr_file = Path(__file__)
         compare_script = curr_file.parent / "slurm_script.sh"
         with compare_script.open() as fp:
@@ -77,8 +79,10 @@ def test_client_with_configfile(self):
             config=cfg,
             username='test'
         )
+        session_dir = client.session_dir
+        physical_graph_file_name = "{0}/{1}".format(session_dir, client._pip_name)
+        job_desc = client.create_job_desc(physical_graph_file_name)
 
-        job_desc = client.create_job_desc(pg)
         curr_file = Path(__file__)
         compare_script = curr_file.parent / "slurm_script.sh"
         with compare_script.open() as fp:
@@ -102,9 +106,14 @@ def test_client_with_slurm_template(self):
             slurm_template=slurm_template,
             username='test'
         )
-        job_desc = client.create_job_desc(pg)
+        session_dir = client.session_dir
+        physical_graph_file_name = "{0}/{1}".format(session_dir, client._pip_name)
+        job_desc = client.create_job_desc(physical_graph_file_name)
+
         curr_file = Path(__file__)
         compare_script = curr_file.parent / "slurm_script_from_template.sh"
         with compare_script.open() as fp:
             script = fp.read()
+            print(job_desc)
+            print(script)
             self.assertEqual(script, job_desc)