diff --git a/daliuge-engine/dlg/deploy/slurm_client.py b/daliuge-engine/dlg/deploy/slurm_client.py index 806848d60..f841e5208 100644 --- a/daliuge-engine/dlg/deploy/slurm_client.py +++ b/daliuge-engine/dlg/deploy/slurm_client.py @@ -263,6 +263,11 @@ def create_job_desc(self, physical_graph_file): return init_tpl.safe_substitute(pardict) + @property + def session_dir(self): + return "{0}/workspace/{1}".format( + self.dlg_root, self.get_session_dirname() + ) def mk_session_dir(self, dlg_root: str = ""): """ @@ -282,9 +287,7 @@ def mk_session_dir(self, dlg_root: str = ""): dlg_root = os.environ["DLG_ROOT"] else: dlg_root = f"{os.environ['HOME']}.dlg" - session_dir = "{0}/workspace/{1}".format( - self.dlg_root, self.get_session_dirname() - ) + session_dir = self.session_dir if not self._remote and not os.path.exists(session_dir): os.makedirs(session_dir) if self._remote: diff --git a/daliuge-engine/test/deploy/slurm_script.sh b/daliuge-engine/test/deploy/slurm_script.sh index 28bc23ccb..48db24b27 100644 --- a/daliuge-engine/test/deploy/slurm_script.sh +++ b/daliuge-engine/test/deploy/slurm_script.sh @@ -3,11 +3,11 @@ #SBATCH --nodes=6 #SBATCH --ntasks-per-node=1 #SBATCH --cpus-per-task=2 -#SBATCH --job-name=DALiuGE-EAGLE_TestSession +#SBATCH --job-name=DALiuGE-SLURM_TestSession #SBATCH --time=00:45:00 #SBATCH --error=err-%j.log export DLG_ROOT=/scratch/pawsey0411/test/dlg source /software/projects/pawsey0411/venv/bin/activate -srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace//home/00087932/github/EAGLE_TestSession --physical-graph "/home/00087932/github/EAGLE_test_repo/eagle_test_graphs/daliuge_tests/engine/graphs/SLURM_HelloWorld_simplePG.graph" --verbose-level 1 --max-threads 0 --app 0 --num_islands 1 --ssid EAGLE_TestSession +srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession --physical-graph "/scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession/SLURM_HelloWorld_simplePG.graph" --verbose-level 1 --max-threads 0 --app 0 --num_islands 1 --ssid SLURM_TestSession diff --git a/daliuge-engine/test/deploy/slurm_script_from_template.sh b/daliuge-engine/test/deploy/slurm_script_from_template.sh index 6c1740050..80c6d6040 100644 --- a/daliuge-engine/test/deploy/slurm_script_from_template.sh +++ b/daliuge-engine/test/deploy/slurm_script_from_template.sh @@ -4,11 +4,11 @@ #SBATCH --ntasks-per-node=2 #SBATCH --cpus-per-task=4 #SBATCH --mem=0 -#SBATCH --job-name=DALiuGE-EAGLE_TestSession +#SBATCH --job-name=DALiuGE-SLURM_TestSession #SBATCH --time=00:45:00 #SBATCH --error=err-%j.log export DLG_ROOT=/scratch/pawsey0411/test/dlg source /software/projects/pawsey0411/venv/bin/activate -srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace//home/00087932/github/EAGLE_TestSession --physical-graph "/home/00087932/github/EAGLE_test_repo/eagle_test_graphs/daliuge_tests/engine/graphs/SLURM_HelloWorld_simplePG.graph" --verbose-level 1 --max-threads 0 --app 0 --num_islands 1 --ssid EAGLE_TestSession \ No newline at end of file +srun -l python3 -m dlg.deploy.start_dlg_cluster --log_dir /scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession --physical-graph "/scratch/pawsey0411/test/dlg/workspace/SLURM_TestSession/SLURM_HelloWorld_simplePG.graph" --verbose-level 1 --max-threads 0 --app 0 --num_islands 1 --ssid SLURM_TestSession \ No newline at end of file diff --git a/daliuge-engine/test/deploy/test_slurm_client.py b/daliuge-engine/test/deploy/test_slurm_client.py index a44048cd4..87c68db47 100644 --- a/daliuge-engine/test/deploy/test_slurm_client.py +++ b/daliuge-engine/test/deploy/test_slurm_client.py @@ -43,16 +43,18 @@ class TestSlurmClient(unittest.TestCase): def test_client_with_cli(self): # Use special graph that also contains file name. See 'create_dlg_job.py' pg = files(test_graphs) / "SLURM_HelloWorld_simplePG.graph" - + client = SlurmClient( facility="setonix", num_nodes=6, job_dur=45, - physical_graph_template_file=str(pg), + physical_graph_template_file=pg, suffix="TestSession", username="test" ) - job_desc = client.create_job_desc(pg) + session_dir = client.session_dir + physical_graph_file_name = "{0}/{1}".format(session_dir, client._pip_name) + job_desc = client.create_job_desc(physical_graph_file_name) curr_file = Path(__file__) compare_script = curr_file.parent / "slurm_script.sh" with compare_script.open() as fp: @@ -77,8 +79,10 @@ def test_client_with_configfile(self): config=cfg, username='test' ) + session_dir = client.session_dir + physical_graph_file_name = "{0}/{1}".format(session_dir, client._pip_name) + job_desc = client.create_job_desc(physical_graph_file_name) - job_desc = client.create_job_desc(pg) curr_file = Path(__file__) compare_script = curr_file.parent / "slurm_script.sh" with compare_script.open() as fp: @@ -102,9 +106,14 @@ def test_client_with_slurm_template(self): slurm_template=slurm_template, username='test' ) - job_desc = client.create_job_desc(pg) + session_dir = client.session_dir + physical_graph_file_name = "{0}/{1}".format(session_dir, client._pip_name) + job_desc = client.create_job_desc(physical_graph_file_name) + curr_file = Path(__file__) compare_script = curr_file.parent / "slurm_script_from_template.sh" with compare_script.open() as fp: script = fp.read() + print(job_desc) + print(script) self.assertEqual(script, job_desc)