Merge pull request #1192 from benfitzpatrick/fix-poll-submit-fail

cylc poll: fix submission failed handling

lib/cylc/task_types/task.py

@@ -1182,6 +1182,14 @@ def process_incoming_message( self, (priority, message) ):
             self.execution_poll_timer.timer_start = None
             self.job_vacated = True
 
+        elif content == "submission failed":
+            # This can arrive via a poll.
+            outp = self.id + ' submitted'
+            if self.outputs.is_completed(outp):
+                self.outputs.remove(outp)
+            self.submission_timer_timeout = None
+            self.job_submission_failed()
+
         else:
             # Unhandled messages. These include:
             #  * general non-output/progress messages

tests/cylc-poll/02-task-submit-failed.t

@@ -0,0 +1,31 @@
+#!/bin/bash
+#C: THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+#C: Copyright (C) 2008-2014 Hilary Oliver, NIWA
+#C:
+#C: This program is free software: you can redistribute it and/or modify
+#C: it under the terms of the GNU General Public License as published by
+#C: the Free Software Foundation, either version 3 of the License, or
+#C: (at your option) any later version.
+#C:
+#C: This program is distributed in the hope that it will be useful,
+#C: but WITHOUT ANY WARRANTY; without even the implied warranty of
+#C: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#C: GNU General Public License for more details.
+#C:
+#C: You should have received a copy of the GNU General Public License
+#C: along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test that polling a submit-failed task sets the task state correctly
+. $(dirname $0)/test_header
+#-------------------------------------------------------------------------------
+set_test_number 2
+#-------------------------------------------------------------------------------
+install_suite $TEST_NAME_BASE $TEST_NAME_BASE
+#-------------------------------------------------------------------------------
+TEST_NAME=$TEST_NAME_BASE-validate
+run_ok $TEST_NAME cylc validate $SUITE_NAME
+#-------------------------------------------------------------------------------
+TEST_NAME=$TEST_NAME_BASE-run
+suite_run_ok $TEST_NAME cylc run --reference-test --debug $SUITE_NAME
+#-------------------------------------------------------------------------------
+purge_suite $SUITE_NAME

tests/cylc-poll/02-task-submit-failed/reference.log

@@ -0,0 +1,41 @@
+2014-10-22T10:14:15+01 INFO - port:7766
+2014-10-22T10:14:15+01 INFO - Suite starting at 2014-10-22T10:14:15+01
+2014-10-22T10:14:15+01 INFO - Log event clock: real time
+2014-10-22T10:14:15+01 INFO - Run mode: live
+2014-10-22T10:14:15+01 INFO - Initial point: 1
+2014-10-22T10:14:15+01 INFO - Final point: 1
+2014-10-22T10:14:15+01 INFO - Cold Start 1
+2014-10-22T10:14:15+01 INFO - [foo.1] -initiate job-submit
+2014-10-22T10:14:15+01 INFO - [foo.1] -triggered off []
+2014-10-22T10:14:16+01 INFO - job 117689 at 2014-10-23 12:00
+2014-10-22T10:14:16+01 INFO - [foo.1] -submit_method_id=117689
+2014-10-22T10:14:16+01 INFO - [foo.1] -submission succeeded
+2014-10-22T10:14:16+01 INFO - [kill_foo_submit.1] -initiate job-submit
+2014-10-22T10:14:16+01 INFO - [kill_foo_submit.1] -triggered off ['foo.1']
+2014-10-22T10:14:17+01 INFO - 11559
+2014-10-22T10:14:17+01 INFO - [kill_foo_submit.1] -submit_method_id=11559
+2014-10-22T10:14:17+01 INFO - [kill_foo_submit.1] -submission succeeded
+2014-10-22T10:14:18+01 INFO - [kill_foo_submit.1] -(current:submitted)> kill_foo_submit.1 started at 2014-10-22T10:14:17+01
+2014-10-22T10:14:23+01 INFO - [kill_foo_submit.1] -(current:running)> kill_foo_submit.1 succeeded at 2014-10-22T10:14:22+01
+2014-10-22T10:14:24+01 INFO - [poll_foo.1] -initiate job-submit
+2014-10-22T10:14:24+01 INFO - [poll_foo.1] -triggered off ['kill_foo_submit.1']
+2014-10-22T10:14:25+01 INFO - 11780
+2014-10-22T10:14:25+01 INFO - [poll_foo.1] -submit_method_id=11780
+2014-10-22T10:14:25+01 INFO - [poll_foo.1] -submission succeeded
+2014-10-22T10:14:25+01 INFO - [poll_foo.1] -(current:submitted)> poll_foo.1 started at 2014-10-22T10:14:25+01
+2014-10-22T10:14:31+01 INFO - [poll_foo.1] -(current:running)> poll_foo.1 succeeded at 2014-10-22T10:14:30+01
+2014-10-22T10:14:31+01 INFO - [foo.1] -initiate job-poll
+2014-10-22T10:14:31+01 INFO - Command succeeded: poll tasks(foo,1,False)
+2014-10-22T10:14:32+01 INFO - polled foo.1 submission failed
+
+2014-10-22T10:14:32+01 INFO - [foo.1] -(current:submitted)> polled foo.1 submission failed
+2014-10-22T10:14:32+01 ERROR - [foo.1] -submission failed
+2014-10-22T10:14:32+01 INFO - [stop.1] -initiate job-submit
+2014-10-22T10:14:32+01 INFO - [stop.1] -triggered off ['foo.1']
+2014-10-22T10:14:33+01 INFO - 11909
+2014-10-22T10:14:33+01 INFO - [stop.1] -submit_method_id=11909
+2014-10-22T10:14:33+01 INFO - [stop.1] -submission succeeded
+2014-10-22T10:14:33+01 INFO - [stop.1] -(current:submitted)> stop.1 started at 2014-10-22T10:14:33+01
+2014-10-22T10:14:34+01 INFO - [stop.1] -(current:running)> stop.1 succeeded at 2014-10-22T10:14:33+01
+2014-10-22T10:14:34+01 INFO - Command succeeded: stop cleanly(False)
+2014-10-22T10:14:35+01 INFO - Suite shutting down at 2014-10-22T10:14:35+01

tests/cylc-poll/02-task-submit-failed/suite.rc

@@ -0,0 +1,27 @@
+[cylc]
+    [[reference test]]
+        allow task failures = True
+        expected task failures = foo.1
+[scheduling]
+    [[dependencies]]
+        graph = """
+            foo:submit => kill_foo_submit => poll_foo
+            foo:submit-fail => stop
+        """
+
+[runtime]
+    [[foo]]
+        [[[job submission]]] 
+            method = at
+            command template = 'echo "%s" | at noon tomorrow'
+    [[poll_foo]]
+        command scripting = sleep 5; cylc poll $CYLC_SUITE_NAME foo 1
+    [[stop]]
+        command scripting = cylc stop $CYLC_SUITE_NAME
+    [[kill_foo_submit]]
+        command scripting = """
+            sleep 5
+            ID=$(sed -n "s/^CYLC_JOB_SUBMIT_METHOD_ID=//p" \
+                 $CYLC_SUITE_RUN_DIR/log/job/1/foo/01/job.status)
+            atrm $ID
+        """