grid_submit: wait until no subjob is active

This fixes a problem in the --wait and --wait-any modes of grid_submit.sh.

- AliEn can briefly report a masterjob as DONE while one of its subjobs in ERROR_A is put back to WAITING.
- --wait read only the masterjob status and returned at that moment, although the job was then retried.
- Both modes now finish only when no subjob is still active, with ERROR_A counted as active.
- A failed or empty status query also counts as active.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: sawenzel

GRID/utils/grid_submit.sh

@@ -454,6 +454,13 @@ EOF
   fi
 
 
+  # true while a subjob is not final; ERROR_A (resubmitted) and a failed query count as active
+  subjobs_active() {
+    jq -e '(.results | length) == 0 or (.results | any(.status | IN("INSERTING", "SPLITTING", "WAITING", "ASSIGNED", "STARTED", "RUNNING", "SAVING", "SAVED", "ERROR_A")))' \
+      <<<"$1" >/dev/null
+    [ $? -ne 1 ]
+  }
+
   # wait here until all ALIEN jobs have returned
   spin[3]="-"
   spin[2]="/"
@@ -488,12 +495,16 @@ EOF
       else
         # check if there are still jobs running/waiting; if not also finish
         # this could happen when all jobs are zombies (in which case we also finish)
-        if ! jq -e '.results | any(.status == "WAITING" or .status == "RUNNING" or .status == "SAVING" or .status == "INSERTING")' \
-            <<<"${DETAILED_STATUS_JSON}" >/dev/null; then
+        if ! subjobs_active "${DETAILED_STATUS_JSON}"; then
               JOBSTATUS="D"  # some job finished successfully
               echo "No remaining good job"
         fi
       fi
+    elif [ "${JOBSTATUS}" == "D" ]; then
+      # the masterjob can briefly show DONE while an ERROR_A subjob is resubmitted
+      if subjobs_active "$(ALIENPY_JSON=true alien.py ps -a -m "${MY_JOBID}")"; then
+        JOBSTATUS="S"
+      fi
     fi
 
     if [ "${JOBSTATUS}" == "D" ]; then