Maxwell : Automatic job requeue

#!/bin/bash -l
#SBATCH --job-name=test-restart
#SBATCH --output=test-restart.out
#SBATCH --time=0-00:03:00
#SBATCH --partition=maxcpu
unset LD_PRELOAD

# the sleep-loop at the end is running for max_iteration*30s
max_iteration=10

# only allow a single restart of the job. 
max_restarts=1

# new partition and timelimit for 2nd and subsequent job runs
alt_partition=all
alt_timelimit=0-01:00:00

# just gather some information about the job
scontext=$(scontrol show job $SLURM_JOB_ID)
restarts=$(echo "$scontext" | grep -o 'Restarts=.' | cut -d= -f2)
outfile=$(echo "$scontext"  | grep 'StdOut='       | cut -d= -f2)
errfile=$(echo "$scontext"  | grep 'StdErr='       | cut -d= -f2)
timelimit=$(echo "$scontext" | grep -o 'TimeLimit=.*' | awk '{print $1}' | cut -d= -f2) 

# term handler
# the function is executed once the job gets the TERM signal
term_handler()
{
    echo "executing term_handler at $(date)"
    if [[ $restarts -lt $max_restarts ]]; then
	   # copy the logfile. will be overwritten by the 2nd run
	   cp -v $outfile $outfile.$restarts
	   # requeue the job and put it on hold. It's not possible to change partition otherwise
	   scontrol requeuehold $SLURM_JOB_ID
	   # change timelimit and partition
	   scontrol update JobID=$SLURM_JOB_ID TimeLimit=$alt_timelimit Partition=$alt_partition
 	   # release the job. It will wait in the queue for 2 minutes before the 2nd run can start
	   scontrol release $SLURM_JOB_ID
    fi
}

# declare the function handling the TERM signal
trap 'term_handler' TERM

# print some job-information
cat <<EOF
SLURM_JOB_ID:         $SLURM_JOB_ID
SLURM_JOB_NAME:       $SLURM_JOB_NAME
SLURM_JOB_PARTITION:  $SLURM_JOB_PARTITION
SLURM_SUBMIT_HOST:    $SLURM_SUBMIT_HOST
TimeLimit:            $timelimit
Restarts:             $restarts
EOF

# the actual "calculation"
echo "starting calculation at $(date)"
i=0
while [[ $i -lt $max_iteration ]]; do
    sleep 30
    i=$(($i+1))
    echo "$i out of $max_iteration done at $(date) "
done

echo "all done at $(date)"