#!/bin/bash
# ---------------------------------------------------------------------------------------------
# orca_retry.sh: run ORCA for OPTIM, and run it again if it fails.
#
# OPTIM calls the program named on the ORCA line of odata as  <program> ORCA.in > ORCA.out
# and then reads ORCA.engrad. Very occasionally ORCA's MPI run dies for no reason to do with the
# geometry (an identical rerun works). Without this wrapper OPTIM then stops, and a PATHSAMPLE
# CHECKMIN/CHECKTS reports that stationary point as "no CONVERGED message".
#
# What it does: runs ORCA; if ORCA exits with an error or leaves no ORCA.engrad, it keeps a copy of
# ORCA.out, the error output and ORCA.in in $ORCA_FAILDIR, moves ORCA.gbw aside (it may be half
# written) and tries again, up to $ORCA_TRIES times. Every failure is logged in
# $ORCA_FAILDIR/failures.log, so check that file after a run: it should not exist.
#
# Settings (environment variables; the defaults are fine for these examples):
#   ORCA_BIN      the ORCA binary, full path (ORCA needs its full path to run in parallel)
#   ORCA_TRIES    attempts per ORCA call
#   ORCA_FAILDIR  where failed calls are kept; by default orca_failures/ in the folder the job was
#                 submitted from (PATHSAMPLE deletes the folders it runs OPTIM in)
# ---------------------------------------------------------------------------------------------
orca=${ORCA_BIN:-/usr/local/chemistry/orca/6.1.1/orca}
tries=${ORCA_TRIES:-3}
faildir=${ORCA_FAILDIR:-${SLURM_SUBMIT_DIR:-$PWD}/orca_failures}

for t in $(seq 1 "$tries"); do
  rm -f ORCA.engrad
  "$orca" "$@" > ORCA.out.run 2> ORCA.err.run
  rc=$?
  if [ $rc -eq 0 ] && [ -s ORCA.engrad ]; then
    cat ORCA.out.run; cat ORCA.err.run >&2; rm -f ORCA.out.run ORCA.err.run
    [ "$t" -gt 1 ] && echo "orca_retry: succeeded on try $t ($(pwd))" | tee -a "$faildir/failures.log" >&2
    exit 0
  fi
  mkdir -p "$faildir"; tag=$(hostname -s)_$(date +%Y%m%d_%H%M%S)_$$_try$t
  cp ORCA.out.run "$faildir/$tag.out"; cp ORCA.err.run "$faildir/$tag.err"; [ -f "$1" ] && cp "$1" "$faildir/$tag.in"
  echo "orca_retry: try $t failed, exit status $rc, ORCA.engrad $([ -s ORCA.engrad ] && echo present || echo missing)," \
       "host $(hostname -s), job ${SLURM_JOB_ID:-none}, folder $(pwd), $(date) -> $tag" | tee -a "$faildir/failures.log" >&2
  [ -f ORCA.gbw ] && mv -f ORCA.gbw ORCA.gbw.failed
done
# all tries failed: pass ORCA's output on, so OPTIM stops as it would have without the wrapper
cat ORCA.out.run; cat ORCA.err.run >&2; rm -f ORCA.out.run ORCA.err.run
exit $rc