#!/bin/bash
# ---------------------------------------------------------------------------------------------
# One OPTIM job; OPTIM calls ORCA for every energy + gradient, and ORCA runs in parallel with MPI.
#
# Slurm numbers for a SINGLE OPTIM run (they mean something else for PATHSAMPLE: see pathsample/):
#   --ntasks        = number of ORCA MPI ranks = NPROCS in ORCA.template   (checked below)
#   --cpus-per-task = 1 core per rank
#   --nodes=1       = keep all ranks on one node (ORCA's ranks talk through shared memory)
# Partition: CPU. Members of the wales group can use --partition=CPU,DEBUG (DEBUG: higher priority).
# ---------------------------------------------------------------------------------------------
#SBATCH --job-name=optim_orca_tspath
#SBATCH --partition=CPU
#SBATCH --nodes=1
#SBATCH --ntasks=2
#SBATCH --cpus-per-task=1
# memory per core: above %maxcore 2500 (MB per rank) in ORCA.template, to leave room for the rest of ORCA
#SBATCH --mem-per-cpu=3000M
# wall time: OPTIM has no restart file, so anything cut off at the limit is lost
#SBATCH --time=12:00:00

module load orca/6.1.1                    # ORCA 6.1.1 ...
module swap openmpi/5.0.5 openmpi/4.1.7   # ... run with Open MPI 4.1.7: the module's default 5.0.5 crashes
                                          # now and then inside ORCA (segfault in its shared-memory transport)

# Settings for the mpirun that ORCA starts itself (Open MPI 4.1 reads OMPI_MCA_... names; it ignores
# the PRTE_MCA_... names that Open MPI 5 uses).
# 1. No core binding: otherwise every ORCA run on a node pins its ranks to the same cores (0, 2, 4, ...)
#    and several jobs sharing a node run 3-5x slower.
export OMPI_MCA_hwloc_base_binding_policy=none
# 2. Let mpirun start NPROCS ranks even if it counts fewer slots in the Slurm allocation;
#    without it ORCA's mpirun can refuse to start ("not enough slots").
export OMPI_MCA_rmaps_base_oversubscribe=1

chmod +x orca_retry.sh

# Stop at once if ORCA would start a different number of ranks than the cores Slurm gave this job.
np=$(awk 'toupper($1)=="%PAL"{for(i=1;i<=NF;i++) if(toupper($i)=="NPROCS") print $(i+1)}' ORCA.template)
[ "$np" = "$SLURM_NTASKS" ] || { echo "ERROR: %PAL NPROCS $np in ORCA.template but --ntasks=$SLURM_NTASKS"; exit 1; }

# OPTIM reads odata (+ ORCA.template, writes path.info, EofS, path.xyz). stdout and stderr both go to OPTIM.out,
# so any MPI crash trace ends up next to the step where it happened.
/sharedscratch/wales/OPTIM.gfortran/OPTIM > OPTIM.out 2>&1

# orca_retry.sh (the program on the ORCA line of odata) reruns any ORCA call that failed and logs it here:
if [ -f orca_failures/failures.log ]; then echo "NOTE: some ORCA calls failed and were retried: see orca_failures/failures.log"; fi