#!/bin/sh
# This wrapper script is intended to be submitted to Slurm to support
# communicating jobs.
#
# This script uses the following environment variables set by the submit MATLAB code:
# MDCE_MATLAB_EXE     - the MATLAB executable to use
# MDCE_MATLAB_ARGS    - the MATLAB args to use
# MDCE_TOTAL_TASKS    - the number of workers to start
#
# The following environment variables are forwarded through mpiexec:
# MDCE_DECODE_FUNCTION     - the decode function to use
# MDCE_STORAGE_LOCATION    - used by decode function
# MDCE_STORAGE_CONSTRUCTOR - used by decode function
# MDCE_JOB_LOCATION        - used by decode function
#
# For backward compatability, this wrapper script uses the following Slurm environment variables:
# SLURM_NODELIST           - instead of SLURM_JOB_NODELIST

# Copyright 2015-2017 The MathWorks, Inc.

# Intel MPI 4.1 implements MPI-2.2
# R2013b ships MPICH2 version 1.4.1p1
# Compatable through at least R2014b (and 15a?)

# Prep module load
source /etc/profile.d/modules.sh
module load profile/eng
# matlab module generally loads intel MPI, but due to the ssh connection do it explicitly
module load intel/pe-xe-2018--binary
module load intelmpi/2018--binary
module load matlab

# This gives MATLAB the best performance
export KMP_AFFINITY='granularity=thread,none'

# Test to make sure IB is being used
export I_MPI_FABRICS=shm:tmi

# Environment variables needed to use omnipath with MDCS
# Not sure which PSM version was used, so using both env. variables.
export IPATH_NO_BACKTRACE=1
export HFI_NO_BACKTRACE=1

MPIEXEC=mpiexec.hydra
MPIEXEC_CODE=0
# For mpiexec.hydra, we need to ensure that we're not using the PMI library
# I_MPI_PMI_LIBRARY must not be defined
unset I_MPI_PMI_LIBRARY

export TZ=Europe/Zurich

runMpiexec() {
    echo Node list: ${SLURM_NODELIST:?"Node list undefined"}

    if [ "X${MDCE_DEBUG}X" == "XtrueX" ]; then
        MPI_VERBOSE=-v
    else
        MPI_VERBOSE=
    fi

    CMD="${MPIEXEC} ${MPI_VERBOSE} -n ${MDCE_TOTAL_TASKS} \"${MDCE_MATLAB_EXE}\" ${MDCE_MATLAB_ARGS}"

    # As a debug stage: echo the command line...
    echo $CMD

    # ...and then execute it
    eval $CMD

    MPIEXEC_CODE=${?}
}

runMpiexec

echo "Exiting with code: ${MPIEXEC_CODE}"
exit ${MPIEXEC_CODE}
