douar.run.mahone

#!/bin/sh
#
# run douar, on mahone
#
# Author: Douglas Guptill
# 2009-06-03: new, from a copy of the one on p690

make_dir() {
  if ! test -d $1;  then mkdir $1; fi
}

if test $# -lt 1 ; then
  echo "Usage: $0 <number-of-processors>"
  exit
fi

PROCS=$1
echo "Will use ${PROCS} processes."
OUT=douar.out

# For pgplot
# on mahone, we have removed the .so, so we get a static link

# For wsmp
export MALLOC_TRIM_THRESHOLD_=-1
export MALLOC_MMAP_MAX_=0

# WSMP needs to know where its license file is.
# export WSMPLICPATH=/home/dguptill/software/wsmp/
# But this doesn't seem to work....)
# It looks like the license must be where we start the job.

# douar needs some directories..
make_dir DEBUG
make_dir DEBUG/mpilogs
make_dir OUT

# define the binary of douar
# BIN=/home/dguptill/software/douar/bin/douar.mpich.02
BIN=/home/dguptill/software/douar/bin/douar

# on mahone, an ldd of the binary (pgi compiler) shows one library not found
# We get it like this...
# There may be an issue here when we start running under SGE
ACML_MV_PATH=/opt/pgi/linux86-64/8.0-1/libso
if test "x$(echo ${LD_LIBRARY_PATH} | grep ${ACML_MV_PATH})" = x ; then
    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ACML_MV_PATH}
fi
echo "LD_LIBRARY_PATH is ${LD_LIBRARY_PATH}"

# For debugging PGI compiled binaries on mahone
# This probably only works while runnin on the head node.
export PGI_TERM=debug,trace
export PGI_TERM_DEBUG="pgdbg -attach %d"

# ulimits
# (I put this in while debugging; not sure it helped anything)
ulimit -s 20480      >>${OUT}
ulimit -a            >>${OUT}


# Here is the run command
#
# MPICH wants the input file on the command line.
# We did not have much luck with MPICH; ace-net no longer supports it.
# COM="/usr/local/mpich.pgi.64/bin/mpirun -arch LINUX -machinefile /home/dguptill/douar/machinefile -np ${PROCS} ${BIN} /home/dguptill/douar/input.txt"
#
# For testing on the head node
COM="mpirun -np ${PROCS} ${BIN}"    
#
# For production runs under SGE
# (more to come here)
echo "COM is <${COM}>"

# start douar and go away....
#
# nohup ${COM} &
# echo "douar has been started.  Look in nohup.out ."

# start douar and watch it........
#
${COM} 2>douar.err | tee -a ${OUT}