latest tweaks from the field

CASCADE/Makefile

@@ -8,7 +8,7 @@
 #              email: Jean.Braun@anu.edu.au
 
 NAME=cascade
-include ../Makefile.p690
+include ../Makefile.deps
 LIBFILE=lib$(NAME)$(BITS).a
 
 # object (all the interesting bits to do the erosion/sedimentation

Makefile

 NAME=douar
-include Makefile.p690
+include Makefile.deps
 BIN=$(NAME)$(BITS)
 
 OBJECTS = \
@@ -76,10 +76,11 @@ write_global_output.o vrm.o
 
 all: $(BIN)
 $(BIN): subdirs $(OBJECTS)
-	echo "$(F90) $(FFLAGS) $(OPTIONS) $(OBJECTS) $(LIBS) -o $(BIN)" \
- 1>$(BIN).link.stdout 2>$(BIN).link.stderr
+	@echo "$(F90) $(FFLAGS) $(OPTIONS) $(OBJECTS) $(LIBS) -o $(BIN)" \
+>$(BIN).link.stdout
 	$(F90) $(FFLAGS) $(OPTIONS) $(OBJECTS) $(LIBS) -o $(BIN) \
- 1>>$(BIN).link.stdout 2>>$(BIN).link.stderr
+1>>$(BIN).link.stdout 2>>$(BIN).link.stderr
+	ldd $(BIN) >>$(BIN).link.stdout
 
 
 .PHONY: install
@@ -122,3 +123,14 @@ more-output:
 	@echo "linker output...."
 	@if test -e $(BIN).link.stdout ; then more $(BIN).link.stdout ; fi
 	@if test -e $(BIN).link.stderr ; then more $(BIN).link.stderr ; fi
+
+.PHONY: mahone
+mahone:
+	cp -f Makefile.mahone Makefile.deps
+
+.PHONY: p690
+p690:
+	cp -f Makefile.p690 Makefile.deps
+
+# end of file
+

NN/Makefile

 NAME=nn
-include ../Makefile.p690
+include ../Makefile.deps
 LIBFILE_f=lib$(NAME)_f$(BITS).a
 LIBFILE_c=lib$(NAME)_c$(BITS).a
 
@@ -13,7 +13,7 @@ $(LIBFILE_f): $(OBJECTS_NN_f)
 
 $(LIBFILE_c): $(OBJECTS_NN_c)
 	$(AR) -rv $(LIBFILE_c) $(OBJECTS_NN_c) >$(LIBFILE_c).ar
-	if test -f $(COMPILE_OUT) ; then mv -f $(COMPILE_OUT) $(COMPILE_OUT)_f; fi
+	if test -f $(COMPILE_OUT) ; then mv -f $(COMPILE_OUT) $(COMPILE_OUT)_c; fi
 
 .PHONY: objclean
 objclean:

OCTREE/Makefile

 NAME=Octree
-include ../Makefile.p690
+include ../Makefile.deps
 LIBFILE=lib$(NAME)$(BITS).a
 
 OBJECTS_OCTREE = \

RESAMPLE/Makefile

 NAME=resample
-include ../Makefile.p690
+include ../Makefile.deps
 LIBFILE=lib$(NAME)$(BITS).a
 
 OBJECTS = \

douar.run.mahone

@@ -16,25 +16,60 @@ fi
 
 PROCS=$1
 echo "Will use ${PROCS} processes."
+OUT=douar.out
+
+# For pgplot
+# on mahone, we have removed the .so, so we get a static link
 
 # For wsmp
 export MALLOC_TRIM_THRESHOLD_=-1
 export MALLOC_MMAP_MAX_=0
+
 # WSMP needs to know where its license file is.
 # export WSMPLICPATH=/home/dguptill/software/wsmp/
 # But this doesn't seem to work....)
 # It looks like the license must be where we start the job.
+
 # douar needs some directories..
 make_dir DEBUG
 make_dir DEBUG/mpilogs
 make_dir OUT
+
 # define the binary of douar
 # BIN=/home/dguptill/software/douar/bin/douar.mpich.02
 BIN=/home/dguptill/software/douar/douar
 
+# on mahone, an ldd of the binary (pgi compiler) shows one library not found
+# We get it like this...
+# There may be an issue here when we start running under SGE
+ACML_MV_PATH=/opt/pgi/linux86-64/8.0-1/libso
+if test "x$(echo ${LD_LIBRARY_PATH} | grep ${ACML_MV_PATH})" = x ; then
+    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ACML_MV_PATH}
+fi
+echo "LD_LIBRARY_PATH is ${LD_LIBRARY_PATH}"
+
+# For debugging PGI compiled binaries on mahone
+# This probably only works while runnin on the head node.
+export PGI_TERM=debug,trace
+export PGI_TERM_DEBUG="pgdbg -attach %d"
+
+# ulimits
+# (I put this in while debugging; not sure it helped anything)
+ulimit -s 20480      >>${OUT}
+ulimit -a            >>${OUT}
+
+
 # Here is the run command
+#
 # MPICH wants the input file on the command line.
-COM="/usr/local/mpich.pgi.64/bin/mpirun -arch LINUX -machinefile /home/dguptill/douar/machinefile -np ${PROCS} ${BIN} /home/dguptill/douar/input.txt"
+# We did not have much luck with MPICH; ace-net no longer supports it.
+# COM="/usr/local/mpich.pgi.64/bin/mpirun -arch LINUX -machinefile /home/dguptill/douar/machinefile -np ${PROCS} ${BIN} /home/dguptill/douar/input.txt"
+#
+# For testing on the head node
+COM="mpirun -np ${PROCS} ${BIN}"    
+#
+# For production runs under SGE
+# (more to come here)
 echo "COM is <${COM}>"
 
 # start douar and go away....
@@ -44,6 +79,5 @@ echo "COM is <${COM}>"
 
 # start douar and watch it........
 #
-OUT=douar.out
 ${COM} 2>douar.err | tee -a ${OUT} 
 

douar.run.p690

@@ -5,6 +5,10 @@
 # Author: Douglas Guptill
 # 2009-05-25: new
 
+make_dir() {
+  if ! test -d $1;  then mkdir $1; fi
+}
+
 if test $# -lt 1 ; then
   echo "Usage: $0 <number-of-processors>"
   exit
@@ -12,10 +16,7 @@ fi
 
 PROCS=$1
 echo "Will use ${PROCS} processes."
-
-make_dir() {
-  if ! test -d $1;  then mkdir $1; fi
-}
+OUT=douar.out
 
 # For poe
 #
@@ -32,8 +33,8 @@ export PGPLOT_DIR=/home/beaumnt1/software/pgplot-5.2.2
 #
 # WSMP needs to know where its license file is.
 # However the export doesn't seem to work.  
-# copy wsmp.lic to the running directory.
 # export WSMPLICPATH=/home/beaumnt1/software/wsmp/wsmp.lic
+# So copy wsmp.lic to the running directory.
 
 # For WSMP on AIX
 export SPINLOOPTIME=200
@@ -50,9 +51,6 @@ export MP_EUILIB=ip
 export MP_CSS_INTERRUPT=yes
 export MP_WAIT_MODE=nopoll
 export_MP_PULSE=0
-# for WSMP on Linux.
-# export MALLOC_TRIM_THRESHOLD_
-# export MALLOC_MMAP_MAX_
 
 # douar needs some directories..
 #
@@ -64,12 +62,14 @@ BIN=/home/beaumnt1/software/douar/bin/douar-q64
 
 # Here is the run command
 COM="poe ${BIN} -procs ${PROCS} -hostfile ${HOME}/host.list"    
+echo "COM is <${COM}>"
 
+# ulimits
+#
 ulimit -d unlimited   >${OUT}
 ulimit -s 2000000    >>${OUT}
 ulimit -m unlimited  >>${OUT}
 ulimit -a            >>${OUT}
-OUT=douar.out
 
 # start douar and go away....
 #
@@ -80,4 +80,9 @@ echo "douar has been started.  Look in nohup.out ."
 #
 # ${COM} 2>douar.err | tee -a ${OUT} 
 
+# start douar under "dbx" and watch it........
+# A nice thught, but we get an error:
+# dbx: fatal error: 1283-008 extraneous argument -procs
+# dbx "${COM} 2>douar.err | tee -a ${OUT} "
+