#!/bin/csh
# script for setting up, submitting, and extracting data from 
# scaling runs Chiba
#
##
## THESE PARAMETERS SHOULD BE EDITED FOR THE RUN YOU
## WANT TO DO
##
#
set procs      = (4 8) 
@ procpernode = 2
set makeflash  = "no"
set setupname  = coronal_loop
set dimension  = 1d
##set user       = mweber
set execname   = flash2
##set site=chiba      #always chiba
##
##


set resultsdir = /homes/mweber/FLASH2.2_init_1d/setups/coronal_loop/scaling_test
set FLASHDIR = /homes/mweber/FLASH2.2_init_1d
unset noclobber

# the name for our set of scaling runs

set runname=`date | awk '{print $2 "_" $3 "_" $4}'`

#variables for the site

set setupsite="chiba.mcs.anl.gov"
set template=scale.template.chiba
set submit="qsub"
set jobdir     = /homes/mweber/FLASH2.2_init_1d/setups/coronal_loop/scaling_test
@ maxprocs = 160


if ($dimension =~ "1d") then
   set maxblocks="1000"
else if ($dimension =~ "2d") then
   set maxblocks="1220"
else if ($dimension =~ "3d") then
   set maxblocks="50"
else
   echo "what on earth does dimension = " $dimension " mean?"
   exit
endif

if ($procpernode > $maxprocs) then
   echo "procspernode = " $procpernode " is too many for this machine!"
   exit
endif

if (! -e $jobdir) mkdir $jobdir
mkdir ${resultsdir}/${runname}
mkdir ${jobdir}/${runname}

if ($makeflash =~ "yes") then
	echo "Setting up FLASH Scaling Test Problem"
	./setup ${setupname} -${dimension} -site=${setupsite} -auto -maxblocks=${maxblocks}
	echo ""
	echo "Compiling....  "
	echo ""
	gmake
endif

cd ~/

cp ${FLASHDIR}/object/${execname} ${jobdir}
cp ${FLASHDIR}/object/helm_table.dat ${jobdir}
cp ${FLASHDIR}/setups/${setupname}/flash.par ${jobdir}


rm -f scale.template.*

#
# create the template for the psub file
# this is for chiba
#


cat > scale.template.chiba <<EOF
#!/bin/csh
#PBS -l walltime=1800
#PBS -A mcs-flash
#ctrans in /homes/mweber/FLASH2.2_init_1d/setups/coronal_loop/asch_100.0_39.8_asym/

echo "*** nodefile ***"
cat $PBS_NODEFILE
echo "*************"

setenv TMPDIR ${jobdir}

echo "*** running mpi program ***"

cd ${jobdir}

mpirun -np @@PROCS@@ -machinefile $PBS_NODEFILE flash2 > flash.out

rcp * chiba:${resultsdir}/${runname}/scale@@PROCS@@

exit 0;
EOF


rm -f scanlog
cat >> ${resultsdir}/${runname}/scanlog <<EOF
#!/bin/csh 
set nprocs=\$1
set FILE=${resultsdir}/${runname}/scale\${1}/flash.log
if (-e \${FILE} ) then
set total=\`grep "monitoring period" \$FILE | awk '{print \$5}'\`
set evol=\`grep "evolution" \$FILE | awk '{print \$2}'\`
set hydro=\`grep "hydro sweep" \$FILE | awk '{print \$3}'\`
set guard=\`grep "guard cell" \$FILE | grep -v tree | awk '{print \$3}'\`
set flux=\`grep "flux conservation" \$FILE | awk '{print \$3}'\`
set ref=\`grep "update refinement" \$FILE | awk '{print \$3}'\`
echo \$nprocs \$total \$evol \$hydro \$guard \$flux \$ref
endif
EOF

rm -f cleanup
rm -f scaling_results
echo "rm scale.template.*" > cleanup
echo "echo '#p total evol hydro guard flux ref'" > ${resultsdir}/${runname}/scaling_results

foreach proc ($procs)
	@ node = $proc / $procpernode
        if ($node == 0) set node = 1
	@ totproc = ($proc +  $procpernode - 1) / $procpernode
	if ($totproc > $node) then
		@ node = $node + 1
	endif
	echo "Using " $proc "processors on " $node " nodes."
        rm -f scale.${proc}
	sed -e "s/@@PROCS@@/${proc}/" -e "s/@@NODES@@/${node}/" -e "s/@@NAME@@/${runname}/" -e "s+@@EXECDIR@@+${FLASHDIR}+" ${template} > scale.${proc}
	echo "rm scale.$proc" >> cleanup
        echo "./scanlog $proc" >> ${resultsdir}/${runname}/scaling_results

        mkdir ${jobdir}/${runname}/scale${proc}/
        mkdir ${resultsdir}/${runname}/scale${proc}/
        cp ${jobdir}/flash.par      ${jobdir}/${runname}/scale${proc}/
        cp ${jobdir}/${execname}    ${jobdir}/${runname}/scale${proc}/
        cp ${jobdir}/helm_table.dat ${jobdir}/${runname}/scale${proc}/

	${submit} scale.${proc}
end

echo "rm -f cleanup" >> cleanup
chmod 755 cleanup ${resultsdir}/${runname}/{scaling_results,scanlog}

hostname          > ${resultsdir}/${runname}/RUN_INFO
echo $procpernode >> ${resultsdir}/${runname}/RUN_INFO

./cleanup