#!/bin/bash

set -xuve
date

#
# General settings
#
arch=$1
model=$2
modver=$3
cupwd=$(pwd)
releases=/cfu/releases
srcm=$releases/models/$model/$modver
srcp=$releases/patches/$model

#
# Prepare hpctmp directory at HPC
#

# Basic settings
tmpname=${arch}_${model}_${modver}
jobname=hpcdep_${tmpname}
header=header_${tmpname}
target=$jobname.cmd
tmptarg=${target}.$$

# Register patch first always; locally
stamp=$(date +%Y%m%d_%H%M)
cd ../patches/$arch/$model/$modver
tar -cvzf $srcp/patch_${tmpname}_${stamp}.tar.gz *
patch=$(ls -1rt $srcp/patch_${tmpname}_*.tar.gz | tail -n 1)
cd -

# Choose hpcdst and hpctmp w.r.t HPC
case $arch in
 ithaca)
  hpcdst=/scratch/cfu/models
  hpctmp=/scratch/cfu/masif/${model}_${modver}
 ;;
 marenostrum)
  hpcdst=/gpfs/projects/ecm86/models
  hpctmp=/gpfs/scratch/ecm86/ecm86010/${model}_${modver}
 ;;
 marenostrum3)
  hpcdst=/gpfs/projects/ecm86/models
  hpctmp=/gpfs/scratch/ecm86/ecm86010/${model}_${modver}
 ;;
 ecmwf)
  hpcdst=/perm/ms/spesiccf/c3m/models
  hpctmp=/scratch/ms/spesiccf/c3m/${model}_${modver}
 ;;
 hector)
  hpcdst=/work/e24/rzg35ene/rzg00gue/models
  hpctmp=/work/e24/rzg35ene/rzg00gue/scratch
 ;;
 lindgren)
  hpcdst=/cfs/klemming/nobackup/a/asifsami/models
  hpctmp=/cfs/klemming/scratch/a/asifsami/${model}_${modver}
 ;;
 jaguar)
  hpcdst=
  hpctmp=
 ;;
 *) $arch is not available; exit 1
 ;;
esac

# Prepare target job's header w.r.t HPC
sed -e "s/HPCJOB/$jobname/" \
    -e "s|HPCTMP|$hpctmp|" \
    ../headers/$arch > $header

# Prepare final target job script
sed -e "s/ARCH/$arch/" \
    -e "s/MODEL/$model/" \
    -e "s/MODVER/$modver/" \
    -e "s|HPCDST|$hpcdst|" \
    -e "s|HPCTMP|$hpctmp|" \
    hpcdep.sh > $tmptarg
cat $header $tmptarg > $target
chmod 775 $target

# Transfer sources, patch and target job script at HPC
case $arch in
 ecmwf)
   set +e
   ecaccess-file-mkdir c2a:$hpctmp
   ecaccess-file-mkdir c2a:$hpctmp/$modver
   set -e
   list=$tmpname.txt
   if [[ ! -a $list ]]; then
      find $srcm -type f > $list
   fi
   for l in $(cat $list); do
       sfn=$(echo $l | rev | cut -d '/' -f 1 | rev)
       ecaccess-file-put $l c2a:$hpctmp/$modver/$sfn
       cat $list | sed -e '1d' > ${list}.tmp
       mv ${list}.tmp $list
   done
   cat $list; rm $list
   pfn=$(echo $patch | rev | cut -d '/' -f 1 | rev)
   ecaccess-file-put $patch c2a:$hpctmp/$pfn
   ecaccess-file-put $target c2a:$hpctmp/$target
 ;;
 *)
   ssh $arch mkdir -p $hpctmp
   rsync -arv $srcm ${arch}:$hpctmp
   rsync -arv $patch ${arch}:$hpctmp
   rsync -arv $target ${arch}:$hpctmp
   rm -f *${tmpname}*
 ;;
esac

#
# Launch target job script at HPC and monitor queue till job finishes
#
case $arch in
 ithaca)
  rjid=$(ssh $arch "cd $hpctmp; qsub $target" | awk '{printf $3}')
  chkst=$(echo ssh $arch "qstat | grep $rjid")
  chkcom=$(echo ssh $arch "ls -1 $hpctmp/compilation")
 ;;
 marenostrum)
  rjid=$(ssh $arch "cd $hpctmp; mnsub $target" | awk '{printf $3}')
  chkst=$(echo ssh $arch "mnq | grep $rjid")
  chkcom=$(echo ssh $arch "ls -1 $hpctmp/compilation")
 ;;
 marenostrum3)
  rjid=$(ssh $arch "cd $hpctmp; bsub < $target" | grep "Job <" | cut -d '>' -f 1 | cut -d '<' -f 2)
  chkst=$(echo ssh $arch "bjobs | grep $rjid")
  chkcom=$(echo ssh $arch "ls -1 $hpctmp/compilation")
 ;;
 ecmwf)
  rjid=$(ecaccess-job-submit -queueName c2a $cupwd/$target)
  chkst=$(echo ecaccess-job-list)
  chkcom=$(echo ecaccess-file-dir c2a:$hpctmp/compilation)
  rm -f *${tmpname}*
 ;;
 hector)
  rjid=$(ssh $arch "cd $hpctmp; qsub $target" | awk '{printf $3}')
  chkst=$(echo ssh $arch "qstat | grep $rjid")
  chkcom=$(echo ssh $arch "ls -1 $hpctmp/compilation")
 ;;
 lindgren)
  rjid=$(ssh $arch "cd $hpctmp; qsub $target" | awk '{printf $3}')
  chkst=$(echo ssh $arch "qstat | grep $rjid")
  chkcom=$(echo ssh $arch "ls -1 $hpctmp/compilation")
 ;;
 jaguar)
  rjid=
  chkst=
  chkcom=
 ;;
esac

# HPC queue monitoring
set +e
while :
do
  sleep 60
  case $arch in
   ecmwf) 
    chkstres=$($chkst | grep $rjid | awk '{printf $3}')
    case $chkstres in
     DONE)
      $chkcom
      if [[ $? -ne 0 ]]; then
         echo "!!!compilation failed!!"
         exit 1
      else
         break
      fi
     ;;
     STOP) break ;;
    esac
   ;;
   *)
    $chkst
    if [[ $? -ne 0 ]]; then
       $chkcom
       if [[ $? -ne 0 ]]; then
          echo "!!!compilation failed!!!"
          exit 1
       else
          break
       fi
    fi
   ;;
  esac
done
set -e

date

