Adding a template folder for JASMIN

cases/template_jasmin/ensemble/template.yaml

@@ -0,0 +1,54 @@
+---
+ensemble:
+  vars:                 {}
+
+  pre_process:          []
+  post_process:         []
+
+  batch_config:
+    templatedir:  TEMPLATE
+    templates:
+      - input/driver.jl.j2
+      - scripts/run_ensemble_member.j2
+    email:        someone@example.com
+    job_file:     scripts/run_ensemble_member
+    cluster:      debug
+    nodes:        1
+    ntasks:       8
+    length:       00:03:00
+    maxruns:      4
+    maxjobs:      4
+
+  batches:
+    - name:       NAME
+      basedir:    ./cases
+      repeat:     True
+      pre_batch:  []
+      pre_run:
+        - name:   check
+          args:
+            cmd:  ../../scripts/jasmin/pre_run.sh /gws/nopw/j04/dit/users/${USER}/WAVIhpc
+            fail: True
+
+      runs:
+      - thickness: 100.0
+        accumulation: 0.1
+      - thickness: 100.0
+        accumulation: 0.2
+      - thickness: 100.0
+        accumulation: 0.3
+
+      post_run:
+        - name:     check
+          args:
+            cmd:    /usr/bin/test -f run/outfile.nc
+            fail:   True
+        - name:     move
+          args:
+            dest:   "/gws/nopw/j04/dit/users/${USER}/WAVIhpc"
+        - name:     remove
+      post_batch:
+        - name:     check
+          args:
+            cmd:    ../scripts/jasmin/post_batch.sh /gws/nopw/j04/dit/users/${USER}/WAVIhpc NAME
+            fail:   True

cases/template_jasmin/input/driver.jl

@@ -0,0 +1,94 @@
+using WAVI
+function driver()
+
+#
+#Grid and boundary conditions
+#
+nx = 640
+ny = 80
+nσ = 4
+x0 = 0.0
+y0 = -40000.0
+dx = 1000.0
+dy = 1000.0
+h_mask=trues(nx,ny)
+u_iszero = falses(nx+1,ny); u_iszero[1,:].=true
+v_iszero=falses(nx,ny+1); v_iszero[:,1].=true; v_iszero[:,end].=true
+grid = Grid(nx = nx,
+            ny = ny,
+            nσ = nσ,
+            x0 = x0,
+            y0 = y0,
+            dx = dx,
+            dy = dy,
+            h_mask = h_mask,
+            u_iszero = u_iszero,
+            v_iszero = v_iszero)
+
+#
+#Bed
+#
+bed = WAVI.mismip_plus_bed #function definition
+
+#
+#solver parameters
+#
+maxiter_picard = 1
+solver_params = SolverParams(maxiter_picard = maxiter_picard)
+
+#
+#Physical parameters
+#
+params = Params()
+
+#
+#make the model
+#
+model = Model(grid = grid,
+              bed_elevation = bed,
+              params = params,
+              solver_params = solver_params)
+
+#
+#timestepping parameters
+niter0 = 0 #CHANGE ME FOR A PICKUP (!! must be niter0 !!)
+step_thickness = false #default = true!
+dt = 0.1
+end_time = 1000.
+chkpt_freq = 1.
+pchkpt_freq = 1.
+timestepping_params = TimesteppingParams(
+                                        niter0 = niter0,
+                                        dt = dt,
+                                        end_time = end_time,
+                                        chkpt_freq = chkpt_freq,
+                                        pchkpt_freq = pchkpt_freq,)
+
+#
+#output parameters
+#
+outputs = (h = model.fields.gh.h,
+           u = model.fields.gh.u,
+           v = model.fields.gh.v)
+output_freq = 50.
+output_params = OutputParams(outputs = outputs,
+                            output_freq = output_freq,
+                            output_format = "mat",
+                            dump_vel = true,
+          zip_format = "nc")
+
+#
+# assemble the simulation
+#
+simulation = Simulation(model = model,
+                        timestepping_params = timestepping_params,
+                        output_params = output_params)
+
+#
+#perform the simulation
+#
+run_simulation!(simulation)
+
+return simulation
+
+end

cases/template_jasmin/input/driver.jl.j2

@@ -0,0 +1,94 @@
+using WAVI
+function driver()
+
+#
+#Grid and boundary conditions
+#
+nx = 640
+ny = 80
+nσ = 4
+x0 = 0.0
+y0 = -40000.0
+dx = 1000.0
+dy = 1000.0
+h_mask=trues(nx,ny)
+u_iszero = falses(nx+1,ny); u_iszero[1,:].=true
+v_iszero=falses(nx,ny+1); v_iszero[:,1].=true; v_iszero[:,end].=true
+grid = Grid(nx = nx,
+            ny = ny,
+            nσ = nσ,
+            x0 = x0,
+            y0 = y0,
+            dx = dx,
+            dy = dy,
+            h_mask = h_mask,
+            u_iszero = u_iszero,
+            v_iszero = v_iszero)
+
+#
+#Bed
+#
+bed = WAVI.mismip_plus_bed #function definition
+
+#
+#solver parameters
+#
+maxiter_picard = 1
+solver_params = SolverParams(maxiter_picard = maxiter_picard)
+
+#
+#Physical parameters
+#
+params = Params()
+
+#
+#make the model
+#
+model = Model(grid = grid,
+              bed_elevation = bed,
+              params = params,
+              solver_params = solver_params)
+
+#
+#timestepping parameters
+niter0 = 0 #CHANGE ME FOR A PICKUP (!! must be niter0 !!)
+step_thickness = false #default = true!
+dt = 0.1
+end_time = 1000.
+chkpt_freq = 1.
+pchkpt_freq = 1.
+timestepping_params = TimesteppingParams(
+                                        niter0 = niter0,
+                                        dt = dt,
+                                        end_time = end_time,
+                                        chkpt_freq = chkpt_freq,
+                                        pchkpt_freq = pchkpt_freq,)
+
+#
+#output parameters
+#
+outputs = (h = model.fields.gh.h,
+           u = model.fields.gh.u,
+           v = model.fields.gh.v)
+output_freq = 50.
+output_params = OutputParams(outputs = outputs,
+                            output_freq = output_freq,
+                            output_format = "mat",
+                            dump_vel = true,
+          zip_format = "nc")
+
+#
+# assemble the simulation
+#
+simulation = Simulation(model = model,
+                        timestepping_params = timestepping_params,
+                        output_params = output_params)
+
+#
+#perform the simulation
+#
+run_simulation!(simulation)
+
+return simulation
+
+end

cases/template_jasmin/scripts/prep_run

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Empty the run directory - but first make sure it exists!
+
+if [ $( basename `pwd` ) != "run" ]; then
+  echo "This should be a directory called run, under the case folder"
+  exit 1
+fi
+
+# Link everything from the input directory
+ln -s ../input/* .
+
+# Proper copy of the driver file
+rm -fv driver.jl
+cp -f ../input/driver.jl .
+
+# Deep copy of any pickups (so they don't get overwritten in input/)
+rm -fv pickup*
+cp -fv ../input/pickup* . 2>/dev/null
+
+ln -sf ../../../envs

cases/template_jasmin/scripts/run_driver

@@ -0,0 +1,34 @@
+#!/usr/bin/env julia
+
+# Overriding println, quite amazing the line buffering doesn't seem to activate
+# when directing to file...
+# https://github.com/JuliaLang/julia/issues/13954
+# https://discourse.julialang.org/t/issues-with-println-buffering-output-when-redirecting-stdout-to-a-txt-file/23738/3
+@eval Base println(xs...) = begin res = println(stdout::IO, xs...); flush(stdout); return res end;
+@eval Base println(io::IO) =  begin res =  print(io, '\n'); flush(io); return res end;
+
+using Dates
+using Pkg
+using Printf
+
+env_name = "WAVI_ENV" in keys(ENV) ? ENV["WAVI_ENV"] : "wavi_test"
+Pkg.activate(joinpath("envs", env_name))
+
+cwd = pwd()
+@printf("\nCurrent working directory: %s\n", cwd)
+@printf("\nStart time: %s\n", Dates.now())
+@printf("\nJulia args: %s\n", join(Base.ARGS, ", "))
+
+#include everything the code directory
+for f in readdir("../code")
+    if f[end-2:end] == ".jl"
+        println("included code: ", f)
+        include(string("../code/", f))
+    end
+end
+
+#run the driver script
+include("../run/driver.jl")
+driver()
+
+@printf("\nEnd time: %s\n", Dates.now())

cases/template_jasmin/scripts/run_ensemble_member

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#SBATCH --output={{ run.dir }}/job.%j.%N.out
+#SBATCH --error={{ run.dir }}/job.%j.%N.err
+#SBATCH --chdir={{ run.dir }}
+#SBATCH --mail-type=begin,end,fail,requeue
+#SBATCH --mail-user={{ run.email }}
+#SBATCH --job-name={{ run.id }}
+#SBATCH --nodes={{ run.nodes }}
+#SBATCH --partition={{ run.cluster }}
+#SBATCH --qos={{ run.cluster }}
+#SBATCH --account=dit
+#SBATCH --cpus-per-task={{ run.ntasks }}
+#SBATCH --mem=8gb
+#SBATCH --signal=B:SIGINT@60
+
+# TODO: this logic is better in the ensemble run items, only to align to
+# previous approach have I ended up implementing it here, but it's growing :(
+
+CWD="`realpath .`"
+NAME="basename $CWD"
+RUNDIR="run"
+OUTFILE="outfile.nc"
+
+cd $RUNDIR
+
+export PATH="`realpath ..`/scripts:$PATH"
+
+if [ ! -f driver.jl ]; then
+  prep_run
+else
+  if ! ls PChkpt_*.jld2 1> /dev/null 2>&1 ; then
+    echo 'job chain: fail, no pickup files'
+    exit 1
+  fi
+
+  for file in PChkpt_*.jld2; do
+    [[ $file -nt $PICKUP_FILE ]] && PICKUP_FILE=$file
+  done
+  # Extract the middle bit of this filename
+  PICKUP=${PICKUP_FILE#PChkpt_}
+  PICKUP=${PICKUP%.jld2}
+
+  # TODO: no check to see if this will actually work
+  re='^[0-9]+$'
+  if [[ $PICKUP =~ $re ]]; then
+    echo 'job chain: pickup from permanent checkpoint'
+
+    # Save the timestep, with any leading zeros removed
+    CURITER=$( egrep '^niter0' driver.jl | sed -r 's/^niter0 = ([0-9]+)/\1/' )
+    NITER0=$(echo $PICKUP | sed 's/^0*//')
+
+    if [ "$CURITER" -eq "$NITER0" ] && [ ! -f $OUTFILE ]; then
+      echo "No progress was made in the last run, we'll assume failure or another
+      termination scenario. Please investigate or contact devs if necessary.
+      We'll be placing an $OUTFILE to prevent further submissions." | mail -s "Failure notification for $NAME " {{ run.email }}
+      # Touch an output file as a marker
+      touch $OUTFILE
+      exit 1
+    fi
+  else
+    echo 'job chain: fail, problem w pickup (WAVI only currently supports pCkpt pickups)' $PICKUP
+    exit 1
+  fi
+
+  #edit the driver namelist: replace the first instance of niter0 = * with appropriate checkpoint number
+  NITER0_LINE="niter0 = $NITER0"
+  echo $NITER0_LINE
+  sed -i '0,/.*niter0.*/s//'"$NITER0_LINE"'/' driver.jl
+fi
+
+run_driver >proc.${PICKUP:-0}.log 2>&1
+
+exit $?