diff --git a/cases/template_jasmin/ensemble/template.yaml b/cases/template_jasmin/ensemble/template.yaml new file mode 100644 index 0000000..4311250 --- /dev/null +++ b/cases/template_jasmin/ensemble/template.yaml @@ -0,0 +1,54 @@ +--- +ensemble: + vars: {} + + pre_process: [] + post_process: [] + + batch_config: + templatedir: TEMPLATE + templates: + - input/driver.jl.j2 + - scripts/run_ensemble_member.j2 + email: someone@example.com + job_file: scripts/run_ensemble_member + cluster: debug + nodes: 1 + ntasks: 8 + length: 00:03:00 + maxruns: 4 + maxjobs: 4 + + batches: + - name: NAME + basedir: ./cases + repeat: True + pre_batch: [] + pre_run: + - name: check + args: + cmd: ../../scripts/jasmin/pre_run.sh /gws/nopw/j04/dit/users/${USER}/WAVIhpc + fail: True + + runs: + - thickness: 100.0 + accumulation: 0.1 + - thickness: 100.0 + accumulation: 0.2 + - thickness: 100.0 + accumulation: 0.3 + + post_run: + - name: check + args: + cmd: /usr/bin/test -f run/outfile.nc + fail: True + - name: move + args: + dest: "/gws/nopw/j04/dit/users/${USER}/WAVIhpc" + - name: remove + post_batch: + - name: check + args: + cmd: ../scripts/jasmin/post_batch.sh /gws/nopw/j04/dit/users/${USER}/WAVIhpc NAME + fail: True diff --git a/cases/template_jasmin/input/driver.jl b/cases/template_jasmin/input/driver.jl new file mode 100644 index 0000000..4d3fe19 --- /dev/null +++ b/cases/template_jasmin/input/driver.jl @@ -0,0 +1,94 @@ +using WAVI +function driver() + +# +#Grid and boundary conditions +# +nx = 640 +ny = 80 +nσ = 4 +x0 = 0.0 +y0 = -40000.0 +dx = 1000.0 +dy = 1000.0 +h_mask=trues(nx,ny) +u_iszero = falses(nx+1,ny); u_iszero[1,:].=true +v_iszero=falses(nx,ny+1); v_iszero[:,1].=true; v_iszero[:,end].=true +grid = Grid(nx = nx, + ny = ny, + nσ = nσ, + x0 = x0, + y0 = y0, + dx = dx, + dy = dy, + h_mask = h_mask, + u_iszero = u_iszero, + v_iszero = v_iszero) + +# +#Bed +# +bed = WAVI.mismip_plus_bed #function definition + +# +#solver parameters +# +maxiter_picard = 1 +solver_params = SolverParams(maxiter_picard = maxiter_picard) + +# +#Physical parameters +# +params = Params() + +# +#make the model +# +model = Model(grid = grid, + bed_elevation = bed, + params = params, + solver_params = solver_params) + +# +#timestepping parameters +niter0 = 0 #CHANGE ME FOR A PICKUP (!! must be niter0 !!) +step_thickness = false #default = true! +dt = 0.1 +end_time = 1000. +chkpt_freq = 1. +pchkpt_freq = 1. +timestepping_params = TimesteppingParams( + niter0 = niter0, + dt = dt, + end_time = end_time, + chkpt_freq = chkpt_freq, + pchkpt_freq = pchkpt_freq,) + +# +#output parameters +# +outputs = (h = model.fields.gh.h, + u = model.fields.gh.u, + v = model.fields.gh.v) +output_freq = 50. +output_params = OutputParams(outputs = outputs, + output_freq = output_freq, + output_format = "mat", + dump_vel = true, + zip_format = "nc") + +# +# assemble the simulation +# +simulation = Simulation(model = model, + timestepping_params = timestepping_params, + output_params = output_params) + +# +#perform the simulation +# +run_simulation!(simulation) + +return simulation + +end diff --git a/cases/template_jasmin/input/driver.jl.j2 b/cases/template_jasmin/input/driver.jl.j2 new file mode 100644 index 0000000..4d3fe19 --- /dev/null +++ b/cases/template_jasmin/input/driver.jl.j2 @@ -0,0 +1,94 @@ +using WAVI +function driver() + +# +#Grid and boundary conditions +# +nx = 640 +ny = 80 +nσ = 4 +x0 = 0.0 +y0 = -40000.0 +dx = 1000.0 +dy = 1000.0 +h_mask=trues(nx,ny) +u_iszero = falses(nx+1,ny); u_iszero[1,:].=true +v_iszero=falses(nx,ny+1); v_iszero[:,1].=true; v_iszero[:,end].=true +grid = Grid(nx = nx, + ny = ny, + nσ = nσ, + x0 = x0, + y0 = y0, + dx = dx, + dy = dy, + h_mask = h_mask, + u_iszero = u_iszero, + v_iszero = v_iszero) + +# +#Bed +# +bed = WAVI.mismip_plus_bed #function definition + +# +#solver parameters +# +maxiter_picard = 1 +solver_params = SolverParams(maxiter_picard = maxiter_picard) + +# +#Physical parameters +# +params = Params() + +# +#make the model +# +model = Model(grid = grid, + bed_elevation = bed, + params = params, + solver_params = solver_params) + +# +#timestepping parameters +niter0 = 0 #CHANGE ME FOR A PICKUP (!! must be niter0 !!) +step_thickness = false #default = true! +dt = 0.1 +end_time = 1000. +chkpt_freq = 1. +pchkpt_freq = 1. +timestepping_params = TimesteppingParams( + niter0 = niter0, + dt = dt, + end_time = end_time, + chkpt_freq = chkpt_freq, + pchkpt_freq = pchkpt_freq,) + +# +#output parameters +# +outputs = (h = model.fields.gh.h, + u = model.fields.gh.u, + v = model.fields.gh.v) +output_freq = 50. +output_params = OutputParams(outputs = outputs, + output_freq = output_freq, + output_format = "mat", + dump_vel = true, + zip_format = "nc") + +# +# assemble the simulation +# +simulation = Simulation(model = model, + timestepping_params = timestepping_params, + output_params = output_params) + +# +#perform the simulation +# +run_simulation!(simulation) + +return simulation + +end diff --git a/cases/template_jasmin/scripts/prep_run b/cases/template_jasmin/scripts/prep_run new file mode 100755 index 0000000..863d342 --- /dev/null +++ b/cases/template_jasmin/scripts/prep_run @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Empty the run directory - but first make sure it exists! + +if [ $( basename `pwd` ) != "run" ]; then + echo "This should be a directory called run, under the case folder" + exit 1 +fi + +# Link everything from the input directory +ln -s ../input/* . + +# Proper copy of the driver file +rm -fv driver.jl +cp -f ../input/driver.jl . + +# Deep copy of any pickups (so they don't get overwritten in input/) +rm -fv pickup* +cp -fv ../input/pickup* . 2>/dev/null + +ln -sf ../../../envs diff --git a/cases/template_jasmin/scripts/run_driver b/cases/template_jasmin/scripts/run_driver new file mode 100755 index 0000000..a217124 --- /dev/null +++ b/cases/template_jasmin/scripts/run_driver @@ -0,0 +1,34 @@ +#!/usr/bin/env julia + +# Overriding println, quite amazing the line buffering doesn't seem to activate +# when directing to file... +# https://github.com/JuliaLang/julia/issues/13954 +# https://discourse.julialang.org/t/issues-with-println-buffering-output-when-redirecting-stdout-to-a-txt-file/23738/3 +@eval Base println(xs...) = begin res = println(stdout::IO, xs...); flush(stdout); return res end; +@eval Base println(io::IO) = begin res = print(io, '\n'); flush(io); return res end; + +using Dates +using Pkg +using Printf + +env_name = "WAVI_ENV" in keys(ENV) ? ENV["WAVI_ENV"] : "wavi_test" +Pkg.activate(joinpath("envs", env_name)) + +cwd = pwd() +@printf("\nCurrent working directory: %s\n", cwd) +@printf("\nStart time: %s\n", Dates.now()) +@printf("\nJulia args: %s\n", join(Base.ARGS, ", ")) + +#include everything the code directory +for f in readdir("../code") + if f[end-2:end] == ".jl" + println("included code: ", f) + include(string("../code/", f)) + end +end + +#run the driver script +include("../run/driver.jl") +driver() + +@printf("\nEnd time: %s\n", Dates.now()) \ No newline at end of file diff --git a/cases/template_jasmin/scripts/run_ensemble_member b/cases/template_jasmin/scripts/run_ensemble_member new file mode 100755 index 0000000..62351c7 --- /dev/null +++ b/cases/template_jasmin/scripts/run_ensemble_member @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +#SBATCH --output={{ run.dir }}/job.%j.%N.out +#SBATCH --error={{ run.dir }}/job.%j.%N.err +#SBATCH --chdir={{ run.dir }} +#SBATCH --mail-type=begin,end,fail,requeue +#SBATCH --mail-user={{ run.email }} +#SBATCH --job-name={{ run.id }} +#SBATCH --nodes={{ run.nodes }} +#SBATCH --partition={{ run.cluster }} +#SBATCH --qos={{ run.cluster }} +#SBATCH --account=dit +#SBATCH --cpus-per-task={{ run.ntasks }} +#SBATCH --mem=8gb +#SBATCH --signal=B:SIGINT@60 + +# TODO: this logic is better in the ensemble run items, only to align to +# previous approach have I ended up implementing it here, but it's growing :( + +CWD="`realpath .`" +NAME="basename $CWD" +RUNDIR="run" +OUTFILE="outfile.nc" + +cd $RUNDIR + +export PATH="`realpath ..`/scripts:$PATH" + +if [ ! -f driver.jl ]; then + prep_run +else + if ! ls PChkpt_*.jld2 1> /dev/null 2>&1 ; then + echo 'job chain: fail, no pickup files' + exit 1 + fi + + for file in PChkpt_*.jld2; do + [[ $file -nt $PICKUP_FILE ]] && PICKUP_FILE=$file + done + # Extract the middle bit of this filename + PICKUP=${PICKUP_FILE#PChkpt_} + PICKUP=${PICKUP%.jld2} + + # TODO: no check to see if this will actually work + re='^[0-9]+$' + if [[ $PICKUP =~ $re ]]; then + echo 'job chain: pickup from permanent checkpoint' + + # Save the timestep, with any leading zeros removed + CURITER=$( egrep '^niter0' driver.jl | sed -r 's/^niter0 = ([0-9]+)/\1/' ) + NITER0=$(echo $PICKUP | sed 's/^0*//') + + if [ "$CURITER" -eq "$NITER0" ] && [ ! -f $OUTFILE ]; then + echo "No progress was made in the last run, we'll assume failure or another + termination scenario. Please investigate or contact devs if necessary. + We'll be placing an $OUTFILE to prevent further submissions." | mail -s "Failure notification for $NAME " {{ run.email }} + # Touch an output file as a marker + touch $OUTFILE + exit 1 + fi + else + echo 'job chain: fail, problem w pickup (WAVI only currently supports pCkpt pickups)' $PICKUP + exit 1 + fi + + #edit the driver namelist: replace the first instance of niter0 = * with appropriate checkpoint number + NITER0_LINE="niter0 = $NITER0" + echo $NITER0_LINE + sed -i '0,/.*niter0.*/s//'"$NITER0_LINE"'/' driver.jl +fi + +run_driver >proc.${PICKUP:-0}.log 2>&1 + +exit $? diff --git a/cases/template_jasmin/scripts/run_ensemble_member.j2 b/cases/template_jasmin/scripts/run_ensemble_member.j2 new file mode 100755 index 0000000..62351c7 --- /dev/null +++ b/cases/template_jasmin/scripts/run_ensemble_member.j2 @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +#SBATCH --output={{ run.dir }}/job.%j.%N.out +#SBATCH --error={{ run.dir }}/job.%j.%N.err +#SBATCH --chdir={{ run.dir }} +#SBATCH --mail-type=begin,end,fail,requeue +#SBATCH --mail-user={{ run.email }} +#SBATCH --job-name={{ run.id }} +#SBATCH --nodes={{ run.nodes }} +#SBATCH --partition={{ run.cluster }} +#SBATCH --qos={{ run.cluster }} +#SBATCH --account=dit +#SBATCH --cpus-per-task={{ run.ntasks }} +#SBATCH --mem=8gb +#SBATCH --signal=B:SIGINT@60 + +# TODO: this logic is better in the ensemble run items, only to align to +# previous approach have I ended up implementing it here, but it's growing :( + +CWD="`realpath .`" +NAME="basename $CWD" +RUNDIR="run" +OUTFILE="outfile.nc" + +cd $RUNDIR + +export PATH="`realpath ..`/scripts:$PATH" + +if [ ! -f driver.jl ]; then + prep_run +else + if ! ls PChkpt_*.jld2 1> /dev/null 2>&1 ; then + echo 'job chain: fail, no pickup files' + exit 1 + fi + + for file in PChkpt_*.jld2; do + [[ $file -nt $PICKUP_FILE ]] && PICKUP_FILE=$file + done + # Extract the middle bit of this filename + PICKUP=${PICKUP_FILE#PChkpt_} + PICKUP=${PICKUP%.jld2} + + # TODO: no check to see if this will actually work + re='^[0-9]+$' + if [[ $PICKUP =~ $re ]]; then + echo 'job chain: pickup from permanent checkpoint' + + # Save the timestep, with any leading zeros removed + CURITER=$( egrep '^niter0' driver.jl | sed -r 's/^niter0 = ([0-9]+)/\1/' ) + NITER0=$(echo $PICKUP | sed 's/^0*//') + + if [ "$CURITER" -eq "$NITER0" ] && [ ! -f $OUTFILE ]; then + echo "No progress was made in the last run, we'll assume failure or another + termination scenario. Please investigate or contact devs if necessary. + We'll be placing an $OUTFILE to prevent further submissions." | mail -s "Failure notification for $NAME " {{ run.email }} + # Touch an output file as a marker + touch $OUTFILE + exit 1 + fi + else + echo 'job chain: fail, problem w pickup (WAVI only currently supports pCkpt pickups)' $PICKUP + exit 1 + fi + + #edit the driver namelist: replace the first instance of niter0 = * with appropriate checkpoint number + NITER0_LINE="niter0 = $NITER0" + echo $NITER0_LINE + sed -i '0,/.*niter0.*/s//'"$NITER0_LINE"'/' driver.jl +fi + +run_driver >proc.${PICKUP:-0}.log 2>&1 + +exit $? diff --git a/scripts/BAS/install_julia.sh b/scripts/BAS/install_julia.sh new file mode 100755 index 0000000..f79b955 --- /dev/null +++ b/scripts/BAS/install_julia.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -eo pipefail + +JULIAURL=${1:-"https://julialang-s3.julialang.org/bin/linux/x64/1.11/julia-1.11.5-linux-x86_64.tar.gz"} +JULIAFILENAME=`echo $JULIAURL | sed -r 's/^.+\/([^\/]+)$/\1/'` +JULIAVERSION=`echo $JULIAFILENAME | sed -E 's/-linux.*//'` + +# Make sure you update the GWS location here +WORKDIR="/data/hpcdata/users/$USER" +BIN_DIR=$WORKDIR/$JULIAVERSION/bin + +cd $WORKDIR + +# Dowload and unpack Julia +echo "Downloading julia, if required" +[ ! -f $JULIAFILENAME ] && wget -nv $JULIAURL -O $JULIAFILENAME + +echo "Unpacking $JULIAFILENAME, this might take a while" +tar xvzf $JULIAFILENAME >/dev/null + +# Cleaning up tar file +echo "Cleaning up tar file" +rm $JULIAFILENAME + +# Add Julia to PATH (example for bash) +echo 'export PATH="$BIN_DIR:$PATH"' >> ~/.bashrc +source ~/.bashrc + +echo "Julia now installed, you should be able to run it" diff --git a/scripts/jasmin/post_batch.sh b/scripts/jasmin/post_batch.sh new file mode 120000 index 0000000..705c2cc --- /dev/null +++ b/scripts/jasmin/post_batch.sh @@ -0,0 +1 @@ +../BAS/post_batch.sh \ No newline at end of file diff --git a/scripts/jasmin/pre_run.sh b/scripts/jasmin/pre_run.sh new file mode 120000 index 0000000..8ca3fd7 --- /dev/null +++ b/scripts/jasmin/pre_run.sh @@ -0,0 +1 @@ +../BAS/pre_run.sh \ No newline at end of file diff --git a/scripts/jasmin/wavi_create_case b/scripts/jasmin/wavi_create_case new file mode 120000 index 0000000..9ce73fb --- /dev/null +++ b/scripts/jasmin/wavi_create_case @@ -0,0 +1 @@ +../local/wavi_create_case \ No newline at end of file diff --git a/scripts/jasmin/wavi_ensemble b/scripts/jasmin/wavi_ensemble new file mode 120000 index 0000000..19fe207 --- /dev/null +++ b/scripts/jasmin/wavi_ensemble @@ -0,0 +1 @@ +../local/wavi_ensemble \ No newline at end of file diff --git a/scripts/jasmin/wavi_execute b/scripts/jasmin/wavi_execute new file mode 120000 index 0000000..3844686 --- /dev/null +++ b/scripts/jasmin/wavi_execute @@ -0,0 +1 @@ +../local/wavi_execute \ No newline at end of file diff --git a/scripts/jasmin/wavi_install b/scripts/jasmin/wavi_install new file mode 120000 index 0000000..c0cf837 --- /dev/null +++ b/scripts/jasmin/wavi_install @@ -0,0 +1 @@ +../local/wavi_install \ No newline at end of file