adding nero example

README.md

@@ -134,9 +134,11 @@ bash start.sh sherlock/containershare-notebook docker://vanessa/repo2docker-juli
 ```
 
 If you would like to request a custom notebook, please [reach out](https://www.github.com/vsoch/containershare/issues).
+See [examples](examples) for Nero and Sherlock in the [examples](examples) folder.
 
-## Usage
+## More Examples
 
+In addition to [examples](examples), here are some quick command examples:
 
 ```bash
 # To start a jupyter notebook in a specific directory ON the cluster resource

end.sh

@@ -22,5 +22,7 @@ NAME=$1
 echo "Killing $NAME slurm job on ${RESOURCE}"
 ssh ${RESOURCE} "squeue --name=$NAME --user=$FORWARD_USERNAME -o '%A' -h | xargs --no-run-if-empty /usr/bin/scancel"
 
-echo "Killing listeners on ${RESOURCE}"
-ssh ${RESOURCE} "/usr/sbin/lsof -i :$PORT -t | xargs --no-run-if-empty kill"
+if [[ "${RESOURCE}" != "nero" ]]; then
+    echo "Killing listeners on ${RESOURCE}"
+    ssh ${RESOURCE} "/usr/sbin/lsof -i :$PORT -t | xargs --no-run-if-empty kill"
+fi

examples/recipe-nero-gpu.md

@@ -0,0 +1,87 @@
+# Jupyter with GPU on Nero
+
+## Nero
+
+If you want to run forward on Nero, you can first generate the login credential for
+your ssh configuration file by doing:
+
+```bash
+/bin/bash hosts/nero_ssh.sh
+```
+It will ask you for your username, and then give a snippet of code to copy into your
+`~/.ssh/config` file. It should prompt you for your password and two step the first time,
+and then work to login with just `ssh nero`. On Nero, you will also need to ensure that 
+you have the nbserverproxy module installed. To install packages from pypi, you'll
+need this pip settings file exists in your $HOME folder:
+
+```bash
+~/.pip/pip.conf
+```
+
+Within in you need to define this mirror:
+
+```
+[global]
+index-url = https://nero-mirror.stanford.edu/pypi/simple
+```
+
+And then install the needed Python3 software for the notebook:
+
+```bash
+pip3 install nbserverproxy --user
+```
+
+Now let's generate our params.sh file that will specify using nero, and wanting a GPU.
+Do this via [setup.sh](setup.sh) to generate a params.sh file that looks like:
+
+```
+FORWARD_USERNAME="vsochat"
+PORT="49153"
+PARTITION="gpu"
+RESOURCE="nero"
+MEM="20G"
+TIME="8:00:00"
+CONTAINERSHARE="/scratch/users/vsochat/share"
+GRES="gpu:1"
+```
+
+You can edit the defaults (e.g., time and memory) to be what fits your needs.
+Now let's run the script to launch our node! By default the notebook home
+will be our home directory, and we will load anaconda 3 and launch a notebook
+on the port specified.
+
+```bash
+bash start.sh nero/py3-jupyter
+```
+
+You'll see information and instructions stream to the screen.
+
+```
+
+== View logs in separate terminal ==
+ssh nero cat /home/vsochat/forward-util/py3-jupyter.sbatch.out
+ssh nero cat /home/vsochat/forward-util/py3-jupyter.sbatch.err
+
+== Instructions ==
+1. Password, output, and error printed to this terminal? Look at logs (see instruction above)
+2. Browser: http://sh-02-21.int:8888/ -> http://localhost:8888/...
+3. To end session: bash end.sh nero/py3-jupyter
+```
+
+In the example above, we are forwarding port 8888 from our local machine. We would
+then open this up to see:
+
+![nero-jupyter.png](nero-jupyter.png)
+
+Remember that we need a password! This is actually echoed into the error stream, so
+we would issue the following command to get it:
+
+```bash
+ssh nero cat /home/vsochat/forward-util/py3-jupyter.sbatch.err
+```
+
+When you are ready to clean up:
+
+```bash
+$ bash end.sh nero/py3-jupyter
+```

helpers.sh

@@ -96,14 +96,17 @@ function get_machine() {
 
     done
 
-    echo $MACHINE
     MACHINE="`ssh ${RESOURCE} squeue --name=$NAME --user=$FORWARD_USERNAME -o "%R" -h`"
+    # Nero must have lookup based on ip addres
+    if [[ ${RESOURCE} == "nero" ]]; then
+        MACHINE=$(ssh ${RESOURCE} scontrol show node $MACHINE | grep NodeAddr | cut -d' ' -f4 | cut -d'=' -f2)
+    fi
     echo $MACHINE
 
     # If we didn't get a node...
-    if [[ "$MACHINE" != "sh"* ]]
+    if [[ "$MACHINE" != "sh"* ]] && [[ "${RESOURCE}" != "nero"* ]]
         then
-        echo "Tried ${ATTEMPTS} attempts!"  1>&2
+        echo "Tried ${ATTEMPT} attempts!"  1>&2
         exit 1
     fi
 }
@@ -137,7 +140,11 @@ function setup_port_forwarding() {
     echo
     echo "== Setting up port forwarding =="
     sleep 5
-    echo "ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N $MACHINE &"
-    ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N "$MACHINE" &
-
+    if [[ "${RESOURCE}" == "nero" ]]; then
+        echo "ssh -N -L${PORT}:${MACHINE}:${PORT} nero  &"
+        ssh -N -L${PORT}:${MACHINE}:${PORT} nero  &
+    else
+        echo "ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N $MACHINE &"
+        ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N "$MACHINE" &
+    fi
 }

hosts/nero_ssh.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Nero at Stanford
+# Prints an ssh configuration for the user, selecting a login node at random
+# Sample usage: bash nero_ssh.sh
+echo
+read -p "Nero username > "  FORWARD_USERNAME
+
+# Randomly select login node from 1..4
+LOGIN_NODE=$((1 + RANDOM % 8))
+
+echo "Host nero
+    User ${FORWARD_USERNAME}
+    Hostname nero.compute.stanford.edu
+    GSSAPIDelegateCredentials yes
+    GSSAPIAuthentication yes
+    ControlMaster auto
+    ControlPersist yes
+    ControlPath ~/.ssh/%C"

sbatches/nero/py3-jupyter.sbatch

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+PORT=$1
+NOTEBOOK_DIR=${2:-${HOME}}
+cd $NOTEBOOK_DIR
+
+export PATH=/share/sw/open/anaconda/3/bin:$PATH
+
+pip3 install nbserverproxy --user
+cat /etc/hosts
+jupyter lab --ip=0.0.0.0 --port=${PORT}

setup.sh

@@ -27,6 +27,15 @@ echo
 read -p "${RESOURCE} partition (default: normal) > "  PARTITION
 PARTITION=${PARTITION:-normal}
 
+echo
+echo "If you are using a gpu partition, you might want to define the next setting,
+--gres to be something like gpu:1 (e.g., setting gpu:1 for this next variable
+will be adding the flag --gres=gpu:1. Otherwise, leave blank to leave this unset"
+echo
+read -p "${RESOURCE} gres (default: unset) > "  GRES
+GRES=${GRES:-}
+
+
 echo
 SHARE="/scratch/users/vsochat/share"
 echo "A containershare (https://vsoch.github.io/containershare is a library of
@@ -43,7 +52,7 @@ MEM=20G
 
 TIME=8:00:00
 
-for var in FORWARD_USERNAME PORT PARTITION RESOURCE MEM TIME CONTAINERSHARE
+for var in FORWARD_USERNAME PORT PARTITION RESOURCE MEM TIME CONTAINERSHARE GRES
 do
     echo "$var="'"'"$(eval echo '$'"$var")"'"'
 done >> params.sh

start-node.sh

@@ -54,11 +54,16 @@ SBATCH_NAME=$(basename $SBATCH)
 command="sbatch
     --job-name=$NAME
     --partition=$PARTITION
-    --output=$RESOURCE_HOME/forward-util/$NAME.out
-    --error=$RESOURCE_HOME/forward-util/$NAME.err
+    --output=$RESOURCE_HOME/forward-util/$SBATCH_NAME.out
+    --error=$RESOURCE_HOME/forward-util/$SBATCH_NAME.err
     --mem=$MEM
-    --time=$TIME
-    $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\""
+    --time=$TIME"
+
+# If we want a gres
+if [[ "${GRES}" != "" ]]; then
+    command="${command} --gres ${GRES}"
+fi
+command="${command} $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\""
 
 echo ${command}
 ssh ${RESOURCE} ${command}

start.sh

@@ -44,6 +44,11 @@ echo
 echo "== Uploading sbatch script =="
 scp $FORWARD_SCRIPT ${RESOURCE}:$RESOURCE_HOME/forward-util/
 
+if [[ "$?" != "0" ]]; then
+    echo "Issue uploading script with scp, trying sftp..."
+    echo "put ${FORWARD_SCRIPT}" | sftp ${FORWARD_USERNAME}@${RESOURCE}:${RESOURCE_HOME}/forward-util/
+fi
+
 # adjust PARTITION if necessary
 set_partition
 echo
@@ -57,8 +62,13 @@ command="sbatch
     --output=$RESOURCE_HOME/forward-util/$SBATCH_NAME.out
     --error=$RESOURCE_HOME/forward-util/$SBATCH_NAME.err
     --mem=$MEM
-    --time=$TIME
-    $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\""
+    --time=$TIME"
+
+# If we want a gres
+if [[ "${GRES}" != "" ]]; then
+    command="${command} --gres ${GRES}"
+fi
+command="${command} $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\""
 
 echo ${command}
 ssh ${RESOURCE} ${command}