diff --git a/README.md b/README.md index d1f44ab..f0238a1 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,11 @@ bash start.sh sherlock/containershare-notebook docker://vanessa/repo2docker-juli ``` If you would like to request a custom notebook, please [reach out](https://www.github.com/vsoch/containershare/issues). +See [examples](examples) for Nero and Sherlock in the [examples](examples) folder. -## Usage +## More Examples +In addition to [examples](examples), here are some quick command examples: ```bash # To start a jupyter notebook in a specific directory ON the cluster resource diff --git a/end.sh b/end.sh index 3398723..c2aed69 100755 --- a/end.sh +++ b/end.sh @@ -22,5 +22,7 @@ NAME=$1 echo "Killing $NAME slurm job on ${RESOURCE}" ssh ${RESOURCE} "squeue --name=$NAME --user=$FORWARD_USERNAME -o '%A' -h | xargs --no-run-if-empty /usr/bin/scancel" -echo "Killing listeners on ${RESOURCE}" -ssh ${RESOURCE} "/usr/sbin/lsof -i :$PORT -t | xargs --no-run-if-empty kill" +if [[ "${RESOURCE}" != "nero" ]]; then + echo "Killing listeners on ${RESOURCE}" + ssh ${RESOURCE} "/usr/sbin/lsof -i :$PORT -t | xargs --no-run-if-empty kill" +fi diff --git a/examples/nero-jupyter.png b/examples/nero-jupyter.png new file mode 100644 index 0000000..91f84de Binary files /dev/null and b/examples/nero-jupyter.png differ diff --git a/examples/recipe-nero-gpu.md b/examples/recipe-nero-gpu.md new file mode 100644 index 0000000..fce4714 --- /dev/null +++ b/examples/recipe-nero-gpu.md @@ -0,0 +1,87 @@ +# Jupyter with GPU on Nero + +## Nero + +If you want to run forward on Nero, you can first generate the login credential for +your ssh configuration file by doing: + +```bash +/bin/bash hosts/nero_ssh.sh +``` +It will ask you for your username, and then give a snippet of code to copy into your +`~/.ssh/config` file. It should prompt you for your password and two step the first time, +and then work to login with just `ssh nero`. On Nero, you will also need to ensure that +you have the nbserverproxy module installed. To install packages from pypi, you'll +need this pip settings file exists in your $HOME folder: + +```bash +~/.pip/pip.conf +``` + +Within in you need to define this mirror: + +``` +[global] +index-url = https://nero-mirror.stanford.edu/pypi/simple +``` + +And then install the needed Python3 software for the notebook: + +```bash +pip3 install nbserverproxy --user +``` + +Now let's generate our params.sh file that will specify using nero, and wanting a GPU. +Do this via [setup.sh](setup.sh) to generate a params.sh file that looks like: + +``` +FORWARD_USERNAME="vsochat" +PORT="49153" +PARTITION="gpu" +RESOURCE="nero" +MEM="20G" +TIME="8:00:00" +CONTAINERSHARE="/scratch/users/vsochat/share" +GRES="gpu:1" +``` + +You can edit the defaults (e.g., time and memory) to be what fits your needs. +Now let's run the script to launch our node! By default the notebook home +will be our home directory, and we will load anaconda 3 and launch a notebook +on the port specified. + +```bash +bash start.sh nero/py3-jupyter +``` + +You'll see information and instructions stream to the screen. + +``` + +== View logs in separate terminal == +ssh nero cat /home/vsochat/forward-util/py3-jupyter.sbatch.out +ssh nero cat /home/vsochat/forward-util/py3-jupyter.sbatch.err + +== Instructions == +1. Password, output, and error printed to this terminal? Look at logs (see instruction above) +2. Browser: http://sh-02-21.int:8888/ -> http://localhost:8888/... +3. To end session: bash end.sh nero/py3-jupyter +``` + +In the example above, we are forwarding port 8888 from our local machine. We would +then open this up to see: + +![nero-jupyter.png](nero-jupyter.png) + +Remember that we need a password! This is actually echoed into the error stream, so +we would issue the following command to get it: + +```bash +ssh nero cat /home/vsochat/forward-util/py3-jupyter.sbatch.err +``` + +When you are ready to clean up: + +```bash +$ bash end.sh nero/py3-jupyter +``` diff --git a/helpers.sh b/helpers.sh index dcb1f32..e3fda58 100644 --- a/helpers.sh +++ b/helpers.sh @@ -96,14 +96,17 @@ function get_machine() { done - echo $MACHINE MACHINE="`ssh ${RESOURCE} squeue --name=$NAME --user=$FORWARD_USERNAME -o "%R" -h`" + # Nero must have lookup based on ip addres + if [[ ${RESOURCE} == "nero" ]]; then + MACHINE=$(ssh ${RESOURCE} scontrol show node $MACHINE | grep NodeAddr | cut -d' ' -f4 | cut -d'=' -f2) + fi echo $MACHINE # If we didn't get a node... - if [[ "$MACHINE" != "sh"* ]] + if [[ "$MACHINE" != "sh"* ]] && [[ "${RESOURCE}" != "nero"* ]] then - echo "Tried ${ATTEMPTS} attempts!" 1>&2 + echo "Tried ${ATTEMPT} attempts!" 1>&2 exit 1 fi } @@ -137,7 +140,11 @@ function setup_port_forwarding() { echo echo "== Setting up port forwarding ==" sleep 5 - echo "ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N $MACHINE &" - ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N "$MACHINE" & - + if [[ "${RESOURCE}" == "nero" ]]; then + echo "ssh -N -L${PORT}:${MACHINE}:${PORT} nero &" + ssh -N -L${PORT}:${MACHINE}:${PORT} nero & + else + echo "ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N $MACHINE &" + ssh -L $PORT:localhost:$PORT ${RESOURCE} ssh -L $PORT:localhost:$PORT -N "$MACHINE" & + fi } diff --git a/hosts/nero_ssh.sh b/hosts/nero_ssh.sh new file mode 100644 index 0000000..ebbfa82 --- /dev/null +++ b/hosts/nero_ssh.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Nero at Stanford +# Prints an ssh configuration for the user, selecting a login node at random +# Sample usage: bash nero_ssh.sh +echo +read -p "Nero username > " FORWARD_USERNAME + +# Randomly select login node from 1..4 +LOGIN_NODE=$((1 + RANDOM % 8)) + +echo "Host nero + User ${FORWARD_USERNAME} + Hostname nero.compute.stanford.edu + GSSAPIDelegateCredentials yes + GSSAPIAuthentication yes + ControlMaster auto + ControlPersist yes + ControlPath ~/.ssh/%C" diff --git a/sbatches/nero/py3-jupyter.sbatch b/sbatches/nero/py3-jupyter.sbatch new file mode 100755 index 0000000..1185356 --- /dev/null +++ b/sbatches/nero/py3-jupyter.sbatch @@ -0,0 +1,11 @@ +#!/bin/bash + +PORT=$1 +NOTEBOOK_DIR=${2:-${HOME}} +cd $NOTEBOOK_DIR + +export PATH=/share/sw/open/anaconda/3/bin:$PATH + +pip3 install nbserverproxy --user +cat /etc/hosts +jupyter lab --ip=0.0.0.0 --port=${PORT} diff --git a/setup.sh b/setup.sh index 729cd91..6c79064 100755 --- a/setup.sh +++ b/setup.sh @@ -27,6 +27,15 @@ echo read -p "${RESOURCE} partition (default: normal) > " PARTITION PARTITION=${PARTITION:-normal} +echo +echo "If you are using a gpu partition, you might want to define the next setting, +--gres to be something like gpu:1 (e.g., setting gpu:1 for this next variable +will be adding the flag --gres=gpu:1. Otherwise, leave blank to leave this unset" +echo +read -p "${RESOURCE} gres (default: unset) > " GRES +GRES=${GRES:-} + + echo SHARE="/scratch/users/vsochat/share" echo "A containershare (https://vsoch.github.io/containershare is a library of @@ -43,7 +52,7 @@ MEM=20G TIME=8:00:00 -for var in FORWARD_USERNAME PORT PARTITION RESOURCE MEM TIME CONTAINERSHARE +for var in FORWARD_USERNAME PORT PARTITION RESOURCE MEM TIME CONTAINERSHARE GRES do echo "$var="'"'"$(eval echo '$'"$var")"'"' done >> params.sh diff --git a/start-node.sh b/start-node.sh index 13e4463..c67ae1b 100755 --- a/start-node.sh +++ b/start-node.sh @@ -54,11 +54,16 @@ SBATCH_NAME=$(basename $SBATCH) command="sbatch --job-name=$NAME --partition=$PARTITION - --output=$RESOURCE_HOME/forward-util/$NAME.out - --error=$RESOURCE_HOME/forward-util/$NAME.err + --output=$RESOURCE_HOME/forward-util/$SBATCH_NAME.out + --error=$RESOURCE_HOME/forward-util/$SBATCH_NAME.err --mem=$MEM - --time=$TIME - $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\"" + --time=$TIME" + +# If we want a gres +if [[ "${GRES}" != "" ]]; then + command="${command} --gres ${GRES}" +fi +command="${command} $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\"" echo ${command} ssh ${RESOURCE} ${command} diff --git a/start.sh b/start.sh index 31cffb6..c9aa82f 100755 --- a/start.sh +++ b/start.sh @@ -44,6 +44,11 @@ echo echo "== Uploading sbatch script ==" scp $FORWARD_SCRIPT ${RESOURCE}:$RESOURCE_HOME/forward-util/ +if [[ "$?" != "0" ]]; then + echo "Issue uploading script with scp, trying sftp..." + echo "put ${FORWARD_SCRIPT}" | sftp ${FORWARD_USERNAME}@${RESOURCE}:${RESOURCE_HOME}/forward-util/ +fi + # adjust PARTITION if necessary set_partition echo @@ -57,8 +62,13 @@ command="sbatch --output=$RESOURCE_HOME/forward-util/$SBATCH_NAME.out --error=$RESOURCE_HOME/forward-util/$SBATCH_NAME.err --mem=$MEM - --time=$TIME - $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\"" + --time=$TIME" + +# If we want a gres +if [[ "${GRES}" != "" ]]; then + command="${command} --gres ${GRES}" +fi +command="${command} $RESOURCE_HOME/forward-util/$SBATCH_NAME $PORT \"${@:2}\"" echo ${command} ssh ${RESOURCE} ${command}