Maxwell : Running jupyter notebooks in batch

#!/bin/bash
#SBATCH --partition=maxcpu
unset LD_PRELOAD

if [[ $# -ne 3 ]]; then
    echo "Usage: batch_notebook <path to input notebook> <path to output notebook> <kernel>"
    exit 1
fi

OUT=$2
export INPUT_NB=$1
export OUTPUT_DIR=$(dirname $OUT)
export OUTPUT_NB=$OUT
export kernel=$3

if [[ ! -e "$INPUT_NB" ]]; then
    echo "Input notebook doesn't exist: $INPUT_NB"
    exit 1
fi

source /etc/profile.d/modules.sh

module load maxwell conda

mkdir -p $OUTPUT_DIR
python3 nbrun.py 

import nbformat
import os

input_nb=os.getenv('INPUT_NB')
output_nb=os.getenv('OUTPUT_NB')
output_dir=os.getenv('OUTPUT_DIR')
kernel=os.getenv('kernel')

from nbconvert.preprocessors import ExecutePreprocessor

nb = nbformat.read(open(input_nb), as_version=4)

ep = ExecutePreprocessor(timeout=600, kernel_name=kernel)

try:
    out = ep.preprocess(nb, {'metadata': {'path': output_dir}})
except CellExecutionError:
    msg = 'Error executing the notebook "%s".\n\n' % input_nb
    msg += 'See notebook "%s" for the traceback.' % output_nb
    print(msg)
    raise
finally:
    nbformat.write(nb, open(output_nb, mode='wt'))
    msg = '\nOutput notebook created as %s \n\n' %output_nb
    print(msg)

srun ./batch_notebook.sh ~/full-example.ipynb ~/notebooks/test_1.ipynb python3 \
     && jupyter notebook --config=~/.jupyter/jupyter_notebook_standalone.py ~/notebooks/test_1.ipynb 

# remarks:
# srun batch_notebook.sh      execute the script as a batch-job
# ~/full-example.ipynb        the input notebook
# ~/notebooks/test_1.ipynb    the output notebook
# python3                     the kernel to execute the notebook

# && jupyter notebook         launch a local notebook server if the previous was successful, and that will open it in a browser
# --config=                   a standalone jupyter config
# ~/notebooks/test_1.ipynb    the freshly generated notebook output
# 
# to run in real batch:
sbatch ./batch_notebook.sh ~/full-example.ipynb ~/notebooks/test_1.ipynb python3