% -*- coding: utf-8 -*-
% 
% Copyright 2007 by Pär Andersson
% National Supercomputer Centre
% Linköpings universitet
% 
% http://www.nsc.liu.se/~paran/
% 
% Licensed by Creative Commons Attribution-ShareAlike 3.0
% http://creativecommons.org/licenses/by/3.0/

\documentclass{beamer}

\mode<presentation>
{
  \usetheme{Darmstadt}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{times}
\usepackage[T1]{fontenc}

% Used to show console commands
\usepackage{alltt}

\usepackage{color}

\title
{SLURM\\Simple Linux Utility for Resource Management}
\author{Pär Andersson}

\institute{National Supercomputer Centre \\
Linköpings universitet}

\date{2009-05-28 / HEPiX Spring 2009}

\subject{Talks}

% If you have a file called "university-logo-filename.xxx", where xxx
% is a graphic format that can be processed by latex or pdflatex,
% resp., then you can add a logo as follows:

% \pgfdeclareimage[height=0.5cm]{university-logo}{university-logo-filename}
% \logo{\pgfuseimage{university-logo}}
% FIXME: Lägg till NSC-logga

% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:
%\AtBeginSubsection[]
%{
%  \begin{frame}<beamer>{Outline}
%    \tableofcontents[currentsection,currentsubsection]
%  \end{frame}
%}


\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\section{SLURM}

\subsection{Introduction}

% What is Resource Management?
% What type of resources does SLURM manage
\begin{frame}{Resource Management?}
  Manage resources within a single cluster
  \begin{itemize}
  \item Nodes
    \begin{itemize}
      \item state (up/down/idle/allocated \dots)
      \item access
    \end{itemize}
  \item Jobs
    \begin{itemize}
      \item queued and running
      \item start/stop jobs
      \item job scheduling
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Introduction}
  \begin{itemize}
    \item Designed and built for scalability
      \begin{itemize}
      \item BlueGene/P @ LLNL 147,456 processors
      \end{itemize}
     \item Easy to install and manage
     \item Easy to use
     \item Very stable
  \end{itemize}
\end{frame}

\begin{frame}{Homepage and development}
  \begin{itemize}
  \item Laurence Livermore National Lab\\
    https://computing.llnl.gov/linux/slurm/
  \item Well supported and actively developed
  \item slurm-dev@lists.llnl.gov
    \begin{itemize}
    \item High SNR
    \item Fast replies
    \item Patches welcome
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Source code}
  \begin{itemize}
  \item GPL licensed
    \begin{itemize}
      \item readable source code
      \item modular design % Mention select/bluegene
    \end{itemize}
  \item \textcolor{red}{No public version control}
  \item \textcolor{red}{No road-map}
    \begin{itemize}
    \item \textcolor{green}{Will be fixed}
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{SLURM support}
  \begin{itemize}
  \item HP - ``XC Cluster''
  \item IBM - ``IBM HPC Open Software Stack''
  \item Sun - ``Sun HPC Software, Linux Edition''
  \end{itemize}
  Also packaged in Debian/Ubuntu as \begin{tt}slurm-llnl\end{tt}
\end{frame}


\begin{frame}{SLURM at NSC 2007}
  2007, we were buying a new 800+ node cluster.
  \begin{itemize}
  \item Was using Torque
    \begin{itemize}
    \item Scalability issues with older 200 node cluster
    \item Bugs
    \end{itemize}
  \item Time to see if there were something better
  \item Found SLURM
  \end{itemize}
\end{frame}

\begin{frame}{SLURM at NSC 2009}
  \begin{tabular}{ l r l l }
    cluster & nodes & SLURM version & Scheduler\\
    \hline
    neolith & 805 & 1.3 & Moab\\
    bore    & 56  & 1.3 & sched/backfill\\
    gimle   & 84  & 1.3 & Moab\\
    vagn    &  6  & 1.3 & Moab\\
  \end{tabular}
\end{frame}

\subsection{Scheduling}

\begin{frame}{Job scheduling}
  \begin{itemize}
  \item Built-in
    \begin{itemize}
    \item sched/builtin - FIFO
    \item sched/backfill - FIFO+backfill
    \item sched/gang - time slicing
    \end{itemize}

  \item SLURM 1.3 needs an external scheduler for advanced job
    prioritization.
    
  \item External schedulers
    \begin{itemize}
    \item sched/wiki - Maui
    \item sched/wiki2 - Moab
      \begin{itemize}
      \item \color{red}{Have caused many problems}
      \item \color{red}{Often bugs in Moab}
      \end{itemize}
    \end{itemize}    
  \end{itemize}
\end{frame}

\begin{frame}{Job scheduling}
  \begin{itemize}
  \item Much improved in SLURM 2.0
    \begin{itemize}
    \item Reservations
    \item priority/multifactor
      \begin{itemize}
      \item Age, Fair-share, Job size, Partition, QoS
      \end{itemize}
    \item Hierarchical Fair-Share
      \begin{itemize}
      \item Accounts, sub accounts
      \item Shares
      \end{itemize}
    \end{itemize}
  \end{itemize}
\end{frame}
    
\subsection{Jobs}

\begin{frame}{What is a job?}
  \begin{itemize}
  \item A job allocation is a set of resources (nodes/cores)
    available to a user for a specified time
  \item Programs started as ``job steps''
  \item Batch scripts just a common special case
  \end{itemize}
\end{frame}

\begin{frame}{Runnig jobs}
  \begin{itemize}
  \item srun - Run a job step, if necessary create allocation first
  \item salloc - Obtain allocation, run command (on current host), release allocation
  \item sbatch - Submit a batch script
  \end{itemize}
\end{frame}

\begin{frame}{srun}  
  \begin{alltt}\small
\ [paran@d2 \~{}]\$ srun -N 2 hostname\\
\ n799\\
\ n798\\
  \end{alltt}
\end{frame}     

\begin{frame}{salloc}  
  \begin{alltt}\small
\ [paran@d2 \~{}]\$ salloc -N2 \\
\ salloc: Granted job allocation 27 \\
\ [paran@d2 \~{}]\$ echo \$SLURM\_NODELIST\ \\
\ n[798-799] \\

\ [paran@d2 \~{}]\$ srun hostname \\
\ n799 \\
\ n798 \\

\ [paran@d2 \~{}]\$ exit \\
\ exit \\
\ salloc: Relinquishing job allocation 27 \\
  \end{alltt}
\end{frame}     

\begin{frame}{sbatch}  
  \begin{alltt}\small
\ [paran@d2 \~{}]\$ cat testjob.sh \\
\ \#!/bin/sh \\
\ \#SBATCH ---nodes 2 \\
\ echo ``Script running on: \$(hostname), allocation: ''$\backslash$\\
\ ``\$SLURM\_NODELIST'' \\

\ [paran@d2 \~{}]\$ sbatch testjob.sh \\
\ sbatch: Submitted batch job 28 \\

\ [paran@d2 \~{}]\$ cat slurm-28.out \\
\ Script running on: n798, allocation: n[798-799] \\
  \end{alltt}
\end{frame}     

\subsection{Daemons}

\begin{frame}{SLURM daemons}
  \begin{itemize}
  \item slurmctld
    \begin{itemize}
    \item Central management daemon
    \item Master/Slave
    \end{itemize}
    
  \item slurmdbd (optional)
    \begin{itemize}
    \item Accounting database system
    \end{itemize}
    
  \item slurmd
    \begin{itemize}
    \item On every compute node
    \end{itemize}
    
  \item slurmstepd
    \begin{itemize}
    \item Started by slurmd for every job step
    \end{itemize}
    
  \end{itemize}
\end{frame}

\begin{frame}{Daemons}
  \begin{itemize}
    
  \item One initscript reads config file and starts slurmctld, slurmd,
    neither or both
    % Can have same config file on login node
  \item Communication is authenticated using MUNGE or OpenSSL
  \item Hierarchial communication
    \begin{itemize}
    \item \textcolor{red}{Hard to debug}
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Configuration}
  \begin{itemize}
  \item One unified configuration
    file \begin{tt}/etc/slurm/slurm.conf\end{tt}
      \begin{itemize}
      \item Always need to be synchronized on all nodes!
      \end{itemize}
    \item scontrol command
  \end{itemize}
\end{frame}

\subsection{Monitoring}

\begin{frame}{Getting information}
  \begin{itemize}
  \item Get the information you need
  \item In the format you like
    \uncover<2->{
    \item Without using \begin{tt}sed\end{tt}
      and/or \begin{tt}awk\end{tt} one-liners
    }
  \end{itemize}
\end{frame}

\begin{frame}{sinfo example}
  Show jobid and allocated nodes for running jobs of the user paran:
  
  \begin{alltt}\small
\$ squeue -t running -u paran -o ``\%i \%u \%D \%N''\\
JOBID USER NODES NODELIST\\
510857 paran 2 n[771-772]\\
510856 paran 4 n[4,52,320,411]\\
  \end{alltt}
\end{frame}     

\begin{frame}{sinfo example}
  Same, without header:
  
  \begin{alltt}\small
\$ squeue -t running -u paran -o ``\%i \%u \%D \%N'' -h\\
510857 paran 2 n[771-772]\\
510856 paran 4 n[4,52,320,411]\\
  \end{alltt}
\end{frame}     

\begin{frame}{sinfo example}
  Show all idle nodes in the partition ``neolith'':
  
  \begin{alltt}\small
\$ sinfo -t idle -p neolith -h -o \%N\\
n[418,773-774,778,794]
  \end{alltt}
\end{frame}     

\begin{frame}{hostlist}
  \begin{itemize}
  \item Hostlist syntax is used everywhere
  \item Same as in pdsh and many other LLNL utilities
  \item \begin{tt}n[1-805]\end{tt} is nicer
    than \begin{tt}n1,n2,n3,n4\dots\dots,n804,n805\end{tt}
  \end{itemize}
  
  python-hostlist\\
  http://www.nsc.liu.se/\~{}kent/python-hostlist/  
\end{frame}

\subsection{Other stuff}

\begin{frame}{Migration}
  \begin{itemize}
  \item When?
    \begin{itemize}
    \item New systems
    \end{itemize}
  \item Think of the users!
    \begin{itemize}
    \item Do they even care?
    \item Non-issue if grid
    \end{itemize}
  \item Batch scripts  
    \begin{itemize}
    \item sbatch parses \begin{tt}\#PBS\end{tt}-lines
    \end{itemize}
  \item Torque/PBS wrappers available
    \begin{itemize}
    \item qstat, qsub, pbsnodes etc
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{SLURM 2.0}
  Used to be 1.4-pre
  \begin{itemize}
  \item 2.0.0 released 2009-05-20
  \item Reservations
  \item Power control
  \item Improved slurmdbd accounting
  \item Fair-share
  \item Topology awareness
  \end{itemize}
\end{frame}

\begin{frame}{Grid}
  \begin{itemize}
  \item SLURM backend for NorduGrid ARC
    \begin{itemize}
    \item Currently used on \begin{tt}ce01.titan.uio.no\end{tt} as part
      of the NDGF Tier1
    \end{itemize}
  \item Creating backends for other middlewares should be easy
  \end{itemize}
\end{frame}

\begin{frame}{Extending SLURM}
  \begin{itemize}
  \item Write a plug-in
  \item SPANK\\SLURM Plug-in Architecture for Node and job (K)control
  \item C API slurm.h
  \item Perl API
  \item Python API
      \begin{itemize}
      \item Separate project, under development
      \end{itemize}
  \end{itemize}
\end{frame}

\subsection{Second Subsection}

\section*{Summary}

\begin{frame}{Summary}
  \begin{itemize}
    \item SLURM 1.3 is working well
    \item SLURM 2.0 looks interesting
  \end{itemize}
\end{frame}

\end{document}
