%       -*- mode: latex; -*-

\documentclass[utf8x,aspectratio=1610,t]{beamer}
% Option "t" makes text on slides top-aligned instead of centered vertically.


\title{
   {\huge Site report} \\
   {\huge National Supercomputer Centre} \\
   {Linköping University, Sweden}
}
\author{
   Thomas Bellman \\
   \texttt{\footnotesize bellman <at> nsc.liu.se}
}
%\institute{National Supercomputer Centre\\Link\"oping, Sweden}
\date{HEPiX autumn 2022}


\setbeamertemplate{navigation symbols}{}


\begin{document}


\begin{frame}
   \hfill\includegraphics[height=8ex]{logo-nsc-red}
   \titlepage
\end{frame}


\begin{frame}{National Supercomputer Centre}

   HPC centre at Linköping University (LiU), Sweden.
   \begin{itemize}
    \item Founded 1989.
    \item 34 employees (and hiring).
    \item Provides high performance compute resources to Swedish
       academia.
    \item And to Swedish, Norwegian and Finnish met-offices for
       weather forecasts and research.
    \item Also collaboration with SAAB in Linköping.
    \item And a little bit WLCG compute and storage.
    \item In total around 3600~worker nodes.
   \end{itemize}
   One out of six similar centres in Sweden (Lund, Gothenburg,
   Linköping, Stockholm, Uppsala, Umeå), but we were the first,
   so we could claim the name ''National''. :-)

\end{frame}


\begin{frame}{Computer rooms}

   Two computer rooms: Hangaren (''The Hangar''), and Kärnhuset
   (''The Core House'', as in CPU cores).

   Separate buildings, located ca~20~meters apart.
   \begin{itemize}
   \item Designed for 1~MW power each.
   \item Currently uses 400-500~kW and $\sim$900~kW, respectively.
   \item District cooling.
   \item Different electrical and cooling feeds.
   \item Solar panels on roof of Kärnhuset: peak 225~kW, average
      25~kW over year.
   \end{itemize}

\end{frame}


\begin{frame}[c]
   \centerline{\Huge Current systems at NSC}
\end{frame}


\begin{frame}{Compute clusters}

   {\large\bf Tetralith / Sigma}
      \begin{itemize}
      \item One physical cluster, split logically into two.
      \item For academia in Sweden (Tetralith) / at LiU (Sigma).
      \item 2006~worker nodes (Tetralith:~1896, Sigma:~110).
      \item Intel Xeon Gold 6130 CPUs (16~cores @ 3.7~GHz / socket)
      \item 96~Gbyte RAM / node (and some fat nodes with more memory)
      \item Intel OmniPath 100~Gbit/s interconnect, with 4:1
         oversubscription.
      \item Purchased and installed in 2018.
      \item Ranked \#74 on Top500 in June~2019 (now:~\#183).
      \item Most common usage: quantum chemistry, molecular
         dynamics, and computational fluid dynamics.
      \end{itemize}

\end{frame}


\begin{frame}{Compute clusters}

   {\large\bf Stratus / Nebula}
      \begin{itemize}
      \item One physical cluster, split logically into two.
      \item Stratus part: Weather forecasting for Sweden, Norway
         and Finland.
      \item Nebula part: Norwegian met-office research.
      \item 656~worker nodes.
      \item Purchased and installed in 2018.
      \item Hardware identical to Tetralith / Sigma.
      \end{itemize}

   {\large\bf Cirrus}
      \begin{itemize}
      \item Weather forecasting for Sweden, Norway and Finland.
      \item 256~worker nodes.
      \item Purchased and installed in 2018.
      \item Hardware identical to Tetralith / Sigma.
      \item Located at Swedish met-office (SMHI) in neighbouring
         city of Norrköping for redundancy.
      \end{itemize}

\end{frame}


\begin{frame}{Compute clusters}

   {\large\bf Bi}
   \begin{itemize}
   \item Swedish met-office research.
   \item 641~worker nodes.
   \item 2 × Intel Xeon E5-2640v3 (8 cores @ 2.6~GHz / socket)
   \item 64~Gbyte RAM / node (and some fat nodes).
   \item Intel/Qlogic Truescale QDR Infiniband (32~Gbit/s)
      interconnect.
   \item Purchased and installed in 2014.
   \item No longer any hardware support
   \item To be replaced sometime next year (we hope); replacement
      will be smaller.
   \end{itemize}

\end{frame}


\begin{frame}{Compute clusters}

   {\large\bf Bluegrass}
   \begin{itemize}
   \item WLCG compute cluster running ARC with SLURM.
   \item Part of NDGF-T1.
   \item 40~worker nodes; see later slide.
   \end{itemize}

\end{frame}


\begin{frame}{Storage clusters}

   {\large\bf Centre Storage}
   \begin{itemize}
   \item Storage for our academic clusters (Tetralith and Sigma).
   \item IBM ESS system (Spectrum Scale, a.k.a. GPFS, ''as an
      appliance'', running on IBM POWER-based servers).
   \item 6200~TiB usable space.
   \item Used to be connected directly onto OmniPath fabric.
   \item IBM have stopped supporting OmniPath when buying new ESS
      building blocks.  And we wanted to expand...
   \item Rebuilt to use 100G Ethernet on storage servers, and
      built routers between Ethernet and OmniPath on 10~worker
      nodes.  Plain Linux servers, forwarding in software.
   \end{itemize}

\end{frame}


\begin{frame}{Storage clusters}

   {\large\bf Accumulus}
   \begin{itemize}
   \item Storage for Swedish met-office research.
   \item Lustre file system, using ZFS-on-Linux as storage backend.
   \item Ca~18~PiB usable space (but compression is used in ZFS,
      so users can actually store more).
   \end{itemize}

   {\large\bf Serenity}
   \begin{itemize}
   \item dCache storage for NDGF-T1, and Swedish national
      storage (which includes the SE-SNIC-T2 WLCG site).
   \item 6400~Tbyte available space ($\sim$3200~TB each for NDGF-T1
      and national storage).
   \end{itemize}

\end{frame}


\begin{frame}{Tape storage}

   \begin{itemize}
   \item IBM TS4500 library with 5 frames.
   \item 10 LTO-8 drives, 6 LTO-7 drives.
   \item 5502 tape slots.
   \item Currently 5097 tapes (mixture of LTO versions).
   \item IBM Spectrum Protect (the software formerly known as ''TSM'',
      Tivoli Storage Manager).
   \item Usage:
      \begin{itemize}
      \item Backups for ourselves, and some of our sister sites in
         Sweden.
      \item WLCG via NDGF-T1.
      \item Swedish met-office archival data (climate data).
      \end{itemize}
   \end{itemize}

\end{frame}


\begin{frame}[c]
   \centerline{\Huge News since autumn 2019}
\end{frame}


\begin{frame}{COVID-19 aftermaths}

   \begin{itemize}
   \item No-one seriously sick from COVID at NSC.
   \item We are now allowed working 40\% from home.
      \begin{itemize}
      \item Some take advantage of that, others don't.
      \item Nice to have the ability/freedom to do so.
      \item But communication challenges: less face-to-face talks.
      \end{itemize}
   \item Zoom is a boon: less travelling for meetings.
   \item And attending conferences you otherwise didn't have time for.
   \end{itemize}

\end{frame}


\begin{frame}{WLCG storage expansion}

   \begin{itemize}
   \item Purchased summer 2020.
   \item 7 × HPE ProLiant DL385 Gen10 Plus
      \begin{itemize}
      \item dCache disk pools for NDGF-T1
      \item 128 Gbyte RAM
      \item 16 × 14 Tbyte disk (224 TB) each
      \item $\sim$1.2~PiB available space across all 7 servers.
      \end{itemize}

   \item 2 × Dell PowerEdge R7515
      \begin{itemize}
      \item dCache tape pools for NDGF-T1
      \item 64 Gbyte RAM
      \item 8 × 1.92 Tbyte (15 Tbyte) SSD
      \end{itemize}

   \item 25~Gbit/s network.
   \item Used for NDGF Tier-1.
   \end{itemize}

\end{frame}


\begin{frame}{New WLCG worker nodes}

   \begin{itemize}
   \item Purchased summer 2020.
   \item 40 × Dell PowerEdge C6525
      \begin{itemize}
      \item 4 nodes in 2U chassis
      \item 2 × AMD EPYC 7302 (Rome)
      \item 16 cores @ 3 GHz / socket
      \item 256 Gbyte RAM
      \item 1.92 Tbyte SATA SSD
      \item 25~Gbit/s Ethernet
      \end{itemize}

   \item Replaces old worker nodes in Bluegrass cluster
      (used for NDGF-T1).
   \end{itemize}
\end{frame}


\begin{frame}{New AI/ML system}

   {\large\bf BerzeLIUs}
   \begin{itemize}
   \item Grant from Knut \& Alice Wallenberg (KAW) foundation.
   \item KAW pointed to NVidia: "buy whatever they tell you"
      \begin{itemize}
      \item ...~and please have it running yesterday...
      \end{itemize}
   \item 60 NVidia DGX worker nodes:
      \begin{itemize}
      \item 2 × AMD Epyc 7742 CPUs (Rome)
      \item 64 cores @ 2.25 GHz / socket
      \item 1 Tbyte RAM
      \item 8 × NVIDIA A100 Tensor Core GPUs, 40~Gbyte memory
      \item 15 Tbyte local NVMe SSD storage
      \item 8 × Mellanox HDR InfiniBand @ 200~Gbit/s (total
         1.6~Tbit/s, per node!)
      \end{itemize}

   \item 1 Pbyte shared NVMe SSD storage
      \begin{itemize}
      \item DDN A³I storage
      \item Dedicated HDR InfiniBand storage network
      \end{itemize}

   \end{itemize}

\end{frame}


\begin{frame}{New AI/ML system}

   \begin{itemize}

   \item Water-cooled rear doors
   \item Bright cluster manager
      \begin{itemize}
      \item Came with the purchase.
      \item Not entirely positive experience...
      \item Need to run Puppet as well, to manage things Bright
         can't manage.
      \end{itemize}
   \item Installed spring 2021.
   \item Ranked \#82 on Top500 in June 2021 (now: \#102).
   \item Largest user is human language modelling.

   \item Expansion of 34~nodes ordered.
      \begin{itemize}
      \item 80~Gbyte GPU memory.
      \item Air-cooled; no water-cooled rear doors.
      \item Expected installation in spring~2023.
      \end{itemize}

   \end{itemize}

\end{frame}


\begin{frame}{New AI/ML system}

   \begin{itemize}

   \item NVidia DGX server not very high quality
      \begin{itemize}
      \item Replacing approx.\ one motherboard per week
         (on just 60 nodes...).
      \item Pro-actively replaced power distribution
         boards in all nodes, after one caught fire.
      \end{itemize}
   \end{itemize}

\end{frame}


\begin{frame}{Core network upgrade}

   \begin{itemize}

   \item Old core (spine) routers:
      \begin{itemize}
      \item 2 × Juniper QFX 5100-48S
         (48 × 10G + 6 × 40G each)
      \item Total 10×10G uplinks to university border routers, which
         have one 100G uplink each to NREN network.
      \item Purchased and installed in 2014.
      \end{itemize}
   \item Finally got around to order new routers this summer:
      \begin{itemize}
      \item 3 × Juniper QFX 5120-48Y
         (48 × 25G + 8 × 100G each)
      \end{itemize}
   \item Increased numer of spines from 2 to 3:
      \begin{itemize}
      \item More bandwidth
      \item More redundancy
      \item More bandwidth in degraded mode
      \end{itemize}
   \item Will initially upgrade uplinks to 3×4×10G uplinks to border
      routers, later hopefully upgrade to 25G (requires new border
      routers from the NREN).
   \item Expected delivery time: 12-20 weeks, according
      to retailer.
   \item Actual delivery time: 3-4 days (arrived same
      week we ordered them...)
   \item Haven't had time to take into production yet.

   \end{itemize}

\end{frame}


\begin{frame}{Data-driven Life Sciences}

   Recently started project building storage and (non-HPC) compute
   system for sensitive data (genetic and medical data about living
   humans).
   \begin{itemize}
   \item Kubernetes-based compute.
      \begin{itemize}
      \item Kubernetes clusters within virtual machines.
      \item VMs spun up and down on-demand.
      \end{itemize}
   \item Each project gets their own, private Kubernetes cluster.
   \item Different projects must not be able to communicate with each
      other.
   \item Ceph-based storage system.
   \item Unholy network machinations to keep projects separate, with
      (encrypted) VPN tunnels between VMs belonging to same project.
      VPN done by the VM \emph{host}, since guests can't be trusted to
      do the right
      thing.
      \begin{itemize}
      \item VXLAN + EVPN?
      \item IPsec / Wireguard / MACsec?
      \end{itemize}
   \end{itemize}

\end{frame}


\begin{frame}{Two-factor authentication}

   Two-factor authentication for users of our compute clusters:
   \begin{itemize}
   \item Introduced in 2021; optional in spring, mandatory after
      summer.
   \item TOTP (Time-based One-Time Passwords).
   \item Users get link to get a TOTP secret emailed to them.
      \begin{itemize}
      \item Need to log in to our user/project portal to use, so not
         enough to steal someones email account.
      \end{itemize}
   \item Problems: Users
      \begin{itemize}
      \item lose or change their smart phones
      \item don't complete all steps in the TOTP registration
      \item use wrong account in their TOTP app
      \item have incorrect time on their phone
      \end{itemize}
   \item A small handful of user problems / TOTP resets each week.
      \begin{itemize}
      \item About the expected amount.
      \end{itemize}
   \end{itemize}

\end{frame}


\begin{frame}{Two-factor authentication}

   Yubikeys for NSC personnel:
   \begin{itemize}
   \item Introduced spring 2022.
   \item SSH keys on Yubikeys.
   \item Uses PIV (Personal Identity Verification) functionality.
      \begin{itemize}
      \item Works with standard SSH servers.
      \item Client tooling easily available in/for all operating
         systems (including EL-7, FreeBSD, MacOS, Windows, Fedora,
         Ubuntu, et.c).
      \end{itemize}
   \item One person is using another kind of smartcard.  Works well,
      and does some things better than Yubikey.
   \item One person also has personal grid certificate on Yubikey (for
      authenticating against web services).  Some issues with Safari
      web browser, but otherwise works.
   \end{itemize}

\end{frame}


\begin{frame}[c]
   \centerline{\Huge UPS woes}
\end{frame}


\begin{frame}{UPS woes, room \#1}
   \begin{itemize}
   \item UPS:es in computer room \#1 getting old (bought when room was
      built in 2007).
   \item Batteries replaced once during lifetime, but
      also getting old.
   \item Bought new UPS system, installed spring 2021:
      \begin{itemize}
      \item 6~×~Eaton 93PM 200~kW for main load.
      \item 1~×~Eaton 93PM 80~kW for cooling system.
      \item 4 power modules @ 50~kW in each unit; can be
         powered off individually to save energy when
         low load; automatically powered on when load
         increases.
      \item Economy mode: run in internal bypass when
         input power is of "good quality".
      \end{itemize}

   \item Room designed for 1~MW, but currently using
      around 400-500~kW.

   \end{itemize}
\end{frame}


\begin{frame}{UPS non-woes, room \#1}

   {\large\bf Battery monitoring system!}

   \vspace{2ex}

   Generex BACS system.  Monitors temperatures, voltages and
   resistance of each individual battery.  Also tries to actively
   ensure all batteries have the same voltage.

   \vspace{2ex}

   Highly recommended to get some way of monitoring temperatures and
   voltages of individual batteries.

\end{frame}


\begin{frame}{UPS woes, room \#1}
   \begin{itemize}
   \item When more than 21~power modules (of total 24) active,
      UPSes generated 450~Hz overtone on output.
   \item Large enough to cause power subcentral to physically,
      and audibly, vibrate...
   \item Took Eaton long time to reproduce in their lab
      (mostly time for them to decide to actually test
      it themselves...).
   \item Solved with a software update.
   \item Now works very well.
   \end{itemize}
\end{frame}


\begin{frame}{More UPS woes, room \#2}
   \begin{itemize}
   \item UPS:es in second computer room had broken down;
      also needed to be replaced.
   \item Since Eaton UPS:es in room~\#1 now worked well,
      we decided to buy identical system for room~\#2.
   \item Installed winter 2021/2022.
   \item Seemed to work well initially.
   \item But...
   \end{itemize}
\end{frame}


\begin{frame}{More UPS woes, room \#2}
   \begin{itemize}
   \item Ten days before planned acceptance test, we had a power
      outage affecting room~\#2.
   \item Noticed that nothing in that room was reachable.
      \begin{itemize}
      \item Including Matrix chat, RT support system, our large
         academic cluster, and more.
      \item Except cooling PLC:s, which for reasons are
         on our landlords network, not our own.
      \item Email server, Nagios, and weather forecast cluster were
         reachable, but they are in computer room~\#1.
      \end{itemize}

   \item Saturday late evening, February, snowy night, and my
      bicycle was broken...
   \item When I arrived, everything was silent, but lights
      (not on UPS) and cooling (separate UPS) were on.
   \item Battery breakers on all six UPSes in main group
      (non-cooling) had been tripped...
   \end{itemize}
\end{frame}


\begin{frame}{More UPS woes, room \#2}
   \begin{itemize}
   \item System designed to run everything for 4 minutes,
      then turn off power to worker nodes, and then
      allow storage, infrastructure servers and network
      to run for one~hour.
      \\
      (Disconnecting worker nodes is done by external
      PLC:s, not the UPS:es themselves.)
   \item According to logs, the UPS:es had been shut
      down by "external command" 10~seconds after
      input power was lost.
   \end{itemize}
\end{frame}


\begin{frame}{More UPS woes, room \#2}
   \begin{itemize}
   \item Turned out to be feature of the network cards...
   \item Network cards can turn off UPS outputs for low-prio
      consumers when batteries are "low", for those that
      don't have external PLC:s doing that.
   \item High current to support worker nodes for a few
      minutes causes battery voltages to drop noticably,
      and network cards decided batteries were low, and
      turned off the UPS:es...
   \item UPS:es in room \#1 had been configured to ignore such
      commands from network cards (this setting only available
      to Eaton service personnel, not customers).
   \end{itemize}
\end{frame}


\begin{frame}{More UPS woes, room \#2}
   More UPS problems in room \#2:
   \begin{itemize}
   \item UPSes didn't want to go into economy mode. Took months
      for Eaton to figure out that the UPSes were configured
      differently than the ones in room \#1.  Despite extracting
      the configs \emph{twice} from them...
   \item After service personnel configured UPSes correctly,
      they now go into economy mode.
   \end{itemize}
\end{frame}


\begin{frame}{More UPS woes, room \#2}
   \begin{itemize}
   \item UPSes now go into economy mode, but voltage drops
      to zero on one phase and ~80V on another phase for
      1/4~period (5~ms) during transition...
   \item Still unsolved...
   \end{itemize}
\end{frame}


\begin{frame}[c]
   \centerline{\Huge A Computer Security Ghost Story}
\end{frame}


\begin{frame}{Security ghost story}

   User support ticket: threatening text appeared in their terminal
   while logged in to our big academic cluster.
   \begin{itemize}
   \item String of $\sim$50 words; not entirely coherent, but not
      complete gibberish either.
   \item Kind of looked like ramblings from a madman, or what you
      would expect from a haunted fax machine in a horror movie.
   \end{itemize}

   \vspace{2ex}
   Contained phrases like {\em ''nobody can come to my home''}, and
   {\em ''if you need help going to kill you okay okay okay yeah''}...

\end{frame}


\begin{frame}{Security ghost story}

   Investigation started:
   \begin{itemize}
   \item Who were logged in at the same time?
   \item Who were logged in earlier using the same tty?
   \item What processes had been run around that time (logged using
      audit system)?
   \item Called user on phone to get more details.  Language barrier:
      user not entirely fluent in English, but after while managed to
      learn:
      \begin{itemize}
      \item Text appeared as if typed, not all at once.
      \item User was using a Macintosh.
      \end{itemize}
   \end{itemize}

\end{frame}


\begin{frame}{Security ghost story}

   After a couple of hours, suggestion from another colleague:
   \begin{quote}
      ''This kind of looks like when I turn on automatic subtitling
        in Youtube.  Could this be some kind of voice typing?''
   \end{quote}
   Hmm...

   \vspace{2ex}
   Off to yet another colleague, someone equipped with a Macintosh.

   \vspace{2ex}
   Learned from local Macintosh user:
   \begin{itemize}
   \item Some work needed to enable voice typing in MacOS.
   \item But when enabled, it can be activated by pressing
      \texttt{Ctrl} button twice.
   \item Icon shows up when voice typing active, but that could be
      overlooked when focusing on other task (and part of screen).
   \end{itemize}

\end{frame}


\begin{frame}{Security ghost story}

   In the meantime, got an actual screenshot from the user, containing
   $\sim$40 more words.  This time containing the phrase {\em ''can you
   take a print screen of this to my home direction''}.

   \vspace{2ex}
   Also learned (via user's PI) that user was sitting in a shared
   office with several people in small room.

   \vspace{2ex}
   And later got confirmation from user that voice typing was very
   likely the culprit.

\end{frame}


\begin{frame}{Security ghost story}

   Conclusions:
   \begin{itemize}
   \item Our biggest system is \emph{not} cracked.
   \item Neither is the user's laptop / workstation.
   \item We \emph{don't} need to take the cluster offline for
      several weeks.
   \item We \emph{can} go home at normal times today.
   \item Voice recognition is not quite ''there'' yet; especially
      when faced with \\ language challenges in a noisy environment.
   \end{itemize}

   \vspace{4ex}
   {\it Whew!}

\end{frame}


\begin{frame}{Security ghost story}

   Questions:
   \begin{itemize}
   \item How would \emph{you} react if this ghost typing happened
      to you?
   \item How long would it take \emph{you} to figure this out if you
      received this support ticket?
   \item Is \texttt{Ctrl-Ctrl} a good user interface for activating
      and deactivating voice typing?
   \end{itemize}
\end{frame}


\begin{frame}[c]
   \vfil
   \centerline{\Huge The End}
   \centerline{\tiny for now...}
   \vfill
   \centerline{\includegraphics[height=6ex]{logo-nsc-red}}
\end{frame}


\end{document}


% 
% Local Variables:
% fill-column: 70
% tex-indent-basic: 3
% tex-indent-item: 3
% End:
