%       File: VTthesis_template.tex
%     Created: Thu Mar 24 11:00 AM 2016 EDT
%     Last Change: Thursday, December 19, 2019
%     Author: Alan M. Lattimer, VT
%	  With modifications by Carrie Cross, Robert Browder, and LianTze Lim. 
%
% This template is designed to operate with XeLaTeX.
%
% All elements in the Title, Abstract, and Keywords MUST be formatted as text and NOT as math.
%
%Further instructions for using this template are embedded in the document. Additionally, there are comments at the end of the file that give suggestions on writing your thesis.  
%
%In addition to the standard formatting options, the following options are defined for the VTthesis class: proposal, prelim, doublespace, draft. 

\documentclass[doublespace,nopageskip]{VTthesis} % nopageskip - Removes arbitrary blank pages.

% Using the following header instead will create a draft copy of your thesis
%\documentclass[doublespace,draft]{VTthesis}

% The lipsum package is just included to put dummy text in the document in order to demonstrate page headers and table of contents behavior. You should remove it once you begin writing your actual thesis or dissertation.
\usepackage{lipsum}
\usepackage{url}
\usepackage{hyperref}
\usepackage{array}
\usepackage{multirow}
\usepackage{graphics}
% Title of your thesis
\title{CS 5604 Migration from Virginia Tech Computer Science Container Cluster to Digital Library Research Laboratory}
% This should be your defense date:
\submitdate{\today} 

 \frontmatter

\acknowledge{We would like to extend our gratitude towards Dr. Edward Fox for his constant guidance and help throughout this project. We would also like to thank our client Xinyue Wang for getting us started on this project, as well as Prashant Chandrasekar for support with the VTCS  clusters. We also owe thanks to the CS5604 Fall 2020 class, whose thorough documentation of the applications and the cluster were invaluable to the migration process.}

% The abstract is required.
\abstract{
  The VTCS container cluster supports an information analysis system that can process and store data on web pages, tweets, and electronic theses and dissertations. This system was developed by previous semesters of Information Storage and Retrieval (CS5604) students and hosted in a Kubernetes cluster within the Virginia Tech Computer Science (VTCS) cloud. Our project goal was to migrate the system from the cloud to a cluster in the Digital Library Research Laboratory (DLRL), a lab dedicated to researching topics in information retrieval, multimedia, and hypertext. \\ 
  The current cluster supports five main projects, each developed by a different team in CS 5604: Tweet Analysis, Electronic Theses and Dissertations, Web Pages, Integration, and Front End. The first three are the main collection management systems, designed to extract and analyze data from each category of written work that they are named after. The Front End project contains the interface that the user interacts with, and the Integration project ensures each of the other projects can communicate with the Front End and any other services they use, such as the search engine Elasticsearch. \\
  Over the course of the project, we have reconstructed the cluster in DLRL as faithfully as possible. We have ensured each of the namespaces within the cluster hosts the same deployments, and these deployments are configured as closely as possible to their VTCS cloud counterparts. All adjustments made to the container configurations are documented herein. The cluster is fully set up for use; any future work on it would be reconfiguration at the discretion of future developers.  }

%\abstract{\lipsum [1-4]}

% The general audience abstract is required. There are currently no word limits.
%\abstractgenaud{You are also required as of Spring 2016 to include a general audience abstract. This should be geared towards individuals outside of your field that may be reading seeking information about your work. You should avoid language that is particular to your field and clearly define any terms that may have special meaning in your discipline.}

\begin{document}
% The following lines set up the front matter of your thesis or dissertation and are required to ensure proper formatting per the VT ETD standards. 

  \maketitle
  \tableofcontents
% The list of figures and tables are now optional per the official ETD standards.  Unless you have a very good reason for removing them, you should leave these lists in the document. Comment them out to remove them.
	\listoffigures
	\listoftables
    \printnomenclature %Creates a list of abbreviations. Comment out to remove it. 

% sample text for abbreviations:
\nomenclature{DLRL}{Digital Library Research Laboratory at Virginia Tech}
\nomenclature{VTCS}{Virginia Tech Computer Science}
\nomenclature{FE}{Front End}
\nomenclature{INT}{Integration and Implementation}
\nomenclature{ETD}{Electronic Theses and Dissertations}
\nomenclature{WP}{Web Pages}
\nomenclature{TWT}{Tweet Analysis}
\nomenclature{NFS}{Network File System}

% The following sets up the document for the main part of the thesis or dissertation. Do not comment out or remove this line.
\mainmatter
	
	%now go ahead and start writing your thesis
	\chapter{Introduction} \label{ch:introduction}
   	   \begin{flushleft}
   	   A container is software that packages the code and dependencies of an application, allowing the application to run in different computer environments \cite{docker}. This technology dates back to 1979 when the $7^{th}$ version of Unix (Unix V7) first introduced chroot, or change root directory \cite{container}. The chroot environment allowed containerization by creating and hosting a virtual copy of the system. From that point, different iterations of containers and container managers were introduced, but it was not until 2013 that Docker emerged and popularized containers. Docker gained such popularity because it offered an entire ecosystem to manage containers, making the deployment and use of containers simpler than previous systems. A year after Docker was introduced, Google released Kubernetes, an open-source system for automating deployment, scaling, and management of containerized applications \cite{kubernetes}. In this project, we will be using Docker and Kubernetes for the migration of Kubernetes managed containers from one container cluster to another. \par
   	    This project aims to transfer container images from Virginia Tech's Computer Science cloud (VTCS cloud) to a local environment in Virginia Tech's Digital Library Research Laboratory (DLRL). The images in question are the outcome from the Fall 2020 CS5604 course. They include the following images: Electronic Theses and Dissertations (ETD), Twitter (TWT), Web Page archives (WP), Front-End (FE), and Integration and Implementation (INT). The intent upon successful migration is for work with those containers to progress in
%   	    the cloud cluster to be deleted in favor of using 
   	    the DLRL. \par
   	    
    		    
    	\end{flushleft}	    
    	\begin{flushleft}
        \section{General Information}
        \begin{flushleft}
        The following information is meant to supplement understanding of our project and define specific terms and technologies that will be helpful for the user to understand our project. 
        \end{flushleft}
        \subsection{Containerization} 
    		    \begin{flushleft}
    		    As mentioned earlier, a container is software that packages the code and dependencies of an application, allowing the application to run in different computer environments \cite{docker}. The process of packaging the code and its dependencies is referred to as containerization.
    		    \end{flushleft}
        \subsection{Docker Containers and Images}
    		    \begin{flushleft}
    		    Docker runs containerized applications using what are known as Docker images. They are created by building a read-only file that contains a set of instructions for creating a container to run on the Docker platform as shown in Figure \ref{fig:docker_c}. 
    		    
        		    \begin{figure}[h]
       	            \centering
       	            \includegraphics[scale=.28]{images/docker image.png}
       	            \caption{Docker Containerization \cite{docker-containerization}}
       	            \label{fig:docker_c}
       	            \end{figure}
    		    \begin{center}
    		    \end{center}
    		   \end{flushleft}
        \subsection{Kubernetes Cluster and Kubernetes CLI} 
    		    \begin{flushleft}
    		    Kubernetes is an open-source system for deploying, scaling, and managing containerized applications. The architecture is illustrated in Figure \ref{fig:kubernetes_arc}.
    		    Generally, anything run with Kubernetes operates on a cluster, i.e.,
    		    a set of node machines that is used for running containerized applications. A Kubernetes cluster contains six main components which can run on Linux or as a Docker container \cite{cluster}.
    		    \begin{enumerate}
    		        \item{API server:} Exposes a REST interface to all Kubernetes resources \cite{cluster}.
    		        \item{Scheduler:} Schedules containers according to resource requirements \cite{cluster}.
    		        \item{Controller manager:} Manages controllers such as node controllers, endpoint controllers, and replication controllers \cite{cluster}.
    		        \item{Kubelet:} Ensures that containers are running in a Pod by interacting with the Docker engine \cite{cluster}.
    		        \item{Kube-proxy:} Manages network connectivity and maintains network rules across nodes \cite{cluster}.
    		        \item{Etcd:} Stores all cluster data \cite{cluster}.
    		    \end{enumerate}
    		     The Kubernetes cluster can be accessed locally using a command line interface. This is done using the Kubernetes command-line tool (CLI), kubectl. Kubectl is powerful enough for deploying applications, inspecting and managing cluster resources, and viewing logs.
    		  
    
		        \begin{figure}[h]
   	            \centering
   	            \includegraphics[scale=.28]{images/kubernetes_architecture.jpg}
   	            \caption{Kubernetes Architecture \cite{kubernetes_architecture}}
   	            \label{fig:kubernetes_arc}
   	            \end{figure}
   	            \begin{figure}[h]
   	            \centering
   	            \includegraphics[scale=.19]{images/Docker-Kubernetes-together.png}
   	            \caption{How Kubernetes uses Docker to run containers \cite{kubernetes+docker}}
   	            \label{fig:kubernetes_doc}
   	            \end{figure}
   	        \end{flushleft}
        \subsection{Rancher and Rancher CLI} 
    		    \begin{flushleft}
    		    Rancher is a cluster management system designed to assist in running and maintaining Kubernetes clusters. Rancher can be accessed locally using a command line interface which is also called the Rancher CLI. It is worth noting that while Rancher was used to configure the CS 5604 cloud cluster, it is not necessary for the functionality of the cluster, and thus will not be used within the DLRL cluster.
    		    \end{flushleft}
        \section{Project-Specific Information}
        \subsection{Computer Science and the Digital Library Research Laboratory} 
    		    \begin{flushleft}
    		    The Virginia Tech Department of Computer Science has as one of its labs the Digital Library Research Laboratory (DLRL).
    		    VTCS has a container cluster that supports its education mission, and was used to support work in Fall 2020 in its course CS5604. Research work with the containers developed in that class is best managed on a container cluster focused on research, i.e., in DLRL.
    		    Accordingly, this project aimed to move the containers working on the VTCS container cluster to the container cluster in DLRL, to be managed there with Kubernetes.
    		    %The Virginia Tech Computer Science Cloud (VTCS) and Digital Library Research Laboratory (DLRL) are two separate entities used by Virginia Tech for various purposes, including housing clusters. Our goal for this project is to move the Kubernetes cluster from the VTCS cloud to DLRL.  
    		    \end{flushleft}
        \subsection{CS 5604 Containerized Applications} 
    	        \begin{flushleft}
    	          The containerized applications we have been tasked with migrating to the DLRL cluster were developed by a group of graduate students taking CS 5604. The applications are divided into sub-projects. Three of these projects are designed to perform different types of natural language processing and analysis (ETD, WP, TWT), one project integrates the functionality of the previous three (INT), and the final project provides a front end for the combined applications (FE). While our focus is mainly on migration, we still must be aware of how these applications are designed to ensure they will function correctly in DLRL. 
    	        \end{flushleft}
    	\end{flushleft}
	\chapter{Requirements} 
	    \begin{flushleft}
	    This project deals with migration from the Kubernetes cluster in the VTCS cloud to DLRL. Our client aims to use the newly created cluster in favor of the original Kubernetes cluster. With that end goal in mind, there are several requirements that must be met by the DLRL cluster to satisfy this need. The following sections are descriptions of these requirements and their details. 
	    \end{flushleft}
	     \section{User Goals}
	     \subsection{Accessibility of Cluster}
	     \begin{flushleft}
	       Any given user must be able to access and successfully run the CS 5604 application containers within DLRL.
	       One type of run involves a user
	       %A successful run is defined by the user having the ability to launch and 
	       interacting with the front end UI.
	       Another involves
	       %However, this may also extend to 
	       a developer running tests on individual containers.
	       Thus, a
	       %In both of these circumstances, the 
	       key goal is to make sure the containers can be run from the front end or from elsewhere in the cluster. 
	     \end{flushleft}
	     \subsection{Container Troubleshooting}
	     \begin{flushleft}
	       If any issues arise while using this cluster, a user must be able to access and debug specific containers as needed to identify the problem. Users in this case will most often be developers for the cluster. The goal for this user is to give them the necessary information to be able to explore the details of the cluster to find and determine any issues therein. 
	     \end{flushleft}
	     \subsection{Usable Documentation}
	     \begin{flushleft}
	       All users need to have access to comprehensive documentation on the DLRL cluster, its structure, and how it is used. This includes details of any changes made during the migration from the cloud to DLRL, a user guide, a manual for new developers, and any other information relevant to fully understanding the project. 
	     \end{flushleft}
	     \section{Project Tasks}
	     \subsection{Transfer Containers to DLRL}
	     \begin{flushleft}
	       The first task to meet these requirements is to transfer the containers to the DLRL cluster. In regards to our project, transferring does not refer to a direct movement of an application from one place to another. Instead, we are rebuilding the containers within DLRL in a way that mirrors the cloud cluster as closely as possible. To do this, we must deploy entirely new containers inside the DLRL clusters, but in such a way that they have the same characteristics as in the cloud cluster. 
	     \end{flushleft}
	     \subsection{Migrate Dependencies to DLRL}
	     \begin{flushleft}
	       Part of this migration process will also include adding any dependencies to the DLRL cluster that are needed for the containers to run properly. This includes external storage, running services that the containers may call on, and environment variables needed to ensure the cluster runs as expected. 
	     \end{flushleft}
	     \subsection{Verify Containers Actively Run}
	     \begin{flushleft}
	       Of course, for a user to interact successfully with the UI, all the containers must be running without error, and thus be able to run their internal applications. Containers that are working as expected will show a running status when checked in the cluster, whereas failing containers will show an error as their status. Those containers must be debugged so that the DLRL cluster is fully functional. 
	     \end{flushleft}
	     \subsection{Compile Documentation Into Final Report}
	     \begin{flushleft}
	       All documentation made by our team throughout this project is compiled in this report. The information herein will provide future users the relevant information on the structure and functionality of the cluster for reference. This provides users with reference material of important information about the cluster that will be needed to correctly run and troubleshoot it.  
	     \end{flushleft}
	     \section{Relevant Workflows}
	     \begin{flushleft}
	       With the knowledge gained from the previous two sections in this chapter, we gathered enough information to construct workflows that guided our process throughout this project. These workflows are detailed as flow charts in Figures \ref{fig:workflow_1}, \ref{fig:workflow_2}, and \ref{fig:workflow_3}.
	       \begin{figure}[h]
	           \centering
	           \includegraphics[scale=0.75]{images/Methodology Goal 1.png}
	           \caption{Workflow to meet the first requirement of the users}
	           \label{fig:workflow_1}
	       \end{figure}
	       
	       \begin{figure}[h]
	         \centering
	         \includegraphics[scale=0.75]{images/Methodology Goal 2.png}
	         \caption{Workflow to meet the second requirement of the users}
	         \label{fig:workflow_2}
	     \end{figure}
	     
	     \begin{figure}[h]
	         \centering
	         \includegraphics[scale=0.75]{images/Methodology Goal 3.png}
	         \caption{Workflow to meet the third requirement of the users}
	         \label{fig:workflow_3}
	     \end{figure}
	     \end{flushleft}
	     
	\chapter{Implementation} \label{ch:implementation}
	\section{Details of the Cloud Cluster} 
	     \begin{flushleft}
	     The VTCS cluster currently holds the containerized applications produced by CS 5604 students. The cloud runs Kubernetes v1.15.5 alongside Rancher to manage several cluster namespaces, each of which has unique deployments of pods within them. Rancher provides a higher level of division on top of Kubernetes namespaces, referred to as Rancher Projects. Table \ref{tab:namespaces} provides a comprehensive list of all the projects registered in the cloud cluster, as well as a brief description of their functions and what namespaces they contain. 
	     \vspace{30pt}
	     \begin{table}[h]
	         \centering
	         \caption{Table of Rancher Projects and their namespaces}
	         \begin{tabular}{| m{8em} | m{12em} | c |}
	               \hline
	               \textbf{Project Name} & \textbf{Description} & \textbf{Kubernetes Namespaces}  \\   
	               \hline
	              CS 5604 ETD 2020 & Electronic Theses and Dissertations analysis & cs5604-etd-db, etd \\
	              \hline
	              CS 5604 FE 2020 & Front End interface & cs5604-fe-db, fe \\
	              \hline
	              CS 5604 INT 2020 & Integration of all projects & cs5604-int-test, cs5604-int-testdb, int \\
	              \hline
	              CS 5604 TWT 2020 & Tweet analysis and processing & cs5604-twt-db, twt \\
	              \hline
	              CS 5604 WP 2020 & Web page archival & cs5604-wp-db, wp-test, wp \\
	              \hline
	         \end{tabular}
	         \label{tab:namespaces}
	     \end{table}
	     
	     \newpage
	     The following namespaces in the VTCS cloud are empty, and will not be included in the DLRL cluster structure: 
	     \begin{itemize}
	         \item fe
	         \item int
	         \item cs5604-int-testdb
	         \item twt
	         \item wp
	     \end{itemize} 
	     Within each namespace there are a series of workloads, otherwise known as Kubernetes deployments. These deployments house a certain number of pods, and each pod within a deployment is running a container of the same image. That way, multiple containers can be running the same application at once within a cluster. \par
	     Containers are not typically designed to store data for long periods of time. Instead, their goal is to be able to deploy a running application that can be quickly deleted once its main task is complete. However, the CS 5604 applications are mainly built for data processing and analysis, so they must be able to access and store data safely. For this reason, most of the containers within the VTCS cloud have been given persistent volume claims to allow them to mount external NFS storage to them, where they can read data from and write their outputs to as needed. \par
	     The majority of the containers within the cluster utilize NFS storage from camelot.cs.vt.edu, a Virginia Tech server. However, some of the containers in the cloud are mounted to a Ceph file system which, at the request of our client, won't be used within the DLRL cluster. 
	     \end{flushleft}
	\section{Project Objectives} \label{se:proj_objectives}
	    \begin{flushleft}
	      We have divided this project into different objectives in order to monitor our progress. They are listed as follows:
    	    \begin{enumerate}
    	        \item Familiarize ourselves with Docker which included the following resources given to us by our client:
    	            \begin{enumerate}
    	                \item \href{https://medium.com/codingthesmartway-com-blog/docker-beginners-guide-part-1-images-containers-6f3507fffc98}{Docker — Beginner’s Guide — Part 1: Images \& Containers} \cite{docker_guide}
                        \item \href{https://www.docker.com/101-tutorial}{Learn how to build and share a containerized app} \cite{docker_learn}
                        \item \href{https://medium.com/google-cloud/kubernetes-101-pods-nodes-containers-and-clusters-c1509e409e16}{Kubernetes 101: Pods, Nodes, Containers, and Clusters} \cite{kubernetes_guide}
                        \item \href{https://kubernetes.io/docs/reference/kubectl/overview/}{Overview of kubectl} \cite{kubectl_overview}
    	            \end{enumerate}
    	        \item Run the images contained in the Virginia Tech Computer Science cloud locally. This was done in the following steps:
    	            \begin{enumerate}
    	                \item Create a Docker file for each image using the yaml configuration of the images.
    	                \item Build the Docker file.
    	                \item Successfully run the Docker file locally.
    	            \end{enumerate}
    	       \item Migrate the Kubernetes cluster to the Digital Library Research Laboratory using the information gathered in step 2. 
    	    \end{enumerate}
	    \end{flushleft}
	    \newpage
	    \section{Setting up the DLRL Cluster} \label{se:set_up_dlrl}
	    \begin{flushleft}
	    The DLRL cluster is made up of a series of nodes running Ubuntu 18.04.5 and using Kubernetes v1.20.4. Table \ref{tab:cluster_nodes} describes all current nodes within the cluster. 
        \vspace{30pt}
        \begin{table}[h]
            \centering
            \caption{DLRL Cluster Nodes}
            \begin{tabular}{|c|c|}
                \hline
                \textbf{Node ID} & \textbf{IP Address} \\
                \hline
                master-node & 128.173.49.22 \\
                \hline
                worker01 & 128.173.49.25 \\
                \hline
                worker02 & 128.173.49.24 \\
                \hline
            \end{tabular}
            \label{tab:cluster_nodes}
        \end{table}
        
        All of our work will be done from the master-node, as it will delegate which of the worker nodes will be assigned which deployments to run upon creation. These nodes had already been created for our use, with Kubernetes and Docker already installed. However, each node was missing the packages necessary in order to mount NFS storage to the containers, which we resolved by running the following command on each node:
        \begin{center}
            \textbf{sudo apt install nfs-common}
        \end{center}
        
        As our team is working remotely, we had to use VT Pulse Secure VPN to access the master node to begin migration. Instructions on how to install and use the VPN can be found at \href{https://onecampus.vt.edu/task/all/installing-pulse-vpn}{https://onecampus.vt.edu/task/all/installing-pulse-vpn}. This step is only needed if you are accessing the cluster from a computer not registered within Virginia Tech's network. \par
        Once connected to the VPN, we are able to ssh into the master node using an account set up for our use. Then, our first setup was to recreate all of the namespaces from the VTCS cloud in the DLRL cluster. We used the following command to create each of the nonempty namespaces referenced in Table \ref{tab:namespaces}: 
        \begin{center}
            \textbf{kubectl create namespace <namespace\_name>}
        \end{center}
        
        We can verify the namespaces were created correctly with the command \textbf{kubectl get namespaces}, as shown in Figure \ref{fig:dlrl_namespace_image}. The namespaces default, kube-node-lease, kube-public, and kube-system are all created automatically with the cluster, and can be disregarded. 
        
        \begin{figure}[h]
            \centering
            \includegraphics[scale=0.75]{images/namespaces.png}
            \caption{View of DLRL Namespaces}
            \label{fig:dlrl_namespace_image}
        \end{figure}
	    \end{flushleft}
	    \subsection{Persistent Volumes and Claims Creation}\label{sec:img_dep_vol}
	    \begin{flushleft}
	      Once we set up the basic skeleton of the DLRL cluster, we could begin to deploy the CS 5604 images into their relevant namespaces and mount the necessary NFS storage to them. Since the containers will only run if they have access to their storage, we began by setting up the persistent volumes that will hold the external storage information. \par
	      To do this, we created a directory in the master node called \textit{volumes} which holds the yaml files that the persistent volumes are created from. Next, we created a yaml file for each persistent volume that the cluster uses, which we based off of the preexisting yaml files within the VTCS cloud. Figure \ref{fig:pv_yaml} shows an example of a persistent volume yaml file. 
	      \begin{figure}[h]
	          \centering
	          \includegraphics[scale=0.5]{images/pv_yaml.png}
	          \caption{Example persistent volume yaml file}
	          \label{fig:pv_yaml}
	      \end{figure}
	      
	      In this file, we have defined the name of the persistent volume, the server and path to be mounted, as well as the access mode of this storage system. The access mode in this case indicates that multiple nodes can read and write to the NFS storage at the same time. Finally, we have given a set of mount options that the containers will need to use for this storage.\par
	      Persistent volumes act as a general reference to any external storage that the cluster may use. In order for a container to be able to access the storage, it must use a persistent volume claim to mount the storage to itself. Thus, we needed to create persistent volume claims within each of the DLRL namespaces, so that containers deployed within them will have access to these claims. Figure \ref{fig:pvc_yaml} shows an example of a persistent volume claim yaml file. \par
	      
	      \begin{figure}[h]
	          \centering
	          \includegraphics[scale=0.5]{images/pvc_yaml.png}
	          \caption{Example of a persistent volume claim yaml file}
	          \label{fig:pvc_yaml}
	      \end{figure}
	      
	      In this case, we reference the related persistent volume so any container that uses this claim can mount that storage to itself, as well as the namespace this claim is attached to. Once again, we utilized the yaml files that were already within the VTCS cloud as a reference point for these claims. \par
	      Once a persistent volume or persistent volume claim yaml file is written, you can run the following command to create it within the cluster:
	      \begin{center}
	          \textbf{kubectl apply -f filename.yaml}
	      \end{center}
	      You can check if these volumes and claims were created successfully using the commands
	      \begin{center}
	          \textbf{kubectl get pv} and \\
	          \textbf{kubectl get pvc -{}-namespace <namespace\_name>}
	      \end{center}
	      respectively. 
	    \end{flushleft}
	    \subsection{Image Deployment into DLRL}\label{sec:deploy_img}
	    \begin{flushleft}
	      Once our external storage has been set up in the cluster in the form of persistent volumes and claims, we can begin deploying the CS 5604 application images into their respective namespaces in DLRL. 
	      
	      Utilizing the yaml files from the workloads listed in the VTCS cloud, we created yaml files for each of the deployments within a directory called \textit{namespaces}. Figure \ref{fig:deployment_yaml} shows what a yaml file of this type looks like. 
	      
	      \begin{figure}[h]
	          \centering
	          \includegraphics[scale=0.65]{images/deployment_yaml.png}
	          \caption{Example yaml file for a single image deployment}
	          \label{fig:deployment_yaml}
	      \end{figure}
	      
	      These yaml files contain all of the information necessary for one or multiple containers to be deployed running the same image. As displayed in the figure, the image is pulled from a private container repository within container.cs.vt.edu.
	      Since this is a private repository, Kubernetes needs to authenticate with the registry in order to access the image, and thus an image pull secret is used, created from one of our teammate's credentials to the registry. The other information within the yaml file describes the volumes that the container needs to mount, as well as any environment variables and other configuration settings that are necessary for the container to run. 
	      
	      As we have for the persistent volumes, we can use the \textbf{kubectl apply} command to deploy the containers from the yaml files. We can see deployments that have successfully launched by the \textbf{kubectl get deployments --namespace <namespace\_name>} command, as shown in Figure \ref{fig:dlrl_deployments}.
	      
	      \begin{figure}
	          \centering
	          \includegraphics{images/deployments.png}
	          \caption{Successful container deployments}
	          \label{fig:dlrl_deployments}
	      \end{figure}
	    \end{flushleft}
	    \subsection{Container Services}
	    \begin{flushleft}
	    In some cases for the cluster, some containers will need to be made accessible to other containers. For example, within the ETD portion of the cluster there is a container running Postgres (i.e., PostgreSQL), which the other containers within ETD are meant to send data to for storage and indexing purposes. However, we must notify Kubernetes that the Postgres container must be discoverable, so that Kubernetes will create an internal IP address for that the other containers will be able to use to connect to Postgres. The way we do this is by setting up Kubernetes services on the necessary containers. 
	    
	    Just like persistent volumes, claims and deployments, we can create services through a yaml file on the master node. Figure \ref{fig:yaml_service} shows an example of a service yaml file. 
	    
	    \begin{figure}[h]
	        \centering
	        \includegraphics[scale=0.75]{images/service_yaml.png}
	        \caption{Example yaml file for a container service}
	        \label{fig:yaml_service}
	    \end{figure}
	    
	    In this yaml file, the service is given a name and a namespace to reside in, and then the owner references section defines what specific container attaches to the service. The uid is the identifier of the container, which you can discover using the command
	    \begin{center}
	    
	    \textbf{kubectl get pods -n <namespace> <pod-name> -o jsonpath='{.metadata.uid}'}
	        
	    \end{center}
	    
	    Within the specification section, we define a port that any client container will be able to attach to, and give it a name and an optional protocol for the port. Then, we apply the service using the \textbf{kubectl apply} command described in previous sections to create the service.
	    
	    Since the service we created essentially exposes a port on the chosen container, we have one extra step we need to take. DLRL is a multi-node cluster, so it may be that some containers deployed on one node will want to communicate with a container deployed on a completely different node. They will only be able to do that if they can connect to the container port through an exposed port on the node. Therefore, we need to create a service that reserves a port on the node where our container resides. Figure \ref{fig:yaml_nodeport} demonstrates the yaml file for a such a service.
	    
	    \begin{figure}[h]
	        \centering
	        \includegraphics[scale=0.75]{images/nodeport_yaml.png}
	        \caption{Example yaml file for a node port service}
	        \label{fig:yaml_nodeport}
	    \end{figure}
	    
	    The major difference to note is that the ports section under the specifications defines the port of the node as well as the port of the container that we defined in the previous yaml file. This effectively links the node port with the container port, giving any containers outside the current node a way to access this Postgres container. 
	    
	    We have created services using this procedure for each container in DLRL that needs to be accessible to other containers. Section \ref{sec:dlrl_services} provides a comprehensive list of all services we have created for DLRL. 
	    \end{flushleft}
	    \newpage
	\chapter{User Guide}
	    \section{Accessing the Front End}
	    \begin{flushleft}
	    At this time, several bugs have been identified within the back end of the CS 5604 applications. These bugs are recorded in Section \ref{sec:current_issues}. Until these issues are resolved, several functions of the applications may not work as intended. 
	    
	    However, a typical user will want to know how to access the front end so they may use the applications. The current home page link is \url{http://10.244.2.123:3000}, which is the exposed API endpoint for the front end. This URL is subject to change if the front end undergoes modifications. Upon accessing this URL, new users may create an account within the system. As this front end deployment is new, any previously registered users should be aware that they will likely need to create a new account within the system, since registration information was not transferred from the VTCS cloud applications. 
	    
	    From the home page and upon successful login, the user will be able to access the various back end applications through the user interface. A comprehensive tutorial on the front end interface and how to navigate it can be found within the user manual section of the CS 5604 Front End team's final report \cite{FE}. 
	    \end{flushleft}
	    \section{General Usage}
	    \begin{flushleft}
	    For users interested in the CS 5604 system, Figure \ref{fig:app_structure} provides a diagram of that system on the DLRL cluster, and how each piece interacts with each other. To sum up, the cluster supports three back end pieces, referred to as TWT \cite{TWT}, WP \cite{WP} and ETD \cite{ETD}, which perform the majority of the data processing. These pieces pull initial data from a VT Camelot server, where they have been provided NFS mounts to persistent volumes. Their work is synchronized and supported by the integration system, INT \cite{INT}, which connects the back end to an ElasticSearch server running on containers on another VM (elasticsearch.cs.vt.edu) for data storage and indexing. Finally, the front end, FE \cite{FE}, provides the user interface wherein a person can pull up the data processed by the back end. 
	    
	    \begin{figure}
	        \centering
	        \includegraphics[scale=0.5]{images/CS 5604 Application Structure.png}
	        \caption{Structure of the DLRL Cluster}
	        \label{fig:app_structure}
	    \end{figure}
	    
	    \newpage
	    
	    Each of these projects that represents a portion of the overall functionality of the cluster is made up of a series of containers within the cluster. These containers are grouped into Kubernetes namespaces for organizational purposes, and the containers are currently housed within one of two worker nodes as detailed in Table \ref{tab:cluster_nodes}.
	    
	    Since our main focus for the project was on setting up the DLRL cluster, the following section provides more insight into using Kubernetes and Rancher to set up containers within a cluster environment. 
	    \end{flushleft}
        
         \section{VT Cloud and General Containers Guide}
    	 \begin{flushleft}
    	  While the new cluster does not reside in the VTCS cloud, the cluster it is modelled off of was hosted there. The VTCS cloud utilized Rancher to provide a user interface and a command line interface to access the cluster from the web or from a terminal. Rancher is not currently set up on DLRL, but if future developers add it for ease of use, this section can act as a guide on how to access a cluster remotely via Rancher CLI. 
          \subsection{Access Cluster Remotely With Rancher} \label{se:rancher access}
          \begin{enumerate}
              \item Step 1: Create API Key in Rancher UI. \par
              On the Rancher website, click on your profile and select API and Keys from the drop down menu.
              	\begin{figure}[h]
						\centering
						\includegraphics{images/create_api.png}
						\caption{API option in Rancher user settings.} 
						\label{fig:api_window}
				\end{figure}
				\item Step 2: Select Add Key
				    \begin{figure}[h]
						\centering
						\includegraphics{images/add_api.png}
						\caption{Add Key button in the API window.} 
						\label{fig:add_key_button}
				\end{figure}
				\newpage
              \item Step 3: Add a description for the key if desired, then click Create. Do not give the key a scope of any kind -- that will prevent the key from being used for authentication with Rancher. 
				    \begin{figure}[h]
						\centering
						\includegraphics{images/add_api2.png}
						\caption{Add Key window.} 
						\label{fig:add_key_window}
				\end{figure}
				\item Step 4: Once the key is created, save all the information it gives you into a Notepad or Word document. You’ll need it later, and it won’t give you the information ever again. 
          \end{enumerate}
          \subsection{Rancher CLI}
          \begin{enumerate}
              \item Download the Rancher CLI to run some commands from your computer. Click the Download CLI button in the bottom right of the cloud webpage and unzip the downloaded folder onto your computer.
              \begin{figure}[htb]
						\centering
						\includegraphics[scale=0.5]{images/rancher_cli.png}
						\caption{Rancher CLI Download button.} 
						\label{fig:rancher_cli_download_button}
				\end{figure}
				\item Open up a terminal and type in the following command: rancher login https://cloud.cs.vt.edu/k8s/clusters/c-l4mcr -{}-token <BEARER\_TOKEN>. Replace the <BEARER\_TOKEN> with whatever your API key gave you, which you saved in step 3 of Section \ref{se:rancher access}.
                \begin{figure}[htb]
						\centering	\includegraphics[scale=0.8]{images/rancher_cli_interface.png}
						\caption{Rancher CLI} 
						\label{fig:rancher_cli_interface}
				\end{figure}
				\item Now you should be able to manipulate the cluster remotely from your machine using Rancher CLI commands - the command \textbf{rancher kubectl} will let you run kubectl commands from Rancher.
          \end{enumerate}
        \end{flushleft}
        \subsection{Using Dockerfiles to Create Containers}
        \begin{flushleft}
        Our team mostly utilized yaml files to deploy containers within DLRL, but you can also use the Docker CLI and Dockerfiles to deploy containers within a cluster. The following is a short guide on how to set up a simple container using a Dockerfile.
          \begin{enumerate}
              \item Get the URL of the application image from the container registry it comes from.
              \vspace{20pt}
              \begin{figure}[htb]
						\centering
						\includegraphics[]{images/image.png}
						\caption{Address of a container image from its Docker registry} 
						\label{fig:image_address}
				\end{figure}
		\newpage
          \item Create a folder for the image to sit in, and create a new file called “Dockerfile” without an extension in the same folder. The Dockerfile should have at minimum the following contents, although more details should be added for environment variables, container commands, and other information to customize the container to your needs: 
            \begin{figure}[htb]
						\centering
						\includegraphics[scale=0.5]{images/docker_file.png}
						\caption{Contents of a Dockerfile.} 
						\label{fig:docker_file_contents}
				\end{figure}
		    \item Open up a terminal, navigate to the folder containing the Dockerfile and enter the following command:
                                \begin{center}
                                    \textbf{docker build -t <image\_name> .} 
                                \end{center}
            
            \item Wait for the image’s layers to download. This might take a while. Once complete, use the following command to deploy the image:
                    \begin{center}
                        \textbf{docker run -dp <your port>:<container port> <image\_name>}
                    \end{center}
            \end{enumerate}
        \end{flushleft}
        \subsection{Working with Kubernetes CLI} \label{sec:kubectl}
        \begin{flushleft}
        The Kubernetes CLI is designed to specifically work with Kubernetes clusters, such as the cluster in DLRL. This section provides an overview of how to work with Kubernetes to deploy, delete, and troubleshoot containers.  
        \end{flushleft}
            \begin{enumerate}
                \item Creating and Deploying
                    \begin{enumerate}
                        \item Create YAML files for each Deployment in each namespace. Information on yaml file creation can be found in Section \ref{sec:deploy_img}. 
                        \item Run the following command to deploy the container
                                \begin{center}
                                    \textbf{kubectl apply -f <name of yaml file>} 
                                \end{center}
                        \item Check container(s) are running using the following command,
                                \begin{center}
                                    \textbf{kubectl get pods -n <namespace of container>} 
                                \end{center}
                        \end{enumerate}
                \item Deleting and troubleshooting
                        \begin{enumerate}
                        \item Delete a container deployment with the following command:
                                \begin{center}
                                    \textbf{kubectl delete deployment <name of deployment> } 
                                \end{center}
                        \item To get container error logs and detailed deployment information for a container, use the following commands
                                \begin{center}
                                    \textbf{kubectl describe pod classify-text-0 -n cs5604-wp-db}
                                    
                                    \textbf{kubectl logs classify-text-0 -n cs5604-wp-db} 
                                \end{center}
                    \end{enumerate}
                    \item Some images are kept within private registries, and therefore need authentication to download them. To authenticate, we need to create a secret with the credentials to the registry, and then add that secret as an imagePullSecret to the deployment's yaml file. Use the following command to create an imagePullSecret:
                     \begin{center}
                         \textbf{kubectl create secret docker-registry <secret-name> -{}-docker-server=<server-url> -{}-docker-username=<username> -{}-docker-password=<password> -{}-docker-email=<email>}
                     \end{center}
            \end{enumerate}
    \chapter{Developer Manual} \label{ch:dev_manual} 
        \section{Cluster Component Guide}
            \subsection{Containers}
            \begin{flushleft}
              
            For information on the CS5604 namespaces and their functionality within the CS container cluster, see Table \ref{tab:namespaces}.
            Describing this CS4624 project,
            Tables \ref{tab:deployments_fe} to \ref{tab:deployments_wp_test} provide a comprehensive list of all of the currently deployed containers (referred to as deployments in Kubernetes nomenclature) within the DLRL cluster, divided out by the namespace they are in. 
            %For information on the current namespaces and their functionality within the cluster, see Table \ref{tab:namespaces}. 
            These tables contain the given names of the deployments, which are taken directly from the VTCS cloud deployments, the number of replicas, or instances, of the given deployment, and its current status. The caption associated with the table lists the namespace that the given deployments reside in. CLBO means CrashLoopBackOff, an error within Kubernetes containers that we discuss more in Section \ref{sec:current_issues}.
            \end{flushleft}
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|}
                    \hline
                     \textbf{Deployment Name} & \textbf{Replicas} & \textbf{Status}\\
                     \hline
                     frontend0 & 1 & Running \\
                     \hline
                \end{tabular}
                \caption{Deployments within cs5604-fe-db}
                \label{tab:deployments_fe}
            \end{table}
            
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|}
                     \hline
                     \textbf{Deployment Name} & \textbf{Replicas} & \textbf{Status}\\
                     \hline
                     crop-image-test & 1 & Running \\
                     \hline
                     mysql & 1 & Running \\
                     \hline
                     postgres-test & 1 & Running \\
                     \hline
                     test-chapter-extract & 1 & CLBO \\
                     \hline
                     test-classification & 1 & Running \\
                     \hline
                     test-image-inference & 1 & Running \\ 
                     \hline
                     test-ingestion & 1 & CLBO \\ 
                     \hline
                     test-pdf-to-image & 1 & Running \\
                     \hline
                     test-text-extraction & 1 & CLBO \\
                     \hline
                \end{tabular}
                \caption{Deployments within cs5604-etd-db}
                \label{tab:deployments_etd_db}
            \end{table}
            
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|}
                     \hline
                     \textbf{Deployment Name} & \textbf{Replicas} & \textbf{Status}\\
                     \hline
                     etd-chapter-extraction & 1 & CLBO \\
                     \hline
                     etd-classification & 1 & Running \\
                     \hline
                     etd-crop-image & 1 & Running \\
                     \hline
                     etd-image-inference & 1 & Running \\
                     \hline
                     etd-metadata-ingestion & 1 & CLBO \\
                     \hline
                     etd-pdf-to-image & 1 & Running \\ 
                     \hline
                     etd-test-extraction & 1 & CLBO \\ 
                     \hline
                     etd-validate & 1 & CLBO \\
                     \hline
                \end{tabular}
                \caption{Deployments within etd}
                \label{tab:deployments_etd}
            \end{table}
            
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|}
                    \hline
                     
                     \textbf{Deployment Name} & \textbf{Replicas} & \textbf{Status}\\
                     \hline
                     airflow & 1 & Running \\
                     \hline
                     api & 1 & Running \\
                     \hline
                     reasoner & 1 & Running \\
                     \hline
                     services-db & 1 & Running \\
                     \hline
                     ubuntu & 1 & Running \\
                     \hline
                \end{tabular}
                \caption{Deployments within cs5604-int-test}
                \label{tab:deployments_int}
            \end{table}
            
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|}
                    \hline
                     \textbf{Deployment Name} & \textbf{Replicas} & \textbf{Status}\\
                     \hline
                     data-parse & 1 & CLBO \\
                     \hline
                     els-index & 1 & CLBO \\
                     \hline
                     filter-merge & 1 & CLBO \\ 
                     \hline
                     geolocation & 1 & CLBO \\
                     \hline
                     hashtags & 1 & CLBO \\ 
                     \hline
                     id & 1 & CLBO \\
                     \hline
                     keywords & 1 & CLBO \\
                     \hline
                     mentions & 1 & CLBO \\
                     \hline
                     timestamp & 1 & CLBO \\
                     \hline
                     twirole & 1 & CLBO \\
                     \hline
                     username & 1 & CLBO \\
                     \hline
                     warc-json & 1 & CLBO \\
                     \hline
                \end{tabular}
                \caption{Deployments within cs5604-twt-db}
                \label{tab:deployments_twt}
            \end{table}
            
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|}
                    \hline
                     \textbf{Deployment Name} & \textbf{Replicas} & \textbf{Status}\\
                     \hline
                     archive-webpage & 2 & CLBO \\
                     \hline
                     classify-text-0 & 1 & CLBO\\
                     \hline
                     extract-url & 1 & CLBO \\
                     \hline
                     index-data & 1 & CLBO \\
                     \hline
                     summarize-text-0 & 1 & CLBO\\ 
                     \hline
                     testnfs & 1 & Running \\
                     \hline
                     testnfs2 & 1 & Running \\
                     \hline
                \end{tabular}
                \caption{Deployments within cs5604-wp-db}
                \label{tab:deployments_wp_db}
            \end{table}
            
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|}
                    \hline
                     \textbf{Deployment Name} & \textbf{Replicas} & \textbf{Status}\\
                     \hline
                     classify-text & 1 & CLBO \\
                     \hline
                     summarize-text & 1 & CLBO \\
                     \hline
                \end{tabular}
                \caption{Deployments within wp-test}
                \label{tab:deployments_wp_test}
            \end{table}
            
            \newpage
            \subsection{Persistent Volumes and Claims}
            \begin{flushleft}
            Persistent volumes are a component of Kubernetes that allow containers to pull in and output data for long term storage. Currently each namespace has two persistent volumes assigned to it and two matching persistent volume claims. These persistent volumes both connect to NFS storage in camelot.cs.vt.edu, at different directory mount points. The claims are referenced within any containers that require a mount point to pull data from. For the purposes of minimizing redundancy, we will not explicitly describe the persistent volumes and their claims here, since they are repeated once per namespace. For details on how to configure these components and how they are mounted to containers, see Section \ref{sec:img_dep_vol}.
            \end{flushleft}
            \subsection{Services}\label{sec:dlrl_services}
            \begin{flushleft}
            Most of the services created for the cluster are ClusterIPs, which expose a port within the container, and NodePorts, which associate those ClusterIPs with ports on the node the container is deployed on. Two of these services have been modified in DLRL: expose-fe and expose-api. In the cloud cluster, these services were LoadBalancers, which asynchronously creates a load balancer for the service using the host cloud's load balancing system. \cite{load_balancing} However, since the new cluster is not hosted in a cloud environment, no load balancer will be created, making the service effectively the same as a NodePort. We changed the type of these services to be reflective of this. Table \ref{tab:services} contains a comprehensive list of every service, the namespace it is housed in, its type, and the ports assigned to it. 
            
            \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|c|c|}
                     \hline
                     \textbf{Namespace} & \textbf{Service Name} & \textbf{Type} & \textbf{Port(s)}  \\
                     \hline
                     \multirow{3}{*}{cs5604-etd-db} & mysql-nodeport & NodePort & 3306:32742/TCP \\
                     \cline{2-4}
                     &postgres-test & ClusterIP & 5432/TCP \\
                     \cline{2-4}
                     & postgres-test-nodeport & NodePort & 5432:31487/TCP \\
                     \hline \hline
                     \multirow{3}{*}{cs5604-fe-db} & frontend0 & ClusterIP & 3000/TCP \\
                     \cline{2-4}
                     & frontend0-nodeport & NodePort & 3000:32577/TCP \\
                     \cline{2-4}
                     & expose-fe & NodePort & 3000:30705/TCP \\
                     \hline \hline
                     \multirow{9}{*} {cs5604-int-test} & airflow & ClusterIP & 5000/TCP \\
                     \cline{2-4}
                      & airflow-nodeport & NodePort & 5000:30335/TCP \\
                      \cline{2-4}
                      & expose-api & NodePort & 5000:32640/TCP \\
                     \cline{2-4}
                      & api & ClusterIP & 5000/TCP \\
                      \cline{2-4}
                      & api-nodeport & NodePort & 5000:31675/TCP \\
                      \cline{2-4}
                      & reasoner & ClusterIP & 5000/TCP \\
                      \cline{2-4}
                      & reasoner-nodeport & NodePort & 5000:32307/TCP \\
                      \cline{2-4}
                      & services-db & ClusterIP & 5432/TCP \\
                      \cline{2-4}
                      & services-db-nodeport & NodePort & 5432:30129/TCP \\
                      \hline
                \end{tabular}
                \caption{All services created for DLRL}
                \label{tab:services}
                
            \end{table}
            \end{flushleft}
            \newpage
        \section{Current Issues} \label{sec:current_issues}
        \begin{flushleft}
        The CS5604 application in DLRL is not fully functional as of the completion of the migration. Many of the containers, as depicted in the deployment Tables \ref{tab:deployments_fe} to \ref{tab:deployments_wp_test} are showing CrashLoopBackOff errors. This particular error, given the name CrashLoopBackOff since it occurs when a container must be perpetually restarted by the cluster, can occur for a variety of reasons. It could range from an issue with the base image's source code down to a simple configuration error. Since there are so many reasons a container may enter a CLBO state, we have compiled a list of all the container deployments currently in this state on DLRL, as well as their most recently traced error log, which we found by running the command 
        \begin{center}
            \textbf{kubectl logs <pod name> -n <namespace of pod> } 
        \end{center}
        Table \ref{tab:deployment_errors} contains the full list of all current deployment errors. Many of the containers producing errors are noted as performing a ``Restart upon completion." We have noted this when the container is not producing a specific error, but instead appears to be functioning normally until the application finishes running. When it finishes, the container is not resetting itself or exiting gracefully. It is instead exiting in a way that Kubernetes assumes is an error, since containers are not meant to exit without being explicitly instructed to do so. 
        
        While this is an issue particularly prevalent in the TWT namespace, we do not currently have a solution for it. These CLBO errors existed within the VTCS cloud before the cluster was migrated, and thus they have appeared again in DLRL. However, it would require some reconfiguration of the inside of the containers to prevent them from exiting inappropriately, which is a fix that we do not have the time to implement.  
        \end{flushleft}
                \begin{table}[h]
                \centering
                \begin{tabular}{|c|c|m{20em}|}
                    \hline
                     \textbf{Namespace} & \textbf{Deployment Name} & \textbf{Logged errors}\\
                     \hline
                     \multirow{3}{*}{cs5604-etd-db} &test-chapter-extract & No module named fitz\\ \cline{2-3}
                     &test-ingestion & Connection to elasticsearch.cs.vt.edu timed out\\
                     \cline{2-3}
                     &test-text-extraction & Resets upon completion\\
                     \hline \hline
                     \multirow{4}{*}{etd} & etd-chapter-extraction & No module named fitz\\
                     \cline{2-3}
                     &etd-metadata-ingestion & Connection to elasticsearch.cs.vt.edu timed out\\
                     \cline{2-3}
                     &etd-text-extraction & Resets upon completion\\
                     \cline{2-3}
                     &etd-validate & SQL: Entry already exists in table\\
                     \hline \hline
                     \multirow{4}{*}{cs5604-wp-db} & archive-webpage & Resets upon completion\\
                     \cline{2-3}
                     &classify-text & Resets upon completion\\
                     \cline{2-3}
                     &extract-webpage & ModuleNotFoundError: No module named 'rouge'\\
                     \cline{2-3}
                     &summarize-text & UnboundLocalError: local variable 'scores' referenced before assignment\\
                     \hline \hline
                       \multirow{2}{*}{wp-test} & classify-text & Resets upon completion\\
                     \cline{2-3}
                     & summarize-text & UnboundLocalError: local variable 'scores' referenced before assignment\\
                     \hline \hline
                      \multirow{12}{*}{cs5604-twt-db}& data-parse & Data Parser of RAW Tweet Logs. Usage: ./data\_parser.py <FILENAME> <hashtag,username> <QUERY>\\
                     \cline{2-3}
                     &els-index & Connection to elasticsearch.cs.vt.edu timed out\\
                     \cline{2-3}
                     &filter-merge & Resets upon completion\\
                     \cline{2-3}
                     &geolocation & Resets upon completion\\
                     \cline{2-3}
                     &hashtags & Resets upon completion\\
                     \cline{2-3}
                     &id & AssertionError\\
                     \cline{2-3}
                     &keywords & Resets upon completion\\
                     \cline{2-3}
                     &mentions & Resets upon completion\\
                     \cline{2-3}
                     &timestamp & Resets upon completion\\
                     \cline{2-3}
                     &twirole & ValueError: Expected object or value\\
                     \cline{2-3}
                     &username & Resets upon completion\\
                     \cline{2-3}
                     &warc-json & Resets upon completion\\
                     \hline
                     
                \end{tabular}
                \caption{Current DLRL Deployment Errors}
                \label{tab:deployment_errors}
            \end{table}
            \newpage
        \section{Troubleshooting Tips}
        \begin{flushleft}
        As we have progressed through this migration process, we have accumulated a set of tips that may be helpful for a future developer on the cluster to keep in mind. These are as follows:
        \begin{enumerate}
            \item Refer to other yaml files for help in container configuration. All yaml files used to create the currently deployed containers are located in the master node, organized into folders based on their type and namespace. Using their setup as a baseline for new components will help ensure that Kubernetes won't throw a formatting error or apply the yaml file incorrectly. 
            \item If you are deploying new images, be sure the imagePullSecret is valid. If the image is in a private container registry, set up a secret using the instructions in Section \ref{sec:kubectl} and use the created secret in the yaml file. The secrets currently in place will eventually become invalid, as they were created with a teammate's credentials, which will expire approximately June 2021. Since the credentials used to grant access to the container registry are created from the login information of a user that has access to the required images, these credentials will remain valid until the user is no longer able to access the registry or the user changes their password. In either of these cases, new credentials must be created for authentication. 
            \item The kubectl commands referenced throughout the report were used many times as we were setting up the containers. In particular, the commands \textbf{kubectl describe deployment <deployment name> -n <namespace of deployment>} and \textbf{kubectl logs <pod name> -n <namespace of pod> } provide a lot of useful information to diagnose issues when the container is being launched and when it isn't running correctly. 
            \item If a cluster component isn't working as anticipated, you can delete it with \textbf{kubectl delete <deployment/service/pv/pvc> -n <namespace of component>}. The yaml file used to create the component will still exist, and can be adjusted as needed. 
        \end{enumerate}
        \end{flushleft}
	\chapter{Summary} \label{ch:discussion}
	    \section{Major Changes from VTCS to DLRL}
	    The transition from the VTCS cloud to DLRL has kept most of the setup intact. The existing namespaces and deployments follow the same naming schemes, as do the services and persistent volumes. All containers that are functional in the VTCS cloud work in DLRL, and the configurations of each are as close as possible to their cloud counterparts. There were, however, a few key changes that were made out of necessity or lack of usefulness, detailed as below:
	    \subsection{Removal of the Ceph Filesystem}
	    \begin{flushleft}
	    The Ceph file system has been fully removed. Several containers in the VTCS cloud were mounted to a Ceph filsystem that fed them input data. However, at the request of our client, we have removed the Ceph filesystem entirely. The cloud containers were already reconfigured to pull data from the Camelot server by the time we began this project, so all we had to do was remove the Ceph persistent volumes and useless mounts on the relevant containers. 
	    \end{flushleft}
	    \subsection{Exclusion of CentOS Containers}
	    \begin{flushleft}
	    Continuing from the previous section, most namespaces within the VTCS cloud had at least one deployed container of the latest version of CentOS. The only customization these containers had were a mount to the now unused Ceph filesystem. Without that mount, the CentOS containers had no use in the cluster, as no other containers spoke to them and they were not performing any tasks. Therefore, they were not migrated to DLRL. 
	    \end{flushleft}
	    \subsection{Exclusion of Unused Services}
	    \begin{flushleft}
	    Almost every container in the VTCS cloud had an associated ClusterIP. However, none of them except for those listed in Table \ref{tab:services} had any active NodePorts attached to them, or were being accessed by other containers. Since they were extra services with no apparent use, they were not migrated to DLRL. 
	    \end{flushleft}
	    \subsection{Reduction of Cluster Nodes}
	    \begin{flushleft}
	    While the VTCS cloud had a total of nine nodes for all of the containers to be deployed across, the current cluster only has three, as described in Table \ref{tab:cluster_nodes}. However, there have not been any noticeable performance issues with this smaller cluster, nor have there been any cluster crashes since all of the containers have been deployed. However, if performance issues do arise as a result of future development, new nodes can be added to the existing cluster without issue. 
	    \end{flushleft}
	    \section{Lessons Learned} \label{sec:lesson_learned}
	    \begin{flushleft}
	    Throughout this project, we had to quickly develop an entirely new set of skills in order to work with a technology that nobody in our team was familiar with at the beginning of the semester. Since we had much to learn, the following section is a brief overview of the major lessons we learned while working on this project. 
	    
	    \begin{enumerate}
	        \item Containerized applications are much different from their non-containerized counterparts. There is much careful planning that needs to be done to properly configure a containerized application, and any problems will almost certainly lead to a CLBO error in the cluster. However, the payoff for the additional setup is that a containerized application is fully platform-independent, so it can be run from any machine that supports virtualization. 
	        \item Failure to carefully document leads to much backtracking. When doing something as procedural as migration, it is vital to be taking consistent notes on what has been moved and its current status. Otherwise, time will consistently be lost by having to manually review previously done work to ensure everything has actually been completed. 
	        \item Prepare for major setbacks. Throughout this semester, we were consistently put behind our planned schedule by various problems that prevented our progress. These issues were often access-related, where we had to wait for a response from an administrator to grant us the privileges we needed to access the images or ssh into the DLRL worker nodes. Administrators are often quite busy, and so responses can be delayed for some time. Therefore, having plans for additional work that can be done while you wait can keep progress moving forward during a stalling moment. 
	        \item Communication is vital in computer science. Whether it be with your assigned team, a mentor, an administrator, or other related group, you are guaranteed to interact with many different people when working on any given project. Having the skills to communicate professionally and concisely are extremely important in these instances, and learning from communication issues even more so. 
	    \end{enumerate}
	    	\end{flushleft}
	    	
	   \section{Future Work}
        \begin{flushleft}
        At this time, the DLRL cluster is not fully functional. The obvious concern is to address the various containers that are currently in a CLBO state, and getting them running. After that is addressed, however, future work could include scaling the cluster to accommodate more users of the front end. Developers may also consider thoroughly testing the back end applications with newer and larger sets of data, and possibly even adding a new back end application that can process a new kind of written work, such as other VT
        %intellectual property 
        documents. 
        \end{flushleft}
        
	\nocite{*}
    \renewcommand{\bibname}{References}	
    \bibliographystyle{plainnat}
    \bibliography{bibliography}

	% This is the standard bibtex file. Do not include the .bib extension in <bib_file_name>.
	% Uncomment the following lines to include your bibliography: 
	%\bibliography{<bib_file_name>}
	%\bibliographystyle{plainnat}   

% 	% This formats the chapter name to appendix to properly define the headers:
% 	\appendix

% 	% Add your appendices here. You must leave the appendices enclosed in the appendices environment in order for the table of contents to be correct.
% 	\begin{appendices}
% 		\chapter{First Appendix} \label{app:appendix_one}
% 			\section{Section one} \label{ase:app_one_sect_1}
% 				\lipsum[1-3]
% 			\section{Section two} \label{ase:app_one_sect_2}
% 				\lipsum[1-3]
% 		\chapter{Second Appendix} \label{app:appendix_two}
% 			\lipsum[2]
% 	\end{appendices}

\end{document}


%****************************************************************************
% Below are some general suggestions for writing your dissertation:
%
% 1. Label everything with a meaningful prefix so that you
%    can refer back to sections, tables, figures, equations, etc.
%    Usage \label{<prefix>:<label_name>} where some suggested
%    prefixes are:
%			ch: Chapter
%     		se: Section
%     		ss: Subsection
%     		sss: Sub-subsection
%			app: Appendix
%     		ase: Appendix section
%     		tab: Tables
%     		fig: Figures
%     		sfig: Sub-figures
%     		eq: Equations
%
% 2. The VTthesis class provides for natbib citations. You should upload
%	 one or more *.bib bibtex files. Suppose you have two bib files: some_refs.bib and 
%    other_refs.bib.  Then your bibliography line to include them
%    will be:
%      \bibliography{some_refs, other_refs}
%    where multiple files are separated by commas. In the body of 
%    your work, you can cite your references using natbib citations.
%    Examples:
%      Citation                     Output
%      -------------------------------------------------------
%      \cite{doe_title_2016}        [18]
%      \citet{doe_title_2016}       Doe et al. [18]
%      \citet*{doe_title_2016}      Doe, Jones, and Smith [18]
%
%    For a complete list of options, see
%      https://www.ctan.org/pkg/natbib?lang=en
%
% 3. Here is a sample table. Notice that the caption is centered at the top. Also
%    notice that we use booktabs formatting. You should not use vertical lines
%    in your tables.
% 
%				\begin{table}[htb]
%					\centering
%					\caption{Approximate computation times in hh:mm:ss for full order 						versus reduced order models.}
%					\begin{tabular}{ccc}
%						\toprule
%						& \multicolumn{2}{c}{Computation Time}\\
%						\cmidrule(r){2-3}
%						$\overline{U}_{in}$ m/s & Full Model & ROM \\
%						\midrule
%						0.90 & 2:00:00 & 2:08:00\\
%						0.88 & 2:00:00 & 0:00:03\\
%						0.92 & 2:00:00 & 0:00:03\\
%						\midrule
%						Total & 6:00:00 & 2:08:06\\
%						\bottomrule
%					\end{tabular}
%					\label{tab:time_rom}
%				\end{table}
% 
% 4. Below are some sample figures. Notice the caption is centered below the
%    figure.
%    a. Single centered figure:
%					\begin{figure}[htb]
%						\centering
%						\includegraphics[scale=0.5]{my_figure.eps}
%						\caption{Average outlet velocity magnitude given an average  
%				        input velocity magnitude of 0.88 m/s.} 
%						\label{fig:output_rom}
%					\end{figure}
%    b. Two by two grid of figures with subcaptions
%					\begin{figure}[htb]
%						\centering
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_1_1.eps}
%							\caption{Subcaption number one}
%							\label{sfig:first_subfig}
%						\end{subfigure}
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_1_2.png}
%							\caption{Subcaption number two}
%							\label{sfig:second_subfig}
%						\end{subfigure}
%
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_2_1.pdf}
%							\caption{Subcaption number three}
%							\label{sfig:third_subfig}
%						\end{subfigure}
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_2_2.eps}
%							\caption{Subcaption number four}
%							\label{sfig:fourth_subfig}
%						\end{subfigure}
%						\caption{Here is my main caption describing the relationship between the 4 subimages}
%						\label{fig:main_figure}
%					\end{figure}
%
%----------------------------------------------------------------------------
%
% The following is a list of definitions and packages provided by VTthesis:
%
% A. The following packages are provided by the VTthesis class:
%      amsmath, amsthm, amssymb, enumerate, natbib, hyperref, graphicx, 
%      tikz (with shapes and arrows libraries), caption, subcaption,
%      listings, verbatim
%
% B. The following theorem environments are defined by VTthesis:
%      theorem, proposition, lemma, corollary, conjecture
% 
% C. The following definition environments are defined by VTthesis:
%      definition, example, remark, algorithm
%
%----------------------------------------------------------------------------
%
%  I hope this template file and the VTthesis class will keep you from having 
%  to worry about the formatting and allow you to focus on the actual writing.
%  Good luck, and happy writing.
%    Alan Lattimer, VT, 2016
%
%****************************************************************************





