%       File: VTthesis_template.tex
%     Created: Thu Mar 24 11:00 AM 2016 EDT
%     Last Change: Thursday, December 19, 2019
%     Author: Alan M. Lattimer, VT
%	  With modifications by Carrie Cross, Robert Browder, and LianTze Lim. 
%
% This template is designed to operate with XeLaTeX.
%
% All elements in the Title, Abstract, and Keywords MUST be formatted as text and NOT as math.
%
%Further instructions for using this template are embedded in the document. Additionally, there are comments at the end of the file that give suggestions on writing your thesis.  
%
%In addition to the standard formatting options, the following options are defined for the VTthesis class: proposal, prelim, doublespace, draft. 

\documentclass[singlespace,nopageskip]{VTthesis} % nopageskip - Removes arbitrary blank pages.



% Using the following header instead will create a draft copy of your thesis
%\documentclass[doublespace,draft]{VTthesis}

% The lipsum package is just included to put dummy text in the document in order to demonstrate page headers and table of contents behavior. You should remove it once you begin writing your actual thesis or dissertation.
\usepackage{lipsum}
\usepackage{lscape} 
\usepackage{enumitem}
% \usepackage{subfig}
\usepackage{outlines}

% Title of your thesis
\title{Final Report \protect\\ CS 5604: Information Storage and Retrieval}

% You should include 3-5 keywords, separated by commas
% \keywords{Instructed by Professor Edward A. Fox}

% Your name, including middle initial(s)
\author{Team 3: Objects \& Topics \protect\\ Nila Masrourisaadat, Raj Sahu, Alan Devera, \protect\\ Nirmal Amirthalingam, Chenyu Mao
\protect\protect\\ Subject Matter Expert: Aman Ahuja}

% Change this to your program, e.g. Physics, Civil Engineering, etc.
% \program{Mathematics} 

% Change this to your degree, e.g. Master of Science, Master of Art, etc.
% \degree{Doctor of Philosophy} 

% This should be your defense date:
\submitdate{January 17, 2023
\protect\\Instructed by Professor Edward A. Fox} 
% Committee members. Only have five readers and one chair available.
% Only use the ones you need and don't include the ones you don't need.
% You can also declare a Co-advisor. If you do, the principal and co-advisors
% will be listed as co-advisors on the title page.  Per the VT ETD standards, 
% you should not include titles or educational qualifications such as PhD or Dr.
% You should, however, include middle initials if possible.
% \principaladvisor{John Q. Williams}
%\coadvisor{Vicente Esparza}
% \firstreader{Janet K. Martin}
% \secondreader{Kara M. Jones}
% \thirdreader{James Smith}
%\fourthreader{Fourth Committee Member}
%\fifthreader{Fifth Committee Member}

% The dedication and acknowledgement pages are optional. Comment them out to remove them.
% \dedication{This is where you put your dedications.}
% \acknowledge{This is where you put your acknowledgments.}

% The abstract is required.
\abstract{The CS 5604: Information Storage and Retrieval class (Fall 2022), led by Dr. Edward Fox, has been assigned the task of designing and implementing a state-of-the-art information retrieval and analysis system that will support Electronic Theses \& Dissertations (ETDs). Given a large collection of ETDs, we want to run different kinds of learning algorithms to categorize them into logical groups, and by the end, be able to suggest to an end-user the documents which are strongly related to the one they are looking for. The overall goal for the project is to have a service that can upload, search, and retrieve ETDs with their derived digital objects, in a human-readable format. Specifically, our team is tasked with analyzing documents using object detection and topic models, with the final deliverable being the Experimenter web page for the derived objects and topics. The object detection team worked with Faster R-CNN and YOLOv7 models, and implemented post-processing rules for saving objects in a structured format. As the final deliverable for object detection, inference on 5k ETDs has been completed, and the refined objects have been saved to the Repository. The topic modeling team worked with clustering ETDs to 10, 25, 50, and 100 topics with different models (LDA, NeuralLDA, CTM, ProdLDA). As the final deliverable for topic modeling, we store the related topics and related documents for 5k ETDs in the Team 1 database, so that Team 2 could provide the related topic and documents on the documents page. By the end of the semester the team was able to deliver the Experimenter web page for the derived objects and topics, and the related objects and topics for 5k ETDs stored in the Team 1 database.\\ \\
% add topic modeling
\textbf{Keywords:} ETD, electronic theses and dissertations, deep learning, object detection, topic modeling}
%\abstract{\lipsum [1-4]}

% The general audience abstract is required. There are currently no word limits.
% \abstractgenaud{You are also required as of Spring 2016 to include a general audience abstract. This should be geared towards individuals outside of your field that may be reading seeking information about your work. You should avoid language that is particular to your field and clearly define any terms that may have special meaning in your discipline.}

\begin{document}
% The following lines set up the front matter of your thesis or dissertation and are required to ensure proper formatting per the VT ETD standards. 
  \frontmatter
  \maketitle
  \tableofcontents

% The list of figures and tables are now optional per the official ETD standards.  Unless you have a very good reason for removing them, you should leave these lists in the document. Comment them out to remove them.
	\listoffigures
	\listoftables
    % \printnomenclature %Creates a list of abbreviations. Comment out to remove it. 


% The following sets up the document for the main part of the thesis or dissertation. Do not comment out or remove this line.
	\mainmatter

	\chapter{Overview} 
        \section{Introduction} 
        In 1997, Virginia Tech was the first university to require its students to submit their theses and dissertations (ETDs) electronically. Recent research at Virginia Tech and Old Dominion University has collected over 500,000 ETDs from around the USA that provides our corpus. Accordingly, we are trying to make them more accessible for searching by extracting the metadata, chapters, abstracts, table of contents, figures, and so on as well as implementing topic modeling algorithms to obtain a list of the most relevant topics associated with the document files. Eventually, users will have access to the ETDs through the web pages developed by the Fall 2022 CS5604 class. In this report, we discuss the pipeline of our work as well as the architecture, design, and implementation of the underlying algorithms that we have employed.
        
        There are 5 teams to achieve this goal. A curator team (Team 1) stores the ETDs and outputs from other teams and provides APIs to access this data. Another team (Team 2) for the front-end user persona allows users to search ETDs and use a recommendation system. Our team will experiment with topic models and object detection which will ultimately allow searching of ETDs through specific topics by a chapter and section level.
        %They would also upload 
        Team 1 enables uploading
        all data contained in an ETD to the database. Another experimenter team (Team 4) would deal with analysis of ETDs through language models, classification, and summarization by a chapter and section level. All of the services would be provided by the DevOps team (Team 5) that would provide CI/CD services through Docker containers.
    
        Our role in this project will be experimenting with topic models and object detection. The refined objects from the object detection pipeline along with the list of topics from the topic modeling pipeline will be fed into Team 1's Repository. Figure \ref{complete} shows the general pipeline that our Objects \& Topics team has been following throughout the semester.
        
	    \begin{figure}[hbt!]
	        \begin{center}
	        \hspace*{-1cm}
	        \includegraphics[width=\linewidth]{pipeline diagram-object detection.png}
            \caption{Complete pipeline diagram} \label{complete}
        \end{center}	    
        \end{figure}
	    
	    The object detection subteam will extract the ETD objects using YOLOv7 and Faster R-CNN (Detectron2), and work on post-processing logic to parse the text and image objects from the detection. Both the list of objects (un-ordered set) and the ETD hierarchy would be stored in separate data structures, which would be used to generate an XML file, and create RDF Triples to be stored in the Knowledge Graph. The outputs can then be accessed from the Repository by other teams for downstream tasks. 
	    
	    The topic modeling team will train the models based on ETDs and segmented chapters generated by the object detection team, and work on topic modeling and clustering algorithms. % add a line here
	    
	    We use Team 1's API to store the ETD objects as well as the list of topics in the Repository. The complete pipeline will be explained in detail in the following sections.
	    
	    \section{Project Management} 
	    Actively managing human resources to get work done efficiently is one of the key requirements in this project. There are a host of project tracking tools like JIRA, Google Tasks, and Trello which can be clubbed with Git integration to track who is working on which feature and what all code changes has been done towards a particular task. 
	    
	    For our team communication, we decided to use Google Tasks owing to its simplicity. We keep track of the progress of this project through weekly meetings, where each of us updates the others with the current progress and challenges faced. Thus whenever some issue comes up, everyone can instantly resolve that issue together. Besides, our project client could assign tasks and set due dates, so the work flow is transparent and efficient. 

    \chapter{Literature Review}
        Research exploration of literature is a time consuming process. A researcher has to go through hundreds of documents, filter out the ones which are of interest and develop further analysis. This iteration has to go multiple times in most cases. Our class is collaborating with the University Library at Virginia Tech \cite{vtechworks} to build an information storage and retrieval system to help speed up that process. They have given us a collection of over 500,000 ETDs \cite{vt2} from different universities across the country. Each ETD has some metadata fields which will be described in the subsequent sections. Along with the data, we have the full text and the PDF version of the ETD document. At present, topic modeling and object detection models are helping literature reviewers by suggesting papers which can be more relevant to the reader.
        
        \section{Dataset}
        Our dataset has more than 500K ETDs ranging from 1845 to 2020, but most of them are published after 1945. 
        \subsection{Object Detection}
        % EVENTUALLY WRITE ABOUT OTHER DATASETS SUCH AS DOCBANK IN THE FUTURE %
        The labeling for the ETD objects was done using Roboflow \cite{roboflow}, a free online labeling tool. There are 24 categories that were identified and labeled which were deemed common in ETDs such as paragraph, figure, and table. Table \ref{etd_categories} contains all the categories that the detection model will identify. There are 20 text-based classes, and 4 image-based classes. 
        
        \begin{table}[!ht]\centering
        \caption{ETD Categories}\label{etd_categories}
        \begin{tabular}{lrrrr}\toprule
        \textbf{Category} &\textbf{\#Instances} \\\midrule
        Title &439 \\
        Author &404  \\
        Date &338  \\
        University &309  \\
        Committee &282  \\
        Degree &279  \\
        Abstract Heading &169  \\
        Abstract Text &183 \\
        List of Contents Heading &512  \\
        List of Contents Text &1059  \\
        Chapter Title &2211  \\
        Section &9337 \\
        Paragraph &30359  \\
        Figure &6359  \\
        Figure Caption &5722  \\
        Table &2654  \\
        Table Caption &2213  \\
        Equation &5092  \\
        Equation Caption &3051  \\
        Algorithm &96  \\
        Footnote &5722  \\
        Page Number &24543  \\
        Reference Heading &313  \\
        Reference Text &2088  \\
        \\\midrule
        Total Objects &99859 \\
        Total Images &25073 \\
        \bottomrule
        \end{tabular}
        \end{table}
        
        The labeling was provided to us from 6 undergraduate students working during the Spring 2022 semester \cite{obj_det_alan}.
        The dataset contains both born digital and scanned ETDs. During the Summer 2022 semester, our Subject Matter Expert (SME) augmented the dataset to add to the final object count of $\sim$25K pages ($\sim$100K objects) to get the dataset we are working with for this project. 
        
        \subsection{Topic Modeling}
        % The dataset contains the basic information of ETDs, such as ETD ID, title, and abstract. We are going to take the title and abstract as our analyzing target, and also segmented chapters generated by the object detection group as extra analyzing targets. 
        The dataset contains the basic information of ETDs, such as ETD ID, title, and abstract. We took the title and abstract as our analyzing target. We also segmented chapters generated by the object detection group as extra analyzing targets.         
        % \section{Object Detection}
        %     \subsection{Faster R-CNN}
             
            
            % \subsection{YOLOv7}
            
        % Merging individual literature reviews to a single review at the beginning of this chapter. 
            
        \section{Object Detection}
        We plan on evaluating our object detection pipeline using two models -- Faster R-CNN and YOLOv7, as each of them has a distinct advantage. 
        
        Faster R-CNN, which was originally published in 2015 \cite{faster_rcnn}, is the third iteration of the R-CNN architecture \cite{rcnn} published by Ross Girshick et al. A second version called Fast R-CNN \cite{fast_rcnn} replaced R-CNN's selective search algorithm with the Region of Interest (RoI) Pooling that enabled sharing expensive computations and made the inference much quicker. Faster R-CNN introduced a Region Proposal Network (RPN) that can be trained end-to-end with the CNN to generate high quality proposals using the feature maps from the CNN. It is to be noted that the original Faster R-CNN implementation used a standard VGG-16 backbone architecture for the CNN, whereas in the framework that we are using, Facebook AI Research's Detectron2 \cite{detectron}, there are object detection models built on top of Faster R-CNN with advanced architectures that perform really well. Though Faster R-CNN remained the state-of-the-art for years, it has been replaced by models with better speed vs. accuracy trade-offs such as the newer iterations of YOLO models. YOLOv7 has the highest accuracy of 56.8 percent average precision \cite{YOLOv7} when trained on the MS COCO dataset \cite{coco} from scratch without using pre-trained weights, and it outperforms both transformer-based object detectors and convolution-based object detectors in real-time. However, it is to be noted that most of our work concerning ETDs is done offline; they are processed in batches. Thus, the inclusion of both YOLO and Faster R-CNN models in the Experimenter web page would be beneficial.
        
        \section{Topic Modeling}
        In recent years, there have been many advances in the fields of text mining and natural language processing (NLP). Topic modeling in the analysis of ETDs aims to extract thematic collections of words that could represent topics from a large corpus of text documents. Several topic modeling algorithms have been proposed over the years. Earlier works such as Latent Dirichlet Allocation (LDA) \cite{1} were based on the idea of probabilistic formulation of topics. Recent works in the domain of topic modeling include neural topic models such as NeuralLDA \cite{2}, ProdLDA \cite{2}, and CTM \cite{3}. The representations learned from topic models can be used for downstream tasks that rely on document representations, such as finding similar documents (document recommendation), finding similar topics, and analyzing the variation of topics over time. 
        
        \section{Tools}
         \subsection{PyTorch}
	        PyTorch \cite{pytorch} is a widely used open-source machine learning framework originally developed by Meta AI. A majority of the deep learning projects that are in deployment now use PyTorch. It supports Python \cite{python} natively while also having a C++ interface. PyTorch and TensorFlow are considered to be the state-of-the-art frameworks for deep learning currently, before which frameworks like Caffe \cite{caffe} and Torch \cite{torch} used to be popular. A major advantage in using PyTorch is that it provides much better GPU acceleration as it relies on Tensor operations at the base level. 
	        
	        \subsection{YOLOv7}
	         YOLOv7 is a real-time object detector that has greatly advanced the computer vision and machine learning world, as it is considered to be the fastest, and most accurate object detector till date. For training the model on the COCO dataset \cite{coco}, pretrained weights were not used in the official YOLOv7 paper \cite{YOLOv7} by Chien-Yao Wang et al. 
 \begin{figure}[hbt!]
  \begin{center}
 \includegraphics[width=16cm]{yolov7.png}
 \caption{Comparison between real-time object detectors \cite{YOLOv7}}
 \end{center}
\end{figure}
           
            
            YOLOv7 uses a convolutional neural network to predict bounding boxes and class probabilities considering the entire image at one step, unlike earlier object detection models which use regions of the image to localize objects and images with high probabilities. The architecture is as follows -- image frames are featured through a backbone (deep neural network composed of many convolutional layers, and that is used to extract certain features) which is then combined and mixed in the neck (additional layers between backbone and the head, which are referred to as a copy of detectors to collect feature maps from different stages) and then it finds the bounding boxes. 
            % The backbones used in YOLOv7 are VoVNET, CSPVoVNET, ELAN, E-LAN. The neck in YOLOv7 uses FPN, RFB, PAN. The Detection happens in the head (dense prediction). Once the detectors make the prediction for localization and classification at the same time, this layer is present at only one stage after detectors like YOLO, SSD, RPN. Eventually, sparse prediction is for two stage detectors, FRCNN and RFCN, which does the class probabilities for the model. 

	        \subsection{Detectron2}
	        Detectron2 \cite{detectron} is a framework for object detection built on top of PyTorch, developed by Facebook AI Research (FAIR). It has support for a number of research projects in computer vision and production applications within Facebook. It is a re-write from 2018's Detectron, which was written using Caffe2, which is now a part of PyTorch \cite{pytorch}.  Besides object detection, Detectron2 also has support for advanced deep learning tasks such as activity recognition, semantic/instance segmentation, and several other tasks. Detectron2 has support for training object detection models on custom datasets using pre-trained weights that are provided in the official repository. All models in Detectron2 are pre-trained on the COCO dataset \cite{coco}. For our experiments, we would be using the Faster R-CNN models \cite{faster_rcnn} present in Detectron2. 
	        
	        \subsection{pdf2image}
	        pdf2image \cite{pdf2image} is a useful Python module that converts a Portable Document Format (PDF) file to PIL (Python Imaging Library) Image objects. The \verb|convert_from_path| function returns a PIL Image object that represents each page of the PDF for which the local path is given as input. We implement this function as a pre-processing step to get the set of page images which would be given as input to the object detection model. Also, we would be storing the page images for ETDs in the Repository.
	        
	        \subsection{Flask}
	        The Flask \cite{flask} framework is a lightweight web framework based on WSGI \cite{wsgi} implemented in Python, so it is convenient and straightforward for us to integrate it with our back end code. WSGI stands for Web Server Gateway Interface, and it describes how the web server communicates with the web application. Due to the reason that it is a micro-framework architecture, it is flexible and highly scalable, with a wide selection of third-party libraries. We can combine it with the Python libraries we need during the project development. In addition, the extension of new web features and functionalities would be easy to build. Besides, we also need the front-end technology, HTML, CSS, and JavaScript. HTML will provide the basic structure of the website, and CSS will control the styling of the website, while JavaScript will increase the interactivity with the users.
	        
	        \subsection{OCTIS}
	        OCTIS (Optimizing and Comparing Topic models) \cite{octis}  is a Python package that provides implementation of various topic modeling algorithms like Latent Dirichlet Allocation (LDA) \cite{1}, CTM \cite{3}, ProdLDA \cite{2}, and NeuralLDA \cite{2}. The library also allows to evaluate the results of an algorithm by using classification metrics, coherence metrics, diversity metrics, and similarity metrics. 
	        
	\chapter{Requirements} 
	    In this chapter, we discuss the requirements of our team: Objects \& Topics. 
	    
	    \section{Data Requirements}
	    Our object detection model is a supervised learning algorithm which needs labeled data, i.e., each of the figures and tables within an ETD has to be explicitly listed for the training algorithms to learn. For the topic modeling algorithm, no labeling is required as it is an unsupervised algorithm. To satisfy our data requirements, Team 1 will be maintaining the Repository with Read/Write APIs using which our model will ingest raw data and give out processed output for catering to the end-users.   
	    
	    Although these were the proposed requirements, Team 1 wanted us to give them the model code for them to run and populate their own Repository. However, after discussing more with Team 1, we decided that it was better to store objects in the Repository using their POST APIs. We will discuss this in more detail in the following sections. 
	    
	    \section{Processing Requirements}
	    Both the object detection and topic modeling experiments are resource intensive. Due to the large volume of input data, we need GPUs (which are specialized hardware having several processors designed to perform one difficult task unlike a CPU designed to do several different tasks). A container on the cloud having the desired hardware will cater to our needs. For this, Team 5 has initialized a Docker instance with GPU enabled for our team. However, since the cloud server does not support Docker images larger than 20 GB in size, we had to scale down the requirements, and update the container to support the frameworks that we are using for both object detection and topic modeling. 
	    
	   % \section{ETD Team Requirements}
	    
	\chapter{Design} 
	    \section{Approach}
	    As is shown in Figure \ref{pipeline}, after creating the page images from the ETDs, we train YOLOv7 to extract objects such as the metadata, figures, tables, chapters, titles, and paragraphs where some are image-based and some are text-based objects. To extract text from the PDF images, we use OCR (for scanned PDFs) or PyMuPDF (for born digital PDFs). Finally, we pass the refined image and text objects to Team 1.
	    \begin{figure}[ht!]
	        \begin{center}
	        \hspace*{-3cm}  
	        \includegraphics[width=1.4\linewidth]{OD pipeline.png}
	        \caption{Object Detection -- pipeline diagram} \label{pipeline}
	        \label{fig:fig}
	        \end{center}
	    \end{figure} 
	    
	    
	   %\newpage
	    \section{Tools}
	        The tools/frameworks that are used in this projects are listed as follows:
	        \begin{itemize}
	            \item VT CS Cloud \cite{cloud}
	            \item GitLab \cite{gitlab-vt}
	            \item Docker \cite{docker}
	            \item Python \cite{python}, PyTorch\cite{pytorch}, Flask \cite{flask}
	            \item Detectron2 \cite{detectron} and YOLOv7 \cite{YOLOv7}
	            \item LDA \cite{1}, Neural LDA \cite{2}, ProdLDA \cite{2}, CTM \cite{3}
	            \item OCTIS \cite{octis}
	        \end{itemize}
	        
	       
	    \section{Methodology}
	        \subsection{Object Detection} \label{rules}
	        From our initial assessment of the object detection models, we were able to conclude that YOLOv7 performs better than Faster R-CNN in terms of speed and accuracy for our use case, which is extracting different objects from ETDs. Therefore, we focus on the YOLOv7 model to implement our post-processing rules, though we give the experimenter the option to use both models (YOLOv7 and Faster R-CNN) as an additional feature with hyper-parameter tuning. 
	        
	        \textbf{Personas:}
	   These include
    students and researchers who want to review work related to their research area, librarians and university administration who want an overview of recent research in their institutions, and experimenters working with object detection. \par
	        \textbf{Goals:} The object detection team is to provide an interface for the experimenter, supporting analysis of the results of an experimental instance of detection, for a specified set of ETDs. They will be able to run experiments with the Detectron2 and YOLOv7 models, with the option to input their own set of hyper-parameters to assess the accuracy of both the model and the post-processing rules. \\
         For other types of personas, various services involving detected objects will be provided by other teams.\\
	       % Try \verbatim{} for pseudo-code. 
	       % \textbf{Pseudo code}
	       %  \subsubsection{Pseudo Code}
	       % % \begin{tabbing}
	       % \begin{verbatim}
	           
        % Load YOLOv7 model
        % Initialize front, body, and back tags for XML generation 
        % Initialize all sub-elements under the front tag (title, author, 
        %      university, degree, committee, date, abs_heading, abs_text, tocs)
        % Initialize all sub-elements under the back tag ref_heading, ref_text)
        % Convert PDF to list of images (pages)
       
        % For image in list of page images do
        % 	    Call  detect function with the image as input
        % 	    Sort  detections from top-bottom
        % 	    For  detection in list of detections do
        %         		 Get  class of detected object
        %         		 Call  parse_front / parse_body / parse_back according to the class
        %     	    	Function - parse_front { 
        %     		        If  class is toc_heading do 
        %                         If  previous toc_heading is not current toc_heading do 
        %      	                    Initialize new toc_heading subelement
        %     		        If class is toc_text do 
        %     			 	    Extract only the text and page numbers of ToC
        %     } 
        %     		Function - parse_body { 
        %     			    If current chapter is None do
        %     				         Initialize null section
        %     			    If class is chapter title or section name do
        %     				        Filter using rules
        %         				    	   First letter should be capitalized
        %         					        Check for duplicate instances
        %         				    	    Check for punctuation overflow from previous page
        %     			    If class is paragraph / footnote / equation number
        %     			    / figure caption / table caption (text-based objects) do
        %     				        Call get_text
        %     				        Set the text attribute of tag to the extracted string
        %     			    If class is figure / table / algorithm (image-based objects) do
        %     				        Call save_image
        %     				        Set the text attribute of tag to the image path
        % 		       Filter and extract page number for each title from ToC
        %         save ToC in a text file or in memory
        %     		   For line in text file do
        %         	        If chapter title and line are >80%  similar do
        %             		        	If extracted page number
        %             		        	is page number object detected do
        %             			            	Write title in a file # the chapter title 
        %             			            	and its page number matched with the title
        %             			            	and page number from ToC
        %         	  If chapter titles is >80% similar to toc titles do
        %         	        	Save the matched chapter titles
        %         	  else
        %         		        Save the unmatched chapter titles
        %         	  If section names is > 80% similar to toc titles do
        %         	            save the matched section names
        %         	  else
        %         	            Save the unmatched section names
        %     } 
        %     		Function - parse_back { 
        %     		    If class is ref_heading do
        %     				        Set ref_heading subelement
        %     		    If class is ref_text do
        % 				            Append text to ref_text subelement
        %     } 
        %     		Common logic for all detected objects { 
        %     			For every object (after initialization) { 
        %                 Get the current chapter / section tag
        %                 Call get_text on the detected object
        %                 Set the corresponding XML tag’s text attribute } 
        %     							} 
            
        %     	Link figures and tables with captions { 
        %         	   Iterate through the list of figures, tables,  figure captions,
        %         	   table captions to determine the pattern  { 
        %             If each figure / table object is followed by a caption
        %             	    Link each image object with the next caption object 
        %             	    in the list
        %             If each figure / table object is preceded by a caption
        %             	    Link each image object with the previous caption object 
        %             	    in the list
                    
        
        % \end{verbatim}
        Next, we discuss the problems that came with the object detection models, and the solutions we implemented post-detection. 
        \subsubsection{False positives}
        Problem: 
        \begin{itemize}
            \item Last line of the paragraph in the previous page is detected as a chapter title as in Figure \ref{fig:fp1}.
        \end{itemize}

        Solution (rule): 
        \begin{itemize}
            \item Check the first letter of the title (capital).
        
        \item Avoid the text overflow for consecutive pages by checking the punctuation.
        \item Check if it is a subset of the table of contents.
        
        \end{itemize}
        \begin{figure}[ht!]
            \centering
            \includegraphics[width=10cm]{1.png}
            \caption{Incorrectly detected chapter title -- false positive}
            \label{fig:fp1}
            % \label{fig:fig}
        \end{figure}
        Problem:
        \begin{itemize}
            \item Image-based objects (figures, tables) and their corresponding captions might not be on the same page as in Figure \ref{fig:fp2}.
        \end{itemize}
        
        Solution (rule):
        \begin{itemize}
            \item Determine the relationship between figure/table and their corresponding captions.
            \item Pair images with their corresponding captions according to the image being on top or bottom of images, or in some cases left or right.
        \end{itemize}
        \begin{figure}[ht!]
            \centering
            \includegraphics[width=14cm]{2.jpg}
            \caption{Incorrect caption detected -- false positive}
            \label{fig:fp2}
        \end{figure}
        Problem:
        \begin{itemize}
            \item Chapter titles appear on multiple pages for the same chapter in some cases.
        \end{itemize}

        Solution (rule):
        \begin{itemize}
            \item Check if two consequent chapter titles are the same.
        \end{itemize}

        \subsubsection{ETD Filtering Rules}
        The rules that we use to filter out incorrect detections, false positives, and map the hierarchy between objects are as listed below: \\ \\
        \textbf{Rules for filtering chapters/sections (delimiters)}
        \begin{itemize}
        \item  The first letter of the detected text object should be capitalized.
        % \item Text overflow punctuation:
        % \setlength{\itemindent}{2cm}
        % \item Whenever a new chapter tag is created (chapter title is detected), the last detected object in the previous page is checked, and if it is a paragraph object and the last character in paragraph is ‘.’ then it is a valid chapter tag.
        \item Whenever a new chapter tag is created (chapter title is detected), the last detected object from the previous page is checked. If it is a paragraph object, and the last character in paragraph is not ‘.’ then it is not considered to be a valid chapter tag.
        % this rule is merged with the sentence above
        % \item If the last detected object in the previous page is a footnote, or page number, then ignore this rule.
        \item If a chapter title is the same as the previous chapter title, then a new chapter tag is not created.
        \item An empty section tag is created when a body object other than section name or chapter title is detected -- this accounts for sections that do not have a chapter title or if the model fails to detect the chapter or section title. 
        \end{itemize}
        
        \textbf{Rules for linking image-based objects with their captions}
        \begin{itemize}
        \item This rule has been implemented for two image-based objects -- figure and table.
        \item {The relationship between figure/table and their corresponding caption is first determined -- captions can either appear on top or bottom of the image-based objects.}
        \item {The images are paired with their corresponding captions according to this pattern, assuming this pattern extends to the entire ETD.}
        \item {Exception to this rule -- If the orientation of a page is is horizontal (landscape) then the pattern might not be consistent with the other pages.
        To avoid this, each page’s orientation is checked, and if found to be horizontal, it is rotated by 90° -- and a new set of detections (sorted by the y-axis) is obtained.}
        \end{itemize}
        % \break
        \textbf{Getting rid of false positives based on matching with table of contents (ToC)}
        \begin{itemize}
        \item {Extract and save all the detected chapter titles, section titles, and page numbers using the text from the ToC.}
        \item {Create two files from comparison of ToC lines and chapter titles, one of which has the chapter titles similar to some lines of ToC, while the other has chapter titles that were not found in ToC. The same goes with section names and page numbers.}
        \item {Filter out the chapter and section titles that have been incorrectly detected.}

        \end{itemize}
        

	        \subsection{Topic Modeling}
	        \begin{figure}[ht!]
    	        % \begin{center}
              \hspace*{-2.5cm}
    	         \includegraphics[width=1.2\linewidth]{TM_pipeline_final.png}
    	        \caption{Topic Modeling -- pipeline diagram}
    	        \label{fig:TM-Pipeline}
    	        % \end{center}
	    \end{figure}
	        Topic Modeling involves clustering documents into list of topics using tokens and finding Nearest Neighbors using vectors to obtain most related documents. Earlier, the input comprised only of the ETD title and the overall abstract. But now, we incorporated the chapters and their summaries from the Team 1 database as input to our model. Figure \ref{fig:TM-Pipeline} shows the complete data pipeline. As a result, our topic browser gives more fine-grained categories. They now link to a list of chapters 
	        and list of ETDs.
	        \newline \break
	        \textbf{Personas:}
         These include
	        students and researchers who want to review work related to their research area, librarians and university administration who want an overview of recent research in their institutions, and experimenters working with topic modeling.  \par
	        
	        \textbf{Goals:}
	        \begin{enumerate}
	            \item Show the document topics based on different analysis models and numbers of desired topics.
	            
	            Right now, the services below are available to all the regular users who have the experimenter page link, but we will require login authorization in the future. The user will first select the model and the number of topics, which could refer to Figure \ref{IndexPage}. Next, the user will press the button for “select topic,” which will lead the user to the topic page that displays several topics with an index (e.g., indication of related digital objects like ETDs and chapters)  of the topic. After the user selects and presses the index of the topic, a page with documents that are topically related will be displayed. 
	            
	            \item Show related documents needed by users through topic models and indexes.
	            
	            The users first need to finish goal 1, then all relevant topics will be shown on a web page. Users can visit all topics and choose what they are interested in.
	            
	            \item Show documents by a timeline search engine on specific topics. The figure example could refer to Figure \ref{FilterDocumentPage}.
	            
	            The user needs to first select the model and the number of topics. Next, the user will press the button for “select topic,” which will lead the user to the topic page that displays several topics. After the user selects and presses the index of the topic, a page with documents that are topics related will be displayed. The user can enter the starting year, end year, and threshold on this page to search on the digital object(s) they want.
                
                \item Show most relevant topics for a search query. 
                
                The user inputs a set of keywords as a search query. Next, the user will select the model name and number of topics. After clicking the search button, a page with the top 10 results will be displayed where each result will indicate a topic (with associated hyperlink). On clicking any of these search results, the users will be redirected to show all of the documents connected with the selected topic. 
	        \end{enumerate}
	      
        %     \begin{figure}[hbt!]
    	   %     \begin{center}
    	   %      \includegraphics[width=\textwidth]{Figs/Goal1.png}
    	   %     \caption{Goal 1 shows document topics based on different analysis models and chosen topics number.}
    	   %     \end{center}
	       % \end{figure}
	       % \begin{figure}[hbt!]
    	   %     \begin{center}
    	   %      \includegraphics[width=\textwidth]{Figs/Goal2.png}
    	   %     \caption{Goal 2 shows related documents needed by users through topic models and indexes.}
    	   %     \end{center}
	       % \end{figure}
	       % \begin{figure}[hbt!]
    	   %     \begin{center}
    	   %      \includegraphics[width=\textwidth]{Figs/Goal3.png}
    	   %     \caption{Goal 3 shows documents by a timeline search engine on specific topics.}
    	   %     \end{center}
	       % \end{figure}
            
            
            
             
            % \begin{landscape}
            % \begin{table}[ht]
            % \centering
            % \caption{\textcolor{red}{Give a caption}} \label{blabla}
            % \begin{tabular}{p{1.5cm} p{5cm} p{6cm} p{3cm} p{3cm}}
            % \toprule
            % \textbf{Service ID}  & \textbf{Service Name}  & \textbf{Input File Names}  & \textbf{Input File IDs} & \textbf{Libraries; Functions; Environments}  \\
            % \midrule \newline
            % G11 & select model & ETDS\_Modelname \_Number & F1 & html,flask,input \\
            % G12 & Select topic number & ETDS\_Modelname \_Number & F1 & html,flask,input\\
            % G13 & Press “Select topics” Button & ETDS\_Modelname \_Number, URI.txt, preprocessed.txt & F1, F2, F3 & html,flask,button \\
            % G21 & Select index by hyperlink & output.p & F4 & html,flask,hyperlink \\
            % G31 & Input start year & Year.txt & F5 & html,flask,input\\
            % G32 & Input end year & Year.txt & F5 & html,flask,input\\
            % G33 & Input threshold & N/A & N/A & html,flask,input\\
            % G34 & Press “Advanced search topic” Button & URI.txt,  Preprocessed.txt, Year.txt & F2, F3, F5 & html,flask,button\\
            % \hline \newline
            % \end{tabular}
            % \end{table}
            % \end{landscape}
	        
	       
	    \section{Deliverables} \label{devliverables}
	        \subsection{Object Detection} \label{oddeliverables}
	       % Our object detection model will support services to upload any PDF of an ETD, extract the data inside it, and upload it to the database as an XML file. The other service would support the front-end user to look up an ETD and see everything contained inside the specific ETD ranging from its metadata elements to its chapters and contained figures. % FIX 
	        Our deliverables for object detection include:
	        \begin{itemize}
	            \item Extracted objects which will be stored inside Team 1's Repository. 
	            \item Experimenter web page which chooses either a Detectron2 (Faster R-CNN) or YOLOv7 model and displays all the extracted objects given an ETD.
	        \end{itemize}
	        
	        \begin{figure}[hbt!]
	        \begin{center}
	        \includegraphics[width=8cm]{xml_skeleton.png}
            \caption{XML Schema} \label{xml}
            \end{center}	    
            \end{figure}
            
            The XML schema to store the refined ETD objects is shown in Figure \ref{xml}.
            
            Each PDF has its ETD ID as the root element, and there are 3 sub-elements -- front, body, and back. There are four image-based objects (figures, tables, equations, and algorithms) whereas the rest are text-based objects. The image-based objects have captions or numbers associated with them. \newline
            \begin{table}[ht]
            \centering
            \caption{Deliverables -- ETD Objects} \label{objects_schema}
            % \begin{tabular}{p{3cm} p{6cm}} % reverse chrono was good
            % \setlength{\tabcolsep}{10pt}
            \renewcommand{\arraystretch}{1.5}
            \begin{tabular}{l p{10cm}}
            \toprule
            \textbf{Output for a given ETD}  & \textbf{Description/Contents}\\
            \midrule
            Page images & Each page of ETD is saved as a JPG file.  \\ 
            Detected objects & Consists of sub-folders for each image-based object class (figure, table, equation, algorithm) saved as JPG files.\\
            JSON object & Each object -- text and image-based -- is saved (unordered set as per the detection sequence).\\
            Parsed XML & Detected objects in a tree structure with XML elements set to the clean text for text-based objects, and the image path for image-based objects.\\
            \bottomrule
            \end{tabular}
            \end{table}
            \newline
            Based on our discussions with Team 1, and assessing the overall requirements of the class, we finalized the schema as shown in Table \ref{objects_schema} to store our objects in the Repository. \newline
            
            An example JSON structure for a saved text-based object is in the following format:
            \begin{verbatim}
            {"type": “university", 
            “text": “University of California, Berkeley", 
            “page_num": 1, “bbox": [587, 1329, 1109, 1391]}
            \end{verbatim} 
            
            An example JSON structure for a saved image-based object is in the following format:
            \begin{verbatim}
            {“type": “figure", 
            "path": "1410_figure_98.jpg", 
            “page_num": 141, “bbox": [565, 170, 1140, 1887]}
            \end{verbatim}
            
            The \verb|type| key contains the class of the detected object (from a total of 24 classes as shown in Table \ref{etd_categories}). The \verb|text| and \verb|path| keys are set to the clean text if it is a text-based object or the saved image path if it is an image-based object, and the \verb|bbox| key consists of the detected bounding box in this format: \verb|[x1, y1, x2, y2]| where \verb|(x1, y1)| represents the top-left corner, and \verb|(x2, y2)| represents the bottom-right corner of the bounding box. \newline
            
             We are using Team 1's \verb|Save detected object| (POST) API to store our objects. 

             \verb|https://team-1-flask.discovery.cs.vt.edu/v1/objects/<etd_id>/<type_name>| \newline
        
             The \verb|<type_name>| is set to \verb|page|, \verb|text|, \verb|image|, and \verb|xml|, to support different file types, and services required by the other teams. \newline
 
            \begin{table}[ht]
            \centering
            \caption{Deliverables -- Objects API} \label{api}
            \renewcommand{\arraystretch}{1.5}
            \begin{tabular}{l p{6.5cm}p{7cm}}
            \toprule
            \textbf{Type}  & \textbf{File} & \textbf{Metadata}\\
            \midrule
            \verb|page| & \verb|<page_number>.jpg| & - \\ 
            \verb|text| & - & \verb|{keys = type, text, page_num, bbox}|\\
            \verb|image| & \verb|<etd_id>_<class>_<count>.jpg| & \verb|{keys = type, path, page_num, bbox}|\\
            \verb|xml| & \verb|<etd_id>.xml| & -\\
            \bottomrule
            \end{tabular}
            \end{table}
            The objects are stored in the Repository as shown in Table \ref{api}. \newline

             The deliverables for the other teams are listed as follows:
            \begin{itemize}
            	\item Team 2: Experimenter web page
            	\item Team 4: Refined objects (with hierarchy) for the chapter segmentation model
            	\item Team 5: The requirements for them to set up the Docker containers for the cloud server, and provide services for workflow automation
            	\end{itemize} 
            	
            We are providing Team 5 with 3 of our services (containerized), which are to be used for workflow automation, and our front-end web page. The services are as follows:
            \begin{itemize}
                \item Generate page images given an ETD ID
                \item Generate outputs given an ETD ID - using YOLOv7
                \item Generate outputs given an ETD ID - using Faster R-CNN (Detectron2)
            \end{itemize}
            
            These services are available through Docker images that generate the aforementioned outputs when an ETD ID that exists in the Repository is specified as input while executing. The first service splits a PDF into multiple pages and creates a sub-directory consisting of the individual pages saved as JPG files. The second and third services generate the required outputs (detected images, JSON objects, and the XML file) using YOLO and Faster R-CNN models, respectively. The detailed instructions on how to use these services are given in Section \ref{developer}.


            % Team 2 will take the extracted objects from our team and the generated summaries and classifications from team 4 to create the document view page after a user clicks on an ETD.
            
            % These teams will access these values from Team 1. To achieve this, we are using an API contract created by Team 1. This contract will give us an etd\_id and uploads a list of extracted object categories, its respective value, and the page number associated with the object in this format:
            % \{'obj\_type': \emph{string} , 'obj\_val': \emph{string}, 'page\_num': \emph{integer}\} \\
            
    %         \textbf{Code Sample}
    	    
    % 	    \hspace{1cm} \textbf{GET} /get\_etd\_objects/\{etd\_id\}

    % 	    \textbf{Responses:} \newline
    % 	    \textbf{Valid Query}
    % 	   %  \hspace{2cm} \textbf{Valid Query }
    % 	     \begin{verbatim}
    %         Status : 200
    %         > {"status":"success", "extracted_objs":[{}, {}, ...]}
    % 	 	\end{verbatim}
    % 	 	\textbf{Invalid Query}
    % 	 	\begin{verbatim}
    %         Status : 400
    %         > {"status":"failure", "message":"Given etd_id not found!"}
    % 	 	\end{verbatim}
    	    \subsection{Topic Modeling}
    	    The deliverables for Topic Modeling include supporting APIs to return related topics and documents for an ETD, and a Topic Browser which visualizes all of the topics generated by the model and their respective documents. Table \ref{deliver_tm} lists the above mentioned points succinctly. 
            
             \begin{table}[ht]
            \centering
            \caption{Deliverables for Topic Modelling} \label{deliver_tm}
            \renewcommand{\arraystretch}{1.5}
            \begin{tabular}{l p{10cm}}
            \toprule
            \textbf{Output}  & \textbf{Description}\\
            \midrule
            Related Topics \& Related Docs & For a given ETD ID or Chapter ID, find the most similar topics and documents.  \\ 
            
            Topic Browser (Experimenter) & Allow a user to browse all topics, click on one and read associated documents (Documents by Topic, Chapters by Topic). \\
            \bottomrule
            \end{tabular}
            \end{table}
         
    	   % get_etd_embeddings
        \subsubsection{APIs}
        We are currently offering 3 kinds of API calls. The ETD embedding API call takes in the model name, number of topics, and ETD ID, and returns a corresponding topic vector which is a vector containing the probability of the document belonging to each of the k different topics generated.  The ETD related documents API takes in the model name, number of topics, and ETD ID, and returns the most related documents, while the parameter top k value is optional and defaults to 5 (related documents). The ETD related topics API takes in the model name, number of topics, and ETD ID, and returns the most related topics, while the parameter top k value is optional and defaults to 5 (related topics).

        
    	    \textbf{Code Sample -- get Topic vector}\\ 
    	    \hspace{1cm} \textbf{GET} /API/get\_etd\_embeddings/\{model\_name\}/\{num\_topics\}/\{etd\_id\} \\
    	   \hspace{1cm} 
    	   \textbf{Path Parameters:}
    	   \begin{enumerate}[leftmargin=2cm]
    	       \item \textbf{model\_name}: String\\
    	       Slug for name of model. Can be either of LDA, CTM, NeuralLDA, or ProdLDA.
    	       \item \textbf{num\_topics}: Integer\\
    	       Replace this with the number of topics to be generated. Has to be either of 10, 25, 50, or 100. 
    	       \item \textbf{etd\_id}: Integer\\
    	       Provide the etd\_id for which you need the results. 
    	   \end{enumerate}
    	   
    	    \hspace{1cm} \textbf{Response (Valid Query):}
    	     \begin{verbatim}
    	     Status: 200
    	     > {"status":"success","vector":[0.0003150326374452561,
    	     0.20719221234321594,0.01724390685558319,0.11466679722070694,
    	     0.00031506028608419,0.000315034732921049,0.00031509360997006297,
    	     0.00031502716592513025,0.00031508112442679703,0.00031509919790551066, 
    	     ........ ]}
    	     \end{verbatim}
    	    \hspace{1cm} \textbf{Response (Invalid Query):}
    	    \begin{verbatim}
    	     Status: 200
    	     > {"status":"failure", "message":"Given etd_id not found!"}
    	 	\end{verbatim}
    	 	
    	 	% get_etd_relatedDocs
    	    \textbf{Code Sample -- Get related Documents}\\ 
    	    \hspace{1cm} \textbf{GET} /API/get\_etd\_relateddocs/\{model\_name\}/\{num\_topics\}/\{etd\_id\}/\{topk\} \\
    	   \hspace{1cm} \textbf{Path Parameters:}
    	   \begin{enumerate}[leftmargin=2cm]
    	       \item \textbf{model\_name}: String\\
    	       Slug for name of model. Can be either of LDA, CTM, NeuralLDA, or ProdLDA.
    	       \item \textbf{num\_topics}: Integer\\
    	       Replace this with the number of topics to be generated. Has to be either of 10, 25, 50, or 100. 
    	       \item \textbf{etd\_id}: Integer\\
    	       Provide the etd\_id for which you need the results. 
    	       \item \textbf{topk}: Integer\\
    	       Provide the top K value, but this path parameter is optional and its default is 5 without specification.
    	   \end{enumerate}
    	 
    	    \hspace{1cm} \textbf{Response (Valid Query):}
    	     \begin{verbatim}
    	     Status: 200
    	     > {"status":"success","body":[218606, 63948, 321952, 91535, 241780]}
    	     \end{verbatim}
    	    \hspace{1cm} \textbf{Response (Invalid Query):}
    	    \begin{verbatim}
    	     Status: 200
    	     > {"status":"failure", "message":"Given etd_id not found!"}
    	 	\end{verbatim}
    	 	
    	 	
    	 	% get_etd_relatedTopic
    	    \textbf{Code Sample -- Get related Topics}\\ 
    	    \hspace{1cm} \textbf{GET} /API/get\_etd\_relatedtopics/\{model\_name\}/\{num\_topics\}/\{etd\_id\}/\{topk\} \\
    	   \hspace{1cm} \textbf{Path Parameters:}
    	   \begin{enumerate}[leftmargin=2cm]
    	       \item \textbf{model\_name}: String\\
    	       Slug for name of model. Can be either of LDA, CTM, NeuralLDA, or ProdLDA.
    	       \item \textbf{num\_topics}: Integer\\
    	       Replace this with the number of topics to be generated. Has to be either of 10, 25, 50, or 100. 
    	       \item \textbf{etd\_id}: Integer\\
    	       Provide the etd\_id for which you need the results. 
    	       \item \textbf{topk}: Integer\\
    	       Provide the top K value, but this path parameter is optional and its default is 5 without specification.
    	   \end{enumerate}
    	   
    	    \hspace{1cm} \textbf{Response (Valid Query):}
    	     \begin{verbatim}
    	     Status: 200
    	     > {"status":"success","body":[11, 46, 70, 3, 22]}
    	     \end{verbatim}
    	    \hspace{1cm} \textbf{Response (Invalid Query):}
    	    \begin{verbatim}
    	     Status: 200
    	     > {"status":"failure", "message":"Given etd_id not found!"}
    	 	\end{verbatim}
    	 	
    	 \subsubsection{Services}
      In order to support workflow automation (Team 5) we have divided our task into 2 services:
      \begin{itemize}
          \item Given a dataset having several ETDs, pre-process and train a Topic Model to generate a list of topics and their associated documents. 
          \item For a given document (ETD/Chapter), return the top 5 most related topics and related documents. 
      \end{itemize}
      These services will provide data to other teams as well as to our experimenter page, i.e., the Topic Browser.
	       
	\chapter{Implementation} 
	    \section{Milestones}
        % This project aims to complete the work that was left by the undergraduate students have left from the Spring 2022 semester. The main tasks to complete are:
    % 	\begin{table}[htb]
    % 	\begin{center}
    % 	\caption{Project timeline}
    %     \begin{tabular}{|| l | p{10cm}||} 
    %      \hline
    %      Date & Task \\ [0.5ex] 
    %      \hline\hline
    %      [] & [] \\ 
    %      \hline
    %     \end{tabular}
    %     \label{tab:project_timeline}
    %     \end{center}
    %     \end{table}
	   % The Interim Report 1 (IR1) is the first milestone that discusses the resources and tools we have, and details the plan for this semester. Simultaneously, we also come up some ideas that could extend functionalities of our previous Topic Modeling website. Interim Report (IR2) will be our second milestone, where we have at least a working prototype that could satisfy the basic functionality of requirements, and this result will be the fundamentals of our final prototype. As for Interim Report 3 (IR3), it will be the clean up and summary report of our project, which includes a ready prototype that will be deployed in the cloud of our university. 
	   \subsection{Object Detection}
	   In Table \ref{OD_Tasks} we list the tasks completed by the Object Detection team.
	   % WE NEED TO TALK MORE ABOUT THE MILESTONES AND UPDATE IT
            \begin{table}[ht]
            \centering
            \caption{Object Detection Milestones} \label{OD_Tasks}
            % \begin{tabular}{p{3cm} p{6cm}} % reverse chrono was good
            \begin{tabular}{cl}
            \toprule
            \textbf{Timeline}  & \textbf{Tasks}\\
            \midrule
            1 Sept & Get familiar with Detectron2 and YOLOv7\\ 
            8 Sept & Set up cloud server and get the model trained\\
            13 Sept & Set up object detection pipeline\\
            19 Sept & PDF to text using YOLOv7\\
            23 Sept & PDF to images using YOLOv7\\
            30 Sept & Parsing by chapters, sections and paragraphs\\
            4 Oct & Linking image based objects to caption / number\\
            6 Oct & API function to return a list of objects for an ETD\\
            15 Oct & Test parser logic and detect false positives\\
            15 Oct & Proposed rules method for removing false positives\\
            15 Oct & Complete object post-processing\\
            21 Oct & HTML generation for YOLOv7 from refined objects in a Flask app\\
            24 Oct & Team 1 deliverables -- DB tasks (read/write)\\
            28 Oct & Added support for Detectron2 (Faster R-CNN)\\ 
            28 Oct & Finalized base models and parser logic for YOLOv7 and Faster R-CNN\\
            1 Nov & Integrated Detectron2 model with the Flask App \\
            % 1 Nov & Front-end UI (Experimenter web page)\\
            4 Nov & Deployed the containerized Flask app on the cloud server\\
            28 Nov & Experiment with a bigger subset of the ETD dataset \\
            Final & Run inference on 5k ETDs using Team 1's API to store into their DB \\
            

            \bottomrule
            \end{tabular}
            \end{table}
	   \subsection{Topic Modeling}
	   In Table \ref{TM_Tasks} we list out the tasks completed by the Topic Modeling team.
	   
            \begin{table}[ht]
            \centering
            \caption{Topic Modeling Milestones} \label{TM_Tasks}
            % \begin{tabular}{p{3cm} p{8cm}}
            \begin{tabular}{cl}
            \toprule
            \textbf{Timeline}  & \textbf{Tasks} \\
            \midrule
            % \textcolor{red}{Change} & \textcolor{red}{dates and correct ordering}\\
            1 Sept & Get familiar with LDA and OCTIS \\
            8 Sept & Run LDA from notebook on 500k dataset \\
            13 Sept & Create a table with etd\_id and topic vector \\
            19 Sept & Set up Nearest Neighbour search \\
            23 Sept & Populate FaisNN with 230k ETD vectors from LDA model \\
            23 Sept & Perform similarity search (topic+document) through FaisNN \\
            30 Sept & Integrate FaisNN to Flask app \\
            3 Oct & Pagination of document results page \\
            9 Oct & Train on complete 500k ETD dataset and integrate with UI\\
            13 Oct & Improve topic filtering and query\\
            13 Oct & Display top-5 topics when topic search query is used \\
            19 Oct & Code refactoring and optimization for reducing  load time \\
            26 Oct & API development and testing for populating Team 1's database \\ 
            28 Oct & Chapter dataset generation, pre-processing and training using \\
                   & XMLs from Object Detection \\
            6 Nov & Finalizing a Topic Browser front-end after trying multiple open source  \\ & options with dummy data\\
            24 Nov & Inference learning of chapters from ETD based model \\
            27 Nov & Topic Bubble data generation complete.\\
            28 Nov & Pushing all chapter's related topics \& docs to DB\\ 
            1 Dec & Push all ETD's related topics \& docs to DB\\
            Final & Set up workflow automation - Team 5\\
            \bottomrule
            \end{tabular}
            \end{table}
            
            \section{Developer Services and Workflow Automation}
            \subsection{Object Detection}
            Since object detection inference is a resource intensive process, we have set up a GPU-enabled container \cite{od_container} (with the required dependencies installed) on the cloud \cite{cloud}. It is to be noted that support for both frameworks (YOLOv7 and Detectron2) has been included in the Docker Image used for this container. Also, we have provided the services for our object detection model (as explained in Section \ref{devliverables}) packaged as separate Docker images to Team 5 for workflow automation, and for other teams to use these services, if required. \\
            
            For Continuous Integration and Continuous Development (CI/CD), we are using the GitLab \cite{gitlab} instance on the cloud, maintained by Team 5. Currently, CI/CD features are only enabled for our Flask application that is running on the cloud. The application essentially converts the generated XML file to HTML, and renders it on the web page. Our front-end web page, with its functionality, is explained in detail in Section \ref{developer}. \\
            
            Workflow automation for the entire object detection pipeline has not been set up by Team 5. However, a basic workflow for the YOLOv7 service that we provided, has been set up, and is functional. 
            % This can be leveraged for the \verb|Add an ETD| service in the future, as is explained in Section \ref{future}.
            
            \subsection{Topic Modeling}
            Since topic modeling training for a dataset is an expensive computation process, we did not  set up a GPU-enabled container for that; that can be done as future work if sufficient resources are available for a suitable container. However, services that show our experimental pages have already been containerized by Team 5. 
            
            For Continuous Integration and Continuous Development(CI/CD), we are using the GitLab instance on the cloud, maintained by Team 5. These features have already been implemented by Team 5 in the GitLab repository. The application basically shows our experimental pages.
            % Its functionality is explained in detail in Section \ref{future}. 
            
            Workflow automation for the entire topic modeling pipeline has been not been set up by Team 5. However, a basic workflow for the Topic Browser application has been set up and is functional. 
            % This can be extended for the new UI service in the future, as is explained in Section \ref{future}.
            
            \section{Inference}
            \subsection{Object Detection}
            Due to time constraints, the original dataset of 50k ETDs was scaled down to 5k ETDs in order to align with the class goals. This collection of ETDs was given to us by Team 4, and the entire class was tasked with using this subset for their experiments. Since workflow automation was not set up to automate the process of running inference using multiple services, it was decided that we run the model on our containers, and use Team 1's API to store objects in the Repository. We are using a temporary directory on the container, that stores the inputs (using the GET API) and outputs (from the model inference), and finally the POST API is used to store the required files in the Repository. The file structure and outputs are as explained in Table \ref{api}. Inference on the 5k ETDs is completed, and the results are saved successfully to the Repository. \newline
            
            The \verb|Get objects by ETD ID| API can be used to get the saved objects, and files, by specifying the \verb|<etd_id>| and the \verb|<type_name>|. \\ \\
            \begin{small}
            \verb|https://team-1-flask.discovery.cs.vt.edu/v1/etds/<etd_id>/objects?type=<type_name>|
            \end{small}
            
            \subsection{Topic Modeling}
            For a dataset size of 50k ETDs each having about 10 chapters, generating a model would have needed much more memory than is available to us during the course. To combat this, we have generated a model using just the 5k ETDs and performed inference for each chapter to generate respective data (related topics and documents).  

            \begin{figure}[hbt!]
	        \begin{center}
	           \includegraphics[width=1.2\textwidth]{Figs/Inference Learning.png}
	        \caption{Related Topics using inference learning}
            \label{inference_topics}
	        \end{center}
            \end{figure}

            \begin{figure}[hbt!]
	        \begin{center}
	           \includegraphics[width=\textwidth]{Figs/Inference Learning - Related ETDs.png}
	        \caption{Related Documents using inference learning}
            \label{inference_docs}
	        \end{center}
            \end{figure}
            Figure \ref{inference_topics} illustrates the inference process we followed for each chapter. We iterated on each available chapter's raw text, performed preliminary data cleaning, and used the cleaned text as a query to perform similarity search on the model's topic\-word\-matrix. This way we get the top 5 related topics. 

            For related ETDs, we use the previously generated result of the top 5 topics to collect their respective top 5 ETDs. The overall list has 25 candidate ETDs which we sort based on their weights to get the top 5 related ETDs for the given chapter. This process is illustrated in Figure \ref{inference_docs}.
            \section{Challenges}
            \subsection{Object Detection}
            A major challenge that we faced while parsing the extracted objects is false positives, as explained below. We implemented rules for the table of contents, chapters, sections, figures, and tables. One of the biggest challenges is that detection is not always in the top-bottom order for pages. 
            This necessitates the creation of an XML tree since a hierarchy of the ETD would be required for other tasks, such as segmenting an ETD into its constituent chapters, and extracting image-based objects along with their corresponding captions.
            The XML tree structure also can be employed to promote accessibility, such as to provide a good order for a screen reader to follow when aiding those with low vision.

            For various document parts and elements, the following lists some of the challenges, and rules to help with the analysis and XML tree construction.
            
            \begin{itemize}
                \item  Front Metadata:
                \begin{outline}[enumerate]
                \1 Abstract is detected as title, and abstract text is detected as paragraph, and in some cases they overflow into the next page as paragraphs.
                \1 Duplicate detections for some objects.
                \end{outline}
                \item  Chapters and Sections (delimiters):
                \begin{outline}[enumerate]
                \1 Finding the right chapter title, since in some cases, chapter titles are being either not detected or wrongly detected.
                \1 Refining chapter title detection by removing outliers.
                \1 Finding if the keyword “chapter” is in the list of all detected chapter titles.
                \1 Difference in font size between chapter titles and other objects.
                \1 Checking the indentation level -- left/center alignment because chapter titles are either on the left side or at the center.
                \1 Chapter titles may appear on multiple pages as a header for the same chapter.
                \end{outline}
                \item  Image-based Objects (algorithms, equations, tables, and figures):
                \begin{outline}[enumerate]
                \1 Linking the captions/numbers with their associated images based on Euclidean distance.
                \1 Linking the captions/numbers with their associated images when they overflow into the next page.
                \end{outline}
            \end{itemize}
            
            \subsection{Topic Modeling}
            The challenges faced in Topic Modeling overlap with the ones faced by the Object Detection sub-team. If the chapters generated from an ETD have errors due to the nature of input ETDs (like scanned images and random PDF layouts), then the output we get after using the erroneous data will also be incorrect.
            
            Other than that, there also is a challenge to fit as much training data into the limited amount of RAM available from the cloud. Finally, we decided to reduce the dataset size to 5000 ETDs to solve it but that is leading to the degradation of cluster output. 

	    \section{Evaluation}
	    \subsection{Object Detection}
	    We randomly sampled a small subset (around 50 ETDs) from the original dataset to experiment with our trained YOLOv7 and Faster R-CNN models, and tested both models with different hyper-parameter settings. Then, we refined the extracted set of ETD objects with our parser logic, and implemented the rules mentioned in Section \ref{rules} to filter out both incorrect detections and false positives. We finalized the set of rules for post-processing and extensively tested our parser logic with the object detection pipeline.\\
	    
	    From running our model on the 5k ETDs, we observed that it takes around 1-2 minutes for each ETD to run our model on a container with GPU (in the cloud).
     If the model is run on a CPU without GPU, it takes about 5 minutes for each ETD. Before implementing the post-processing rules for our model, about 20\% of the chapter titles and 5 percent of the captions were detected incorrectly, whereas after imposing the ETD filtering rules as specified in Section \ref{rules}, we brought down the incorrect detections to about 3-4\%. 
	    
	    \subsection{Topic Modeling}
	    We initially processed 2000 ETDs and trained the model in order to save time and increase efficiency. However, this also caused some unsatisfactory results in training, such as repeated topics and inaccurate topics while describing documents. Therefore, in the subsequent large-scale data processing, we were able to record diversity and coherence to ensure the acceptability of models, and we provided tables of diversity and coherence for users to browse as a tip to choose different models and set the number of topics according to their criteria. Below is the code snippet that shows how we determined the diversity and coherence. 
	    \begin{verbatim}
        # Initialize metrics
        npmi=Coherence(texts=dataset.get_corpus(), topk=10, measure='c_npmi')
        topic_diversity = TopicDiversity(topk=10)
        # Retrieve metrics score
        topic_diversity_score = topic_diversity.score(output)
        print("Topic diversity: "+str(topic_diversity_score))
        npmi_score = npmi.score(output)
        print("Coherence: "+str(npmi_score))
        \end{verbatim}
        
    \section{Future Work} \label{future} %Although there is a lot of work done regarding both object detection and topic modeling, there is always room for improvement. 
    In the interest of continuous improvement, we recommend future work to extend our efforts with both object detection and topic modeling.
    In this section we highlight the different plans of action post-class which could be implemented in the future.
        \subsection{Object Detection}
        The object detection group has mostly worked on the model, post-processing rules, and Experimenter UI. The biggest room for improvement was to be able to integrate better with the overall information system that the class has worked on. Here are a few highlighted downstream tasks that can be implemented to extend this project.
        \begin{itemize}
            \item Integrate the Flask app/Experimenter UI with the front-end and workflow services.
            \item Update our pipeline to support the \verb|Add an ETD| service once Team 1 provides the API for saving metadata.
            \item Set up more workflow automation.
            \item Add support for scanned documents, and modify the existing pipeline accordingly.
            \item Add more post-processing rules based on the results of the 5k dataset.
            \item Improve UI layout, and add more functionality such as choosing hyper-parameters. 
            \item Add test routines to the CI/CD pipeline.
        \end{itemize}
        \subsection{Topic Modeling}
        In the future, it would be helpful to implement more models and integrate them into the topic browser. Besides that, the user interface can be further extended and improved. First and foremost would be to connect the Topic Bubble UI and our Topic Browser, so that users can switch between different applications. It also would help to further automate the workflow of the Topic Bubble application. Secondly, it would help to integrate our UI with Team 2 services, so that users who log in with accounts could access the experiment pages.
    
        

	    
	\chapter{User Manual}
	\section{Object Detection}
    The front-end web page that the object detection team provides is the web page for experimenters. Figures \ref{experimenter_wireframe} and \ref{parsed_etd_wireframe} are the proposed wireframes that the team has come up with. \par
    % model selector page
    \subsection{Proposed Wireframes}
    \begin{figure}[hbt!]
	        \begin{center}
	         \includegraphics[width=\textwidth]{experimenter_wireframe.png}
	        \caption{Object detection experimenter wireframe (Model selector page)}
            \label{experimenter_wireframe}
	        \end{center}
    \end{figure}
    
    As is shown in Figure \ref{parsed_etd_wireframe}, the model selector page can support both the Detectron2 and YOLOv7 models. This also gives customization for different model weights and hyper-parameters. The Experimenter will be able to access this page by the sidebar menu, after logging in, and choosing object detection in the menu.
     % etd viewer page
    \begin{figure}[hbt!]
	        \begin{center}
	         \includegraphics[width=\textwidth]{parsed_etd_wireframe.png}
	        \caption{Object detection parsed ETD wireframe (Extracted ETD view page)}
         \label{parsed_etd_wireframe}
	        \end{center}
    \end{figure}    
    
    After choosing a model, the experimenter will be redirected to the extracted ETD view page. As is shown in Figure \ref{parsed_etd_wireframe}, the generated XML from object detection will be used to generate an HTML page. This page will show all objects extracted from an ETD using the order of the XML schema, as discussed in Section \ref{oddeliverables}. 

    \subsection{Current Model}
    In order to visualize how the different models are producing the XML output on an ETD, we developed a Flask application that produces an HTML web page from the object detection model. In the future, we recommend that this be connected to the front-end website.
    % is this fixed?
    \newpage

    \begin{figure}[hbt!]
	        \begin{center}
	         \includegraphics[width=0.9\linewidth]{etd_viewer_model_page.png}
	        \caption{ETD viewer model selector page}
         \label{etd_viewer_model_page}
	        \end{center}
    \end{figure}    
     In Figure \ref{etd_viewer_model_page}, the user is given a choice to choose between the Detectron2 and YOLOv7 models for object detection. 
    
    % \newpage
    \begin{figure}[hbt!]
	        \begin{center}
	         \includegraphics[width=0.9\linewidth]{etd_viewer_upload_page.png}
	        \caption{ETD viewer upload ETD page}
            \label{etd_viewer_upload_page}
	        \end{center}
    \end{figure}
    
    After a user selects a model for object detection, they are given an option to upload a PDF of the ETD they want to perform the detection on. This is demonstrated in Figure \ref{etd_viewer_upload_page}.

    \begin{figure}[hbt!]
	        \begin{center}
	         \includegraphics[width=0.9\linewidth]{etd_viewer_metadata_page.png}
	        \caption{ETD viewer with metadata page}
	        \label{etd_viewer_metadata_page}
	        \end{center}
    \end{figure}    
    % \newpage
    \begin{figure}[hbt!]
	        \begin{center}
	         \includegraphics[width=0.9\linewidth]{etd_viewer_image_page.png}
	        \caption{ETD viewer with extracted image page} 
	        \label{etd_viewer_image_page}
	        \end{center}
    \end{figure}   
    
    As is shown above, Figures \ref{etd_viewer_metadata_page} and \ref{etd_viewer_image_page} both show the produced HTML page that was generated from the object detection model. Figure \ref{etd_viewer_metadata_page} highlights the metadata fields while Figure \ref{etd_viewer_image_page} shows the detected chapters and figures with linked captions extracted from the ETD. 
%    There is also a demo video uploaded to VTechWorks, associated with the Team 3 presentation slides, as well as a standalone video, ETDViewerDemo.mp4, regarding the ETD viewer. 
VTechWorks has two files associated with this report that illustrates the object detection results.
One is the final class presentation.
The other is an associated standalone video, ETDViewerDemo.mp4, regarding the ETD viewer.
    In Table \ref{OD_ETDViewer}, we give the timeline of events in ETDViewerDemo.mp4:

        \begin{table}[ht]
        \centering
        \caption{Object Detection ETD viewer} \label{OD_ETDViewer}
        \begin{tabular}{|l|p{0.8\linewidth}|}
        \toprule
        \textbf{Time Stamp}  & \textbf{Description} \\
        \midrule
        00:00 & Shows how one would upload an ETD to the application.\\
        00:10 & Loading ETD and running object detection. The user will be able to see the progress of the model.\\
        00:41 & Goes through the detected objects. The user is able to navigate to different chapters and sections of the ETD.\\
        00:43 & Demonstrates the functionality of the sidebar.\\
        \bottomrule
        \end{tabular}
        \end{table}
        
	\section{Topic Modeling}
	The experimental front-end user interface (UI) encapsulates multiple downstream tasks and services for users of a digital library. Figure \ref{IndexPage} shows the entry point for an end-user. Once the user selects a model and number of topics, the UI will lead the user to the topic list page. Figure \ref{TopicPage} gives the topic list where a user could select their interest.
 It will lead to the documents page, shown in Figure \ref{TopicSelectedPage}, which gives an overview of ETDs related to that topic. Users can expand the abstract by clicking the `show more button' as in Figure \ref{ShowMoreButton}. Besides, users can click the title of the ETD and go to its documentation (see Figure \ref{DocumentPage}) which has related topics and similar documents. The full text page in Figure \ref{URIPage} includes the full text URI button. The experimental system also supports filtering documents within a time range and university.
 That will lead the user to the result as shown in Figure \ref{FilterDocumentPage}. In order to provide a more intuitive and interactive UI, we made a Topic Modeling Bubble chart as shown in Figure \ref{Topic_Bubble}, where each topic bubble represents a topic. The size and color of the topic bubble are determined by the number of documents related to that topic. Thus the larger and more red a topic bubble is, the higher is the number of related documents. After the user clicks a topic bubble, it will give a word cloud of the top 10 words in that topic, as seen in Figure \ref{Topic_Bubble_Word_Cloud}. The topic bubble will expand to a larger rounded square when the user clicks on the blue expand button.
 That leads to the documents view and sources view as shown in Figure \ref{Topic_Bubble_Full_View}. The documents view gives the top 20 ETDs related to that topic, and shows the title and author information, while the sources view gives the author distribution. Clicking a word in the word cloud view, works as a filter to only show the topic bubble that contains the selected word.
 Meanwhile, it also presents the word weight in that topic as a pie chart as shown in Figure \ref{Topic_Bubble_Pie_View}.
In order to provide a more vivid presentation of the interactive UI, we made a video, TopicBubbleDemo.mp4.
The key times and associated descriptions are shown in Table \ref{TM_Bubble}.
	   
            \begin{table}[ht]
            \centering
            \caption{Topic Modeling Topic Bubble UI} \label{TM_Bubble}
            % \begin{tabular}{p{3cm} p{8cm}}
            \begin{tabular}{|l|p{0.8\linewidth}|}
            \toprule
            \textbf{Time Stamp}  & \textbf{Description} \\
            \midrule
            % \textcolor{red}{Change} & \textcolor{red}{dates and correct ordering}\\
            00:16 & Word cloud view after clicking the topic bubble \\
            00:23 &  Documents view and sources view after clicking the blue expandable button\\
            00:34 &  Filter words after clicking a word in the word cloud view\\
            00:51 &  Locating the topic bubble by entering the topic number\\
            \bottomrule
            \end{tabular}
            \end{table}
            
        \begin{figure}[hbt!]
    	        \begin{center}
    	         \includegraphics[width=\textwidth]{Figs/FigureIndex Page.png}
    	        \caption{Index Page shows searching topic by selecting topic model and number of topics}\label{IndexPage}
    	        \end{center}
	    \end{figure}
	    
        \begin{figure}[hbt!]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Topic Page.png}
    	         \caption{Topic Page shows browsing topic which is ranked by number of documents}
    	        \label{TopicPage}
    	        \end{center}
	    \end{figure}
	    \begin{figure}[hbt!]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Figs/Topic Selected Page.png}
    	        \caption{Topic Selected Page shows browsing documents in a selected topic}
    	        \label{TopicSelectedPage}
    	        \end{center}
	    \end{figure}
        \begin{figure}[hbt!]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Figs/Shore More Button.png}
    	        \caption{Show more Button Page shows full text of abstract}
    	        \label{ShowMoreButton}
    	        \end{center}
	    \end{figure}
     
	    \begin{figure}[hbt!]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Figs/Document Page.png}
    	        \caption{Document Page shows specific information about the selected document}
    	        \label{DocumentPage}
    	        \end{center}
	    \end{figure}
     
	    \begin{figure}[hbt!]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Figs/Full Text URI.png}
    	        \caption{Full Text URI Page shows full text source of the selected document}
    	        \label{URIPage}
    	        \end{center}
	    \end{figure}
	    
	    \begin{figure}[ht]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Filter Document Page.png}
    	        \caption{Filter Document Page shows filtering documents by start year, end year, threshold, and university}
    	        \label{FilterDocumentPage}
    	        \end{center}
	    \end{figure}

        \begin{figure}[ht]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Figs/Topic_Bubble.png}
    	        \caption{Topic Bubble page shows topic bubble where each bubble is represented as a topic}
    	        \label{Topic_Bubble}
    	        \end{center}
	    \end{figure}

        \begin{figure}[ht]
    	        \begin{center}
    	         \includegraphics[width=0.9\linewidth]{Figs/Topic_Bubble_Word_Cloud.png}
    	        \caption{Topic bubble gives the word cloud view of the top 10 words when selecting a topic bubble}
    	        \label{Topic_Bubble_Word_Cloud}
    	        \end{center}
	    \end{figure}

         \begin{figure}[ht]
                \begin{center}
                 \includegraphics[width=0.9\linewidth]{Figs/Topic_Bubble_Full_View.png}
                \caption{Topic bubble gives the documents view and sources view after clicking the expand button}
                \label{Topic_Bubble_Full_View}
                \end{center}
        \end{figure}

        \begin{figure}[ht]
                \begin{center}
                 \includegraphics[width=0.9\linewidth]{Figs/Topic_Bubble_Pie_View.png}
                \caption{Clicking the word in the word cloud works as a filter to show only the topic bubble that contains the word}
                \label{Topic_Bubble_Pie_View}
                \end{center}
        \end{figure}

	
	\chapter{Developer Manual} \label{developer}
    In this chapter, we describe the onboarding steps which any interested developer can go through in order to be able to contribute to this project.
    \section{Folder Structure}
        Navigating to the Team 3 folder in the cloud through “/tree/camelot/team3/" (see Figure \ref{folders}) will show the user the top level topic modeling and object detection directories, which contain the code used to perform the respective tasks. For object detection, the team has two directories where object\_detection is the folder with the APIs and Flask application, which contains the post-processing rules that we have come up with over the semester, as well as the code which utilizes Team 1's API to upload detected objects. The other folder, object\_detection\_refactored, contains the cleaned and refactored versions of both the Faster R-CNN and YOLOv7 models, which are to be used going forward. The code repository is organized such that it is easier to add support for other object detection models in the future. There are still filtering rules and false positives that have not been addressed with the refactored model. In the future it would help to add the post-processing rules that were implemented in the original model (explained in Section \ref{rules}), as there are only the necessary rules to detect objects and build an XML structure, in the refactored version.  Both folders have the capability to run inference with either Detectron2 or YOLOv7.  
         \begin{figure}[ht]
    	        \begin{center} 
    	         \includegraphics[width=\textwidth]{team3_cloud_folder_structure.png}
    	        \caption{Folder structure for Team 3}
    	        \label{folders}
    	        \end{center}
	    \end{figure}
	    
    \section{Object Detection}
            \subsection{Access}
    A new user needs to have access to the following:
        \begin{enumerate}
            \item VT Computer Science Cloud \cite{cloud}
            \item Team 3 container cluster \cite{od_container}
            \item The \href{https://code.vt.edu/nirmal/cs5604-f22-team-3-flask}{GitLab} repository for our Flask application with CI/CD 
        \end{enumerate}
            \subsection{Steps }
To access the container cluster, you must first get the access code from the VT CS Cloud website \cite{cloud}, illustrated in Figure \ref{manual} and explained below.
         \begin{figure}[ht]
    	        \begin{center} 
    	         \includegraphics[width=\textwidth]{od_cloud_folder_structure.png}
    	        \caption{Folder structure for object detection}
    	        \label{manual}
    	        \end{center}
	    \end{figure}
	    \begin{enumerate}
	        \item 5k\_ETDs: This folder contains the files used to perform the batch upload of objects to the Team 1 database.
	        \item detectron2: This folder contains the Detectron2 model.
	        \item Flask: This folder contains the Experimenter UI application.
	        \item flask\_inputs: This folder contains the inputs that the Experimenter UI application uses.
	        \item yolov7: This folder contains the YOLOv7 model.
	    \end{enumerate}
\subsubsection{Saving Objects}
\begin{enumerate}
\item Open the “/tree/camelot/team3/object\_detection/5k\_ETDs" directory in the container.\\
    \item Run the code below in a notebook. The “etd\_id" can be replaced by the PDFs from Team 1's database by running the curl command and GET request using Team 1's API.   \\
    \begin{verbatim}
        !curl --output tmp/{etd_id}.pdf --request GET
        https://team-1-flask.discovery.cs.vt.edu/v1/etds/{etd_id}/pdf
        
        etd = f"tmp/{etd_id}.pdf"
        
        !python ETDExtraction.py --source $etd --output "tmp"  
    \end{verbatim}
    \item After you run the cell, the objects are extracted and tagged, and they will be saved in the “tmp" folder mentioned in the code above.
\end{enumerate}
\subsubsection{Running Flask}
\begin{enumerate}
\item Navigate to the “/tree/camelot/team3/object\_detection/Flask/xml\_parser/" directory in the container.\\
    \item Run the following commands to run the Flask app:  \\
        \begin{verbatim}
        $ export FLASK_APP=flask_main.py
        $ flask run --host=0.0.0.0 -p 5001
    \end{verbatim}
\end{enumerate}
\subsubsection{Services}
\begin{enumerate}
\item We have packaged our code as multiple services provided to Team 5, to be used for workflow automation, and for our front-end web page. A sample service (for our YOLOv7 model) is
%linked here, to show how the service it is to be used -- \href{https://hub.docker.com/layers/nirmal2519/yolo/0.4/images/sha256-64754d9407c042dc6f277e3261fc46fbeb8bc0f9fc78ecd79417f51927d173c3?context=explore}{Docker Hub link}
available at Docker Hub. Use the following URL to see how the service is to be used.
\begin{verbatim}
https://hub.docker.com/layers/nirmal2519/yolo/0.4/images/
sha256-64754d9407c042dc6f277e3261fc46fbeb8bc0f9fc78ecd79417f51927d173c3?
context=explore
\end{verbatim}

\item Run the Docker image, and execute the following command:
    \begin{verbatim}
        $ python ETDExtraction.py \
                  --source test/CMU-ISR-16-113.pdf \
                  --output test
    \end{verbatim}
\item There is a sample ETD (\verb|CMU-ISR-16-113.pdf|) in the Docker image for testing. Once this command is executed, the outputs as shown in Table \ref{objects_schema} would be generated in the root directory for the sample ETD.
\end{enumerate}
    \section{Topic Modeling}
    For Topic Modelling, we will be using the ETD Dataset to train various models in a Jupyter notebook which will generate pickle files corresponding to each model. Now, these model files will be referred to by the Flask application to serve APIs or display the results to a user's queries. For a developer to start contributing, one needs to follow the steps mentioned in the following sections. 
    \subsection{Access}
        A new user needs to have access to the following tools:
        \begin{enumerate}
            \item VT Computer Science Cloud \cite{cloud}
            \item Team 3 Topic Modeling container cluster \cite{tm_container}
            \item The \href{https://github.com/mchenyu1218/ETD-web}{Git} repository for our Flask application 
        \end{enumerate}
    
    \subsection{Steps} 
    \begin{enumerate}
    % change modelling --> modeling
        \item Look for the ETD dataset stored in the “/tree/camelot/team3/topicmodeling/5k\_ETDs" directory in the container. This file will be available in a CSV format. In this work we used the dataset containing 5000 entries which was available both locally at the above path and through API call from Team 1. Note that in the future, a bigger dataset can be used as appropriate. 
        \item Open the topic\_model.ipynb Python notebook location at path “data/team3/topic\_ modelling/TextMining/Code" in the container as shown in Figure~\ref{fig:tmipynb}. \newline  
        \begin{figure}
            \centering
            \includegraphics[width=16cm]{Figs/TM_notebook.png}
            \caption{A snapshot of topic\_model.ipynb }
            \label{fig:tmipynb}
        \end{figure}
        
        \item By running all the cells, all the pre-requisites will be downloaded. Then the pre-processing steps and clustering will be done to generate the required output. 
        
        \item To make changes, git checkout to a new branch and add your code to the notebook. 
        
        \item After validating your code, add the new changes to ETD.py, which is the Flask application running the backend logic for the webview. Once the model is generated, it's time to see whether the Flask application is giving the correct webpages. To verify, use the command \textbf{flask run} to check on your local host. If no errors are found, the command line output will look like what appears in Figure~\ref{fig:flask}.\newline
        \begin{figure}
            \centering
            \includegraphics[scale=0.5]{Figs/TM_flask.png}
            \caption{Command line output of Flask application}
            \label{fig:flask}
        \end{figure}
        
        \item Once the complete pipeline (from the model training code through the Flask application code) is working as intended, raise a pull request on the Git with detailed commit messages. 

    \end{enumerate}
        
	% This is the standard bibtex file. Do not include the .bib extension in <bib_file_name>.
	% Uncomment the following lines to include your bibliography: 
	%\bibliography{<bib_file_name>}
	%\bibliographystyle{plainnat}   

    \begin{thebibliography}{references.bib}
    

    
    \bibitem{vtechworks}
    Virginia Tech. “VTechWorks.” \textit{ETDs: Virginia Tech Electronic Theses and Dissertations}. 2011. Last Accessed December 12, 2022. \url{https://vtechworks.lib.vt.edu/handle/10919/5534} 
    
    \bibitem{vt2}
    Uddin, Sami, et al. “Building a Large Collection of Multi-Domain Electronic Theses and Dissertations.”\textit{ 2021 IEEE International Conference on Big Data (Big Data)}. 2021. 
    
    \bibitem{roboflow}
    Alexandrova, Sonya, et al. “RoboFlow: A flow-based visual programming language for mobile manipulation tasks.” \textit{2015 IEEE International Conference on Robotics and Automation (ICRA)}. 2015.
    
    \bibitem{obj_det_alan}
    % Ahuja, Devera, et al. “Parsing Electronic Theses and Dissertations Using Object Detection." \textit{Workshop on Information Extraction from Scientific Publications (WIESP)}. 2022. Manuscript accepted for publication.
    Ahuja, Aman, et al. “Parsing Electronic Theses and Dissertations Using Object Detection." \textit{The First Workshop on Information Extraction from Scientific Publications, WIESP 2022}. Taipei and Online, November 20, 2022, ACL.
    
    \bibitem{faster_rcnn}
    Ren, Shaoqing, et al. “Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks.” \textit{IEEE Transactions on Pattern Analysis and Machine Intelligence, vol. 39, no. 6, 2017, pp. 1137–1149.} 2017.
    
    \bibitem{rcnn}
    Girshick, Ross, et al. “Rich feature hierarchies for accurate object detection and semantic segmentation.” \textit{Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition.} 2014.
    
    \bibitem{fast_rcnn}
    Girshick, Ross. “Fast R-CNN.” \textit{Proceedings of the IEEE International Conference on Computer Vision.} 2015.
    
    \bibitem{detectron}
    % Wu, Yuxin, et al. “Detectron2” \textit{GitHub}, https://github.com/facebookresearch/detectron2. 
    Wu, Yuxin, et al. “Detectron2." 2019.  Last Accessed December 12, 2022. \url{https://github.com/facebookresearch/detectron2}
    
    
    \bibitem{YOLOv7}
    Wang, Chien-Yao, et al. “YOLOv7: Trainable bag-of-freebies sets new state-of-the-art for real-time object detectors.” \textit{arXiv preprint arXiv:2207.02696.} 2022.
    
    \bibitem{1}
    Blei, David M., et al. “Latent Dirichlet Allocation.” \textit{Journal of Machine Learning Research. 3 Jan (2003): 993-1022.} 2003.
    
    \bibitem{2}
    Srivastava, Akash, et al. “Autoencoding Variational Inference for Topic Models.” \textit{arXiv preprint arXiv:1703.01488} (2017).
    
    \bibitem{3}
    Bianchi, Federico, et al. “Pre-Training Is a Hot Topic: Contextualized Document Embeddings Improve Topic Coherence.” \textit{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing. (Volume 2: Short Papers)}. 2021.
    
    \bibitem{pytorch}
    Paszke, Adam, et al. “PyTorch: An Imperative Style, High-Performance Deep Learning Library.” \textit{Proceedings of the 33rd International Conference on Neural Information Processing Systems}. 2019.  
    
    \bibitem{coco}
    Lin, Tsung-Yi, et al. “Microsoft COCO: Common Objects in Context.” \textit{European Conference on Computer Vision.} 2014.
    
    \bibitem{pdf2image}
    Belval. “Belval/pdf2image: A Python Module That Wraps the pdftoppm Utility to Convert PDF to PIL Image Object.” \textit{GitHub}. Last Accessed December 12, 2022. Last Accessed December 12, 2022. \url{https://github.com/Belval/pdf2image}
    
    \bibitem{flask}
    Grinberg, Miguel. “Flask Web Development: Developing Web Applications with Python.” \textit{O'Reilly Media, Inc.}, 2018.

    \bibitem{wsgi}
    Eby, P. J. “Python Enhancement Proposals.” \textit{PEP 3333 – Python Web Server Gateway Interface v1.0.1.} 2010. Last Accessed December 12, 2022. \url{https://peps.python.org/pep-3333/}
    
    \bibitem{octis}
    Terragni, Silvia, et al. “OCTIS: Comparing and Optimizing Topic models is Simple!" \textit{Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: System Demonstrations}. 2021.
 
    \bibitem{caffe}
    Jia, Yangqing, et al. “Caffe: Convolutional Architecture for Fast Feature Embedding.” \textit{Proceedings of the 22nd ACM International Conference on Multimedia}. 2014.
    
    \bibitem{docker}
    Docker, Inc. “Docker." 2013. Last Accessed December 12, 2022. \url{https://www.docker.com/}
    
    \bibitem{python}
    Python Software Foundation. “Python." 2001. Last Accessed December 12, 2022. \url{https://www.python.org/}
    
    \bibitem{torch}
    Collobert, Ronan. “Torch." 2017. Last Accessed December 12, 2022. \url{https://github.com/torch/torch7}
    
    \bibitem{cloud}
    Virginia Tech. “Computer Science Cloud." 2022. Last Accessed December 12, 2022. \url{https://cloud.cs.vt.edu/}
    
    \bibitem{gitlab-vt}
    Virginia Tech. “VT GitLab Repository." 2022. Last Accessed December 12, 2022. \url{https://code.vt.edu/}
    
    \bibitem{od_container}
    Virginia Tech. “Object Detection Container." 2022. Last Accessed December 12, 2022. \url{https://team3-container-1.discovery.cs.vt.edu/}
    
    \bibitem{tm_container}
    Virginia Tech. “Topic Modeling Container." 2022. Last Accessed December 12, 2022. \url{https://team3-container-2.discovery.cs.vt.edu/}
    
    \bibitem{gitlab}
    GitLab. “GitLab CI/CD." 2011. Last Accessed December 12, 2022. \url{https://docs.gitlab.com/ee/ci/}
    
    \end{thebibliography}

    \bibliography{references}

\end{document}


%****************************************************************************
% Below are some general suggestions for writing your dissertation:
%
% 1. Label everything with a meaningful prefix so that you
%    can refer back to sections, tables, figures, equations, etc.
%    Usage \label{<prefix>:<label_name>} where some suggested
%    prefixes are:
%			ch: Chapter
%     		se: Section
%     		ss: Subsection
%     		sss: Sub-subsection
%			app: Appendix
%     		ase: Appendix section
%     		tab: Tables
%     		fig: Figures
%     		sfig: Sub-figures
%     		eq: Equations
%
% 2. The VTthesis class provides for natbib citations. You should upload
%	 one or more *.bib bibtex files. Suppose you have two bib files: some_refs.bib and 
%    other_refs.bib.  Then your bibliography line to include them
%    will be:
%      \bibliography{some_refs, other_refs}
%    where multiple files are separated by commas. In the body of 
%    your work, you can cite your references using natbib citations.
%    Examples:
%      Citation                     Output
%      -------------------------------------------------------
%      \cite{doe_title_2016}        [18]
%      \citet{doe_title_2016}       Doe et al. [18]
%      \citet*{doe_title_2016}      Doe, Jones, and Smith [18]
%
%    For a complete list of options, see
%      https://www.ctan.org/pkg/natbib?lang=en
%
% 3. Here is a sample table. Notice that the caption is centered at the top. Also
%    notice that we use booktabs formatting. You should not use vertical lines
%    in your tables.
% 
%				\begin{table}[htb]
%					\centering
%					\caption{Approximate computation times in hh:mm:ss for full order 						versus reduced order models.}
%					\begin{tabular}{ccc}
%						\toprule
%						& \multicolumn{2}{c}{Computation Time}\\
%						\cmidrule(r){2-3}
%						$\overline{U}_{in}$ m/s & Full Model & ROM \\
%						\midrule
%						0.90 & 2:00:00 & 2:08:00\\
%						0.88 & 2:00:00 & 0:00:03\\
%						0.92 & 2:00:00 & 0:00:03\\
%						\midrule
%						Total & 6:00:00 & 2:08:06\\
%						\bottomrule
%					\end{tabular}
%					\label{tab:time_rom}
%				\end{table}
% 
% 4. Below are some sample figures. Notice the caption is centered below the
%    figure.
%    a. Single centered figure:
%					\begin{figure}[htb]
%						\centering
%						\includegraphics[scale=0.5]{my_figure.eps}
%						\caption{Average outlet velocity magnitude given an average  
%				        input velocity magnitude of 0.88 m/s.} 
%						\label{fig:output_rom}
%					\end{figure}
%    b. Two by two grid of figures with subcaptions
%					\begin{figure}[htb]
%						\centering
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_1_1.eps}
%							\caption{Subcaption number one}
%							\label{sfig:first_subfig}
%						\end{subfigure}
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_1_2.png}
%							\caption{Subcaption number two}
%							\label{sfig:second_subfig}
%						\end{subfigure}
%
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_2_1.pdf}
%							\caption{Subcaption number three}
%							\label{sfig:third_subfig}
%						\end{subfigure}
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_2_2.eps}
%							\caption{Subcaption number four}
%							\label{sfig:fourth_subfig}
%						\end{subfigure}
%						\caption{Here is my main caption describing the relationship between the 4 subimages}
%						\label{fig:main_figure}
%					\end{figure}
%
%----------------------------------------------------------------------------
%
% The following is a list of definitions and packages provided by VTthesis:
%
% A. The following packages are provided by the VTthesis class:
%      amsmath, amsthm, amssymb, enumerate, natbib, hyperref, graphicx, 
%      tikz (with shapes and arrows libraries), caption, subcaption,
%      listings, verbatim
%
% B. The following theorem environments are defined by VTthesis:
%      theorem, proposition, lemma, corollary, conjecture
% 
% C. The following definition environments are defined by VTthesis:
%      definition, example, remark, algorithm
%
%----------------------------------------------------------------------------
%
%  I hope this template file and the VTthesis class will keep you from having 
%  to worry about the formatting and allow you to focus on the actual writing.
%  Good luck, and happy writing.
%    Alan Lattimer, VT, 2016
%
%****************************************************************************




