%       File: VTthesis_template.tex
%     Created: Thu Mar 24 11:00 AM 2016 EDT
%     Last Change: Thursday, December 19, 2019
%     Author: Alan M. Lattimer, VT
%	  With modifications by Carrie Cross, Robert Browder, and LianTze Lim. 
%
% This template is designed to operate with XeLaTeX.
%
% All elements in the Title, Abstract, and Keywords MUST be formatted as text and NOT as math.
%
%Further instructions for using this template are embedded in the document. Additionally, there are comments at the end of the file that give suggestions on writing your thesis.  
%
%In addition to the standard formatting options, the following options are defined for the VTthesis class: proposal, prelim, doublespace, draft. 

\documentclass[doublespace,nopageskip]{VTthesis} % nopageskip - Removes arbitrary blank pages.
\usepackage{multicol} % Used for the two-column layout of the document
\usepackage{amsmath}
\usepackage{float}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{hyperref}


% for table
\usepackage[table]{xcolor}
\usepackage{longtable}
\usepackage{multirow}
\usepackage{geometry}

\geometry{
    letterpaper,
    left=1in,
    right=1in,
    top=1in,
    bottom=1in
}

% Using the following header instead will create a draft copy of your thesis
%\documentclass[doublespace,draft]{VTthesis}

% The lipsum package is just included to put dummy text in the document in order to demonstrate page headers and table of contents behavior. You should remove it once you begin writing your actual thesis or dissertation.
\usepackage{lipsum}
\usepackage{natbib}
\usepackage{graphicx}
\bibliographystyle{plainnat}



% Title of your thesis
\title{
Final Report

CS 5604: Information Storage and Retrieval

\large Team 3: Object Detection and Topic Modelling
}

% You should include 3-5 keywords, separated by commas
%\keywords{Some Keywords, Subject matter, etc.}

% Your name, including middle initial(s)
\author{Amr Ahmed Aboelnaga, Anushka Sivakumar, Jayanth Narla, Pradyumna Upendra Dasu, Ragul Seetharaman, Sahana Bhaskar, Shankar Srinidhi Srinivas }


% Change this to your program, e.g. Physics, Civil Engineering, etc.
%\program{Mathematics} 

% Change this to your degree, e.g. Master of Science, Master of Art, etc.
%\degree{Doctor of Philosophy} 

% This should be your defense date:
\submitdate{January 8, 2024} 

% Committee members. Only have five readers and one chair available.
% Only use the ones you need and don't include the ones you don't need.
% You can also declare a Co-advisor. If you do, the principal and co-advisors
% will be listed as co-advisors on the title page.  Per the VT ETD standards, 
% you should not include titles or educational qualifications such as PhD or Dr.
% You should, however, include middle initials if possible.
% \principaladvisor{}
% \coadvisor{}
\firstreader{Guided by}
\secondreader{Instructor: Dr. Edward. A. Fox}
\thirdreader{Subject Matter Expert: Chenyu Mao}
%\fourthreader{Fourth Committee Member}
%\fifthreader{Fifth Committee Member}

% The dedication and acknowledgement pages are optional. Comment them out to remove them.
%\dedication{This is where you put your dedications.}
%\acknowledge{This is where you put your acknowledgments.}

% The abstract is required.
\abstract{Under the guidance of Dr. Edward A. Fox, the CS 5604: Information Storage and Retrieval class (Fall 2023) was tasked with developing a cutting-edge information retrieval system to facilitate Electronic Theses and Dissertations (ETDs).
We used learning algorithms on a large ETD collection to classify closely related documents. The project's overarching objective is to enhance the already available service, which enables users to upload, search, and retrieve ETDs along with their associated digital objects in a human-readable format. Our team's specific assignment is to use object detection and topic modeling to analyze documents and thereby assist in building a system that supports searching and retrieving documents using topics and user defined digital objects, and enables experimenters to conduct further research into objects and topics. To achieve this effort we have implemented object detection on 200 segmented ETDs and topic modeling using BERTopic (BERT embeddings) and LDA (Latent Dirichlet Allocation) on nearly 334k ETDs. The object detection and topic modeling pipelines have been modified to utilize APIs (Application Programming Interfaces) for populating database tables related to ETDs. Each ETD page is converted into an image and stored in the file system, with corresponding entries in the database. Additionally, all detected objects are stored both in the database and the file system. The generated XMLs now include an object ID for each detected object, facilitating the capture of structural relationships using knowledge graphs (Team 1). Efforts have also been invested in enhancing chapter segmentation in XMLs. This involves exploring and experimenting with the LLaMA 2 model, ResNet model, and clustering approaches to accurately identify the start and end pages of chapters.The topic modeling results using BERTopic were not satisfactory, leading to exploration of the LDA model. Switching to the LDA model has provided promising outputs. The topics generated using LDA were refined using various pre-processing techniques and given to team 6 to be used in the sign-up page, and to team 2 for indexing.
%which will be discussed in the following chapters.
\\ 
\\
Keywords: ETD, electronic theses and dissertations, deep learning, object detection, topic
modeling, chapter segmentation, probabilistic models}
%\abstract{\lipsum [1-4]}

% The general audience abstract is required. There are currently no word limits.
%\abstractgenaud{}

\begin{document}
% The following lines set up the front matter of your thesis or dissertation and are required to ensure proper formatting per the VT ETD standards. 
  \frontmatter
  \maketitle
  \tableofcontents

% The list of figures and tables are now optional per the official ETD standards.  Unless you have a very good reason for removing them, you should leave these lists in the document. Comment them out to remove them.
	\listoffigures
	\listoftables
    %\printnomenclature %Creates a list of abbreviations. Comment out to remove it. 

% sample text for abbreviations:

%\nomenclature{NLP}{Natural Language Processing}
 
%$\sigma$ is the eighteenth letter of the Greek alphabet, and carries the 's' sound. In the system of Greek numerals, it has a value of 200. 
 
%\nomenclature{$\sigma$}{The total mass of angels per unit area}

% The following sets up the document for the main part of the thesis or dissertation. Do not comment out or remove this line.
	\mainmatter

	%now go ahead and start writing your thesis
	\chapter{Overview} \label{ch:introduction}
   			%\lipsum[1]
      
%Copy/paste the code below to add sections and subsections to each chapter. Add your own text to the chapter and (sub)section labels to create custom headings.          
    \section{Introduction} \label{se:one_section}
    A thesis or dissertation is a scholarly documentation of the original research conducted by students, typically as a requirement to obtain their graduate degrees. In the past, the theses and dissertations were documented and stored on paper which made them inaccessible to other scholars and researchers interested in the field. In the year 1997, Virginia Tech became the first university to mandate the recording of these precious artifacts electronically. The shift in the modality of storage has contributed greatly to the scholarly community by improving the accessibility of the information in the artifacts. 

Our goal over the course of the semester was to build on top of the existing architecture for improving the efficiency of storing the Electronic Theses and Dissertations (ETDs) and make them more accessible by improving the organization of the information. To accomplish this, the CS5604 class strived to build auxiliary systems to model topics of ETDs for better searching of related work, extract and segment relevant information like metadata, figures, chapters, table of contents, etc., and also provide summarizations of chapters in support of streamlining different tasks for different personas like researchers, experimenters, and general users. There are six teams responsible for different components and services for the overall system. Team 1 is responsible for generating knowledge graphs that model the structural/relational information for the ETD based on the information extracted during object detection by Team 3. Team 2 is responsible for indexing, searching, and ranking ETDs. Team 3 is responsible for extraction of image-based and text-based objects from ETDs, and topic modeling. Team 4 is responsible for summarizing and classifying the chapters and their contents to effectively describe them. Team 5 is responsible for maintaining the file system and database, providing APIs (Application Programming Interfaces) to perform actions on the data, and managing the overall infrastructure for the system. Team 6 is responsible for the User Interface to support all personas and define how the user interacts with the system.

In this report, we discuss the progression of our team’s work (Team 3).
The primary focus was to develop the auxiliary services for the experimenter and curator personas to analyze the documents by performing object detection and topic modeling which will enable searching of ETDs through chapters or specific topics. We also discuss the architecture, and plans for implementation of the algorithms we utilized. Team 5 provided the support to store and retrieve the segmented objects from the ETDs, as well as the list of topics generated by the topic modeling pipeline, by utilizing a sequence of API calls. Figures \ref{fig: Object Detection} and \ref{fig: Topic Modeling} shows the overall pipeline that our team has been following. 


The object identification system utilized the YOLO (YOLOv7) and Faster R-CNN (Detectron2) models to leverage their capabilities. Additionally, post-processing techniques were employed to analyze and interpret the text and image objects. The topic modeling system used methods like LDA and BERTopic to associate ETDs to different topics. The complete pipeline will be made clear in the following sections.

			%\lipsum[2]
			%\subsection{A sub-section} \label{ss:this_subsection}
				%\lipsum[1-4]
		\section{Project Management} \label{se:another_section}
  Since our team has responsibilities of two distinct components, team members are often required to be involved in both. Effective task management has become essential for enhancing productivity. Our project utilizes Trello's functionalities to improve team member collaboration and expedite operations. We have created a Trello board with labels such as ``To-Do,'' ``In Progress,'' and ``Completed'' to represent the various stages of our Endeavour. Each task or deliverable is represented by a card that is moved across these categories to indicate its current status. To ensure timely completion, team members are assigned individual cards with plainly marked deadlines. Further, to track and manage changes made to the software by the team members, we are using GitLab as our version control system. We conduct weekly meetings to track project progress. During these meetings, team members provide updates on their progress, discuss future tasks, and address any challenges while planning steps for issue mitigation. We also hold discussions with the Subject Matter Experts in class to get feedback on our progression towards the goals, and insights into the existing implementation of the systems. 

        \section{Challenges Faced and Solutions}
        During the course of this work, we encountered several challenges that required solutions.The professor, the subject matter experts, as well as the teaching assistant for the course
have assisted us in laying out the process while removing barriers in each step. The following
are some of the solutions for the challenges faced.
        \begin{enumerate}
            
         \item Setting up of the previous code base in the Endeavour cluster, especially due to version mismatch of various packages and inconsistencies between pip installed packages and conda installed packages. Here, the previous code base was successfully set up in the Endeavour cluster by cross-verifying the package versions installed in the conda environment as well as in Python by referring to error logs and online forums to debug installation errors.
        \item  Uncertain erasure of conda environments in the container as a result of rebooting each node to fix the failed updates to the underlying kubernetes system, and container crashes that led to problems with cluster
networking. To deal with the uncertain erasure of the conda environment, we documented the conda environment set up to ensure speedy set-up in case the environment was erased again. Additionally, on the advice of Dr. Fox and Chris Arnold, we worked on avoiding package updates to a running container on Kubernetes, and focused on installation outside of the cluster to the Docker image itself.
        \item  Issue with the initialization of the Faster-RCNN Detectron2 trained model.We were unable to resolve the installation issues with respect to the config file for the Detectron2 model. From the report by the Fall 2022 CS 5604 team working on object detection \cite{OD-cs5604-f22}
, YOLOv7 showed better performance than Detectron2 on the collection of ETDs. Hence, we decided to implement only the YOLOv7 model for the scope of this work.
        \item  Stalled efforts to populate the database and file system, and generate XMLs for the ETDs processed through the object detection pipeline due to issues with API implementations. Since then, we continued to work closely with Team 5 to ensure complete integration of the APIs into the object detection pipeline.
        \item  Stalled efforts in implementing Kafka due to the prioritization of other more important tasks and infrastructure setup. We worked on continually improving the proposed architecture for the Kafka implementation of the object detection model based on discussions with Dr. Fox, the subject matter experts, and the TA so that it could be ready to be implemented when the architecture allows for it.
        \item Long running jobs like topic modeling training on the ETD dataset used a Virtual Machine (hostname: dlrl) provided by Satvik Chekuri, an SME as part of the CS 5604 course who is also a Ph.D. student in the DLRL (Digital Library Research Laboratory). This was done because of the model training stopping in between and sometimes not starting at all. Since the root cause was not known, we have decided to train the model and move all the results and scripts that can be used to the Endeavour cluster
        \end{enumerate}

        \section{Future Work}
        This project is a work in progress and has large scope for improvements in accuracy and scalability.
        Following are brief descriptions of some of the improvements in the scope of the object detection and topic modeling pipeline in this work. The Future Work is elaborated in Chapter \ref{future_work}.
        \begin{enumerate}
            \item Improve the object detection model accuracy and generalizability. 
            \item Experiment with newer object detection models for ETD data.
            \item Explore a top-down object detection approach and its benefits in generating the XML for the ETD. A top-down object detection approach which provides a global view of an image to identify objects, can be beneficial for generating XML for an ETD. The structured data in XML format, resulting from this approach, is both machine-readable and human-readable, enhancing the usability and accessibility of the ETD.
            \item Ensure accurate handling and processing of scanned PDF ETDs and born PDF ETDs.
            \item Incorporate the chapter segmentation code with the object detection pipeline. An XML with well-segmented chapters will then be passed on to Team 4 for classification and summarization tasks.
            \item Set up Kafka for batch processing.
            \item Populate both the database and the file system with the extracted data and files generated by passing the ETD through the object detection pipeline using the integrated APIs.
            \item Complete processing 500,000 ETDs through the object detection pipeline and the topic modeling pipeline.
            \item For topic modeling, analyze and improve the usage of BERTopic on the cleaned ETD dataset; this is an important future goal.
            \item Analyze and improve the keywords formed by the LDA model by updating custom stopwords while also trying different vectorizers other than tf-idf. Further, we want to explore different lemmatization methods.
            \item Explore other topic modeling approaches such as ProdLDA and NeurLDA on the large ETD dataset.
            \item While signing up, scanning through 70 generated topics is not a user friendly approach. A better approach like hierarchical topic modeling needs to be considered.
        \end{enumerate}



The report is organized as follows: Chapter \ref{ch:lit_review} performs a literature review, highlighting the tools and methodologies used for object detection and topic modeling. Chapter \ref{requirements} provides details on the project's requirements concerning the dataset, training models, usage, and processing capabilities. The current designs for object detection and topic modeling are elaborated in Chapter \ref{ch:design}. This chapter also includes information about the previous architecture, proposed improvements, and deliverables aimed at enhancing the model.

Chapter \ref{ch:Implementation} outlines the milestones achieved during this course and delves into the experiments, including both failures and successes, as well as the implementations made in CS 5604 for object detection, chapter segmentation, and topic modeling. Our future plans for improving and expanding the current model are described in Chapter \ref{future_work}.

Finally, Chapter \ref{user manual} and Chapter \ref{developer_manual} detail the execution of the object detection model and topic modeling model from the perspective of users and its navigation from the perspective of developers, respectively.

			%\lipsum[1-2]

    \chapter{Literature Review} \label{ch:lit_review}
    The field of information storage and retrieval has seen significant advancements. Yet, there remain gaps in its application to large Electronic Theses and Dissertations (ETDs) datasets. This literature review aims to explore current methods in object detection and topic modeling, focusing on their relevance and limitations for handling ETD collections. We conducted a methodical search of recent studies using databases like Google Scholar and the ACM Digital Library. The review is organized into sections that cover the fundamental object detection technologies, and the various topic modeling approaches. Through this examination, we seek to inform the design of an advanced ETD retrieval system in collaboration with Virginia Tech's University Libraries.
    
    \section{Object Detection}
    We implement the object detection process with two models, namely YOLOv7 \cite{wang2022yolov7} and Faster R-CNN \cite{ren2016faster}.
The third version of Ross Girshick et al.'s R-CNN architecture is called Faster R-CNN, and it was first released in 2015. R-CNN's Region of Interest (RoI) Pooling was used in a second version of the algorithm known as Fast R-CNN to replace the selective search technique. This allowed for the pooling of costly calculations and significantly accelerated inference.
With the use of the CNN's feature maps, the Region Proposal Network (RPN), which was introduced by Faster R-CNN, can be trained in tandem with the CNN to produce high-quality suggestions. It should be noted that the original Faster R-CNN implementation used a standard VGG-16 backbone architecture for the CNN; however, the framework we are using, Facebook AI Research's Detectron2, contains object detection models built on top of Faster R-CNN with sophisticated architectures that perform incredibly well \cite{wu2019detectron2}. Even though Faster R-CNN was the most advanced model for many years, newer YOLO models and other models with improved speed vs. accuracy trade-offs have supplanted it. When YOLOv7 is compared to other real-time object detectors, it outperforms both transformer-based and convolution-based object detectors, with the highest accuracy of 56.8 percent average precision \cite{wang2022yolov7}. 
    \section{Topic Modelling}
    
Natural language processing (NLP) and text mining have both made significant strides in recent years. The goal of topic modeling in ETD analysis is to identify thematic clusters of words from a large corpus of text documents that may be used to represent themes.
Over the years, numerous topic modeling algorithms have been put forth. Previous research, such as with Latent Dirichlet Allocation (LDA) \cite{LDA}, relied on the concept of topic formation based on probability. In the field of topic modeling, recent developments include neural topic models like CTM (that allows for correlation between topics) \cite{10.5555/2976248.2976267}, NeuralLDA (to model topics as continuous vectors), ProdLDA (using 
Variational Autoencoders (VAEs)) \cite{srivastava2017autoencoding}, and BERTopic
(using transformers and c-TF-IDF to create dense clusters allowing for easily interpretable topics) \cite{bertopic}. The representations that are acquired through topic models can be applied to downstream tasks that depend on document representations, like document recommendation, locating documents by topic, topic by university, and examining topic fluctuation over time.

  \section{Tools}
   Below is a list of the tools used for our work. This is followed by a subsection for each of those tools giving more detail.
    \begin{enumerate}
  \item PyTorch\\
  \item Nvidia GPU\\
  \item Docker Engine\\
  \item Detectron and YOLOv7, OCR\\
  \item LDA, Neural LDA, ProdLDA, CTM, BERTopic\\
  \item OCTIS
    \end{enumerate}
    
\subsection{PyTorch}
The Facebook AI Research lab (FAIR) created PyTorch, an open-source deep learning framework \cite{paszke2019pytorch}. It is employed for numerous machine learning and artificial intelligence tasks, specifically in the area of deep learning. The core data structures of PyTorch are tensors. They are the basic units for constructing and modifying neural network models. Tensors can be used on GPUs to enable faster processing. With PyTorch, the graph is constructed dynamically as operations are carried out. This dynamic nature enables more flexible and understandable model development, making it simpler to troubleshoot and test out various network structures.
\subsection{Detectron2}
The newest library from Facebook AI Research, Detectron2, offers cutting-edge object segmentation and detection algorithms. It is the replacement for the original Detectron framework and is well known for its computer vision applications, adaptability, modularity, and excellent performance. Detectron2 is developed on top of PyTorch and it provides a full suite of tools and pre-made components for object detection. It enables faster training of models. Another benefit of Detectron2 is that it allows models created with it to be deployed in TorchScript or Caffe2 format. Detectron2 offers pre-trained models on expansive datasets like COCO (Common Objects in Context). As a starting point for transfer learning, these pre-trained models enable you to fine-tune models on your datasets with only a minimal quantity of data. We will be using this library for carrying out object detection and classifying the pages of the ETD into multiple text and image based objects. An example of the object detection using Detectron 2 can be seen in Figure \ref{fig: Detectron2}.
\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{Detectron2.png}
    \caption{Object detection using Detectron2 \cite{detectronImage}}
    \label{fig: Detectron2}
\end{figure}

\subsection{YOLOv7}
YOLOv7 is one of the state of the art real time object detection models, excelling at detecting various objects in real time. As mentioned in the official paper written by Chien-Yao Wang, Alexey Bochkovskiy, and Hong-Yuan Mark Liao \cite{wang2022yolov7},  YOLOv7 operates efficiently in a frame rate range of 5 FPS to 160 FPS and boasts the highest accuracy score of 56.8\% AP among real-time object detectors with 30 FPS or higher on a GPU V100. The model significantly outperforms other object detectors in both speed and accuracy metrics, even when compared to both transformer-based and convolutional-based models like SWINL Cascade-Mask R-CNN and ConvNeXt-XL Cascade-Mask R-CNN. Notably, YOLOv7 was trained exclusively on the MS COCO dataset without leveraging any other datasets or pre-trained weights. YOLOv7 performance compared to other object detection models can be seen in Figure \ref{fig: yolov7}.

YOLOv7 employs a convolutional neural network to simultaneously generate bounding boxes and class likelihoods by analyzing the entire image in a single go. This is in contrast to traditional object detection algorithms, which focus on specific regions of the image to pinpoint objects and their associated probabilities. The YOLOv7 architecture consists of several layers: firstly, the image frames pass through a `backbone,' which is essentially a deep neural network made up of numerous convolutional layers designed to extract specific features. These features then flow through the `neck,' an intermediary set of layers that integrates and collates the feature maps extracted at various stages by the backbone. This is followed by the `head,' where actual object detection takes place via dense prediction methods. The backbones implemented in YOLOv7 include Vision-Only Volumetric Networks (VoVNET), Cross-Stage Partial VoVNET (CSPVoVNET), Efficient LAttice Networks (ELAN), and Extended-Efficient LAttice Networks (E-LAN). As for the neck, it utilizes Feature Pyramid Networks (FPN), Receptive Field Block (RFB), and Path Aggregation Network (PAN). Finally, the detection head makes simultaneous localization and classification predictions, setting it apart from two-stage detectors like Faster R-CNN and Region-based Fully Convolutional Networks (RFCN), which perform sparse predictions to establish class probabilities.

\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{yolov7.png}
    \caption{YOLOv7 performance compared to other object detectors \cite{yolov7Image}}
    \label{fig: yolov7}
\end{figure}

There is an unofficial new version of YOLOv7, called YOLOv8. Future work includes experimenting object detection with the YOLOv8 model due to its improved performance capabilities as seen in Figure \ref{fig: yolov8}.

\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{yolov8.png}
    \caption{YOLOv8 performance compared to older versions \cite{yolov8Image}}
    \label{fig: yolov8}
\end{figure}

\subsection{LDA}
Latent Dirichlet Allocation (LDA) \cite{LDA} is a probabilistic generative model used in machine learning and natural language processing for document grouping and topic modeling. It envisions a generative process in which subjects and words are chosen at random from a set of topics to produce documents. Reverse engineering this generative process is the aim of LDA. LDA determines the subjects that produced a collection of documents and the distribution of words within each topic. Usually, probabilistic inference widths like variational inference or Gibbs sampling are used for this. The distribution of document topics and the distribution of topic words are the two basic groups of parameters in LDA \cite{LDA}. 

\subsection{ProdLDA}
A probabilistic topic modeling approach for massive amounts of text data is called ProdLDA, \cite{srivastava2017autoencoding} or Product of Latent Dirichlet Allocation. By constructing numerous LDA-based ``experts,'' each modeling a subset of topics, it divides the topic modeling challenge into more manageable, smaller subproblems. By merging the findings from multiple experts through a product operation, the final topic distribution is generated. With the help of this width, scalability and parallelization are improved, making it effective for processing big-text corpora and identifying relevant subjects in vast datasets. Several natural language processing applications benefit greatly from ProdLDA's hierarchical structure and complexity reduction.

\subsection{NeuralLDA}
Neural LDA (Latent Dirichlet Allocation) is an extension of conventional LDA.
It integrates neural networks into the topic modeling procedure. The representation of documents and subjects is improved by the use of neural networks, typically deep learning models. By capturing semantic linkages in text data, this width enables more intricate and context-aware topic modeling. Compared to classic LDA, neural LDA is more adaptable because it can automatically learn a variety of themes. It is frequently utilized in contemporary text analysis jobs as it can improve performance and adaptability.
\subsection{CTM}
Correlated Topic Model (CTM) \cite{10.5555/2976248.2976267}, a probabilistic topic modeling method, is used to capture correlations between themes in a group of texts. Although CTM assumes that documents are produced using a variety of topics, it goes a step further by enabling topics to be associated using a common topic correlation matrix. Due to the ability to model topic interdependence, CTM is appropriate for jobs in which topics are not independent. To calculate topic-word distributions and topic correlations, CTM uses probabilistic inference techniques. It has uses in a number of NLP tasks, including content recommendation, document classification, and information retrieval, where topic relationships must be captured for better modeling and comprehension of document collections.
\subsection{BERTopic}
BERTopic \cite{bertopic} is a topic modeling technique that uses the transformer-based language model -- BERT (Bidirectional Encoder Representations from Transformers) -- for topic modeling. It could overcome some of the shortcomings of traditional topic modeling methods by leveraging the power of contextual word embeddings provided by pre-trained transformer models. It has the capability to create  highly context-aware word embeddings and capture the meaning of words in the context of the entire text. BERTopic applies dimensionality reduction techniques like UMAP (Uniform Manifold Approximation and Projection) to reduce the high-dimensional BERT embeddings into a lower-dimensional space. It performs clustering using methods like HDBSCAN (Hierarchical Density-Based Spatial Clustering of Applications with Noise, to find clusters of varying shapes \cite{McInnes2017}, which effectively groups the documents into distinct topics.
\subsection{pdf2image}
This is a Python library that offers a quick and easy approach to turn PDF files into a series of images to perform operations like document processing, automation, content extraction, OCR (Optical Character Recognition), or image processing. This library provides an API that makes it simple to incorporate into our Python programs. The output path, other parameters, and the input PDF file can all be specified as needed like in Figure \ref{fig: PDF2Image}. We are incorporating this tool in the data pre-processing step where we will be extracting images from the ETDs.
\begin{figure}[H]
    \centering
    \includegraphics[width=0.7\linewidth]{pdf2Image.png}
    \caption{Sample Code Snippet for Converting PDF to Image using PDF2Image library.}
    \label{fig: PDF2Image}
\end{figure}

\subsection{Flask}
Integrating the Flask framework with our back-end code is easy and convenient because it is a lightweight, Python-based web framework based on WSGI \cite{grinberg2018flask}.
Because it is a micro-framework design, we may combine it with any wheels or Python libraries we require for the project development, and it is highly scalable and versatile thanks to the large range of third-party libraries. Furthermore, it would be simple to develop new online features and functionalities. In addition, front-end technologies like React \cite{reactGitHub} are required. The fundamental framework of the website, its styling, and user interaction will be managed by Team 6.
\subsection{OCTIS}
The acronym OCTIS \cite{terragni2020octis} refers to optimizing and comparing topic models. It is a Python program that makes this possible and facilitates the analysis of ETD topics. LDA is now the most widely used topic model, however it is insufficient. As a result, OCTIS provides us with an embedding model, including CTM, ProdLDA, and NeuralLDA. 



    \chapter{Requirements} \label{requirements}
    In this chapter, we will be discussing our requirements,
%    as of the time of this report, 
    as well as some potential requirements for considerable improvements that we might utilize. 
    
    \section{Data Requirements}
    Our data collection consists of 500,000 Electronic Theses and Dissertations (ETDs) covering the time period from 1845 to 2020. 
    
    \subsection{Dataset Cleaning}
    Our initial analysis of the dataset revealed that it was not clean, and had errors including improper characters, having data in the wrong fields, or not having data in important fields. Many rows had no abstracts or used placeholder words like ``Abstract'' or ``None''. After cleaning such data, we have 333,867 rows that we can use.

    
    \subsection{Object Detection}
    Roboflow \cite{roboflow}, a free online labeling tool, was previously used for the labeling. Twenty-four categories—such as paragraphs, figures, and tables—that were judged common in ETDs were recognized and named.
We received the labeling from students in the spring semester of 2022 \cite{OD_cs4624}, which included both scanned and born digital ETDs.  The dataset we are using for this research has a final object count of approximately 25K pages and 100K objects. 

    
    \subsection{Topic Modelling}
   The collection includes the title, abstract, and ETD ID, among other basic information about ETDs.
As additional analyzing targets, the segmented chapters produced by the object detection algorithm in addition to the title and abstract can be used.

    \section{Training}
    We are utilizing a pre-trained object detection model, which eliminates the need for initial training data for its operation. However, to enhance the existing model's performance, perform further fine-tuning, and improve generalization capabilities and accuracy, the model needs to be trained on a larger dataset. Model improvement is highlighted in the future scope.
    As for our topic modeling algorithm, it operates on an unsupervised learning paradigm, meaning there is no requirement for labeled training data in its case.
    \section{Usage}
    For inference and usage, some protocols were established with teams 5 and 6. Team 5 plays a critical role in data retrieval and storage; they provided us with efficient Read/Write APIs to fetch the raw ETD data. Once the data is in our possession, our team applies object detection and topic modeling algorithms to process it. This processed information needs to be stored back securely, for which we again rely on Team 5's data storage APIs. On the other end, Team 6 is in charge of front-end user interactions.
    \vspace{-\baselineskip} % Remove the vertical gap
    \section{Processing Requirements}
    Both the topic modelling and object detection tasks require significant computational power. Image processing and object detection tasks require specialized GPUs (Graphics Processing Unit), which are optimized for high-performance calculations. To meet these demands, we have access to Endeavour, a computing cluster equipped with the necessary GPUs, which serves as the backbone for our computationally intense operations.
    \let\cleardoublepage
    
\chapter{Design} \label{ch:design}
This chapter will cover the aim of our project, and explain the approach and deliverables we planned to complete during the semester.

\textbf{Personas} \\
The system supports three types of personas.
\begin{enumerate}
  \item User: Students, researchers, and other users interested to view work related to their research as well as university administration, and librarians interested in an overview of recent research in their institution.
  \item Curator: Persona that can process one document at a time and view the result at the end of each process in the workflow pipeline, or can choose to batch process multiple documents at once.
  \item Experimenter: Persona that can modify available model parameters and evaluate performance and efficiency of the existing system on different parameter combinations.
\end{enumerate}
\textbf{Aim}\\
The aim of this work is to extend and improve upon the previous architecture (designed by Team 3, CS 5604 Fall 2022 \cite{OD-cs5604-f22}) for object detection and topic modeling. The core design and implementation of the previous architecture remain the same. The current scope of this project deals with both born-digital and scanned PDFs.  The extensions to the work are detailed for object detection and topic modeling in this section.
\section{Approach} 
\subsection{Object Detection}
\textbf{Goals}\\ 
The curator or experimenter can upload an ETD to be processed using the object detection model, and view the ETD after it has been segmented and rendered as a web page.

\textbf{Previous architecture} \\
The previous architecture is explained with respect to the user persona. The user begins by uploading an ETD file in PDF format. After the user uploads the ETD file, they can choose between two object detection models YOLOv7 and Detectron2 (based on Faster RCNN). Each page in the PDF file is converted into an image using the pdf2image module. Then, each page image is processed one by one based on the chosen object detection module. Features such as Abstract, Metadata, Table of Contents, List of Contents, and Chapters are extracted from the ETD as objects. These objects are parsed and image based objects such as figures, tables, and equations are saved to the file system and their path is returned after saving. Similarly, text objects extracted are parsed using the PyMuPDF module to extract relevant sections of text. The returned path of image objects as well as the text extracted after object parsing is stored in an XML schema. Post-processing rules are applied to the XML schema to enhance chapter segmentation and to correct false positives. The XML schema can then be used to be display the ETD on the final webpage.

The architecture diagram for object detection with improvements (detailed in Section 4.2.1) is represented in Figure \ref{fig: Object Detection}.
    \begin{figure}[H]
    \centering
    \includegraphics[width=1.4\linewidth]{ETD.png}
    \caption{ Object Detection - from ETDs to the XML/JSON}
    \label{fig: Object Detection}
\end{figure}

\subsection{Topic Modeling}
\textbf{Goals}\\ 
Users of our framework can choose topics of interest while registering to use the system. The services offered to the user include Topic List, Documents per Topic Distribution, and Similar topics. 

\textbf{Previous architecture} \\
The extracted ETD text for the chosen ETD undergoes tokenization and pre-processing.
%to create tokens. These t
Tokens are fed to topic modeling algorithms (LDA, NeuralLDA, ProdLDA, CTM, BERTopic) to retrieve topic words and document vectors. For an ETD, related topics and documents are returned via APIs, and all of the topics produced by the model and the corresponding documents are displayed in a Topic Browser. The user can get the displayed data using Documents per Topic Distribution, Topic List, and similar topics services.

The previous architecture diagram for topic modeling is represented in Figure \ref{fig: topic_modeling_f22}.
 \begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth, height = 7cm]{topic_modeling.png}
    \caption{Previous Architecture: Topic Modeling pipeline diagram \cite{AAhuja}}
    \label{fig: topic_modeling_f22}
\end{figure}


    \section{Deliverables}
\subsection{Object Detection}
\textbf{Deliverables completed}
\begin{enumerate}
    \item Establishing connection with the database/file system infrastructure managed by Team 5: The previous architecture was a stand-alone end-end application handling retrieving, uploading, and saving of ETDs and objects into a dedicated file server. This work includes establishing connection to the main file server on the Endeavour cluster handled by Team 5 which will hold all the necessary data required for any task in the pipeline in the larger scope of the information retrieval and analysis system for ETDs. The connection is established by defining APIs for the following tasks:
  \begin{itemize}
      \item Retrieve the ETD to be processed.
      \item Save the image of each PDF page of the ETD. 
      \item Retrieve the image of PDF page stored  for the purpose of running Object Detection on it.
      \item Save the detected objects from the pages processed (text, image objects).
      \item Populate the ETD metadata database table with the metadata extracted from the ETD.
      \item Save the XML file generated for the ETD after running the object detection pipeline on it.
  \end{itemize}
  \item Discuss the intended UI with team 6 to display the results of the parsed ETD.
  \item Add Object IDs for each object in the XML file. Team 1, working on the knowledge graph, requires an Object ID to be associated with each object stored, to be able to record structural relationships and construct an efficient and accurate knowledge graph.
  \item Chapter Segmentation: The current work aims to improve the accuracy of chapter segmentation by using the page numbers from Chapters and Sections from the Table of Contents as heuristics to identify the start and end of chapters and subsections. Additionally, the page numbers extracted during object detection can be mapped to the information extracted from the Table of Contents for the segmentation process to occur during object detection thereby aiming to incorporate both processes into one as opposed to having separate architectures for the two. This heuristic approach will include two steps, parsing the PDF page by page and passing the text into an LLM (Large Language Model) such as LLaMA 2-13b in order to determine the page indices of the table of contents. Then, we will use a heuristic approach to parse the table of contents and extract the chapters and their corresponding page numbers. We discuss the second step further in the implementation section.
  % However, we are still experimenting with step 1 and the usage of the LLM. 
  \item Implement Batch Processing: The current workflow for object detection processes only one ETD at a time, page by page. To scale the system and handle approximately 500,000 ETDs, the object detection code should be executed in batch processing mode. This effort aims to enhance the object detection pipeline by incorporating Kafka, enabling it to efficiently process batches. While preliminary experiments on identifying optimal batch sizes have been conducted, further experiments on the same with Kafka fall under the future scope.
  \item Refining post-processing rules for the XML file: Building upon the post-processing rules from the previous architecture, this work aims to add or modify existing post-processing rules after implementing the proposed improvements to the object detection architecture. This is to ensure a more accurate representation of the ETD in the XML file.
  \item Making Figure Captions Searchable: This work aims to accurately map the figure captions to images and make them searchable when the ETD is rendered on the webpage.
\end{enumerate}

    \subsection{Topic Modeling}

\textbf{Deliverables completed}
\begin{enumerate}
    \item Implemented data cleaning and pre-processing steps on 500k ETDs to utilize good quality data to train LDA and BERTopic models.
    \item Selected LDA model with 70 topics to be the default model being used. 
    \item Establishing connection with the database managed by Team 5: The previous architecture was a stand-alone end-end application handling the retrieving and saving the data into a dedicated file server. This work included establishing connection to the main file server on the Endeavour cluster handled by Team 5 which will hold all the necessary data required for any task in the pipeline in the larger scope of the information retrieval and analysis system for ETDs.
        \item We have APIs suitable for Team 6 to have the UI make use of all topic modeling related functions. 
    \item Modified the previous architecture to make use of the BERTopic model \cite{bertopic} and LDA model for topic modeling. Initial inference revealed that the fixed size of 100 topics is not producing good results when we use the BERTopic model as it is not able to reduce the original 2800 topics generated for 330k ETDs down to 100 topics. Hence, we needed to improve the code implemented in previous courses. However, LDA model provided much better results.
    \item Overall, make code changes to optimize processing of a large ETD dataset. This includes a data cleaning script along with pre-processing steps that need to be updated and also writing scripts to use the trained models efficiently.
\end{enumerate}

\chapter{Implementation} \label{ch:Implementation}
\section{Milestones}
The Interim Report 1 (IR1) was the first milestone that where the team worked together to understand the objective, get familiar with the code, sort out the resources and tools available, upload the code to the Endeavour cluster, and made a semester plan to finish the whole project. Simultaneously, we also devised some improvements to extend the previous architecture for object detection and topic modeling. Interim Report (IR2) is our second milestone, where we have a working prototype satisfying the basic functionality of the code tested on 200 ETDs. As of Interim Report 3 (IR3), the API requirements for object detection have been defined, and we began integrating them into our object detection code. The Kafka framework for object detection was outlined, and tests on the chapter segmentation model were performed. Regarding topic modeling, the BERTopic model was tested on 333,867 ETDs, and we began working on implementing the LDA model to improve topic modeling results. We also started running tasks from the object detection pipeline on the set of 500,000 ETDs. The Final Report encompasses the complete code integration, clean-up, and a summary of our project, including a fully developed prototype that will be deployed in our university's cloud.

\section{Object Detection} 

      \begin{table}[ht]
            \centering
            \caption{Object Detection and Chapter Segmentation Milestones} \label{TM_Tasks}
            % \begin{tabular}{p{3cm} p{8cm}}
            \begin{tabular}{cl}
            \toprule
            \textbf{Timeline}  & \textbf{Tasks} \\
            \midrule
            % \textcolor{red}{Change} & \textcolor{red}{dates and correct ordering}\\
            1 Sept & Get familiar with the Fall 2022 Object Detection system. \\
            8 Sept & Understand the YOLO V7 model.  \\
            13 Sept & Familiarize with the Detectron2 model implemented in the previous semester. \\
            19 Sept & Explore the object detection system and set it up  on the Endeavour cluster \\
            23 Sept & Perform object detection on 200 ETDs (Electronic Theses and Dissertations).  \\
            23 Sept &  Modify the code to add (pseudo) Object IDs to the XML file. \\ &
            Begin exploring Chapter Segmentation methodologies\\
            9 Oct & Experiment with multiprocessing to determine the optimal batch size.\\
            13 Oct & Perform object detection on 200 ETDs (Electronic Theses and Dissertations)\\ & to 
            generate XML with the IDs.\\ &
            Test LLaMA 2 model and heuristic model for chapter segmentation
            \\
            14 Oct & Collaborate to plan and implement the architecture for Kafka. \\
            19 Oct & Discussion with Team 1 on the XML file requirements.\\ &
            Refine chapter segmentation code.\\
            26 Oct & Discussion with Team 5 on API definitions \\ 
            7 Nov & Draft 1 of the APIs required for integration to the database and file system. \\ & 
            Explore RESNET for chapter segmentation. \\
            14 Nov & API definition and integration. \\ &
            Refine chapter segmentation code.\\ 
            26 Nov & Modularize the code files for Kafka and plan to run the object \\ &
            detection pipeline for 500,000 ETDs\\ 
            2 Dec & Work on running the object detection\\ & pipeline for 500,000 ETDs\\ 
            Final & Documentation of work done in final report\\
            \bottomrule
            \end{tabular}
            \end{table}
\textbf{XML Generation}
\\

As of IR2, the object detection (YOLOv7) model has been successfully run on 200 ETDs that were uploaded to the Endeavour cluster.
The object detection algorithm returns an XML file as the output of parsing the ETD. The XML schema, as seen in Figure \ref{fig: xml_schema}, contains the root element as the ETD ID and three sub-elements:
\begin{enumerate}
\item Front: The front section includes ETD metadata, such as document title, author, university, degree, committee, etc., as shown in Figure \ref{fig: xml_front}.
\item Body: The body contains the main ETD data, including chapters, sections, and subsection content, such as text, tables, figures, equations, and algorithms, along with the associated page numbers, as shown in Figure \ref{fig: xml_body1}. Image-based objects have corresponding file paths and associated captions/numbers as seen in Figure \ref{fig: xml_body2}. 
\item Back: The back section provides details on the reference section of the ETD, as shown in Figure \ref{fig: xml_back}.
\end{enumerate}

\begin{figure}[H]
    \centering
    \includegraphics[width=0.5\linewidth]{xml_schema.png}
    \caption{An example snippet of the XML schema}
    \label{fig: xml_schema}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{xml_front.JPG}
    \caption{An example snippet of the front section of the XML schema of ETD \cite{brammer_2010}}
    \label{fig: xml_front}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{xml_body1.JPG}
    \caption{An example snippet of the body of the XML schema of ETD \cite{brammer_2010}}.
    \label{fig: xml_body1}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{xml_body2.jpg}
    \caption{An example snippet of details associated with the image-based objects in the body of the XML schema of ETD \cite{brammer_2010}}.
    \label{fig: xml_body2}
\end{figure}

\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{xml_back.jpg}
    \caption{An example snippet of the back section of the XML schema of ETD \cite{brammer_2010}}.
    \label{fig: xml_back}
\end{figure}



As of IR3, it has been discussed with Team 1 who are working on the knowledge graph, to have the generated XMLs have obj\_id denoting the Object ID associated with every object detected. The Object ID is received as a response from the API call after saving the objects in the database system. A snippet of the modified XML can be seen in Figure \ref{fig: xml_obj_id}. 

\begin{figure}[H]
    \centering
    \includegraphics[width=0.9\linewidth]{xml_obj_id.jpg}
    \caption{An example snippet of the XML schema of ETD \cite{Huang2017} with Object ID highlighted for each object detected.}
    \label{fig: xml_obj_id}
\end{figure}



\textbf{APIs} 
\\
The main purpose of the APIs is to access the centralized resources and store intermediate results in the database as well as the file system. This enables speedy retrieval of details, be it for rendering in the front-end or for further processing tasks in the pipeline of the ETD search system. Having a centralized data store would also ensure security and authorized access to the data store.

As of IR3, the APIs required for interacting with the database and file system have been defined and provided to Team 5 for implementation, and consequently integration into the current object detection code. As a result of the object detection pipeline, the intermediary outputs (such as ETD page converted into page image, detected text-based object, etc.) would be used to populate the database tables namely the ETD\_metadata, ETD\_pages, and the Objects table.

The GET APIs for object detection are defined as seen in Table \ref{tab:api_get}. These APIs are used to get the ETDs, and the saved page images of the ETD for further processing in the pipeline. The POST APIs for object detection are defined as seen in Table \ref{tab:api_post}. These APIs are used to populate the database tables and the file system with the intermediate results of the tasks of the object detection pipeline (such as ETD page (.pdf) to page image, objects detected in the page image, the .xml file generated for the ETD).
The exact request parameters or body of the API are subject to modification based on Team 5’s finalized API implementation.

\begin{center}
\begin{longtable} {|p{1.5cm}|p{3cm}|p{7cm}|p{4cm}|}
% \centering
% \begin{tabular}{|p{1.5cm}|p{3cm}|p{7cm}|p{4cm}|}

\hline
\textbf{Request Method} & \textbf{Description} & \textbf{Request Parameters} & \textbf{Database/File System} \\ 
\hline

GET* & Get ETD IDs of the .pdf file & - & Database Table: etds \\
\hline
GET & GET ETD with ETD ID & - & File System \\
\hline
GET & Retrieve page image with pageID & \begin{tabular}[c]{@{}l@{}}data = \{\\ pageID = page ID received after \\saving the page.\\ \}\end{tabular} & File System \\
\hline

\caption{GET APIs for Object Detection}
\label{tab:api_get}
\end{longtable}
\end{center}
\small *The implementation of this API may be temporary until the integration of Kafka, after which the ETD IDs will be available as Kafka topics.

\begin{center}
\begin{longtable} {|p{1.5cm}|p{3cm}|p{7cm}|p{4cm}|}
% \centering
% \begin{tabular}{|p{1.5cm}|p{3cm}|p{7cm}|p{4cm}|}

\hline
\textbf{Request Method} & \textbf{Description} & \textbf{Request Parameters} & \textbf{Database/File System and API Response (if any)} \\ 
\hline

POST & Save page image (page of .pdf) & \begin{tabular}[c]{@{}l@{}}data = \{\\ etds\_id = ETD ID,\\ page\_no = page number of the page,\\ page\_image = .jpg image\\ \}\end{tabular} & Database Table: etd\_pages and File System. Responds with pageID\\
\hline


POST & Store the detected objects & \begin{tabular}[c]{@{}l@{}}data = \{\\ object\_type = Text (paragraph,\\ abstract, etc.) or\\ Image type (figure, equation, etc),\\ object\_format = Text or Image,\\ object\_text = Text extracted from \\ text-based objects,\\ object\_image = .jpg of image-based \\object,\\ETD\_ID: etd ID of the image\\ \}\end{tabular} & Database Table: objects and File System. Responds with objectID \\
\hline

POST & Store the generated XML file & \begin{tabular}[c]{@{}l@{}}data = \{\\ File = .xml file (same etd\_id.xml)\\
etds\_id = ETD ID \\
landing = provide landing url \\\}\end{tabular} & Filesystem \\
\hline


\caption{POST APIs for Object Detection}
\label{tab:api_post}
\end{longtable}
\end{center}


\textbf{Multiprocessing}
\\
To estimate an approximate batch size that uses anywhere between 50\% - 75\% of the maximum GPU utilization resources, we made use of the Python multiprocessing and concurrent.futures library to execute multiprocessing. With a cap of maximum 4 worker nodes being utilized, we tested batch sizes of 10, 50, and 75 and monitored utilization rates using the 'nvidia -smi' command for CUDA to print an output in 1 second intervals. It was found that for a batch size of 10, the maximum resource utilization observed was 42\% (Figure \ref{fig:mp1}), for a batch size of 50, the maximum resource utilization observed was 51\% (Figure \ref{fig:mp2}), for a batch size of 75, the maximum resource utilization observed was 55\% (Figure \ref{fig:mp3}). The average resource utilization increased minimally (approximately 5\%-10\%) with an increase in batch size.

\begin{figure}[!htb]\includegraphics[width=\linewidth]{multi2.JPG}
\caption{Maximum resource utilization for batch size = 10}\label{fig:mp1}
\end{figure}

\begin{figure}[!htb]\includegraphics[width=\linewidth]{multi4.JPG}
  \caption{Maximum resource utilization for batch size = 50}\label{fig:mp2}
\end{figure}
 
\begin{figure}[!htb]\includegraphics[width=\linewidth]{multi5.JPG}
  \caption{Maximum resource utilization for batch size = 75}\label{fig:mp3}
\end{figure}

The experiments performed were performed on a very preliminary level and were just to get a rough estimate of the batch size we can implement for optimal batch processing. Further testing is required to fix an appropriate batch size.
\\

\textbf{Batch Processing using Kafka} 
\label{kafka}
\\

The depicted architecture (Figure \ref{fig: Bulk processing}) illustrates our Kafka cluster configuration, integral to our data processing workflow. This cluster encompasses multiple Kafka Brokers, Topics, and a ZooKeeper instance for systematic coordination and configuration management. We will be using the Kafka service provided by Team 5. In this setup, the Kafka Producer operates within a Docker container, tasked with generating the IDs of the ETDs for subsequent processing.

ZooKeeper plays a pivotal role in maintaining Kafka configurations. It aids in the establishment of a Kafka Topic, where the ETD IDs are enqueued for processing. The initial processing stage involves the PDF to image conversion, for which a dedicated container continuously queries the Kafka Broker, specifically targeting the 'PDF to Image Conversion' topic. Utilizing the ETD IDs queued, the container retrieves the relevant ETDs through API calls.

Each ETD, typically in PDF format, undergoes a conversion process where every page is transformed into an image format. This process meticulously accounts for the total number of pages, alongside capturing the unique ETD ID and individual page IDs. Subsequent to this conversion, these details are forwarded to the next queue in our pipeline, namely the 'Object Detection' queue.

In the object detection phase, the container, functioning as a Kafka Consumer, polls the Kafka Broker for messages on the 'Object Detection' topic. Post processing, this container transitions into a Producer role for the subsequent stage, pushing the data onto the next Kafka Topic, which is 'XML Generation'.

During the 'XML Generation' stage, the container resumes its role as a Consumer, polling the Kafka Broker for messages on the 'XML Generation' topic. The primary output from this phase is the generation of XML files, concluding the data transformation process in our Kafka-based architecture. The generated XML can be polled by consumer tasks of other teams that process the generated XML.

 \begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth, height = 10cm]{kafka.drawio.png}
    \caption{Current architecture for bulk processing}
    \label{fig: Bulk processing}
\end{figure}



\section{Heuristic Chapter Segmentation}
We now move on to the segmentation of the TOC (Table of Contents). The objective is to identify the pages of the table of contents and parse them in a manner that allows us to determine where chapters begin and end. 

The first algorithm, illustrated in Algorithm~\ref{alg:tocDetermine}, goes a step further by using a powerful model, \texttt{LLaMA 2-13b-chat-hf}, to determine if a particular page (or a segment of it) belongs to the TOC. The essence of this algorithm lies in the prompt generation, which couples the extracted text with a specific question. The model is then tasked to determine if the provided text is part of the TOC. By continuously assessing pages until a negative response is received, we can efficiently bound the range of pages that make up the TOC.

The second algorithm, as detailed in Algorithm~\ref{alg:tocParse}, is designed to parse a document's TOC to determine chapter titles and their corresponding page numbers. By using the \texttt{pymupdf} library, we extract text blocks from specified pages, adjust their y-coordinates, and group lines by their y-values. Heuristics are then applied based on the indentation and presence of page numbers to build a hierarchical structure. This structure captures the chapters and their associated sub-chapters or sections.

The third approach as detailed in Algorithm \ref{alg:tocParse2} is to detect only numbers on the page, and then create clusters of bounding boxes for these numbers to form a column. The rightmost column is assumed to be the page numbers of the chapters or subchapters. However, this method does not differentiate between chapters and subchapters.


Together, these algorithms provide a robust and efficient method for segmenting chapters and identifying the TOC, essential to the segmentation of ETDs.


\begin{algorithm}
\caption{Determining Table of Contents Pages from PDF}
\label{alg:tocDetermine}
\begin{algorithmic}[1]
\Procedure{FindTOCPages}{pdf\_file}
    \State \textbf{open} \textit{doc} from pdf\_file
    \State \textbf{initialize} toc\_pages as empty list
    \State \textbf{initialize} model with \textit{LLaMA 2-13b-chat-hf}
    \State continuous\_yes\_count $\gets$ 0
    \For{each page in doc}
        \State page\_text $\gets$ Extract first 4000 tokens from page
        \State prompt $\gets$ CreatePrompt(page\_text)
        \State answer $\gets$ model.\Call{Predict}{prompt}
        \If{answer is ``yes''}
            \State continuous\_yes\_count $\gets$ continuous\_yes\_count + 1
            \State Add page to toc\_pages
        \ElsIf{answer is ``no'' and continuous\_yes\_count > 0}
            \State \textbf{break} \Comment{Stop when we get a ``no'' after continuous ``yes'' answers}
        \EndIf
    \EndFor
    \State \Return toc\_pages
\EndProcedure

\Function{CreatePrompt}{page\_text}
    \State question $\gets$ ``Can you tell me if the above text is a part of the table of contents or not? Answer only with yes or no.''
    \State prompt $\gets$ ``[INST] '' + page\_text + " " + question + " [INST]"
    \State \Return prompt
\EndFunction
\end{algorithmic}
\end{algorithm}


\begin{algorithm}
\caption{Parse Table of Contents in PDF}
\label{alg:tocParse}
\begin{algorithmic}[1]
\Procedure{ParseTOC}{pdf\_file}
    \State \textbf{initialize} \textit{doc} from pdf\_file
    \State \textbf{initialize} blocks as empty list
    \For{each page in doc}
        \For{each text block in page}
            \State Adjust y-coordinates of block based on page number
            \State Add block to blocks
        \EndFor
    \EndFor
    \State grouped\_lines $\gets$ \Call{ExtractGroupedLines}{blocks}
    \State hierarchy $\gets$ \Call{ProcessLinesImproved}{grouped\_lines}
    \State cleaned\_hierarchy $\gets$ \Call{CleanHierarchy}{hierarchy}
    \State \textbf{write} cleaned\_hierarchy to output.json
\EndProcedure

\Function{ExtractGroupedLines}{blocks}
    \State \textbf{initialize} grouped\_lines as empty list
    \For{each block in blocks}
        \State Combine text spans in a line by y-coordinate
        \State Add to grouped\_lines
    \EndFor
    \State \Return grouped\_lines
\EndFunction

\Function{ProcessLinesImproved}{grouped\_lines}
    \State \textbf{initialize} hierarchy, stack as empty lists
    \State \textbf{initialize} prev\_indent as None
    \For{each line in grouped\_lines}
        \If{line contains page number}
            \State Determine indent based on x-coordinate and leading spaces
            \State Organize into hierarchy based on indent compared to prev\_indent
            \State Update prev\_indent
        \EndIf
    \EndFor
    \State \Return hierarchy
\EndFunction

\Function{CleanHierarchy}{hierarchy}
    \State \textbf{initialize} cleaned as empty list
    \For{each item in hierarchy}
        \If{item has page number}
            \State Add to cleaned
        \EndIf
    \EndFor
    \State \Return cleaned
\EndFunction
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
\caption{Numeric Detection and Clustering for TOC Segmentation}
\label{alg:tocParse2}
\begin{algorithmic}[1]

\Procedure{ExtractAllSpansWithBBox}{$pdf\_path$, $page\_number$}
    \State Open PDF at $pdf\_path$ and load $page\_number$
    \State Initialize $all\_spans\_bboxes$ as empty list
    \For{each span in each line of each block on the page}
        \State Append (span text, bbox) to $all\_spans\_bboxes$
    \EndFor
    \State Close the PDF and return $all\_spans\_bboxes$
\EndProcedure

\Procedure{ClusterAndAverage}{$y\_coords$, $threshold$}
    \State Sort and cluster $y\_coords$ based on $threshold$
    \State Calculate and return average y-coordinate for each cluster
\EndProcedure

\Procedure{ClusterSpansByUniqueY}{$spans\_bboxes$}
    \State Extract y-coords from $spans\_bboxes$ and find unique values
    \State Cluster y-coords using KMeans if more than one cluster
    \State Group spans by their cluster labels and return
\EndProcedure

\Procedure{ExtractLinesFromClusters}{$clustered\_spans$}
    \State Concatenate texts of spans in each cluster to form lines
    \State Return the list of formed lines
\EndProcedure

\end{algorithmic}
\end{algorithm}





\newpage

Following the application of our heuristic-based algorithms, we wanted to show some of the initial results. In Figures \ref{fig:output1}, \ref{fig:output2}, and \ref{fig:toc_output}, the results are displayed from the application of the second heuristic algorithm. This process involved initially identifying the pages of the table of contents manually and subsequently submitting them for algorithmic processing.


\begin{figure}[H]
  \centering
  \begin{minipage}{.5\textwidth}
    \centering
    \includegraphics[width=.9\linewidth]{input1.png}
    \caption{Input page 1 for Algorithm 2}
    \label{fig:output1}
  \end{minipage}%
  \begin{minipage}{.5\textwidth}
    \centering
    \includegraphics[width=.9\linewidth]{input2.png}
    \caption{Input page 2 for Algorithm 2}
    \label{fig:output2}
  \end{minipage}
\end{figure}

\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{output.png}
  \caption{Segmented Table of Contents Output}
  \label{fig:toc_output}
\end{figure}


\section{Automated Identification of TOC Pages Using ResNet Model}
Upon a detailed review of Algorithm \ref{alg:tocDetermine}, it became evident that an extensive dataset is required to effectively fine-tune the LLaMA 2 model for the accurate identification of TOC material. Additionally, the procedure proved to be time-consuming, primarily due to the substantial hardware demands associated with both inference and the fine-tuning process. Consequently, it was imperative to devise an alternative strategy that could seamlessly identify and extract TOC pages for processing, addressing these constraints.
\subsection{Description}
Instead of relying on manual identification of TOC pages or the previously mentioned approach, a new method involving a ResNet model has been developed. This model was trained on approximately 24,000 data points, consisting of 12,000 TOC page images and 12,000 non-TOC images from the ETD dataset available on ARC (Advanced Reasearch Computing) center at Virginia Tech. For TOC images, the process involved iterating through ETDs and identifying ``table of contents'' or ``contents'' pages. Conversely, non-TOC images were obtained by taking random samples from the documents.

The algorithm is designed to scan the first 50 pages of any document to identify TOC pages. The model runs for 10 iterations, with pages identified as TOC in more than 50\% of the runs being selected for further processing. These pages are then subjected to the heuristic algorithm for detailed segmentation.

\subsection{Pseudocode for ResNet Model Application}
\begin{algorithm}[H]
\caption{Automated Identification of TOC Pages Using ResNet}
\label{alg:resnetTOC}
\begin{algorithmic}[1]
\Procedure{IdentifyTOCPages}{pdf\_file}
    \State \textbf{initialize} resnet\_model trained on ETD dataset
    \State \textbf{initialize} toc\_page\_candidates as empty dictionary
    \For{run from 1 to 10}
        \For{each page in first 50 pages of pdf\_file}
            \State page\_image $\gets$ Convert page to image
            \State prediction $\gets$ resnet\_model.\Call{Predict}{page\_image}
            \If{prediction is TOC}
                \State Increment count of page in toc\_page\_candidates
            \EndIf
        \EndFor
    \EndFor
    \State final\_toc\_pages $\gets$ Filter pages in toc\_page\_candidates with count > 5
    \State \Return final\_toc\_pages
\EndProcedure
\end{algorithmic}
\end{algorithm}

\subsection{Evaluation and Results}
The ResNet-based algorithm for identifying TOC pages was subjected to an evaluation on a segmented sample of ETDs from the dataset, comprising 56 documents. Remarkably, the algorithm achieved a 100\% success rate in correctly identifying the TOC pages within these documents. This high level of accuracy underscores the effectiveness of the algorithm in handling real-world data and scenarios.

\subsubsection{Results Visualization}
The following figures, Figure \ref{fig:result1} and Figure \ref{fig:result2}, illustrate some of the successful outcomes of the algorithm. They represent a subset of the ETDs where the TOC pages were accurately identified, demonstrating the practical utility and robustness of the proposed method.

% Placeholder for Image 1
\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{resnetresults1.jpg}
  \caption{Examples of correctly identified TOC pages}
  \label{fig:result1}
\end{figure}

% Placeholder for Image 2
\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{resnetresults2.jpg}
  \caption{Examples of correctly identified TOC pages}
  \label{fig:result2}
\end{figure}

\subsection{Application on the ETD 500 Dataset}
In a further phase of evaluation, the identification model was applied to the ETD 500 dataset, encompassing 500 manually annotated ETDs. This dataset is notable for its comprehensive range of labels that categorize different sections commonly found in academic theses and dissertations. The labels include Chapters, Appendices, Reference Lists, Table of Contents, Title Pages, Abstracts, Lists of Figures, Acknowledgments, Lists of Tables, Curriculum Vitae, Dedications, Chapter Abstracts, and other miscellaneous sections. The application of our model to this diverse and annotated dataset was instrumental in assessing its capability to accurately identify and categorize various document elements, particularly focusing on the effective recognition of TOC pages.

\subsubsection{Results of Binary Classification Approach}
In an effort to streamline the classification process, we adapted our approach to convert it into a binary classification problem. This was achieved by amalgamating the labels ``Table of Contents'', ``List of Figures'', and ``List of Tables'' into a singular label termed ``TOC''. Conversely, all other labels were consolidated under the ``non-TOC'' category. This binary categorization significantly simplified the identification process, focusing primarily on distinguishing between TOC-related content and other document elements.

Our results post this reclassification revealed notable insights into the model's efficiency and accuracy in discerning TOC elements from non-TOC content as seen in Figure \ref{fig:result3}. The binary classification approach not only reduced the complexity of the task but also enhanced the model's ability to generalize across different document structures, making it more robust in handling a variety of ETDs. The subsequent sections detail the specific outcomes and metrics achieved with this revised methodology.


\begin{figure}[H]
  \centering
  \includegraphics[width=0.9\linewidth]{resnetmetrics.jpg}
  \caption{Metrics for our ResNet model on ETD 500 dataset}
  \label{fig:result3}
\end{figure}

\subsection{Combining ResNet Approach with Algorithm 2 for Chapter Extraction}
To further enhance the efficiency of our document processing system, we integrated the ResNet-based TOC identification approach with Algorithm \ref{alg:tocParse}, which focuses on extracting the number of chapters and their corresponding starting pages. This combination aims to create a more streamlined and automated process for segmenting ETDs.
\subsubsection{Success and Failure Scenarios}
The integration of these methods has led to a variety of outcomes, demonstrating both successful applications and areas needing improvement.
\paragraph{Success Scenarios}
In many instances, this combined approach successfully identified and extracted chapter information, as illustrated in Figure \ref{fig:successScenario}.

% Placeholder for Success Scenario Image
\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{fullalgosuccess1.jpg}
  \caption{Example of a successful chapter extraction}
  \label{fig:successScenario}
\end{figure}

\paragraph{Failure Scenarios}
However, there were scenarios where the heuristic algorithm failed, particularly when chapter titles were split across two lines as seen in Figure \ref{fig:failureScenario2}, disrupting the extraction process. An example of such a failure is depicted in Figure \ref{fig:failureScenario1}.

% Placeholder for Failure Scenario Image
\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{fullalgofail1.jpg}
  \caption{Example of a failure in chapter extraction due to multi-line titles}
  \label{fig:failureScenario1}
\end{figure}
\begin{figure}[H]
  \centering
  \includegraphics[width=0.9\linewidth]{fullalgofail1reason.jpg}
  \caption{Multi-line titles resulting in the failure of the proposed algorithm}
  \label{fig:failureScenario2}
\end{figure}
\subsubsection{Proposed Solutions and Ongoing Work}
In response to the challenges encountered, we are currently exploring two potential solutions:

\begin{enumerate}
    \item \textbf{Heuristic and Bounding Box Approach:} This solution involves the use of heuristics and bounding box analysis to identify titles that span multiple lines. By enhancing the algorithm's ability to recognize and concatenate broken lines, we aim to improve the accuracy of chapter extraction.

    \item \textbf{BERT Model Integration:} We are also experimenting with the incorporation of a BERT model to determine if consecutive lines are complementary parts of a single title. This approach leverages the model's natural language processing capabilities to discern contextual relationships between lines.
\end{enumerate}

Both solutions are currently under testing and development, with the goal of addressing the identified shortcomings and optimizing the overall performance of the document segmentation process.
As part of our future work, we plan to fuse the second and third approaches into a single heuristic method. This hybrid approach aims to enhance accuracy in cases where reliable JSON objects are produced. Additionally, we intend to use the refined data from this hybrid approach to fine-tune a \texttt{LLaMA 2} model, which will be designed to generate the desired outputs autonomously.


\section{Topic Modeling}

\subsubsection{Milestones}

In Table \ref{TM_Tasks} we list out the tasks completed by the Topic Modeling team. 

      \begin{table}[ht]
            \centering
            \caption{Topic Modeling Milestones} \label{TM_Tasks}
            % \begin{tabular}{p{3cm} p{8cm}}
            \begin{tabular}{cl}
            \toprule
            \textbf{Timeline}  & \textbf{Tasks} \\
            \midrule
            % \textcolor{red}{Change} & \textcolor{red}{dates and correct ordering}\\
            1 Sept & Get familiar with Fall 2022 system \\
            8 Sept & Get familiar with the BERT model, OCTIS, LDA and other topic modeling approaches  \\
            13 Sept & Set up the code in Endeavour cluster \\
            19 Sept & Train BERT model on 40k smaller ETD dataset for initial analysis \\
            23 Sept & Analyze BERTopic results  \\
            23 Sept &  Finalizing topic modeling wire frame\\
            9 Oct & Train BERTopic on complete 500k ETD dataset\\
            13 Oct & Analyze results of the BERT model on large dataset\\
            14 Oct & Experimentation on BERT model with parameter tuning \\
            19 Oct & Code refactoring and optimization for reducing training time\\
            26 Oct & Writing new data cleaning and pre-processing script for BERTopic \\ 
            7 Nov & Unsatisfactory results of BERTopic led to exploring the LDA model \\
            14 Nov & Implement LDA on the 334k cleaned dataset \\ 
            26 Nov & Improve pre-processing and experimentation of LDA model\\ 
            2 Dec & Finalize the list of topics to be given to team 6\\ 
            Final & Documentation of work done in final report\\
            \bottomrule
            \end{tabular}
            \end{table}

\textbf{APIs} 
\\
The main purpose of the APIs is to access the centralized resources and store intermediate results in the database as well as the file system. This enables speedy retrieval of details, be it for rendering in the front-end or for further processing tasks in the pipeline of the ETD search system. Having a centralized data store would also ensure security and authorized access to the data store.

As of IR3, the APIs required for interacting with the database and file system have been defined and provided to Team 5 for implementation, and consequently integration into the current topic modeling code. The APIs would be used to populate the database tables namely the Topic\_Models, Collection\_topics, and the ETD Topics table as seen in Table \ref{tab:api_tm}.

The exact request parameters or body of the API are subject to modification based on Team 5’s finalized API implementation.

\begin{center}
\begin{longtable} {|p{1.5cm}|p{3cm}|p{7cm}|p{4cm}|}
% \centering
% \begin{tabular}{|p{1.5cm}|p{3cm}|p{7cm}|p{4cm}|}

\hline
\textbf{Request Method} & \textbf{Description} & \textbf{Request Parameters} & \textbf{Database/File System} \\ 
\hline

POST & Add to Topic\_models to add one row for a topic modeler &
\begin{tabular}[c]{@{}l@{}}data = \{\\    name = Model name,\\    description= Model description,\\   type= Model type,\\   parameters= num\_topics=\\<number of topics>,\\  passes= number of passes,\\  alpha= alpha value (eg: auto),\\   path= path to model\\ \}\end{tabular} &
  Database: Topic\_Models table \\
 \hline 
  
  POST &
  Add to Collection\_topics to store the set of topics from that modeler &
  \begin{tabular}[c]{@{}l@{}}data =\{\\  collections\_id= Collection ID,\\  term\_list = List of terms,\\  probability\_list= List of probabilities\\ \}\end{tabular} &
  Database: Collection\_topics \\
\hline
  POST &
  Topic Vector for an ETD &
  \begin{tabular}[c]{@{}l@{}}data=\{\\ etdId = ETD ID,\\ topics= List of dictionaries of topic \\values as string and score values\\ as float\\ \}\end{tabular} &
  Database: ETD Topics \\
\hline
  POST &
  Store Topic Modeling Results for ETDs &
  \begin{tabular}[c]{@{}l@{}}data = \{\\  etds\_id= ETD ID,\\  collections\_id= Collection ID,\\  topic\_models\_id= ID for Topic Model,\\  Topic\_list= List of Topics (strings) ,\\  probability\_list= List of probabilities\\ \}\end{tabular} &
  Database: ETD Topics \\
\hline
  
% \end{tabular}
\caption{POST APIs for Topic Modeling}
\label{tab:api_tm}
\end{longtable}
\end{center}

\subsubsection{Current Architecture}

Figure \ref{fig: Topic Modeling} depicts our approach for topic analysis of ETDs using two topic modeling methods: BERTopic and LDA. Initially, the ETD data undergoes cleaning and pre-processing to prepare for analysis. Subsequently, the refined data is processed through both BERTopic, which utilizes BERT for contextual embeddings, and LDA, a probabilistic model emphasizing word frequencies. These methods yield a comprehensive list of topics that represent the thematic essence of the documents. The derived topics are then leveraged to categorize the documents, assist in user registration by aligning user interests with the topics, and serve other downstream tasks that might benefit from a deep thematic understanding of the ETDs.

 \begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth, height = 20cm]{Mod.jpg}
    \caption{Current Architecture: Topic Modeling}
    \label{fig: Topic Modeling}
\end{figure}


Implementing BERTopic and LDA on a large dataset of ETDs is a significant task. In our case, we have applied LDA to 333,867 ETDs, which is a considerable volume of scholarly work. Finally, after choosing the LDA model as the default approach, assigning each ETD to one of the 73 topics formed has created a structured way to navigate and analyze this extensive collection. This approach allows for the identification of major themes and areas of research within the corpus. The decision to limit each topic to a list of 10 key words is a strategic choice that balances detail with conciseness. These keywords serve as a concise summary of each topic, making it easier to understand the primary focus of each thematic category at a glance. They can be instrumental in helping researchers and students to quickly find relevant ETDs in their area of interest.

Furthermore, this structure could facilitate comparative studies across different academic fields. By analyzing the distribution and frequency of topics across various departments or time periods, we could uncover trends in academic research, shifts in focus areas, or emerging disciplines within the scholarly community. This kind of analysis might reveal valuable insights into the evolution of research topics over time and highlight the interconnectivity of different fields of study.


\subsubsection{Experiments, Results and Learnings}

In our analysis of topic modeling on the cleaned ETD dataset, it is evident that LDA has shown more promising results compared to BERTopic. As illustrated in Figure \ref{fig:LDADist}, the distribution of topics generated by LDA demonstrates a more balanced and comprehensive categorization of ETDs than that of Figure \ref{fig:BERTDist}, which represents the BERTopic model's output.

A notable observation is the dominance of the ``-1'' topic in the BERTopic model, encompassing over 169,000 ETDs. This skewness is likely attributable to the model's configuration, wherein the number of topics is capped at 100, a threshold that appears insufficient for the dataset's complexity and size. BERTopic's topic reduction technique, designed to minimize redundancy, might be overly aggressive for extensive datasets like ours, leading to an over-consolidation of diverse documents into a single, dominant topic.

Conversely, LDA's approach seems to align better with the intricacies of the ETD dataset. LDA, a generative probabilistic model, assumes each document is a mixture of a small number of topics and that each word's presence is attributable to one of the document's topics. This methodology is inherently suited for datasets with a wide range of themes and subtopics, as it allows for a more nuanced and granular topic distribution. The result is a more evenly spread representation of topics, reflecting the diverse nature of academic research covered in ETDs.

The superior performance of LDA in this context also underscores the importance of model selection based on dataset characteristics. It highlights that while advanced models like BERTopic, with their neural network foundations, offer powerful tools for natural language processing, traditional models like LDA can sometimes provide better results, especially when dealing with specialized or highly varied text corpora.

Moreover, this comparison sheds light on the need for careful tuning of model parameters. In the case of BERTopic, revisiting the topic limit and exploring ways to refine its topic reduction algorithms could potentially improve its performance for large datasets like ETDs.

\begin{table}[H]
\centering
\resizebox{\textwidth}{!}{%
\begin{tabular}{llllllll}
Experiment ID & Model & Dataset size (ETDs) & Stopword removal & Stemming & Lemmatization & Dataset cleaning & Number of Topics \\
1 & BERTopic & 40k ETDs  & No  & No  & No  & Yes & 100  \\
2 & BERTopic & 524k ETDs & No  & No  & No  & No  & 100  \\
3 & BERTopic & 334k ETDs & No  & No  & No  & Yes & 100  \\
4 & BERTopic & 334k ETDs & No  & No  & No  & Yes & 500  \\
5 & BERTopic & 334k ETDs & No  & No  & No  & Yes & 1000 \\
6 & LDA      & 334k ETDs & Yes & No  & No  & Yes & 100  \\
7 & LDA      & 334k ETDs & Yes & Yes & Yes & Yes & 100  \\
8 & LDA      & 334k ETDs & Yes (Custom Stopwords) & No  & Yes & Yes & 100  \\
9 & LDA      & 334k ETDs & Yes (Custom Stopwords added) & No  & Yes & Yes & 100 \\
10 & LDA      & 334k ETDs & Yes (Custom Stopwords added 2) & No  & Yes & Yes & 100 
\end{tabular}%
}
\caption{Summary of Topic Modeling experiments}
\label{tab:tmExperiments}
\end{table}

Table \ref{tab:tmExperiments} summarizes the outcomes of nine experiments comparing the performance of two topic modeling algorithms— BERTopic and LDA— across various datasets and preprocessing techniques. The table lists experiments with BERTopic on datasets size ranging from 40k to 334k ETDs, with a number of topics set at 100, 500, and 1000, and varying levels of dataset cleaning, but without employing stopword removal, stemming, or lemmatization. In contrast, the LDA model is applied to a 334k ETDs dataset with stopword removal, and variations include stemming, lemmatization, and custom stopwords.

From our observation, the LDA model provided better results particularly in experiments 8, 9 and 10 where custom stopwords were utilized, and comprehensive data cleaning, including lemmatization, was performed. This suggests that LDA, a more traditional statistical approach for topic modeling, may benefit more from classical text preprocessing methods than BERTopic, which relies on contextual embeddings from transformer models that may not require such preprocessing to capture the semantic nuances of text.

In conclusion, our findings suggest that while both LDA and BERTopic are capable tools for topic modeling, their effectiveness is significantly influenced by the nature of the dataset and the configuration of the model parameters.

We will continue to explore additional approaches and improvements to enhance the results of topic modeling.

\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{LDADistributionDec3.png}
  \caption{Distribution of documents per topic using LDA}
  \label{fig:LDADist}
\end{figure}

\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{Bert_distribution.png}
  \caption{Distribution of documents per topic using BERTopic}
  \label{fig:BERTDist}
\end{figure}


\chapter{Future Work}
\label{future_work}
While Team 3 has put in a considerable amount of effort in improving the object detection and topic modeling pipelines for the information retrieval system to facilitate ETDs, there is still scope for exploration and experimentation in improving the current approach.
In this section we highlight the different plans of
action post-class which could be implemented in the future.

\section{Object Detection}
\begin{enumerate}

            \item Improve the object detection model by training on a larger set of training data, and optimizing hyperparameters to improve the accuracy of the model. Additionally, improve the granularity of categories of objects extracted from the ETDs. For example, explicitly extract and store the keywords detected from the abstract of the ETD and store it along with the ETD metadata. The lower accuracy of the object detection model can be attributed to the limited number of training samples (elements such as degree, date, and algorithm
have very few instances in our dataset). 
Furthermore,  object detection models tend to
struggle with localization of smaller objects such as page numbers and equation numbers. 
            \item Experiment with newer models such as the YOLOv8 model and chart the performance comparison between YOLOv7 and YOLOv8 for ETD data. 
            \item Explore a top-down object detection approach and its benefits in generating the XML for the ETD. One of the biggest challenges is that detection is not always in
the top-bottom order for pages, and this necessitates the creation of an XML tree since a
hierarchy of the ETD would be required for other tasks, such as segmenting an ETD into
its constituent chapters, and extracting image-based objects along with their corresponding
captions. Exploring top-bottom object detection can help in easier correlation between captions and figures and tables, as well as identifying continuing paragraphs and tables that span multiple pages in the ETD.
            \item Improve the performance of the object detection model with a focus on scanned PDF files. Additionally, explore data augmentation (along with applying some sort of image transformation) to the training dataset to reduce manual annotation efforts and improve model performance (especially for scanned PDFs). More can be read on this in ``Analyzing and Navigating Electronic Theses and Dissertations'' \cite{AAhuja}.
            \item Incorporate the chapter segmentation code with the object detection pipeline to ensure accurately detected chapters in the XML file generated for the ETD. An XML with well-segmented chapters will then be passed on to Team 4 for classification and summarization tasks.
            \item Set up an automated workflow to use Kafka as explained in the object detection Section \ref{kafka} so that it can speedily handle batches. The code files for each task have been segmented but the API calls have not been defined for each task. After integrating the same, work needs to be done to set up the code to poll and publish topics from the Kafka broker and perform each task parallelly.
            \item Populate both the database and the file system with the extracted data and files generated by passing the ETD through the object detection pipeline using the integrated APIs. The test\_api.py file (as seen in the Developer's Manual) holds the codes with the API definitions for object detection. The code may need to be modified slightly to fit to the final API implementation defined by Team 5. Following that, the methods can integrated into the existing code and the API calls can be initiated in the code at given placeholders (the placement of each API call in the code is given by the comments in the code files).
            \item Complete processing 500,000 ETDs through the object detection pipeline and populate the database with the intermediary results of the same.
            \item Clearly outline and implement the workflow for the tasks performed by each persona (the user, the experimenter, and the curator). Ensure the smooth working and incorporation of the object detection pipeline with the front-end for each of these personas. The experiments that can be performed by the experimenter and the processing of one or more ETDs by the curator (task wise) is yet to be precisely defined.
            
            
\end{enumerate}

\section{Topic Modeling}
\begin{enumerate}
    \item For topic modeling, analyzing and improving the usage of BERTopic on the cleaned ETD dataset is needed; this is an important future goal. Integrating BERTopic for topic modeling on ETD data promises a better semantic understanding and contextual relevance, which is crucial for academic texts. It has an ability to handle large datasets efficiently and adapt to evolving research trends makes it an ideal tool for in-depth, meaningful analysis.
    \item Analyze and improve the keywords formed by the LDA model by updating custom stopwords while also trying different vectorizers other than tf-idf. Further, exploring different lemmatization methods.
    \item Explore other topic modeling approaches such as ProdLDA and NeurLDA on the large ETD dataset.
    \item Clearly define and implement the workflow for the tasks performed by each persona (the user, the experimenter, and the curator).
\end{enumerate}


\chapter{User Manual} \label{user manual}

\section{Object Detection}

The experimenter and the curator are the personas focused on in the user manual for object detection.

The experimenter can navigate to the front-end page to upload a PDF file for an ETD of their choice and run object detection on it. Subsequently, the experimenter will be redirected to the extracted ETD view page (HTML page). This page displays all objects extracted from the ETD in a top-to-bottom order, including metadata, chapter text, figures, captions, tables, and equations. The web layout and navigation for the same can be viewed in Team 6's report.

The user layout for the curator is yet to be defined and is a future scope of this project. The current proposal aims to enable the curator to process more than one ETD file simultaneously. Here, we will utilize the Kafka framework, as defined in Section \ref{kafka}, to batch process the uploaded ETDs. The curator will have the option to perform object detection and generate XML files as separate tasks in the pipeline. Additional ideas regarding the different tasks the curator can perform are yet to be discussed.



\section{Topic Modelling}

\subsection{Prerequisites}

These guidelines outline the process of configuring the environment to support the operation of both the backend and frontend. In order to optimize performance, it is recommended to create a new Conda virtual environment in order to prevent any potential package conflicts. It is imperative to underscore the necessity of a Python version 3.7 or later in order to successfully install all the necessary utilities.

We can set up and activate the conda environment as follows:
\begin{enumerate}
    \item Make sure you have Anaconda installed and ready.
    \item \texttt{conda create --name <env> --file requirements\_conda.txt}
    \item \texttt{conda activate <env>}
\end{enumerate}

By adhering to these steps, the user can make sure the environment is appropriately configured, thereby enabling the backend and frontend to function without any problem.

\subsection{Website}

The frontend user interface is composed of four main pages: the homepage, a page for searching documents, a page for exploring topics, and a page for viewing documents. The user is either an experimenter or a curator.

\begin{enumerate}
    \item \textbf{Home Page}

The landing page functions as the primary gateway to the website, presenting an all-encompassing synopsis of the functionalities and attributes of the system. As depicted in Figure \ref{fig:homepage}, users can then effortlessly navigate to the document searching page and topic browsing page, which are both intended to assist them in locating and exploring relevant data in the shortest possible time.

\begin{figure}[h!]
\centering
\includegraphics[width=1\linewidth]{Homepage.jpg}
\caption{The Home Page Interface}
\label{fig:homepage}
\end{figure}

    \item \textbf{ETD Search Page}


An important component of the website, the ETD searching page enables users to browse and explore an extensive collection of documents sourced from the database. As depicted in Figure \ref{fig:etdsearch}, users are able to refine their document searches using the following modifiable parameters and conduct keyword-based searches.


\begin{figure}[h!]
\centering
\includegraphics[width=1\linewidth]{search page.jpg}
\caption{The ETD Search Page Interface}
\label{fig:etdsearch}
\end{figure}

    \item \textbf{Sorting and Filtering}

    
By default, search results are sorted by estimated relevance. However, users can choose to sort results by publishing date and order the results in ascending or descending order as shown in Figure \ref{fig:sortingfiltering}. Additionally, users can filter their search results by various options generated from the config file, such as university, department, or language. This feature allows users to narrow down their search to more specific categories and find the exact documents they need.


\begin{figure}[h!]
\centering
\includegraphics[width=1\linewidth]{sort.jpg}
\caption{Sorting Options}
\label{fig:sortingfiltering}
\end{figure}



    \item \textbf{Topic Browsing Page}

As illustrated in Figure \ref{fig:topicbrowsing}, the topic browsing page enables users to navigate and study various subjects associated with the documents housed in the database. When a user selects a particular list of topics of interest, a page containing a collection of relevant documents will appear. Seventy topic categories are generated by default when the database contains ~334,000 documents. Furthermore, in order to present the most relevant topics, the topic catalogs will be dynamically updated in response to user queries for particular subjects. Users who wish to uncover and investigate novel subjects that pertain to their areas of interest or inquiry must utilize the topic browsing page. Related documents for each topics are displayed when clicked on a particular topic. Please refer to Figure \ref{fig:docs}.


\begin{figure}[h!]
\centering
\includegraphics[width=1\linewidth]{browsing.jpg}
\caption{The Topic Browsing Page Interface}
\label{fig:topicbrowsing}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[width=1\linewidth]{docs.jpg}
\caption{The Topic Browsing Page Interface}
\label{fig:docs}
\end{figure}

\newpage
\item \textbf{Document Page}

The document page exhibits the entirety of the selected document's content, in addition to important metadata. As illustrated in Figure \ref{fig:documentpage}, for a collection of text documents, the following information is required: author's name, publication date, university affiliation, title, abstract, advisor, URL, and language. Additionally, it provides users with suggestions for relevant subjects and documents, depending on the current document's content. This functionality facilitates effortless navigation to related subjects and documents.


\begin{figure}[h!]
\centering
\includegraphics[width=1\linewidth]{docpage.jpg}
\caption{The Topic Browsing Page Interface}
\label{fig:documentpage}
\end{figure}



\end{enumerate}

When integrating the topic modeling toolkit into the larger system, it is crucial to consider both its features, and how it will interact with the system's other components. This involves ensuring that the toolkit's capabilities are compatible with the overall system's requirements. Additionally, any UI (User Interface) provided by the toolkit should be adapted to align with the system's design language, offering a seamless and intuitive experience for both curators and end users. It's important that these UI changes do not compromise the toolkit's core functionalities, ensuring that it remains as effective and seamless as before the integration. 




\chapter{Developer Manual} \label{developer_manual}

\section{Object Detection}

\textbf{Conda Environment Set-Up}
We can set up the conda environment for the object detection pipeline by following the steps below.
\begin{enumerate}
\item Make sure you have Anaconda installed and ready.
\item Run \texttt{./install-structure.sh}
\item Run \texttt{./install-dependencies.sh}
\item create new environment with the \texttt{.yml} file
\item check if the latest version of Torch is installed
\item Run \texttt{pip install torchvision}
\item  Run \texttt{apt-get update \&\& apt-get-install ffmpeg libsm6 libxext6 -y}
\item Run \texttt{pip install pdf2image}
\item Run \texttt{conda install -c conda-forge poppler}
\end{enumerate}

\textbf{config.yml}
The config.yml in the config folder is used to configure the relative paths of folders in the Endeavour cluster as shown in Figure \ref{fig:config}.
If the generated intermediary outputs need to be stored in the local file system, the relative paths can be rewritten here.
\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{config.JPG}
  \caption{config.yml file}
  \label{fig:config}
\end{figure}

\textbf{Python Code Files}
All the code files are accessible through Team 3's container on the Endeavour cluster \cite{od-cluster}.
Figure \ref{fig:container} shows the directory containing all the code folders and some of the code files required for the object detection pipeline.

\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{Endeavour1.JPG}
  \caption{Team 3 container on the Endeavour cluster}
  \label{fig:container}
\end{figure}

The flask\_app folder as shown in Figure \ref{fig:flask_app}, contains 200etdsscript.py, the executable which takes in the path to the folder containing ETD files and runs the object detection pipeline by invoking the task method from the flask\_main.py. 
\\
The xml\_parser as shown in Figure \ref{fig:xml_parser}, contains the xml\_utils.py and the pdf\_utils.py which are auxiliary files crucial to the tasks of XML generation and convertion of ETD pages into images. The layout\_models folder contains the YOLOv7 model along with files containing the model training weights, and a file initializing the model. The test\_api.py holds definitions for each of the APIs for the object detection pipeline and need to be invoked in the flask\_main.py file as well as in xml\_util.py once implemented to populate the database with the details of the ETD.

\begin{figure}[!htb]
  \includegraphics[width=\linewidth]{Flask_app.JPG}
  \caption{flask\_app}\label{fig:flask_app}
\end{figure}

  \begin{figure}[!htb]\includegraphics[width=\linewidth]{xml_parser.JPG}
  \caption{xml\_parser}\label{fig:xml_parser}

\end{figure}

\section{TOC Segmentation}

In order to run the table of contents segmentation algorithm, navigate to the main project folder's directory and run the script of resnet.py. If the model is further trained for better image classification, then simply replace the resnet\_toc\_classifier2.pth with the new model weights, then rename the file path in the code in the part mentioned in Figure \ref{fig:modelweightscode}.
\begin{figure}[H]
  \centering
  \includegraphics[width=0.7\linewidth]{modelweights.png}
  \caption{Team 3 container on the Endeavour cluster with model path}
  \label{fig:modelweightscode}
\end{figure}

Run the script by using the Python command on the script ``resnet.py''. The script will run on the default directory for the ETDs that needs to be processed. However, if that path needs to be changed, it can be changed at the end of the script by replacing the main directory that contains the ETDs as shown in Figure \ref{fig:etddirectory}.
\begin{figure}[H]
  \centering
  \includegraphics[width=0.9\linewidth]{etdmaindirectory.png}
  \caption{Team 3 container on the Endeavour cluster with root directory}
  \label{fig:etddirectory}
\end{figure}


\section{Topic Modeling}

\subsection{Prerequisites}

These guidelines outline the process of configuring the environment to support the operation of both the back-end and front-end. In order to optimize performance, it is recommended to create a new Conda virtual environment in order to prevent any potential package conflicts. It is imperative to underscore the necessity of a Python version 3.7 or later in order to successfully install all the necessary utilities.

We can set up and activate the conda environment as follows:
\begin{enumerate}
    \item Make sure you have Anaconda installed and ready.
    \item \texttt{conda create --name <env> --file requirements\_conda.txt}
    \item \texttt{conda activate <env>}
\end{enumerate}

By adhering to these steps, the user can make sure the environment is appropriately configured, thereby enabling the back-end and front-end to function without any problem.

In this section, we will provide steps to implement the topic modeling scripts and also use the trained LDA model. Click on the links to access files in the Endeavour cluster.

Refer to Figure \ref{fig:container_topicmodeling} for the topic modeling folder structure. Figure \ref{fig: Topic Modeling} gives an idea of the topic modeling flow that is used in the scope of this project.

To reach the topic modeling repository, refer to code in the Endeavour cluster \cite{tm-cluster}.




\begin{enumerate}
    \item First, run the topicmodeling.yml file to set up the conda environment for topic modeling. You can also refer to the README.txt file to set up your environment.
    \item Make changes in the config.ini to either train a model or load a model by changing the arguments.
    \item The config.ini is also used to pass the path to the database.
    \item Following this, make changes in \texttt{/team3/fall2023/TopicModelling/\\DocumentBrowsingToolkit-main/back\_end/server.py} file to either run the BERTopic model or the LDA model.

    \item To access already saved models, please refer to 
  \texttt{/team3/fall2023/TopicModelling/\\DocumentBrowsingToolkit-main/back\_end/\\TopicModelingKit/src/models/bertopic\_model\_bk} 
  file for the BERT model or 
  \texttt{/team3/fall2023/\\TopicModelling/DocumentBrowsingToolkit-main/back\_end/\\TopicModelingKit/src/models/LDA\_model} 
  for the LDA model.
    \item To analyze and use the saved LDA and BERTopic models, go to 
  \texttt{/team3/fall2023/TopicModelling/\\DocumentBrowsingToolkit-main/back\_end/TopicModelingKit/TopicModeling scripts/\\Topic Modeling scripts}.
\item The \texttt{test\_LDA.py} file contains APIs to POST into the database tables with topic modeling results.


\end{enumerate}

\begin{figure}[H]
  \centering
  \includegraphics[width=1\linewidth]{TopicmodelingEndeavour.png}
  \caption{Team 3 container on the Endeavour cluster for Topic Modeling.}
  \label{fig:container_topicmodeling}
\end{figure}

The outputs are stored in CSV, JSON and TSV folders and have been passed along to other teams. The code for generating outputs can be found in the topic modeling scripts. 

In summary, this section outlines a guide for implementing topic modeling using LDA and BERTopic models within a specified project scope. It provides steps for setting up the environment, configuring models, and integrating them with a database, as well as accessing and utilizing saved models. The process culminates in generating outputs in various formats, facilitating collaboration and data sharing with other teams.









   % \chapter{Discussion} \label{ch:discussion}
   % \chapter{Conclusions} \label{ch:conclusions}
   % \chapter{Summary} \label{ch:summary}

 

	% This is the standard bibtex file. Do not include the .bib extension in <bib_file_name>.
	% Uncomment the following lines to include your bibliography: 
	%\bibliography{<bib_file_name>}
	%\bibliographystyle{plainnat}   

	% This formats the chapter name to appendix to properly define the headers:
	%\appendix

	% Add your appendices here. You must leave the appendices enclosed in the appendices environment in order for the table of contents to be correct.
% 	\begin{appendices}
% 		\chapter{First Appendix} \label{app:appendix_one}
% 			\section{Section one} \label{ase:app_one_sect_1}
% 				\lipsum[1-3]
% 			\section{Section two} \label{ase:app_one_sect_2}
% 				\lipsum[1-3]
% 		\chapter{Second Appendix} \label{app:appendix_two}
% 			\lipsum[2]
% 	\end{appendices}
\bibliography{citations}
\end{document}


%****************************************************************************
% Below are some general suggestions for writing your dissertation:
%
% 1. Label everything with a meaningful prefix so that you
%    can refer back to sections, tables, figures, equations, etc.
%    Usage \label{<prefix>:<label_name>} where some suggested
%    prefixes are:
%			ch: Chapter
%     		se: Section
%     		ss: Subsection
%     		sss: Sub-subsection
%			app: Appendix
%     		ase: Appendix section
%     		tab: Tables
%     		fig: Figures
%     		sfig: Sub-figures
%     		eq: Equations
%
% 2. The VTthesis class provides for natbib citations. You should upload
%	 one or more *.bib bibtex files. Suppose you have two bib files: some_refs.bib and 
%    other_refs.bib.  Then your bibliography line to include them
%    will be:
%      \bibliography{some_refs, other_refs}
%    where multiple files are separated by commas. In the body of 
%    your work, you can cite your references using natbib citations.
%    Examples:
%      Citation                     Output
%      -------------------------------------------------------
%      \cite{doe_title_2016}        [18]
%      \citet{doe_title_2016}       Doe et al. [18]
%      \citet*{doe_title_2016}      Doe, Jones, and Smith [18]
%
%    For a complete list of options, see
%      https://www.ctan.org/pkg/natbib?lang=en
%
% 3. Here is a sample table. Notice that the caption is centered at the top. Also
%    notice that we use booktabs formatting. You should not use vertical lines
%    in your tables.
% 
%				\begin{table}[htb]
%					\centering
%					\caption{Approximate computation times in hh:mm:ss for full order 						versus reduced order models.}
%					\begin{tabular}{ccc}
%						\toprule
%						& \multicolumn{2}{c}{Computation Time}\\
%						\cmidrule(r){2-3}
%						$\overline{U}_{in}$ m/s & Full Model & ROM \\
%						\midrule
%						0.90 & 2:00:00 & 2:08:00\\
%						0.88 & 2:00:00 & 0:00:03\\
%						0.92 & 2:00:00 & 0:00:03\\
%						\midrule
%						Total & 6:00:00 & 2:08:06\\
%						\bottomrule
%					\end{tabular}
%					\label{tab:time_rom}
%				\end{table}
% 
% 4. Below are some sample figures. Notice the caption is centered below the
%    figure.
%    a. Single centered figure:
%					\begin{figure}[htb]
%						\centering
%						\includegraphics[scale=0.5]{my_figure.eps}
%						\caption{Average outlet velocity magnitude given an average  
%				        input velocity magnitude of 0.88 m/s.} 
%						\label{fig:output_rom}
%					\end{figure}
%    b. Two by two grid of figures with subcaptions
%					\begin{figure}[htb]
%						\centering
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_1_1.eps}
%							\caption{Subcaption number one}
%							\label{sfig:first_subfig}
%						\end{subfigure}
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_1_2.png}
%							\caption{Subcaption number two}
%							\label{sfig:second_subfig}
%						\end{subfigure}
%
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_2_1.pdf}
%							\caption{Subcaption number three}
%							\label{sfig:third_subfig}
%						\end{subfigure}
%						\begin{subfigure}[h]{0.45\textwidth}
%							\centering
%							\includegraphics[scale=0.4]{figure_2_2.eps}
%							\caption{Subcaption number four}
%							\label{sfig:fourth_subfig}
%						\end{subfigure}
%						\caption{Here is my main caption describing the relationship between the 4 subimages}
%						\label{fig:main_figure}
%					\end{figure}
%
%----------------------------------------------------------------------------
%
% The following is a list of definitions and packages provided by VTthesis:
%
% A. The following packages are provided by the VTthesis class:
%      amsmath, amsthm, amssymb, enumerate, natbib, hyperref, graphicx, 
%      tikz (with shapes and arrows libraries), caption, subcaption,
%      listings, verbatim
%
% B. The following theorem environments are defined by VTthesis:
%      theorem, proposition, lemma, corollary, conjecture
% 
% C. The following definition environments are defined by VTthesis:
%      definition, example, remark, algorithm
%
%----------------------------------------------------------------------------
%
%  I hope this template file and the VTthesis class will keep you from having 
%  to worry about the formatting and allow you to focus on the actual writing.
%  Good luck, and happy writing.
%    Alan Lattimer, VT, 2016
%
%****************************************************************************





