 %       File: VTthesis_template.tex
%     Created: Thu Mar 24 11:00 AM 2016 EDT
%     Last Change: Thursday, December 19, 2019
%     Author: Alan M. Lattimer, VT
%	  With modifications by Carrie Cross, Robert Browder, and LianTze Lim. 
%
% This template is designed to operate with XeLaTeX.
%
% All elements in the Title, Abstract, and Keywords MUST be formatted as text and NOT as math.
%
%Further instructions for using this template are embedded in the document. Additionally, there are comments at the end of the file that give suggestions on writing your thesis.  
%
%In addition to the standard formatting options, the following options are defined for the VTthesis class: proposal, prelim, doublespace, draft. 

\documentclass[doublespace,nopageskip]{VTthesis} % nopageskip - Removes arbitrary blank pages.

% Using the following header instead will create a draft copy of your thesis
%\documentclass[doublespace,draft]{VTthesis}
\usepackage{natbib}
\usepackage{dirtytalk}
\usepackage{graphicx}
\usepackage{tabularx}
\setcounter{tocdepth}{4}
\setcounter{secnumdepth}{4}
\usepackage{float}
\usepackage{hyperref}
\usepackage [english]{babel}
\usepackage [autostyle, english = american]{csquotes}
\MakeOuterQuote{"}



% Title of your thesis
\title{Final Report}
% \title{CS 5604: Information Storage and Retrieval}
% You should include 3-5 keywords, separated by commas
% \keywords{Some Keywords, Subject matter, etc.}

% Your name, including middle initial(s)
\author{Kaushik Ganesan, Dharneeshkar Jayaprakash, Deepak Nanjundan, Abhilash Neog, Aditya Shah, Deval Srivastava }

% Change this to your program, e.g. Physics, Civil Engineering, etc.


% Change this to your degree, e.g. Master of Science, Master of Art, etc.
% \degree{Doctor of Philosophy} 

% This should be your defense date:
\submitdate{January 11, 2023} 

% Committee members. Only have five readers and one chair available.
% Only use the ones you need and don't include the ones you don't need.
% You can also declare a Co-advisor. If you do, the principal and co-advisors
% will be listed as co-advisors on the title page.  Per the VT ETD standards, 
% you should not include titles or educational qualifications such as PhD or Dr.

%\fourthreader{Fourth Committee Member}
%\fifthreader{Fifth Committee Member}

% The dedication and acknowledgement pages are optional. Comment them out to remove them.
% \dedication{This is where you put your dedications.}
% \acknowledge{This is where you put your acknowledgments.}

% The abstract is required.
\abstract{{\large{\textbf{Abstract}}}\\
Under the guidance of Dr. Edward A. Fox, the class of CS5604 Fall 2022 semester at Virginia Tech was assigned the task of building an Information Retrieval and Analysis System that can support a collection of at least 5000 Electronic Theses and Dissertations (ETDs). The system would act as a search engine, supporting a number of features, such as searching, providing recommendations, ranking search results, and browsing. In order to achieve this, the class was divided into five teams, each assigned separate tasks with the intent of collaborating through CI/CD. The roles can be described as follows: Content and Representation, End-user Recommendation and Search, Object Detection and Topic Models, Classification and Summarization with Language Models, and Integration and Coordination. The intent of this report is to outline the contribution of Team 4, which focuses on language models, classification, summarization, and segmentation. In this project, Team 4 was successful in reproducing Akbar Javaid Manzoor’s pipeline to segment ETDs into chapters, summarize the segmented chapters using extractive and abstractive summarizing techniques, and classify the chapters using deep learning and language models. Using the APIs developed by Team 1, Team 4 was also tasked with storing the outcomes of 5000 ETDs in the file system and database. Team 4 containerized the services and assisted Team 5 with workflow automation to help automate the services. The project's main lessons were effective team collaboration, efficient code maintenance, containerization of services, upkeep of a CI/CD workflow, and finally effective information storage and retrieval at scale. The report describes the goals, tasks, and achievements, along with our coordination with the other teams in completing the higher-level tasks concerning the entire project.
}
% \abstract{\textbf{Abstract}\\

%  Under the guidance of Dr. Edward A Fox, the class of CS5604 for the Fall 2022 semester at Virginia Tech has been assigned the task of building an Information Retrieval and Analysis system that can support a data collection of at least 30,000 Electronic Theses and Dissertations (ETDs). This system should act as a search engine, which supports a number of features, such as searching, providing recommendations, ranking search results, and browsing. In order to achieve this task, the class has been divided into five teams, each assigned separate tasks with the intent of collaborating using CI/CD in order to build the complete system. The persona of each team can be described as follows: Content and Representation Curators, End-user Recommendation and Search System, Object Detection and Topic Models, Classification and Summarization with Language Models, Integration and Coordination. The intent of this report is to outline the contribution of Team 4, which focuses on language models, classification, summarization & segmentation. Our primary responsibilities include experimenting with language models, as well as with knowledge distillation and prompt tuning for classification, as well as building an end-to-end system for segmentation and summarization. Our report hence describes our required goals and tasks, as well as our coordination with the other teams in order to complete the higher-level tasks concerning the entire project.   



%\abstract{\lipsum [1-4]}


\begin{document}
% The following lines set up the front matter of your thesis or dissertation and are required to ensure proper formatting per the VT ETD standards. 
  \frontmatter
  \maketitle
  \tableofcontents

% The list of figures and tables are now optional per the official ETD standards.  Unless you have a very good reason for removing them, you should leave these lists in the document. Comment them out to remove them.
	\listoffigures
	\listoftables
    \printnomenclature %Creates a list of abbreviations. Comment out to remove it. 

% sample text for abbreviations:
% NLP is a field of computer science, artificial intelligence, and linguistics concerned with the interactions between computers and human (natural) languages.
 
% \nomenclature{NLP}{Natural Language Processing}
 

% The following sets up the document for the main part of the thesis or dissertation. Do not comment out or remove this line.
\nomenclature[]{NLP}{Natural Language Processing }
\nomenclature[]{ETDs}{Electronic Theses and Dissertations}
\nomenclature[]{BERT}{Bidirectional Encoder Representations from Transformers}
\nomenclature[]{PDF}{Portable Document Format}
\nomenclature[]{HTML}{HyperText Markup Language}
\nomenclature[]{XML}{Extensible Markup Language}
\nomenclature[]{NLTK}{Natural Language Toolkit}
\nomenclature[]{API}{Application Programming Interface}
\nomenclature[]{CI/CD}{Continuous Integration and Continuous Deployment}
\nomenclature[]{UI}{User Interface}
\nomenclature[]{SVM}{Support Vector Machine}
	\mainmatter

	%now go ahead and start writing your thesis
	\chapter{Overview} \label{ch:overview}
   			
      
%Copy/paste the code below to add sections and subsections to each chapter. Add your own text to the chapter and (sub)section labels to create custom headings.          
    \section{Introduction} \label{se:introduction}
	{As part of this project, we were tasked with building a state-of-the-art Information Retrieval and Analysis system that functions effectively and reliably for over 5000 ETDs. In order to accomplish this, Team 4 undertook the role of working and experimenting with language models. This primarily involved segmentation, summarization, and classification of ETDs. One of our primary roles was also to collaborate with Team 2 (which is responsible for designing the end-user recommendation and search system), by building an end-to-end segmentation pipeline, with output generation in the format required for front-end representation. In order to accomplish this, we explored the existing datasets and examined ways we can segment the ETDs into chapters. Given an ETD PDF as an input, the ideal output would be a set of PDFs, corresponding to each chapter of the ETD.}

    {Along with this, we were also responsible for building a summarization system. As part of this work, we generated three different summaries using different models and chose the best one by qualitatively evaluating the summaries while also considering the time constraints to run the model. Our third role involved chapter classification. Based on the performance achieved, we have proceeded with the most suitable method for the classification of ETDs. Finally, we built a front-end application that allows users to run and experiment with the trained models, through a user interface.}
			
				
		\section{Discussions with Subject Matter Experts} \label{se:another_section}
		Before settling on our individual objectives and deadlines, we spoke with Bipasha and Sara, our Subject Matter Experts on the Classification, Segmentation, and Summarization of ETDs. In addition to the expectations of the end users who may use our system in the future, we also got an overview of a few general facts about the dataset.
		
		Around 500,000 ETDs from more than 42 universities have been gathered by the team working with Dr. Fox. The collection of ETDs spans a large number of domains and a considerable amount of time (from 1875 to 2022).
		
		The two main categories of ETDs that were gathered are: 
		\begin{itemize}
		    \item Born-Digital ETDs: These are ETDs that were uploaded digitally to university repositories. The majority of the new ETDs are digitally born.
		    \item Non-Born-Digital ETDs: ETDs that weren't originally submitted digitally but were later scanned and submitted. Some older ETDs weren't always digital. A few of the scanned ETDs have also had their text extracted using OCR (Optical Character Recognition). However, there are issues with the recovered text that OCR produced.

		\end{itemize}	
		Earlier work on chapter segmentation was recommended to us as a starting point, and we were encouraged to extend it to segment ETDs into chapters. Additionally, we were advised to use pre-trained versions of models such as Longformer \cite{longformer} to summarize the chapters of the ETDs and establish a baseline for future works. It was also suggested that we build a benchmark dataset to assess the summaries produced by the models. If the ETDs are associated with journal papers, summaries of the ETDs can be found in the related publications' abstracts. To acquire a background on the classification tasks to be completed, it was suggested that we refer to Palakh's thesis \cite{Palakh}.
  
\chapter{Review of Literature} \label{ch:lit_review}
    \section{Segmentation}
    \subsection{Segmenting Electronic Theses and Dissertations By Chapters}
    {In his work \cite{Javaid}, Javaid has discussed a method for segmenting a born-digital ETD PDF into chapters. A labeled dataset is required to train any machine learning model to automatically partition large publications into chapters. However, there are no datasets that include ETDs as well as information about the underlying document structures. Therefore, arXiv's repository \cite{arxiv} is used to create a dataset that contains 1459 ETDs' segmented chapters.  This is accomplished by altering the LaTeX files according to chapter boundaries. The dataset obtained is used to train a deep learning model, which may then be used to segment other born-digital ETDs.}
    \subsection{Chapter Captor: Text Segmentation in Novels}
    {This paper \cite{captor} explores a hybrid approach combining BERT inference and Regex rule matching for chapter segmentation. The dataset used for training the BERT model consists of 8400 English fiction books. Initially, when a text is given as input, the custom BERT model tokenizes the text, and using the sliding window approach it outputs the confidence score for each token. Max pooling is performed to get a single confidence score for each token. Then, de-tokenization is performed, and potential headers are extracted. The output is next given to the Regex rule matcher where they have a list of the common chapter heading keywords. This rule matcher tries to find the chapters that the BERT model misses. Finally, the chapter outputs are refined and each chapter is saved in a separate XML file.}
    
    \section{Summarization}
    Text summarization is the process of converting a large body of text into smaller yet concise text containing the key points and the overall meaning of the original document. Though text summarization started gaining attention as early as the 1950s, with the rapid growth of textual data and large documents like ETDs in recent times, it has become even more imperative to summarize large volumes of text for better navigation and retrieval of relevant information. Text summaries not only reduce reading time but also improve the effectiveness of indexing{} \cite{autotext}. 

    Generally, there are two approaches to text summarization -- Abstractive and Extractive Summarization. Abstractive Summarization involves generating completely new text data to capture the meaning of the source{} \cite{introsummary}. Extractive Summarization, on the other hand, aims to select the most relevant (to the meaning of the document) phrases and sentences within the source to create the new summary{} \cite{introsummary}. Although Abstractive Summarization generates more human-like and potentially optimal solutions, it is quite challenging. While most of the works in the past have mainly focused on Extractive Summarization, the emergence of deep learning has greatly advanced the progress of Abstractive Summarization. 
    Nallapati et al. {}\cite{seq2seqsummary} in 2016 introduced a sequence-to-sequence model with attention to solve the problem of Abstractive Summarization. They proposed a model that can freely read and generate text and achieved state-of-the-art performance in two different English corpora. Although this led to a rise in research involving sequence-to-sequence models, those generally suffered from two limitations -- inaccurate and redundant information generation{} \cite{pointersummary}. To address these limitations, See et al.{} \cite{pointersummary} proposed a modified sequence-to-sequence model with a hybrid pointer-generator network and coverage mechanism. The pointer generator network consists of a pointer that can copy text from the source while retaining the capability to generate new text using the generator. The coverage technique helps keep track of the text already summarized, thus reducing redundancy. 

    Our problem area involves text summarization of Electronic Theses and Dissertations (ETDs). Previously, Ingram et al.{} \cite{etdsummary} applied deep learning techniques to summarize ETDs. Their work makes use of transfer learning (with standard models for Abstractive Summarization) to address the issue of lack of ETD training data. In the current project, we explored this idea, however, owing to certain constraints, stuck to pre-trained transformer models.


    \section{Classification}
    Classification is one of the major tasks that can be performed on text.
    %and is used to assign labels to a piece of text. 
    When we classify a piece of text, we assign it certain labels, that in some way describe the document. In classification terminology, we can either assign a single label or multiple labels to text input. In our research, we will be performing multi-label classification over documents. Specifically, we will be classifying long text pieces which can be either chapters or documents. In the next section, we will outline some of the previous work that has been done in this space. 
    
    \subsection{Multi-label Classification for Documents}
    Since each chapter could be associated with several labels, the chapter-level classification we perform in this work is multi-label in nature. An ETD may fall under more than one discipline. This is confirmed by the practice of ProQuest (we have used the dataset of ProQuest -- now a part of Clarivate -- to create the initial baseline), which enables authors to pick numerous secondary categories in addition to a primary category.
    
    The majority of conventional algorithms are designed for classification with a single label. Therefore, the multi-label classification problem can be split up into several single-label classification problems in order to use the pre-existing algorithms. Some of the common techniques used to convert multi-label classification problems to multiple single-label classification problems are (i) the OneVsRest method and (ii) the Label Powerset method. The \textbf{OneVsRest} method builds numerous independent binary classifiers, and when a test instance that has not yet been observed needs to be classified, the class labels can be chosen from among those that have the highest confidence. The underlying correlation between the class labels is ignored in this method \cite{Multilabel}. In the \textbf{label powerset} approach, each label combination is treated as a separate class, converting a multi-label dataset into a single multi-class dataset. By adding an instance to a class made up of a number of labels, it is capable of performing multi-label classification \cite{Multilabel}.
    
    \subsection{Text Classification using BERT}
       Since the seminal paper from Vaswani et al. \cite{attention}, proposing the transformer architecture, the NLP landscape has changed and current methods have leapfrogged in performance. The same can be said for text classification approaches. Before the development of transformer-based methods, researchers would use Word2Vec \cite{word2vec}, etc. to generate representations for text and then apply classification methods to it. Now BERT \cite{bert} can be used to classify text pieces effectively by first generating a contextual representation of the input text and then applying a logistic regression style classifier on top of it. In studies, it has been shown that BERT models are able to outperform previously known best methods. Inspired by this progress, we also experiment with BERT-based methods. 
 
    \subsection{Long Text Classification using Doc2Vec}
    A few machine learning algorithms exist that convert words into numeric vector representations. Some well-known ones include Word2Vec \cite{word2vec} and GloVe \cite{glove}.  However, these are insufficient for paragraph-level or document-level representation. Lee et al. \cite{doc2vec} proposed generating paragraph vectors and an unsupervised algorithm that generates fixed-length representations from the variable-length text. Their experiments show that this method is very effective in classifying text documents. To do that you generate the embeddings and then send them to a logistic regression-style classifier. 
    
    \subsection{Long Text Classification using Transformer Based Networks}
    In a prior section, we discuss BERT for text classification. A limiting factor in this method is the max sequence length of 512 tokens in BERT and similar models. This implies that most documents will have to be truncated, leading to a loss of information. Hierarchical transformers \cite{hierarchial} attempt to address this issue by incorporating either a recurrence unit or a transformer layer to obtain the BERT representation of input by splitting it into shorter segments. This method performed better than many of the approaches proposed until then. Apart from this, Park et al. \cite{longtextclassification} first selected the most important sentences from the long text using TextRank \cite{textrank} and then applied BERT style classification on the reduced text. This method proved effective in their testing. More recently advancements have been made in improving attention methods. Longformer \cite{longformer} introduces an attention mechanism that scales linearly. Due to this advancement, Longformer is able to accept texts of lengths up to 4096 tokens, which allows it to retain significantly more information compared to BERT; it performs well on long text classification. Another work, BigBird \cite{bigbird}, also introduces a new sparse attention mechanism and accepts texts of lengths up to 4096 tokens. This method also does well on long text classification.

    \subsection{SciBERT}
    As per “SciBERT: A Pretrained Language Model for Scientific Text” by Iz Beltagy et al. \cite{SciBERT}, SciBERT is a language model that has been trained on various scientific corpora to carry out various downstream scientific NLP tasks. Sequence tagging, sentence categorization, dependency parsing, and many other tasks are among them. On a significant subset of these downstream tasks, SciBERT has produced brand-new state-of-the-art outcomes.
    The BERT architecture is the foundation of SciBERT. The only difference between it and BERT is that it has been pretrained on a corpus of scientific text. Tokenizing input text and creating a model's vocabulary are both performed by BERT using WordPiece \cite{bert} tokenization. The vocabulary includes the most common word and subword combinations. Tokenization and vocabulary generation for SciBERT are done using a lexicon of scientific words called SciVOCAB \cite{SciBERT}.
    SciBERT was trained using 1.14 million papers from Semantic Scholar \cite{semanticscholar}. The papers' complete texts, including their abstracts, are used. The papers are typically 154 sentences long.
    The results of SciBERT are very encouraging. SciBERT has outperformed BERT on seven biomedical datasets.
    It has performed
    %relatively equally to 
    about the same as
    BioBERT \cite{biobert} on biomedical datasets. SciBERT surpassed BERT and attained state-of-the-art outcomes on all five datasets tested by the authors for computer science and other disciplines.

    
    \section{Knowledge Distillation}
    Deep learning models have been performing better and better on many tasks, but alongside the increase in performance, the computational cost required to run these models has also grown a lot. A few methods have been designed to optimize the models and reduce the costs, Knowledge Distillation being one of them. Initially proposed by Hinton et al. \cite{kd}, this method involves firstly training a large model with a huge number of parameters on the target dataset. Once this model has been trained it's called the teacher model. Now, a smaller model with fewer parameters is designed. This model will be called the student model. The student model is trained in such a manner that it learns to replicate the results of the parent model. Taking an example of classification we would compute the class probabilities for every item in the dataset using the teacher model and then use these probabilities as the target for the student model. DistillBERT \cite{distillbert} is the distilled version of the BERT model, which has 40\% fewer parameters when compared to the teacher BERT model.  
    % \section{Prompt Tuning And Adapter Based Prompt Tuning}
    % The effectiveness of prompt tuning was demonstrated by (Brown et al., 2020) where the authors showed that GPT-3 model could handle wide variety of tasks using only a few training examples. The use of prompts was first proposed by (Radford and Narasimhan, 2018). The authors showed that language models can perform well in few-shot and zero-shot settings through these natural language prompts. More recently, Jiang et al. (2020) proposed an approach to automatically discover better prompts in order to improve the factual knowledge retrieval from these language models.
    
\chapter{Requirements} \label{ch:requirements}
    
    \section{Overall Project Requirements}
    
    As the overarching goal of the project is to build a state-of-the-art information retrieval and analysis system, the main requirements can be listed as follows:
    
    \begin{itemize}
    
    \item Either a complete document or a sub-document derived from another document through tasks such as segmentation or extraction should be used for processing.
    \item Both the metadata as well as the entire text should support searching.
    \item Any aspects related to information derived from both the data and metadata should dictate the searching and browsing functionalities.
    \item Any aspects related to information derived from documents -- through processing techniques such as summarization, classification, and analysis -- should dictate the searching and browsing functionalities.
    \item Actions such as clicks and user queries should be logged as well as analyzed to increase user support. User-provided recommendations and feedback should also be considered and implemented.
    \item Techniques such as weighting or indexing should be selected in a manner that ensures the effectiveness of operations.
    \item The most effective method should be used to rank search results.
    \item  The content collection should dictate how pre-processing is handled, for both the managing of linguistic issues (e.g., word sense disambiguation, lemmatization, sentence identification, and object recognition), as well as for managing input formats (e.g., Text, PDF, JSON).
    \item In order to ensure that further progress and improvements can be made, any produced software and data should be released to all members of the project.
    \end{itemize}
    
    \section{Team 4 Requirements}
    
    As our team acts as experimenters with machine learning based on language models, our expected tasks can be outlined as follows:
    
    \begin{itemize}
    
    \item Developing a front-end application to access our services and run the models through a UI.
    
    \item Building an end-to-end system for segmentation -- implement Javaid's segmentation model into the general front-end layout in collaboration with Team 2, and compare this implementation with Team 3's segmentation technique. 
    
    \item Implementing a PDF parsing and clean-text generating script as a standalone service.
    
    \item Building an end-to-end system for summarization.
    
    \item Performing chapter classification by experimenting with language models to get the discipline labels.
    
    \end{itemize}
    
    \section{Collaboration}
    The other teams collaborated with us as follows.
    \begin{itemize} 
    
        \item Team 1: Finalized API contracts. 
        Collaborated to run our clean and parse script, as well as the segmentation pipeline in their container.
        
        \item Team 2: Developed front-end (experimenter and document view pages). Used chapter summaries for indexing.
    
        \item Team 3: 
        %Explored the possibility of generating summaries and classification labels from their XML output.
        They provided digital objects and XML files that could be used for classification and summarization.
        
        \item Team 5: GPU setup for training, containerizing our services, CI/CD, and workflow automation.
    
    \end{itemize}
    
\chapter{Design} \label{ch:methods}
    \section{Overall Flow}	
    The key aim of the project is to segment, summarize, and classify a subset of the ETD dataset. Figure \ref{fig:overall} shows the overall flow diagram of the system. For this project, only born-digital documents from the ETD dataset were considered. This is because Team 4's segmentation pipeline only works with born-digital type documents. Initially, the digital ETD document is given as an input to the segmentation pipeline and the segmented chapter output is generated. These generated chapters are passed into summarization and classification pipelines. Finally, after post-processing, all outputs are integrated into the front-end of the system. 
	
	\begin{figure}[h]
						\centering
						\includegraphics[scale=0.51]{Info Ret Diagrams-updated.png}
						\caption{Overall flow} 
						\label{fig:overall}
	\end{figure}

    In the sections below we elaborate further on the functioning and implementation of each component defined within the overall flow.
	
   \section{Segmentation}
   \subsection{Approach}
   
The segmentation task is performed to divide the ETD into chapters. 
The generated segmented chapters can then be given as input to services for chapter summaries, chapter topics, and chapter searches.

Our approach was to build on the M.S. thesis work of Akbar Javaid Manzoor, which was one approach to ETD segmentation into chapters.
His work is summarized as follows.
Initially, Javaid’s segmentation pipeline was implemented 
 using two datasets \cite{Javaid}. The main dataset contains 1459 ETD documents in LaTeX format which were obtained from the open-access arXiv repository. 
%Also, there is another 
The second dataset containing 150 ETD documents with manually labeled chapter boundaries.
%The input dataset is . 

The pipeline consists of two parts. In the first part, the main .tex file is identified for documents in the first dataset. Then, the chapter start is found and delimiters are inserted into the chapter start location. Additionally, the chapters are saved as separate PDF files. In the second part, the generated chapter PDFs are used as ground truth to train a deep learning model. This approach is useful for scanned and born-digital documents, but does better with the latter (according to an evaluation with the second dataset). 

	\subsection{Methodology}
 \begin{figure}[h]
	    \centering
	    \includegraphics[width=16.5cm]{Segmentation.png}
	    \caption{High-level view of Segmentation task }
	    \label{fig: segmentation}
    \end{figure}
    The high-level view of the segmentation module is shown in Figure \ref{fig: segmentation}. When a PDF is
given as input to the segmentation pipeline, the model predicts the chapter boundaries and
saves each chapter as a separate PDF.\\
First, Javaid’s latest deep learning model checkpoint is loaded, and the model architecture
is initialized. The model consists of two networks that try to extract both the image and
the text features from each page and predict a classification label (0: “FM PAGE”, 1: “FM
START”, 2: “CH START”, 4: “CH PAGE”, 3: “EM START”, 5: “EM PAGE” ). The predicted labels for each page are stored in an array. Next, the ideal way to extract the chapter boundaries from the prediction array is to traverse the array and find the page numbers from one “CH START” to the next one. Also, we check if the pages in between two “CH START” occurrences are “CH PAGE”. But this approach is over-dependent on the “CH START” label. If one “CH START” prediction goes wrong the entire chapter is not detected. When testing this approach on more than 100 ETDs, an average of only 2 chapter PDFs were saved. Due to the above shortcomings, we came up with a slightly modified approach to extract chapter boundaries.

In this approach, we leverage the “CH PAGE” information. From Javaid’s model metrics, it's clear that “CH PAGE” has a very good F1 score when compared to “CH START”. The prediction array is traversed and subarrays with more than 6 consecutive “CH PAGE” entries are saved as a chapter.  The start and end of the subarray’s labels are checked and a few other edge cases are taken into consideration. Finally, this new approach is also tested on more than 100 ETDs and an average of 4 chapters PDFs are detected. Therefore, the new approach has a better performance, comparatively.

Finally, the segmentation code is packaged with all requirements and model weights. The packaged service is moved to the Team 1 container. This makes it easier to access the ETD data and also saves a few API calls. The segmentation service is then integrated with Team 1’s database and file system saving functionality. The segmentation is run on the 5000 born-digital ETDs subset. All ETDs in this subset belongs to the “000” folder. The generated chapter PDFs are saved in a folder named “Team4\textunderscore chapters” inside each corresponding ETD folder, and entries are made in the objects table.


    \subsection{Evaluation}
    The segmentation model is evaluated and the results are shown in Table \ref{tab: Eval}. The model is evaluated on a test set created by Javaid with 100 sample data. We have also used Javaid's evaluation script to calculate the   Precision, Recall, and F1 scores for all 6 classes.  The “CH Page” class has the best F1 score. 
    
\begin{table}[H]
\centering

\caption{Evaluation for Segmentation Model \label{tab: Eval}}

\begin{tabular}{|l|l|l|l|} 
\hline
Page Type                                           & Precision     & Recall & F1 Score           \\ 
\hline
FM Start & 64\% & 47\% & 54\% \\
\hline
FM Page & 65\% & 44\% & 52\% \\
\hline
Chapter Start & 82\% & 77\% & 79\% \\ 
\hline
Chapter Page & 91\% & 97\% & 94\% \\
\hline
EM Start & 78\% & 68\% & 73\% \\
\hline
EM Page & 83\% & 64\% & 72\% \\
\hline

    \end{tabular}
    \end{table}
\subsection{Future Work}

We recommend additional work on chapter segmentation.
We suggest testing three different approaches, evaluating the results, and then tuning the best of the methods.

The first and easiest new method would be to use an updated version of Javaid's segmentation software.
After he provided the software that was used during the semester, he continued work, and developed an improved deep learning model.
That is likely to give better results than the earlier version.

The second method would build on the work of another class team.
Team 3 implemented an object detection-based approach to segment ETDs into parts, such as chapters. This approach is useful for both scanned and born-digital documents. This can be leveraged to perform summarization for scanned as well as born-digital documents.

%As a future work, 
Finally,
chapter segmentation can be performed using the Chapter Captor \cite{captor} approach. The first step for this approach is to extract the text from the PDF. Tools like PDFMiner, PyPDF2, and PyMuPDF can be used to convert the PDF to an HTML or XML format. The extracted text is then given as input to the BERT model. The BERT model predicts the chapter headers. Furthermore, regex rules are applied to find the chapter headers that the BERT model misses out on. Finally, based on the predicted chapter headers, unique delimiters are inserted which serve as positional information for chapters. Furthermore, to improve the model performance, the dataset size can be increased and extensive model tuning can be performed.

Regarding determining the best approach, evaluation is essential.
For any system, testing is a crucial task. When it comes to machine learning systems, it's a bit trickier to do testing. One good approach is to first validate the traditional software codes surrounding the machine learning model which includes input data, data preparation, feature engineering, configurations, and pipeline integration.

%\\
%Furthermore, Team 3 implemented an object detection-based approach to segment the chapters. This approach is useful for both scanned and born-digital documents. This can be leveraged to perform summarization for scanned as well as born-digital documents.\\
    \section{PDF Parsing and Cleaning}
    Segmented chapter PDFs are required to be parsed and converted into text files before utilizing them for downstream tasks like summarization and classification. Hence, instead of performing PDF parsing and text cleaning at every stage, this task has been separated out, to be operated independently, such that this service gets called every time after segmentation but before the generated clean text files are stored for the other services. 
    
    In the current implementation, the parsed PDF is cleaned by removing only the figures and tables. It is, however, desirable to remove equations from the text as well. This would be an important part of future work.
	\section{Summarization}
	\subsection{Approach}

	ETD text summarization is a challenging task primarily due to the size of the input text, and the lack of suitable training data. As reported by \cite{Palakh}, summarizing the complete ETD results in a loss of key concepts present within the chapters. Hence, the proposed methodology deals with summarization at the chapter level as shown in Figure~\ref{fig: TextSummarization}. 
    \begin{figure}[h]
	    \centering
	    \includegraphics[width=16.5cm]{TextSummarization.png}
	    \caption{Summarization flow diagram}
	    \label{fig: TextSummarization}
    \end{figure}
    \\
    In the proposed pipeline (Figure~\ref{fig: TextSummarization}), the ETD first passes through the ‘Chapter Segmentation’ module, which extracts every chapter from the ETD and stores them as chapter PDFs. Each chapter is then passed through the parsing and cleaning service (which is an independent service) which parses the chapter PDFs, cleans the extracted text, and stores the chapters as text files.
    
    The chapter text is then passed to the text summarizer model, which generates a summary of the complete chapter. Repeating the steps for every chapter input, we get a list of chapter summaries, which can either be concatenated to produce a summary of the entire ETD or utilized separately based on the end-user requirements.
    
    As part of the project, although we planned to research new techniques for text summarization, the majority of the work focused on experimenting with various abstractive and extractive summarization algorithms, like TextRank{} \cite{textrank}, LexRank{} \cite{lexrank}, Luhn’s Algorithm based summarizer{} \cite{luhn}, Latent Semantic Analysis based summarizer{} \cite{lsa}, and transformer-based models (like T5{} \cite{t5}, BART{} \cite{bart}, etc.).
    
    \subsection{Methodology}
    The summarization pipeline is built as explained in the previous section. It has been containerized and is integrated into the workflow developed by Team 5. The current work is described as follows:
    \begin{itemize}
        \item Set the required parameters -- model, summary length, etc.
        \item Pass a chapter text as input to the pipeline
        \item Summarize the chapter text using the model selected
        \item Generate a summary JSON (Figure~\ref{fig: summary_json})
    \end{itemize}
    
    \begin{figure}[h]
	    \centering
	    \includegraphics[width=16.4cm]{summary_json.png}
	    \caption{Chapter summary JSON}
	    \label{fig: summary_json}
    \end{figure}
    Initially, the summarization pipeline included steps like parsing and cleaning chapter PDFs. However, as part of the work completed earlier for Interim Report 3, this was separated into an independent cleaning module, hence only the path to the cleaned text needs to be passed to the pipeline. The generated JSON has two keys -- summary and the algorithm used for summarization.
    
    The summarization pipeline supports both abstractive (BigBird \cite{bigbird} pre-trained on the booksum dataset) and extractive summarization (TextRank \cite{textrank}, LexRank \cite{lexrank}, and LSA \cite{lsa}) models. However, it is to be noted that no model training is performed for summarization due to a lack of labeled training data. The pipeline only works in the inference mode. 
    
    In the current version of the IR system, summaries stored in the database are generated using TextRank \cite{textrank} owing to its low processing time. 
    
    \subsubsection{Summarization using Extractive models}
    The extractive models supported in the pipeline are: TextRank \cite{textrank}, LexRank \cite{lexrank}, and LSA \cite{lsa}. As mentioned in the earlier section, these models do not generate new summaries, rather they pick the most important text(s) from the chapters as summaries. In the current workflow, the following are the primary steps.
    \begin{itemize}
        \item Read the chapter text file
        \item Perform word stemming
        \item Remove stop words
        \item Feed the processed text into the model
        \item Write the generated summaries into a JSON file
    \end{itemize}
    
    \subsubsection{Summarization using Abstractive models}
    The abstractive model currently being used (although this can be switched at any time before a run) is the Big Bird model \cite{bigbird} pre-trained on the booksum dataset \cite{booksum}. The library used for implementing these pre-trained models is Hugging Face \cite{hugginface}. The advantage of this library is that it automatically performs tokenization and text embedding based on the transformer model selected. Hence, no manual pre-processing is performed while using Hugging Face transformer models, and the chapter text is simply fed into their model pipeline. Besides, the Big Bird \cite{bigbird} model, BART \cite{bart}, and T5 \cite{t5} have also been tested and are supported by the pipeline.
    
    \subsection{Challenges}
    As mentioned in the above subsections, there is no training being done for summarization. Multiple models have been experimented with and integrated into the pipeline. Hence, for the purpose of selecting a final model for generating the summaries to be stored in the database, model evaluation was planned. However, it was not carried out, owing to a few challenges as mentioned below. 
    
    Summaries generated by extractive and abstractive models vary based on their way of working. Abstractive models generate new text, whereas extractive models, on the other hand, pick out the most important lines/sentences from the text. Given that most of the ground-truth data consists of newly generated summaries, the evaluation results would be biased toward abstractive models. Hugging Face transformer models, particularly Big Bird, were used as abstractive models for the project. Big Bird \cite{bigbird} needs between five and ten minutes to compose a chapter summary. Therefore, the estimated time for 5000 ETDs (with an average of five to six chapters each) would be enormous.
    
    The end-to-end workflow integration was the main focus due to time constraints. As a result, as the first version of the ETD summaries, TextRank \cite{textrank} was used to build the summaries and store them in the database for further indexing.
    % In the evaluation dataset, there are 39 ETDs with ground-truth summaries for almost all of their chapters. The planned model evaluation to dictate the final model selection, involved evaluation metrics, like, BLEU\cite{bleu}, ROUGE\cite{rouge} and BERT-score \cite{bertscore}.

    \subsection{Future work}
    As mentioned in the previous section, owing to time constraints, an extractive summarization model has been used currently to generate the summaries. However, such models do not always produce correct summaries, as the text might not contain any sentence(s) that can correctly signify the summary of the text, or if one exists, the algorithm might fail to extract it. Hence, abstractive models like transformer models are preferred. As the current summarization pipeline already supports transformer models, producing accurate summaries using these models can be an important future goal/task. Moreover, instead of directly using pre-trained transformer models, it would be useful if they can be further fine-tuned on some ETD-relevant dataset. Furthermore, the granularity can be varied, i.e., experiments at different granularity levels could be carried out as well, like at the section and paragraph levels. 
    
	\section{Classification}

    \subsection{Approach}
    
    In Figure \ref{fig:classification_meta_data_1},  the steps for building classification models are highlighted. 
    
        \begin{figure}[h]
        \centering
        \includegraphics[width = 17.4cm]{Classification_Flow.jpg}
        \caption{Classification flow diagram for chapter classification}
        \label{fig:classification_meta_data_1}
    \end{figure}
    
    Chapter classification is supported by our classification pipeline. The chapter classification's purpose is to produce discipline labels. Before we can build models, we must first create a training dataset, which is done by selecting a subset of the larger dataset based on the disciplines we are focusing on. Initially, chapters from the full PDF are segmented using the segmentation model. The dataset will then need to be preprocessed before models can be trained on it. This step may include tokenization and label creation. For the classification models, we have employed Deep Transformer based models as well as a simple statistical model with fewer parameters. We have considered BERT, SciBERT, and Longformer for deep transformer-based models. For the statistical model, we have used a Support Vector Machine or Random Forest that is trained on word embeddings of text. In the following sections, we will first discuss the dataset creation and then describe the modeling approaches. 
    \subsection{Chapter Classification: Methodology}
	One of our primary goals is to experiment with language models and traditional NLP methods for the task of classifying chapters into disciplines. The model will be fed segmented chapters in text format as input, and the output will be discipline labels. We assign at least two disciplines to each chapter because each chapter might relate to many disciplines. In contrast to the methods outlined in the metadata classification section, this problem presents some additional challenges due to the length of text in chapters. We have anticipated that, compared to abstracts, chapters will generally be significantly more verbose. Therefore, our approach must be capable of efficiently consuming and comprehending information from the input.
	
        \subsubsection{Data Collection and Preprocessing}
        We need the chapter-by-chapter split of text from the ETDs to classify the chapters. We have extracted the chapter texts from the ETDs in the segmentation ground truth dataset gathered by Javaid \cite{Javaid}. 132 chapter segmented ETDs from arXiv make up the segmentation ground truth dataset that Javaid gathered. We have gathered the chapter information for ETDs that are included in the ETD database as well as Javaid's dataset.
        Overall, 40 ETDs in the ETD database were found in Javaid's dataset. Only 30 of the 40 have metadata with discipline/department labels. So,  we have compiled a dataset using chapters from 30 ETDs that have been segmented.
        Additionally, we have gathered data by running our segmentation pipeline on nearly 1500 ETDs. The segmentation pipeline yielded 4000 chapters in total. 
        
        After that, additional preprocessing was done to remove the tables from the text file that was created. Chapter-by-chapter PDF files were parsed and cleaned using the standalone service discussed in Section 4.3. However, for a few chapters, the parsed output was not ideal because of restrictions and flaws in the PDF Plumber library, which we presently use to remove graphics and tables from the chapter text.  Occasionally, albeit not very often, the PDF Plumber failed to convert tokens from its character dictionary to ASCII values, leaving characters that are unintelligible. In the future, this error can be rectified by iterating over the output and converting the characters that weren't converted to ASCII. After eliminating the subpar chapter texts, we had 3742 chapters left with which we fine-tuned our models. 

        The "Discipline" feature in the metadata was used to generate the classification labels for the chapters in the ETDs. It should also be noted that when the discipline labels were not available we used the department field as the label for the chapter. We observed that in many cases when both fields were available, they were quite similar to each other. Hence we used one as a substitute for the other. Due to the small dataset, we utilized a training set to test set ratio of 85\% to 15\%, to train and then evaluate the models that were trained. For the test set and training set, samples were randomly chosen from the full dataset. However, the stratified splitting preserved the distribution of labels in the test set. The chapter classification model trained can predict 27 different classes.
        
        % Finally, we used 2993 chapters for the training set and 749 chapters for the evaluation of the trained models. We've increased the number of classes from 13 to 27.
        

	
	
	% \subsubsection{Chapter Classification using BERT}
 %    BERT has a sequence limit of 512 tokens, meaning that any input over that limit will get truncated and information will be lost. And as was discussed above, the chapter texts are fairly long which means simply using BERT would result in a loss of information. So now we first use TextRank as a preprocessing method on our chapter texts to obtain a short summary of the most important sentences within the chapter. Through this method, we reduce the input length to a much more manageable 512 tokens. These selected sentences will be the new input to the BERT model. After this processing, we apply a model architecture similar to the method described in metadata classification. The architecture is illustrated in Figure \ref{fig:bert-textrank}. We have developed pipelines for this method and also completed testing on the PQDT dataset.
    
 %     \begin{figure}[h]
 %        \centering
 %        \includegraphics[width = 15cm]{BERT+TextRank.jpeg}
 %        \caption{Chapter Classification using BERT and TextRank}
 %        \label{fig:bert-textrank}
 %    \end{figure}
        \subsubsection{Chapter Classification using BERT and SciBERT}
       BERT is a transformer-based language model that has been trained on a huge general English text corpus with a strong understanding of language. We believe that it can be appropriately applied here for the use case of chapter classification. We have developed a pipeline for classifying chapter texts based on disciplines. The chapter texts in the dataset amassed by our team were preprocessed to eliminate tables and figures by our clean and parse service. While storing the results of 5000 ETDs in the database,  we have used the cleaned text already present in the database rather than reading and preprocessing the PDF file each time the classification pipeline is executed. Since BERT has a sequence limit of 512, we have selected and classified the first 512 words in the chapter. We felt this was not problematic because the average length of the chapter text was 255 tokens (in the dataset used for training). 
       
       We have also used the summaries generated by our team to fine-tune BERT on the summaries rather than the entire text.  We didn't notice any significant improvement in the performance of models tuned on summaries over the entire text as the mean length of the entire text was well within the maximum sequence limit of BERT.  We have developed a pipeline to train the model and gather the results after training. 
       
       Since the results presented in a SciBERT paper \cite{SciBERT} are very encouraging for a variety of disciplines, we have also decided to use the SciBERT language model. SciBERT, which is used to understand scientific texts, is actually quite similar to BERT but has the additional advantage of being trained in scientific corpora. A pipeline for training and evaluating SciBERT has also been created. Due to the SciBERT pretraining on scientific text, we observe that it is able to outperform BERT on our evaluation set. The model architecture used for this task is the same as the model design described in Section 4.5.4.2, with the input changed to chapter text.
       
       
	\subsubsection{Chapter Classification using Longformer}
	Longformer is an alternative larger transformer model, which has been trained on a language modeling task similar to BERT and SciBERT but with a key difference in architecture. It uses a different attention mechanism allowing it to increase the sequence limit from 512 to 4096. Now with this  higher sequence limit, we do not need to truncate the chapters to 512 during prepossessing. We can assume that the first 4096 tokens within the chapter would provide enough information to accurately classify the entire chapter. In terms of architecture, we apply a similar architecture to simple BERT-based classification, but now we use Longformer to provide contextual representations. The architecture is illustrated in Figure \ref{fig:longformer}. We have developed pipelines for this method to train the model.  The model architecture used for this task is the same as the model design described in Section 4.5.4.2 with the model changed to Longformer. 
 \begin{figure}[h]
        \centering
        \includegraphics[width = 17cm]{Longformer.png}
        \caption{Chapter Classification using Longformer}
        \label{fig:longformer}
    \end{figure}
	
	\subsubsection{Chapter Classification based on chapter summary}
	Before building the models, we hypothesized that the majority of chapter texts in our dataset would have more tokens than 512. This is problematic because, as previously indicated, most language models have a 512 tokens restriction. Using larger models, such as Longformer, is one method to remedy this. However, these models are inefficient and need a lot of memory to execute. A more resource-efficient approach would involve creating chapter summaries first, then classifying those summaries. This way, the summaries would condense all the material into 512 words. Once the summaries are generated we can use a BERT or SciBERT based classification pipeline that has been described above. Since we have generated chapter summaries, it makes sense to follow this approach.  
	 A pipeline for this method has been created. This approach has been illustrated in Figure \ref{fig:summary_classifier}.
	
	 \begin{figure}[h]
        \centering
        \includegraphics[width = 17cm]{summerizer approach.png}
        \caption{Chapter Classification based on summaries}
        \label{fig:summary_classifier}
    \end{figure}

	
	\subsubsection{Chapter Classification using SVM and Random Forest}
	 \begin{figure}[H]
        \centering
        \includegraphics[width = 17.5cm]{SVM_RF_Diagram.jpg}
        \caption{Classification using SVM and Random Forest}
        \label{fig:svm_and_rf}
    \end{figure}
    In Figure \ref{fig:svm_and_rf}, we highlight the major steps followed during the replication of Palakh's thesis \cite{Palakh} to set up a baseline for other models.

    First, the data was preprocessed. General preprocessing steps performed during NLP tasks such as stop word removal, punctuation removal, lemmatization, and conversion of all the text into the lower case were completed, and the results stored. 
    
    Word embeddings of the full text were generated using two models, namely FastText and Doc2Vec. The word embeddings are not ideal for long documents such as ETDs as the order of words is not considered, resulting in the loss of important information. But for baseline implementation, word embeddings were generated. 
    
    Since we want multiple discipline labels for each chapter, as each chapter in a thesis may represent multiple disciplines,  we considered the classification task at hand as a multi-label classification problem. Most of the classifiers implemented in Scikit-learn are built for single-label classification. Thus to solve the multi-label classification problem, the multi-label classification problem can be transformed into multiple single-label classification problems so that we can use the pre-existing algorithms to solve it. As was done by Palakh, we have considered the LabelPowerset approach for the dataset. We have finally developed a pipeline to train SVM (Support Vector Machine) and Random Forest to set us up with a baseline.
     
    \subsection{Model Evaluation}
    
    %At this stage of our project, w
    We have been able to train the BERT and SciBERT models described above on our current ETD chapter classification dataset, and a dataset of chapter summaries. The summaries contain the condensed versions of the documents and are useful in cases when the number of tokens exceeds the sequence limit of 512 tokens. All our data pipelines and training pipelines were developed in PyTorch. 
    For the Language Models, we used Hugging Face's Transformers library. Using this library, we were able to obtain the pre-trained weights for BERT, SciBERT, and Longformer. All of our models have been trained on the following hyperparameters:
    \begin{itemize}
        \item Optimizer: AdamW
        \item Learning Rate: 5e-5
        \item Batch Size: 8 
        \item Epochs: 8 
    \end{itemize}

    Once the training was complete, we performed and evaluated both the summary and chapter models on a small evaluation test set consisting of 749 chapters. The performance metrics of Accuracy and F1 have been recorded and are mentioned in Table \ref{tab:classification_results}. 
%     \\
%     \textbf{BERT-base trained on chapters}
%     \begin{itemize}
%         \item Accuracy on Test set: 64.75 \%
%         \item F-1 score:  56.28 \%
%     \end{itemize}

%     \textbf{SciBERT-base trained on chapters}
%     \begin{itemize}
%         \item Accuracy on Test set: 80.77 \%
%         \item F-1 score: 77.5 \%
%     \end{itemize}

% \textbf{BERT-base trained on summaries}
%     \begin{itemize}
%         \item Accuracy on Test set: 63.01 \%
%         \item F-1 score: 54.41 \%
%     \end{itemize}

%     \textbf{SciBERT-base trained on summaries}
%     \begin{itemize}
%         \item Accuracy on Test set: 70.89 \%
%         \item F-1 score:  67\%
%     \end{itemize}
    \begin{table}[H]
    \caption{\label{tab:classification_results} Evaluation of classification models}
    \centering
    \begin{tabular}{|l|l|l|l|}
    \hline
    Model Name & Accuracy (\%) & F-1 Score (\%) & Chapter/Summary \\ \hline
    BERT-base  & 64.75         & 56.28          & Chapter         \\ \hline
    SciBERT    & 80.77         & 77.5           & Chapter         \\ \hline
    BERT-base  & 63.01         & 54.41          & Summary         \\ \hline
    SciBERT    & 70.89         & 67             & Summary         \\ \hline
    \end{tabular}
    \end{table}



    % It should be noted that due to the dataset being very small, our test set is also very small, so the performance of the model on the test set may not be very representative of the actual performance. Once we collect a larger dataset, we should be able to get a more accurate reading of the model performance. Keeping this in mind, we can observe that the BERT-base model gets 90\% accuracy on the  evaluation set and SciBERT gets 95\% accuracy. The higher performance of the SciBERT model can be explained by the model having a better understanding of Scientific text which is common in our test set, through our experimenter API we plan to expose both the models to the user so depending on the type of document the model can be switched. Discussing the results qualitatively we observed that even though the models have been trained on a small dataset they are fairly competent even on new documents. 
    When we look at the results of the chapter and summary models we can see that the chapter models perform better than the summary models, across the board. For both of the models, i.e., SciBERT and BERT, their summary-trained counterparts are worse than the chapter models. This result was unexpected, because generally one would assume the chapters to be so long that once we use transformer-based models on this dataset, information would be truncated. In practice, however, that is not the case.
    We see that the chapter texts are generally within 512 tokens for most documents and the average number of tokens in the chapters is around 300. This suggests that when we are generating summaries from the chapter text we are effectively reducing the amount of information available. Due to these reasons, it can be understood that the summary models are worse on the test set when compared to the chapter models. From the results, it can also be noted that the chapter models are quite good at classifying the chapters.
    In our qualitative experiments we tested the code with some new documents and the output of the models was satisfactory. 
    \subsection{Inference on ETD subset}
    As is clearly shown in Section 4.5.3, SciBERT fine-tuned on chapter text performed the best of all. The performance of the models fine-tuned on chapter summaries wasn't as impactful as we hypothesized, because the average length of the chapter text was less than the maximum permissible token limits of BERT.
    Classification performance not only depended on the performance of the segmentation model but also on the model generating summary instead of just segmentation models. Thus we have used the  SciBERT model fine-tuned on the chapter text to store the classification labels on 5000 ETDs subset.

    \subsection{Future Work}
    \subsubsection{Metadata Classification}
	\paragraph{Methodology}\mbox{}\\
 
	 Library metadata is a very important source describing a crucial piece of information through which information can be stored and retrieved easily. Metadata ensures that the data can be found, accessed, and interpreted easily.
	The absence of information in metadata can be considered a barrier to information retrieval. While combing through the metadata of 500k ETDs, we found out that a lot of fields in the metadata are missing.
 %As per the paper published last year, 
 According to one of our prior studies,
 65,955 fields of year data, 232,653 fields of department data, and 166,690 fields of discipline data were missing out of 450k ETDs collected by Uddin et al. \cite{ETDs_Last_Year}. Since the absence of metadata is very detrimental to efficient information storage and retrieval, we were planning on developing traditional NLP methods and deep transformer-based methods to classify the discipline given an abstract. We are posing this problem as a multi-label classification problem where each abstract may be assigned multiple disciplines. 
% 	\subsubsection{Traditional NLP methods}
	

	
	
	\paragraph{BERT-based Metadata Classification}\mbox{}\\
 
	We have been able to develop a BERT-based pipeline for classifying the metadata into disciplines, since the input to the models, i.e., the abstracts, are generally short texts. We do not need to worry about information loss due to the text length limit of BERT models. This pipeline is able to consume a dataset of ETD abstracts, convert them into batches, and perform necessary preprocessing which for transformers includes tokenization. Tokenization breaks the sentence down into smaller tokens which may be words or sub-words. After this, we split the dataset into training and validation so the model can be thoroughly evaluated. We can explain the model design used for this task.
 First, the text is tokenized using a BERT tokenizer. This step is equivalent to converting each compound word in the text to a number that the model can interpret. Within this step, we also truncate the text content to 512 words. When this tokenized text is provided as input, the BERT model processes the text and generates a 768-dimensional representation of the entire sentence. We take this vector and pass it through a linear layer that returns logits that have the same length as the number of classes. On these logits, we apply the BinaryCrossEntropy loss which can be used in multi-label classification. This loss essentially treats each value in the logits as a logistic regression problem, i.e., 1 if the class is present, otherwise 0. Once the model is trained, we can round the logits and get the class predictions. The architecture is illustrated in Figure \ref{fig:abstract-bert}. 
	
	 \begin{figure}[h]
        \centering
        \includegraphics[width = 17cm]{Meta data classification.png}
        \caption{Metadata classification using BERT}
        \label{fig:abstract-bert}
    \end{figure}
	
    The input to chapter classification (when applied to chapter text) and metadata classification (when applied to the abstract) are very similar, so we can also use the chapter classification model to classify the missing disciplines in metadata. 

    \subsubsection{Knowledge Distillation}
    Knowledge distillation is the process through which a smaller network is trained to leverage the knowledge gathered by large networks. Through knowledge distillation, we are effectively training a smaller network to replicate the performance of the larger network. This is highly advantageous because fine-tuning and getting inferences from a large model is computationally very expensive and time-consuming. Since the distilled models are lighter than the original models, further fine-tuning can be done quite easily. In the future, the inference time of our classification system can be further improved by distilling SciBERT into a smaller model.

    \subsubsection{Citation Intent Classification}

	Citing research papers referred to is an integral part of scientific works such as ETDs. The authors are required to cite the work done by others previously because the idea used is not from them. Citations give your work more credibility, advise readers of where to get a more in-depth explanation of it that is pertinent to the content, and acknowledge the person whose work or concept you are employing. 
	
    For the analysis of scientific works and the relationships between them, determining the citation's intent is essential. Finding the purpose of a citation helps to determine the publication's influence and can also point interested readers in the direction of the kinds of additional readings they might find interesting. 
	
	The development of a Citation Intent Classification dataset from ETDs is the initial stage in the classification of citation intent. The purpose of citations can be marked in a manner that is analogous to the SciCite dataset \cite{SciCite}. This allows citations to be classified into the following three classification labels: Background, Method, and Result Comparison. To assure the quality of the dataset, at least two separate individuals can verify the prepared dataset. The generated dataset can be used to evaluate how well the trained and tuned models function.
	
	The approach's second stage would involve investigating various deep learning or language models to classify the citation intent. After classifying the citation intent using the model that performed the best out of all those that   would be trained,  prompt tuning \cite{prompt} can be investigated to effectively fine-tune the model.  Knowledge Distillation \cite{kd} of the best larger model trained to a smaller model can also be attempted to decrease the training duration and also to fine-tune the learning of the larger model to the domain-specific datasets.

    \subsubsection{Investigating Chapter Lengths}
    The average chapter length of the training set was less than 512 tokens. This runs counter to our assumption that chapter text would typically be longer than 512 tokens. We believe that improvements to the clean and parse pipeline will resolve this issue. Currently, the clean and parse service is not able to extract all the chapter tokens from the chapter PDF. However, this matter may be looked upon in the future.
	
%     	\subsubsection{Adapter based Prompt Tuning}
%     	 Pretrained language models improve performance on a wide range of natural language understanding (NLU) tasks. A widely-used method, fine-tuning, updates the entire set of model parameters for a target task. While fine-tuning may provide good performance, it is memory-consuming during training because gradients and optimizer states for all parameters must be stored. Moreover, keeping a copy of model parameters for each task during inference is inconvenient since pre-trained models are usually large. 
    
%         The concept of prompt-tuning was introduced to improve parameter efficiency in the downstream tasks. In prompt tuning, there’s no need for new parameters as we convert our problem into a language modeling task.
    
%         A prompt is usually a sequence of words or parameters that are appended or prepended to the input so that the given downstream task can be constructed as a language modeling problem. 
    
%         However, prompt-tuning for smaller language models does not perform well when compared to traditional fine-tuning. This approach only works for significantly larger models (BERT large, RoBERTa large, etc). We propose a further method called \textbf{Adapter-based Prompt tuning (APT)} for smaller language models.
        
%         In Adapter based Prompt tuning, We insert adapter modules in the language model, then jointly fine-tune the parameters of this adapter module along with soft prompts while keeping the rest of the model frozen. 
    
%         We will investigate this approach for ETD-based datasets. We would like to see if this approach can be beneficial for efficiently tuning smaller language models like BERT base, RoBERTa base, SciBERT, etc for ETD datasets. We will evaluate the effectiveness of our proposed APT approach in comparison to fine-tuning and see if it can match the performance of traditional fine-tuning while using fewer parameters.
    
    
    
    



    
    

	\section{User Interface}
 The Figure \ref{fig:UI_1} experimenter portion of the front-end accepts a new PDF as input. First, the uploaded PDF is added as a new ETD using Team 1’s add new ETD API. The API returns the newly added ETD’s ID as a response.  Next, Team 5’s workflow is triggered. The workflow is designed in such a way that it calls the services in the following order:
\begin{enumerate}
\item Get ETD PDF using ETD ID
\item Segmentation service
\item Clean and parse service
\item Summarization service
\item Classification service
\item Save results to the database
\end{enumerate}


 




The status of the workflow can be checked using the “get workflow status” API. Once the workflow execution is complete, the generated results are fetched from the database and displayed in the front-end. \\
\newpage
 % The system's pipeline flow is shown in Figure \ref{fig:UI_Pipeline}. The front-end is developed using React and the back end uses Flask. All three models, namely summarization, classification, and segmentation, are connected to the Flask app. The model-generated outputs are stored in the database using APIs.   \\ 
	% \begin{figure}[H]
	%     \centering
	%     \includegraphics[width=15cm]{pipeline.png}
	%     \caption{Integrated pipeline flow}
	%     \label{fig:UI_Pipeline}
	% \end{figure}
	
	% Figure \ref{fig:UI_1} shows the experimenter's user interface of the system. The PDF runs through the segmentation pipeline and generates the chapter outputs. Also, the user has the option to select the summarization and classification models. With the click of the “Go” button, the segmented chapters,  summaries, and classification labels are displayed to the user. Furthermore, the PDF’s metadata can be accessed by the user.  
       
	    \begin{figure}[H]
	    \centering
	    \fbox{\includegraphics[width=16cm]{WireFrame_2.jpg}}
	    \caption{Experimenter User Interface wire-frame}
	    \label{fig:UI_1}
    \end{figure}

	
    \section{Tools}
    To successfully complete the project we worked with the following tools and technologies.
    
    

    \textbf{Python:} Our primary language for development is Python as it allows for fast prototyping when researching different solutions. Furthermore, Python offers the best support for various kinds of NLP and Deep Learning/Machine Learning libraries overall, making it an ideal choice. 
    
    \textbf{Docker:} Docker \cite{docker} provides a completely focused environment that is catered towards running your specific solution in any kind of environment. It contains all the necessary prerequisites libraries and system-level binaries required for functioning. For our project, we build containers for each of our services, which are summarization, segmentation, and classification. 
    
    
    \textbf{Python utility libraries:} We use the PyPDF2 \cite{pypdf} and PDF Plumber \cite{pdfplumber} libraries to parse the PDFs. Through these libraries, we are extracting text information that is used for the classification and summarization pipelines. We also use the Requests library to fetch and send data through APIs \cite{Requests}.

    \textbf{GitLab:} GitLab will host the code for the team and allows for easy maintenance of various feature branches. Furthermore, through GitLab, CI/CD pipelines can be set up for deployment \cite{vt_git}.
    
    \textbf{PyTorch:} We are working with PyTorch \cite{torch} which is a machine learning library providing methods to build layers, models, and data pipelines for this project. PyTorch will accelerate development time and also allows for hardware-accelerated training. 
    
    \textbf{Hugging Face:} Hugging Face \cite{hugginface} is a library built around PyTorch that provides support for state-of-the-art Transformer models. Hugging Face provides easy access to pre-trained weights for transformer architectures like BERT, etc., for faster experimentation.
    
    \textbf{NLP libraries:} We use libraries like spaCy \cite{spacy} and NLTK \cite{nltk} which provide APIs to perform some common NLP tasks like lemmatization, removing stop words, etc. We are also using sumy \cite{sumy} for performing text summarization. We rely on other libraries like NumPy \cite{numpy} for working with arrays and Pandas \cite{pandas} for working with dataframes.


	\chapter{Implementation} \label{ch:deliverables}
% 	\chapter{Summary} \label{ch:summary}
\section{Tasks and Timeline}
Tables \ref{tab:segmentation_timeline}, \ref{tab:summarization_timeline}, and  \ref{tab:classification_timeline} detail the activities completed and the completion timeline for segmentation, summarization, and classification, respectively.

\subsection{Segmentation Tasks and Timeline}

Tasks Assigned: Dharneeshkar
\begin{table}[H]
\centering
\caption{\label{tab:segmentation_timeline}Tasks and Timeline for Segmentation}
\begin{tabular}{|l|l|l|l|} 
\hline
Task                                           & Due Date     & Remarks            \\ 
\hline
Literature Review                              & IR2          & Complete              \\ 
\hline
Setting up Javaid's
  pipeline                  & IR3          & Complete               \\ 
\hline
Setting up segmentation code
  in Team 1's container    & Final Report          & Complete       \\ 

\hline
Storing segmentation results for 5000 ETDs
  to the database    & Final Report           & Complete       \\ 

\hline
Collaborating with Team 5 to automate our service
           & Final Report & Complete       \\

\hline
Integrating with the User
  Interface            & Final Report & Complete       \\ 
%\hline
%Model Tuning                                   & Stretch Goal & To be Completed       \\
\hline
\end{tabular}
\end{table}

\newpage
\subsection{Summarization Tasks and Timeline}

Tasks Assigned: Abhilash and Deepak

\begin{table}[H]
\centering
\caption{\label{tab:summarization_timeline}Tasks and Timeline for Summarization }
\begin{tabular}{|l|l|l|l|} 
\hline
Task                                                                                                             & Due Date & Remarks                                                                                                                         \\ 
\hline
A Basic Summarization pipeline                                                                                            & IR2               & Complete                                                                  \\ 
\hline
Data Preprocessing/Cleaning                                                                                               & IR3               & Complete \\ 
\hline
\begin{tabular}[c]{@{}l@{}}Implementing at least 3 different models\end{tabular}               & IR3               & Complete                                                                \\ 
%\hline
%\begin{tabular}[c]{@{}l@{}}Evaluation on ground-truth data of around\\~25-30 ETDs\end{tabular} & Final Report      & In Progress                                                                       \\ 
\hline
\begin{tabular}[c]{@{}l@{}} Qualitative evaluation and experimentation \\for final model selection\end{tabular}                  & Final Report      & Complete                                                                        \\ 

\hline
\begin{tabular}[c]{@{}l@{}}Storing summaries of 5000 ETDs in the database \end{tabular}                  & Final Report      & Complete                                                                       \\

\hline
Collaborating with Team 5 to automate our service
           & Final Report & Complete       \\

\hline
\begin{tabular}[c]{@{}l@{}}Integrating with User Interface\end{tabular}                  & Final Report      & Complete                                                                       \\
%\hline
%\begin{tabular}[c]{@{}l@{}}Fine-tuning Pre-trained models \\on relevant dataset\end{tabular}                              & Final Report      & Stretch Goal                                                                                                                                       \\
\hline
\end{tabular}
\end{table}
% \vspace{-30pt}

% \newpage
\subsection{Classification Tasks and Timeline }

Tasks Assigned: Kaushik, Deval, and Aditya
\begin{table}[H]
\centering
\caption{\label{tab:classification_timeline}Tasks and Timeline for Classification}
\begin{tabular}{|l|l|l|} 
\hline
Task                                                                                                                                                                  & Due Date       & Remarks                                                             \\ 
\hline
\begin{tabular}[c]{@{}l@{}}Establishing a baseline by classifying disciplines \\in the full text of PQDT dataset\end{tabular}                                            & IR2            & Complete                                                            \\ 
\hline
\begin{tabular}[c]{@{}l@{}}Establishing a baseline for metadata classification \\by classifying the disciplines in the 500k metadata dataset.\end{tabular}            & IR2            & Complete                                                            \\ 
\hline
\begin{tabular}[c]{@{}l@{}}Creating a labelled dataset for classification of \\disciplines from ETD repository\end{tabular}                                             & IR3            & Complete  \\ 
\hline
\begin{tabular}[c]{@{}l@{}}Training chapter classification models \\including BERT, SciBERT and Longformer\end{tabular}                                             & IR3            & Complete  \\ 
%\hline
%\begin{tabular}[c]{@{}l@{}}Classify missing disciplines of ETD 500k metadata \\dataset and perform chapter classification on the \\newly created dataset\end{tabular} & IR3            & Complete          \\ 

\hline
Fine-tuning the models for best performance                                                                                                                             & Final
  Report & Complete          \\ 

\hline
\begin{tabular}[c]{@{}l@{}}Storing the chapter classification results of 5000 ETDs\\ in the database\end{tabular}                                                                                                                            & Final
  Report & Complete          \\ 

\hline
Collaborating with Team 5 to automate our service
           & Final Report & Complete       \\
           
\hline
Integrating
  with User Interface                                                                                                                                       & Final
  Report & Complete          \\ 
%\hline
%Distill
%  knowledge from fine tuned models~                                                                                                                           & Final
%  Report & Stretch
 % Goal                                                      \\
\hline
\end{tabular}
\end{table}

% \subsection{Evaluation of Methods}

% For each of our services, we first designed some initial methods based on prior literature. For each of our methods, we experimented with two approaches, one, which involves utilizing simple and less resource-intensive models like SVM, etc. The second approach was comparatively expensive as it involved deep neural networks, especially transformer models. We evaluated both approaches to figure out the trade-off between complexity and accuracy gains. 


% \section{User Stories}

% \begin{table}[h]
% \centering
% \caption{User Stories}
% \begin{tabularx}

% \begin{tabular}{|m{3cm}|m{12cm}|} 
% \hline
% User story 1  & As an experimenter, I want to segment the ETD into chapters. The entire ETD PDF is the input and the generated chapter PDFs are the output.                                                                                       \\ 
% \hline
% User story 2  & As an experimenter, I want to provide a chapter text and obtain 3 different summarized versions.                                                                                                                                  \\ 
% \hline
% User story 3  & As an experimenter, I want to use work with prompt tuning and apply that to an ETD citation dataset. Does the number of parameters needed reduce significantly with prompt tuning approaches?                                     \\ 
% \hline
% User story 4  & As an experimenter, I want to understand how knowledge distillation works in language models. How well does knowledge transfer from a model trained on larger parameters to a smaller model constituting of a specialized domain  \\ 
% \hline
% User story 5  & Front-end application to access our services and run the models through a UI.                                                                                                       
% \\
% \hline
% \end{tabular}
% \end{tabularx}
% \end{table}
\chapter{User Manual} 
\section{Experimenter Page}
\begin{figure}[h]
	    \centering
	    \fbox{\includegraphics[width=16cm]{Exp_1a.png}}
	    \caption{Experimenter User Interface web page }
	    \label{fig:EXP_UI1}
    \end{figure}
Figure \ref{fig:EXP_UI1} shows the page where users can experiment with different summarization and classification models. React \cite{react} is used for front-end development. Initially, to access this page the user needs a login. To experiment with a new ETD, the user can upload an ETD PDF file. The uploaded ETD is saved in the repository using an API call.  When the user clicks the submit button, Team 5's workflow is triggered using an API call. The workflow runs all the required services and saves the results to the database and file system.\\


The generated results are displayed in Figure \ref{fig:EXP_UI2}. The summaries for each chapter and the top two classification labels are presented to the user.\\
    \begin{figure}[H]
	    \centering
	    \fbox{\includegraphics[width=16cm]{Exp_2.png}}
	    \caption{UI displaying segmented chapters and summaries}
	    \label{fig:EXP_UI2}
    \end{figure}

The models for summarization and classification can be selected by the user using the drop-down box shown in Figures \ref{fig:summarydropdown} and \ref{fig:classificationdropdown}, respectively.
    \begin{figure}[H]
	    \centering
	    \fbox{\includegraphics[width=16cm]{Summarization_UI_dropdown.jpg}}
	    \caption{Dropdown box to select summarization model}
	    \label{fig:summarydropdown}
    \end{figure}
    
    \begin{figure}[H]
	    \centering
	    \fbox{\includegraphics[width=16cm]{Classification_UI_dropdown.jpg}}
	    \caption{Dropdown box to select classification model}
	    \label{fig:classificationdropdown}
    \end{figure}

However, since the back-end of the experimenter page is running the workflow automation done by Team 5, it currently supports only default models for summarization and classification. The default models supported are TextRank for summarization and SciBERT for classification. The default models for summarization and classification were selected by analyzing the trade-offs based on the time taken for the model to run and the performance of the models.


We have attached two videos, \textbf{team4\_experimenter\_demo.mp4}
\newline
and \textbf{team4\_documentView\_demo.mp4}. The description of each is as shown in Tables \ref{tab:vid1} and \ref{tab:vid2}, respectively.

\newpage

% \textbf{team4\_experimenter\_demo.mp4:}

\begin{table}[H]
\centering
\caption{\label{tab:vid1}Description of video team4\_experimenter\_demo.mp4 }
\begin{tabular}{|l|l|}
\hline
Timestamp & Description                                                                                                                                                                                                                                                                                                                      \\ \hline
0:00      & The home page of the UI.                                                                                                                                                                                                                                                                                         \\ \hline
0:05      & Navigating to the Experimenter page.                                                                                                                                                                                                                                                                                             \\ \hline
0:06      & \begin{tabular}[c]{@{}l@{}}Uploading ETD PDF, and selecting summarization and classification\\ models.\end{tabular}                                                                                                                                                                                                                                                        \\ \hline
0:20      & \begin{tabular}[c]{@{}l@{}}Receiving alerts indicating the successful addition of an ETD into the\\ file system, as well as that the workflow has successfully been\\ generated and is now running.\end{tabular}                                                                                                                  \\ \hline
0:24      & Switching over to the back-end to show Team 5's workflow.                                                                                                                                                                                                                                                                         \\ \hline
0:25      & \begin{tabular}[c]{@{}l@{}}We can observe Team 5's 4 APIs here as indicated:\\ Generate Grammar- Generates context-free grammar representation.\\ Generate workflow- Creates workflow for our services.\\ Run workflow- Starts a workflow with ETD ID.\\ Get workflow status- Returns the current status of the workflow.\end{tabular} \\ \hline
0:27      & \begin{tabular}[c]{@{}l@{}}A key is inputted to check the workflow status; \\ we can observe that the process is still running.\end{tabular}                                                                                                                                                                                     \\ \hline
0:39      & Switching back to the front-end.                                                                                                                                                                                                                                                                                                  \\ \hline
0:43      & \begin{tabular}[c]{@{}l@{}}We can observe the output in the form of the chapter title, \\ classification labels, and chapter summary.\end{tabular}                                                                                                                                                                                \\ \hline
0:49      & Receiving an alert indicating the completion of the workflow.                                                                                                                                                                                                                                                                    \\ \hline
\end{tabular}
\end{table}

% \newpage 

% \textbf{team4\_documentView\_demo.mp4:}

\begin{table}[H]
\centering
\caption{\label{tab:vid2}Description of video team4\_documentView\_demo.mp4 }
\begin{tabular}{|l|l|}
\hline
Timestamp & Description                                                                                                                                                                                                                                                         \\ \hline
0:00      & The home page of the UI.                                                                                                                                                                                                                            \\ \hline
0:03      & Searching for a topic.                                                                                                                                                                                                                                              \\ \hline
0:06      & \begin{tabular}[c]{@{}l@{}} Since our ETD subset is from UCB, we have filtered the search result\\ by UCB.\end{tabular}                                                                                              \\ \hline
0:11      & Selecting an ETD.                                                                                                                                                                                                                                                    \\ \hline
0:15      & We can observe the document view page output designed by Team 2.                                                                                                                                                                                                     \\ \hline
0:18      & \begin{tabular}[c]{@{}l@{}}We can observe the results generated by Team 4, in the form of the\\  chapter title, classification labels and chapter summary. \\ It should be noted that this type of output is only generated for the\\ first 5000 ETDs.\end{tabular} \\ \hline
0:34      & \begin{tabular}[c]{@{}l@{}}We navigate back to the Home page and select an ETD with an ID\\ of more than 5000 to show the different generated output.\end{tabular}                                                                                                  \\ \hline
0:43      & \begin{tabular}[c]{@{}l@{}}We can observe that no table is shown for this ETD since the results\\ for ETDs having an ID of more than 5000 are not stored.\end{tabular}                                                                                             \\ \hline
\end{tabular}
\end{table}

\section{Future Work}
We have currently used Team 5's workflow automation as the back-end for our experimenter page. As explained, when a new ETD is uploaded, the workflow automation spins up the required containers sequentially based on the Docker images given by Team 4 and stores the results in the file system and database. The user must wait for a short period of time until the workflow automation is completed and the user interface is populated. This latency can be reduced in the future. Due to a lack of time, we haven't integrated other models into the workflow automation even though the Docker images compiled by us are capable of running multiple models for summarization and classification. We have chosen a default model for summarization and classification by considering trade-offs between the time taken by the individual model and the performance of the model.  The additional integration of models other than default models can be taken up in the future.
% At present, only the summarization model is integrated with the Flask app. 


\chapter{Developer Manual} 
In this section, we describe how to set up, develop, and contribute to our existing services and pipelines. We cover the installation setup, and then for each of the services, we explain how to run the code and train models.
The API endpoints are listed for each of the services. 

\section{Installation Setup}
We have already performed this setup within our containers but for the sake of completeness we describe the setup. Most of our code relies on GPU acceleration, so we would suggest gaining access to a GPU-accelerated container. Then to enable acceleration we would suggest installing Cuda Toolkit and CuDNN. Once that is done, we can check if it is working as expected or not by running "nvidia-smi". On a system level we would like to ensure the following libraries and binaries are installed.

\begin{itemize}
    \item Cuda Toolkit\cite{cuda}
    \item CuDNN\cite{cuda}
    \item Nvidia Drivers
    \item Python 3.8\cite{Python} 
\end{itemize}
Next install the necessary libraries which are the dependencies for each of our services. This includes the following:

\begin{enumerate}
    \item Hugging Face \cite{hugginface}
    \item PyTorch \cite{torch}
    \item Tensorflow-gpu \cite{tensorflow}
    \item scikit-learn \cite{scikit-learn}
    \item Pandas \cite{pandas}
    \item Matplotlib \cite{matplotlib}
    \item Numpy \cite{numpy}
    \item spaCy \cite{spacy}
    \item NLTK \cite{nltk}
    \item PyPDF \cite{pypdf}
    \item PDF Plumber \cite{pdfplumber}
    \item sumy \cite{sumy}
\end{enumerate}

\section{APIs}
We have used the following APIs to access the objects from the database and file system and store the objects produced by us (summary and classification labels) in the database.
% \textbf{Segmentation}
% \begin{enumerate}
%     \item GET: path to the full-text PDF in the full-text database.
%     \item SAVE: save the text/PDF chapters in the database.
% \end{enumerate}
 
\textbf{Summarization}
\begin{enumerate}
    \item Getting Object IDs and Object metadata: \\
    https://team-1-flask.discovery.cs.vt.edu/v1/etds/<etd-id>/objects?type=\\cleaned\_text  
    \item Get the object:
    https://team-1-flask.discovery.cs.vt.edu/v1/objects/<object-id>/\\file
    \item Uploading summary to the database: 
    https://team-1-flask.discovery.cs.vt.edu/v1/\\objects
    /<object-id>/summarisation
\end{enumerate}
\textbf{Classification}
\begin{enumerate}
    \item Getting Object IDs and Object metadata: \\
    https://team-1-flask.discovery.cs.vt.edu/v1/etds/<etd-id>/objects
    ?type=cleaned\_\\text
    \item Get the object:
    https://team-1-flask.discovery.cs.vt.edu/v1/objects/<object-id>/\\file
    \item Uploading classification results to the Database:
    https://team-1-flask.discovery.cs.vt.\\edu/v1/
    objects/<object-id>/classification
\end{enumerate}

\section{Instructions}

\subsection{Classification}
To run the classification code to store classification results in the database, navigate to and clone \url{https://code.vt.edu/ano22/cs5604-team4-classification} GitLab repository and run the Final\_Classification\_Store\_1.ipynb jupyter notebook.


% Currently our classification service is hosted on the cloud.cs server as a Flask API container. If one wishes to check results of a single chapter, it can be directly accessed as an API. The API expects input in the following form: 
% \begin{itemize}
%     \item \textbf{POST:} http://team4.classifyflask.discovery.cs.vt.edu/upload 
%     \begin{itemize}
%         \item Body: Expects form data with a '.pdf' file attached to the key 'file'. 
%         \\
%         Note that we will ingest a clean text ('.txt') file instead of '.pdf' in the future.
%     \end{itemize}
% \end{itemize}

% The API will return a JSON file with the following fields. 
% \begin{itemize}
%     \item \textbf{label}: Array of detected labels
%     \item \textbf{text}: Cleaned text just for reference  
% \end{itemize}

To run our classification model pipelines and training, access the Team 4 folder on cloud.cs and navigate to data/Team4/Classification. Then run the following files with GPU support: 
\begin{itemize}
    % \item Run the train\textunderscore multi-label\textunderscore classifier\textunderscore nogrid\textunderscore lp \textunderscore doc2vec\textunderscore fasttext\textunderscore svm\textunderscore rf.ipynb.
    \item To train the BERT model, run the Deep\textunderscore classifier.ipynb.
    \item To train Longformer model, run the Deep\textunderscore classifier\textunderscore longformer.ipynb.
    \item To train SciBERT, run the BERT\textunderscore based\textunderscore chapter\textunderscore classifier\textunderscore SciBERT.ipynb.
    \item To gather inferences of BERT and SciBERT models, run test inference.ipynb
    \item While running test inference.ipynb, give the correct chapter path, choose the right model type -- either bert-base-uncased or allenai/scibert\_scivocab\_uncased (SciBERT) -- and choose the right model weights file:
    \begin{itemize}
    \item /mnt/data/Team4/Classification/saved\_models/new\_chapter\_bert\_5.pt --\\ for BERT-base trained on chapter text
    
    \item /mnt/data/Team4/Classification/saved\_models/chapter\_scibert\_8\_final\_\\weights \_chapter.pt -- for SciBERT trained on chapter text
    
    \item /mnt/data/Team4/Classification/saved\_ models/new\_Summary\_bert\_5.pt --\\ for BERT trained on summaries
    
    \item /mnt/data/Team4/Classification/saved\_ models/new\_summary\_scibert\_8.pt -- for SciBERT trained on summaries
    \end{itemize}
\end{itemize}

 We have also containerized the classification service which can be pulled by a developer to run the models and code in a pre-defined environment. 
 The name of our image is \textbf{dexuiz/classification:0.0.3}.
 This image contains our latest code and models for classification. We can now follow the steps below to get a running model. 
\begin{itemize}
    \item First we need to pull the image from the Docker repository. 
    \begin{itemize}
        \item \texttt{docker pull dexuiz/classification:0.0.3}
    \end{itemize}
    \item Now we want to run this image.
    The command below runs the container and starts an interactive container with bash access.
    If you wish to attach a volume to this container you can use the flag -v, specifically add the following to the command below \texttt{-v PATH\_TO\_FOLDER:/data}.
    This will mount the folder specified in the local machine to the folder /data in the Docker container. 
    \begin{itemize}
        \item \texttt{docker run --rm -it dexuiz/classification bash}
    \end{itemize}
    \item Once the above command successfully executes we will have a bash command line within the Docker container. Now we can run the classification script within the container. In the command below for the flag \texttt{--input-path} you need to provide the path to the input file which can be either a cleaned text file or a PDF file. The default location for the output is /out but if you need to change that folder you can provide that using the \texttt{--output-path} flag. 
    \begin{itemize}
        \item \texttt{python3 classifier.py --input-path ./data/chap.txt --output-path\\ ./out}
    \end{itemize}
    \item Now the script will compute the labels for this file and place the output JSON in the provided output folder. 
\end{itemize}

% \begin{figure}[H]
% 	    \centering
% 	    \fbox{\includegraphics[width=15cm]{Team_4_Classification_Workspace.jpg}}
% 	    \caption{Classification Files of Team-4}
% 	    \label{fig:screenshot1}
% \end{figure}

\subsection{Summarization}
% Currently the summarization service is  hosted on cloud.cs server as a Flask API container and can be used to generate summaries for a document. The API expects the input in the following format: 

% \begin{itemize}
%     \item \textbf{POST:} http://team4.summflask.discovery.cs.vt.edu/upload 
%     \begin{itemize}
%         \item Body: Expects form data with a ".pdf" file attached to the key "file". 
%         \\
%         Note that we will ingest a clean text (".txt") file instead of ".pdf" in the future.
%     \end{itemize}
% \end{itemize}

% The API will return a JSON file with the following fields. 
% \begin{itemize}
%     \item \textbf{summarization}: Text summary
%     \item \textbf{model\_used}: the model used to generate the summary. 
% \end{itemize}

To run the summarization code directly on your machine, navigate to and clone \url{https://code.vt.edu/harishbabu/cs5604-f22-team-4/-/tree/Summarization} GitLab repository.
% \\
% \\
Running instructions are:
\begin{itemize}
    \item Add required parameter values (model, summary max length, etc.) to the internal\_config.json (Figure \ref{fig:internalconfig}). All the required parameters and hyper-parameter values need to be specified in this JSON file.
    \item Run >> python summarization.py \texttt{-{}-}path <path\_to\_chapter\_text> \texttt{-{}-}path : mandatory argument
\end{itemize}

\begin{figure}[H]
	    \centering
	    \includegraphics[width=15cm]{internalconfig.png}
	    \caption{Summarization Internal Config}
	    \label{fig:internalconfig}
\end{figure}

Similar to the Classification service we have also containerized the summarization service in DockerHub.
In the following section we have included instructions to quickly set it up and run it. 

\begin{itemize}
    \item First we need to pull the image from the Docker repository. 
    \begin{itemize}
        \item \texttt{docker pull dharneeshkar/summarization:0.9}
    \end{itemize}
    \item Now we want to run this image; we can do that using the command provided below. 
    If you wish to attach a volume to this container you can use the flag -v, specifically add the following to the command below \texttt{-v PATH\_TO\_FOLDER:/data}.
    This will mount the folder specified in the local machine to the folder /data in the container.
    \begin{itemize}
        \item \texttt{docker run --rm -it dharneeshkar/summarization:0.9 bash}
    \end{itemize}
    \item Once the above command successfully executes we will have a bash command line within the Docker container. Now we can run the classification script within the container. In the command below for the flag \texttt{--input-path} you need to provide the path to the input file which has to be a text file containing cleaned text.
    The default location for the output is /out but if you need to change that folder you can provide that using the \textttP{--output-path} flag. 
    \begin{itemize}
        \item \texttt{ python summarization.py --input-path ./data/ --output-path ./out}
    \end{itemize}
    \item Now the script will generate a summary for this file and place the output JSON file in the provided output folder. 
\end{itemize}
\subsection{Summarization API Driver}
The summarization driver script can be found here: \url{https://code.vt.edu/harishbabu/cs5604-f22-team-4/-/tree/Summarization/driver.py}.
The driver script encapsulates all of the API calls to run the summarization pipeline: fetch cleaned chapter text, summarize, and save in the database. The working of the driver can be summarized as follows:
\begin{enumerate}
    \item Read ETD IDs from text file containing ETD IDs
    \item For each ETD ID get the list of object IDs using the get\_obj\_by\_etd\_id API
    \item For each object ID fetch the object (in this case, chapter text) using get\_obj\_file\_by\_\\obj\_id API
    \item For each object/file fetched, perform summarization and save the summary JSON into the database using the save\_summary API
\end{enumerate}

To run the driver script, use: 
\\
>> python driver.py
\\
\textbf{Note}: Keep the etd\_id.txt file in the same directory as the driver script
\subsection{Segmentation}

We developed a segmentation service to split an entire ETD into chapters. The code for the service is also available at \url{https://code.vt.edu/ano22/cs5604-team-4-segmentation}. 
You can run it following the instructions specified in the readme file. 

Apart from using the code directly, you can also use it as a container as we have containerized the segmentation service. In the following section, we have included instructions to quickly set it up and run it. 

\begin{itemize}
    \item First we need to pull the image from the Docker repository. 
    \begin{itemize}
        \item \texttt{docker pull dharneeshkar/segmentation:0.4}
    \end{itemize}
    \item Now we want to run this Docker image.
    We can do that using the command provided below.
    If you wish to attach a volume to this container you can use the flag -v, specifically add the following to the command below \texttt{-v PATH\_TO\_FOLDER:/data}.
    This will mount the folder specified in the local machine to the folder /data in the container.
    \begin{itemize}
        \item \texttt{docker run --rm -it dharneeshkar/segmentation:0.4 bash}
    \end{itemize}
    \item Once the above command successfully executes we will have a bash command line within the Docker container. Now we can run the classification script within the container. In the command below for the flag \texttt{--input-path} you need to provide the path to the input file which has to be a PDF file containing the entire ETD.
    The default location for the output is /out but if you need to change that folder you can provide that using the \textttP{--output-path} flag. 
    \begin{itemize}
        \item \texttt{ python segmentation.py --input-path ./data/ --output-path ./out}
    \end{itemize}
    \item Now the script will read the entire ETD and segment it into smaller chapters.
    Once that is done it will write each chapter back as PDFs in the output folder. So, after completion, the output folder will be populated with chapter PDFs. 
\end{itemize}

\subsection{Parsing and Cleaning}
The Summarization and Classification services require cleaned text input to properly generate their respective outputs, so to prevent the repetition of code  we decided to create a parse and clean service. This service reads a chapter PDF after it has been segmented and then reads all the text present in it after filtering all the tables and images. The final output of this service is a chapter-cleaned text that can be used in downstream tasks.

To run the parsing and cleaning code directly on your machine, navigate to and clone \url{https://code.vt.edu/harishbabu/cs5604-f22-team-4/-/tree/Summarization} GitLab repository.

Apart from running the code directly, we have also deployed it as a container similar to other services so you can directly pull that and start using the parse and clean service directly. The following instructions can be followed to do that. 

\begin{itemize}
    \item First we need to pull the image from the Docker repository. 
    \begin{itemize}
        \item \texttt{docker pull dharneeshkar/parse\_and\_clean:0.1}
    \end{itemize}
    \item Now we want to run this image.
    We can do that using the command provided below.
    If you wish to attach a volume to this container you can use the flag -v, specifically add the following to the command below \texttt{-v PATH\_TO\_FOLDER:/data}.
    This will mount the folder specified in the local machine to the folder /data in the container.
    \begin{itemize}
        \item \texttt{docker run --rm -it dharneeshkar/parse\_and\_clean:0.1 bash}
    \end{itemize}
    \item Once the above command successfully executes we will have a bash command line within the Docker container. Now we can run the classification script within the container. In the command below for the flag \texttt{--input-path} you need to provide the path to a folder that contains all the PDF file chapters of the ETD that have to be cleaned. The default location for the output is /out but if you need to change that folder you can provide that using the\textttP{--output-path} flag. 
    \begin{itemize}
        \item \texttt{ python parse\_and\_clean.py --input-path ./data/chapters\\ --output-path ./out}
    \end{itemize}
    \item Now the script will read the folder, which will have a set of chapters in it, and then create the cleaned text versions of these chapters, and write them to the output folder. 
\end{itemize}

\section{WorkFlow Automation APIs}

In collaboration with Team 5, we performed workflow automation to generate a flow of processes that will be completed automatically once we obtain a new ETD. This workflow is controlled by a few APIs which have been described below:

\begin{itemize}
    \item \textbf{Generate Workflow: } First, you need to run Generate Workflow to create a workflow for the services. Input to the API is the ETD ID of the file that has to be processed. Along with this, in the body of the request, you need to add the environment of the services. An example of the body and the URL has been included below. Once you run this script with the correct environment file you will get a workflow ID in return and that can be used in the next stage. 
    \begin{itemize}
        \item \textbf{URL:} https://reasoner.discovery.cs.vt.edu/generateWorkflow/:workflowId \\\\
        
        \item \textbf{Body:}\begin{lstlisting}
'{
    "env":[
       {
           "service_id":"all",
           "service_env":[
            "ETDID={{etd_id}}"
            ]
       }
       ]
}'
            \end{lstlisting}
    \end{itemize}
    \item \textbf{Run Workflow:} After running the previous command you would obtain a workflow ID and you can use that to start the workflow.
    The command for that has been provided below. Once this command completes you will obtain a workflow key and you can use that to run the API provided below to look at the status of the request. 
    \begin{itemize}
        \item\textbf{URL: }https://team2020-airflow.discovery.cs.vt.edu/workflow/run
        \item\textbf{Body:}  \begin{lstlisting}
{
    "args":["{{workflowId}}","2020-10-10"], 
    "service_names":[
        "curl-get-etd", "segmentation", "curl-save-chapters"
        ]
}
        \end{lstlisting}
    \end{itemize}

    \item \textbf{Get Workflow status:} To monitor the status of the workflow and see how far along is it, you can use the get workflow status API. The API URL requires a workflow key that would have been generated by the previous API. This API will return the logs of the workflow that can used for monitoring.
    \begin{itemize}
        \item \textbf{URL: }http://team2020-airflow.discovery.cs.vt.edu/workflow/run?key={{key}}            
    \end{itemize}
\end{itemize}
% % % % % % 

	% This is the standard bibtex file. Do not include the .bib extension in <bib_file_name>.
	% Uncomment the following lines to include your bibliography: 
	%\bibliography{<bib_file_name>}
	%\bibliographystyle{plainnat}   
	

\begin{thebibliography}{}
\bibitem{SciCite}
Cohan, Arman,  Waleed Ammar, Madeleine van Zuylen, Field Cady. “Structural Scaffolds for Citation Intent Classification in Scientific Publications.” Proceedings of the 2019 Conference of the North, Association for Computational Linguistics, 2019, pp. 3586–96. https://doi.org/10.18653/v1/N19-1361.

\bibitem{longtextclassification}
Hyunji Park, Yogarshi Vyas, and Kashif Shah. 2022. "Efficient Classification of Long Documents Using Transformers." In Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers), pages 702–709, Dublin, Ireland. Association for Computational Linguistics.

\bibitem{ETDs_Last_Year}
Sami Uddin, Bipasha Banerjee, Jian Wu, William A. Ingram, Edward A. Fox. "Building A Large Collection of Multi-domain Electronic Theses and Dissertations".  2021 IEEE International Conference on Big Data.

\bibitem{captor}
Pethe, Charuta, Allen Kim, and Steven Skiena. "Chapter Captor: Text Segmentation in Novels." arXiv preprint arXiv:2011.04163 (2020).

\bibitem{word2vec}
Mikolov, T., Sutskever, I., Chen, K., Corrado, G. S., & Dean, J. (2013). "Distributed Representations of Words and Phrases and their Compositionality." Advances in neural information processing systems, 26.

\bibitem{glove}
Pennington, Jeffrey, Richard Socher, and Christopher D. Manning. "GloVe: Global Vectors for Word Representation." In Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP), pp. 1532-1543. 2014.

\bibitem{kd}
Hinton, Geoffrey, Oriol Vinyals, and Jeff Dean. "Distilling the Knowledge in a Neural Network." arXiv preprint arXiv:1503.02531 2, no. 7 (2015).

\bibitem{hierarchial}
Pappagari, Raghavendra, Piotr Zelasko, Jesús Villalba, Yishay Carmiel, and Najim Dehak. "Hierarchical Transformers for Long Document Classification." In 2019 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU), pp. 838-844. IEEE, 2019.

\bibitem{bert}
Devlin, Jacob, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding." arXiv preprint arXiv:1810.04805 (2018).

\bibitem{longformer}
Beltagy, Iz, Matthew E. Peters, and Arman Cohan. "LongFormer: The Long-Document Transformer." arXiv preprint arXiv:2004.05150 (2020).

\bibitem{bigbird}
Zaheer, Manzil, Guru Guruganesh, Kumar Avinava Dubey, Joshua Ainslie, Chris Alberti, Santiago Ontanon, Philip Pham et al. "Big Bird: Transformers for Longer Sequences." Advances in Neural Information Processing Systems 33 (2020): 17283-17297.

\bibitem{doc2vec}
Le, Quoc, and Tomas Mikolov. "Distributed Representations of Sentences and Documents." In International conference on machine learning, pp. 1188-1196. PMLR, 2014.

\bibitem{distillbert}
Sanh, Victor, Lysandre Debut, Julien Chaumond, and Thomas Wolf. "DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter." arXiv preprint arXiv:1910.01108 (2019).

\bibitem{attention}
Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, Ł. and Polosukhin, I., 2017. "Attention Is All You Need". Advances in neural information processing systems, 30.

\bibitem{autotext}
Torres-Moreno, Juan-Manuel, ed. "Automatic Text Summarization". John Wiley \& Sons, 2014.

\bibitem{introsummary}
Jason Brownlee. "A Gentle Introduction to Text Summarization". 2017. \url{https://machinelearningmastery.com/gentle-introduction-text-summarization}. Accessed on December 3, 2022.

\bibitem{seq2seqsummary}
Nallapati, Ramesh, Bowen Zhou, Caglar Gulcehre, and Bing Xiang. \say{Abstractive Text Summarization Using Sequence-to-Sequence RNNs and Beyond.} arXiv preprint arXiv:1602.06023 (2016).

\bibitem{pointersummary}
See, Abigail, Peter J. Liu, and Christopher D. Manning. \say{Get To The Point: Summarization with Pointer-Generator Networks} arXiv preprint arXiv:1704.04368 (2017).

\bibitem{etdsummary}
Ingram, William A., Bipasha Banerjee, and Edward A. Fox. \say{Summarizing ETDs with deep learning.} Cadernos BAD 1 (2019): 46-52.

\bibitem{textrank}
Mihalcea, Rada, and Paul Tarau. \say{TextRank: Bringing Order into Texts.} In Proceedings of the 2004 conference on empirical methods in natural language processing, pp. 404-411. 2004.

\bibitem{lexrank}
Erkan, Günes, and Dragomir R. Radev. \say{LexRank: Graph-based Lexical Centrality as Salience in Text Summarization.} Journal of artificial intelligence research 22 (2004): 457-479.  

\bibitem{luhn}
Luhn, Hans Peter. \say{The Automatic Creation of Literature Abstracts.} IBM Journal of research and development 2, no. 2 (1958): 159-165.

\bibitem{lsa}
Dumais, Susan T. \say{Latent semantic analysis.} Annu. Rev. Inf. Sci. Technol. 38, no. 1 (2004): 188-230.

\bibitem{t5}
Raffel, Colin, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J. Liu. \say{Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer.} J. Mach. Learn. Res. 21, no. 140 (2020): 1-67.

\bibitem{bart}
Lewis, Mike, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov, and Luke Zettlemoyer. \say{BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension.} arXiv preprint arXiv:1910.13461 (2019).

\bibitem{Palakh}
Palakh Mignonne Jude. "Increasing Accessibility of Electronic Theses and Dissertations (ETDs) Through Chapter-level Classification". June 2020, MS thesis, Computer Science, Virginia Tech, Blacksburg, VA 24061, http://hdl.handle.net/10919/99294.

\bibitem{torch}
Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas Kopf, Edward Yang, Zachary DeVito, Martin Raison, Alykhan Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu Fang, Junjie Bai, Soumith Chintala. "PyTorch: An Imperative Style, High-Performance Deep Learning Library." Advances in neural information processing systems 32 (2019).

\bibitem{hugginface}
Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pierric Cistac, Tim Rault, Rémi Louf, Morgan Funtowicz, Joe Davison, Sam Shleifer, Patrick von Platen, Clara Ma, Yacine Jernite, Julien Plu, Canwen Xu, Teven Le Scao, Sylvain Gugger, Mariama Drame, Quentin Lhoest, Alexander M. Rush. "HuggingFace's Transformers: State-of-the-art Natural Language Processing." arXiv preprint arXiv:1910.03771 (2019).

\bibitem{Javaid}
Javaid Akbar Manzoor. "Segmenting Electronic Theses and Dissertations By Chapters." Virginia Tech, Computer Science, MS thesis defended September 23, 2022, Blacksburg, VA 24061 USA, available late 2022 from \url{https://vtechworks.lib.vt.edu}.

\bibitem{arxiv}
Cornell University. "arXiv.org e-Print archive". 2022. Retrieved December 3, 2022, from \url{https://arxiv.org}.

\bibitem{Multilabel}
Asma Aldrees and Azeddine Chikh and Jawad Berri. "Comparative Evaluation Of Four Multi-label Classification Algorithms In Classifying Learning Objects", 2016, Computer Science \& Information Technology, 10.5121/csit.2016.60210.

\bibitem{SciBERT}
Iz Beltagy and Arman Cohan and Kyle Lo. "SciBERT: Pretrained Contextualized Embeddings for Scientific Text". 2019, CoRR, \url{http://arxiv.org/abs/1903.10676}.

\bibitem{rouge}
Chin-Yew Lin. 2004. "ROUGE: A Package for Automatic Evaluation of Summaries." In Text Summarization Branches Out, pages 74–81, Barcelona, Spain. Association for Computational Linguistics.

\bibitem{bleu}
Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. "BLEU: a method for automatic evaluation of machine translation." In Proceedings of the 40th Annual Meeting on Association for Computational Linguistics (ACL '02). Association for Computational Linguistics, USA, 311–318. \url{https://doi.org/10.3115/1073083.1073135}.

\bibitem{bertscore}
Zhang, Tianyi, Varsha Kishore, Felix Wu, Kilian Q. Weinberger, and Yoav Artzi. "BERTScore: Evaluating Text Generation with BERT." arXiv preprint arXiv:1904.09675 (2019).

\bibitem{cs.vt.edu}
Virginia Tech. "Computer Science Cloud at VT". Retrieved December 3, 2022, from  \url{https://cloud.cs.vt.edu/}.

\bibitem{docker}
Merkel, D. (2014). "Docker: lightweight Linux containers for consistent development and deployment". Linux Journal, 2014(239).

\bibitem{vt_git}
Virginia Tech. "Virginia Tech GitLab". 2022. Retrieved December 3, 2022, from \url{https://code.vt.edu/}.

\bibitem{sumy}
Mišo Belica. "Automatic Text Summarizer". 2022. Retrieved December 3, 2022, from \url{https://github.com/miso-belica/sumy}.

\bibitem{pdfplumber}
Hailei. "PDF Plumber Library Documentation". 2020. Retrieved December 3, 2022, from \url{https://gitee.com/hailei\_yan/pdfplumber}.

\bibitem{Python}
Van Rossum, G., & Drake, F. L. (2009). "Python 3 Reference Manual". Scotts Valley, CA: CreateSpace.

\bibitem{cuda}
Vingelmann, P., & Fitzek, F. H. P. (2020). "CUDA release: 10.2.89". Retrieved December 3, 2022, from \url{https://developer.nvidia.com/cuda-toolkit}.

\bibitem{Requests}
Kenneth Reitz, Cory Benfield, Ian Stapleton Cordasco, Nate Prewitt et al. "Requests Library Documentation. Requests". 2022. Retrieved December 3, 2022, from \url{https://requests.readthedocs.io/en/latest/}. 

\bibitem{tensorflow}
Martín Abadi, Ashish Agarwal, Paul Barham, Eugene Brevdo,
Zhifeng Chen, Craig Citro, Greg S. Corrado, Andy Davis,
Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Ian Goodfellow,
Andrew Harp, Geoffrey Irving, Michael Isard, Rafal Jozefowicz, Yangqing Jia,
Lukasz Kaiser, Manjunath Kudlur, Josh Levenberg, Dan Mané, Mike Schuster,
Rajat Monga, Sherry Moore, Derek Murray, Chris Olah, Jonathon Shlens,
Benoit Steiner, Ilya Sutskever, Kunal Talwar, Paul Tucker,
Vincent Vanhoucke, Vijay Vasudevan, Fernanda Viégas,
Oriol Vinyals, Pete Warden, Martin Wattenberg, Martin Wicke,
Yuan Yu, and Xiaoqiang Zheng.
"TensorFlow: Large-scale machine learning on heterogeneous systems",
2015. Software available from tensorflow.org.

\bibitem{scikit-learn}
Pedregosa, F., Varoquaux, Gael, Gramfort, A., Michel, V., Thirion, B., Grisel, O., et al. (2011). "Scikit-learn: Machine learning in Python. Journal of Machine Learning Research", 12(Oct), 2825–2830.

\bibitem{pandas}
McKinney, Wes. "Data Structures for Statistical Computing in Python." Proceedings of the 9th Python in Science Conference. Vol. 445. No. 1. 2010.

\bibitem{matplotlib}
J. D. Hunter, "Matplotlib: A 2D Graphics Environment" in Computing in Science & Engineering, vol. 9, no. 3, pp. 90-95, May-June 2007, doi: 10.1109/MCSE.2007.55.

\bibitem{numpy}
Harris, C. R., Millman, K. J., Van Der Walt, S. J., Gommers, R., Virtanen, P., Cournapeau, D., \& Oliphant, T. E. (2020). "Array programming with NumPy". Nature, 585(7825), 357-362.

\bibitem{spacy}
Honnibal, Matthew, and Ines Montani. "spaCy 2: Natural language understanding with Bloom embeddings, convolutional neural networks and incremental parsing." 
January 2017, https://sentometrics-research.com/publication/72/
%To appear 7, no. 1 (2017): 411-420.

\bibitem{nltk}
Bird, S., Klein, E.,\& Loper, E. (2009). "Natural language processing with Python: analyzing text with the natural language toolkit". Reilly Media, Inc.

\bibitem{pypdf}
Mathieu Fenniak. "pyPdf 1.13". 2010. Retrieved December 3, 2022, from \url{https://pypi.org/project/pyPdf/}.


\bibitem{semanticscholar}
Allen Institute for Artificial Intelligence. "Semantic Scholar". 2022. Retrieved December 12, 2022, from \url{https://www.semanticscholar.org/}.

\bibitem{biobert}
Lee, J., Yoon, W., Kim, S., Kim, D., Kim, S., So, C. H., & Kang, J. (2019). "BioBERT: a pre-trained biomedical language representation model for biomedical text mining". arXiv. https://doi.org/10.1093/bioinformatics/btz682.

\bibitem{booksum}
Kryściński, W., Rajani, N., Agarwal, D., Xiong, C., & Radev, D. (2021). "BookSum: A Collection of Datasets for Long-form Narrative Summarization". arXiv. https://doi.org/10.48550/arXiv.2105.08209.

\bibitem{react}
Jordan Walke. "React A JavaScript library for building user interfaces". 2022. Retrieved December 13, 2022, from \url{https://reactjs.org/}.

\bibitem{prompt}
Lester, Brian, Al-Rfou, Rami, and Constant, Noah. (2021). "The Power of Scale for Parameter-Efficient Prompt Tuning". "https://aclanthology.org/2021.emnlp-main.243". "10.18653/v1/2021.emnlp-main.243". 

\end{thebibliography}{}
\end{document}

