import os
from config import *
import PyPDF2

dirname = "/mnt/ceph/cme/dissertation"

def getAllPDFFilesPath(base_path):
    pdfFiles = []
    for root, dirs, files in os.walk(base_path):
        for dir_ in dirs:
            for fname in os.listdir(os.path.join(base_path,dir_)):
                path = os.path.join(base_path,dir_)
                if (os.path.splitext(fname)[-1] == ".pdf"):
                    pdfFiles.append(os.path.join(path, fname))
        for file in files:
            if 'pdf' in file:
                pdfFiles.append(os.path.join(NEW_ETDs_UPLOAD_PATH, file))
    return pdfFiles

def extractTextForAllPDFs(pdfFilePaths, etdTextPath):
    for pdf in pdfFilePaths:
        pdfFileObj = open(pdf, 'rb')
        print("Processing:"+str(pdf))
        pdfReader = PyPDF2.PdfFileReader(pdfFileObj)
        text = ""
        count = 0
        num_page = pdfReader.numPages
        while count<num_page:
            pageObj = pdfReader.getPage(count)
            count = count + 1
            text = text + pageObj.extractText()
        dir_ = str(pdf).split("/")[-2]
        fname = str(pdf).split("/")[-1]
        folder = os.path.exists(etdTextPath + '/' + dir_)
        if not folder:              
            os.makedirs(etdTextPath + '/' + dir_)  
        text_file = open(etdTextPath + '/' + dir_+"/"+fname+".txt", "w",encoding='utf-8')
        text_file.write(text)
        text_file.close()
        pdfFileObj.close()

def extractTexts(new_etds_upload_path, etd_text_path):
    pdfFilePaths = getAllPDFFilesPath(new_etds_upload_path)
    extractTextForAllPDFs(pdfFilePaths, etd_text_path)

if __name__ == "__main__":
        extractTexts(NEW_ETDs_UPLOAD_PATH, ETD_TEXT_PATH)


