#!/usr/bin/env python
# coding: utf-8
from config import *
import json
import glob
import os


path = '/mnt/ceph/cme/'
file1 = METADATA_JSON_FILE_PATH
file2 = NEW_METADATA_JSON_FILE_PATH

info = []
file = open(file1, "r", encoding="utf8")
for line in file:
    jsonObject = json.loads(line)
    info.append(jsonObject)

file.close()

with open(path + file2, 'w', encoding='utf-8') as outfile:
    json.dump(info, outfile, sort_keys=False, indent=4)


with open(path + file2) as f:
    oldJson = json.load(f)


finalData = []
for metadata in oldJson:
    try:
        url = path + "txt_etd/txt_" + metadata['type-none'].strip().lower() + "/" + metadata['handle'].strip()
        filename = os.path.basename(glob.glob(url + '/*.txt', recursive=True)[0])
        file = open(url + "/" + filename, "r", encoding="utf8")
        text = file.read()
        file.close()
        metadata['text'] = text
        finalData.append(metadata)
    except:
        finalData.append(metadata)


with open(path + file2, 'w', encoding='utf-8') as outfile:
    json.dump(finalData, outfile, sort_keys=False, indent=4)

def getAllPDFFilesPath(base_path):
    pdfFiles = []
    for root, dirs, files in os.walk(base_path):
        for dir_ in dirs:
            for fname in os.listdir(os.path.join(base_path,dir_)):
                path = os.path.join(base_path,dir_)
                if (os.path.splitext(fname)[-1] == ".pdf"):
                    pdfFiles.append(os.path.join(path, fname))
        for file in files:
            if 'pdf' in file:
                pdfFiles.append(os.path.join(NEW_ETDs_UPLOAD_PATH, file))
    return pdfFiles

def addAllTextToMetadata():
    print("Yes") #Implement several small methods and this would be the driver method

