import spacy
import json

print('spaCy: %s' % (spacy.__version__))
spacy_nlp = spacy.load('en_core_web_sm')

from pathlib import Path
import sys

# mypath = "/home/ri/Desktop/CEPH_data"
mypath="tobacco\mnt\ceph\shared\\tobacco\data\\1million_raw"

source_dir = Path(mypath)
# print(source_dir)

files = source_dir.iterdir()
# print(files)

#CReate a directory called 'NER' in the location where this python script is saved. All
#the text files containing ner results are saved in this directory
import os

with os.scandir(mypath) as root_dir:
    for path in root_dir:
        f=open(path, 'r')
        nerlist=[]
        file_handle=f.readlines()
        for line in file_handle:

            document = spacy_nlp(line)
            for element in document.ents:
                nerlist.append((element.label_, element))

        respath="NER/"+path.name+".txt"

        f2= open(respath,"w+")
        f2.write(str(nerlist))
        f2.close()
        f.close()

