#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import os
import re
import string
import nltk
import io
import csv

def clean(inputDir, outputFile):

    # print("Cleaning "+path)
    from nltk.corpus import stopwords
    from nltk.stem import PorterStemmer
    from nltk.tokenize import sent_tokenize, word_tokenize
    ps = PorterStemmer()
    #stop_words = set(stopwords.words('english'))
    stop_words = nltk.corpus.stopwords.words('english')
    #stop_words.append('heart')
    #stop_words.append('attack')
    #print(stop_words)
    #stop_words.append('heart attack')
    WRITE_HANDLER = open(outputFile, 'w')
    tweets = dict()
    with open('./HeartAttack_7000_Uncleaned.csv',encoding="ISO-8859-1") as csvfile:
            readCSV=csv.reader(csvfile,delimiter=',')
            for row in readCSV:
                line=row
                #line= yield [unicode(cell, 'ISO-8859-1') for cell in row]
                #print(line)
                line='\n'.join(line)
                #line=' '.join(line)
                
                line = re.sub(r'[.,"!*;]+', '', line, flags=re.MULTILINE)  # removes the characters specified
                line = re.sub('&lt;', '', line, flags=re.MULTILINE)  # removes &lt;
                line = re.sub('RT', '', line, flags=re.MULTILINE)  # removes RT
                line = re.sub('&gt;', '', line, flags=re.MULTILINE)  # removes &gt
                line = re.sub('OMG', 'oh my god', line, flags=re.MULTILINE)  # repair OMG to oh my god
                line = re.sub('omg', 'oh my god', line, flags=re.MULTILINE)  # repair omg to oh my god
                line = re.sub('-__-', '', line, flags=re.MULTILINE)  # remove -__-
                line = re.sub('Omgggg', 'oh my god', line, flags=re.MULTILINE)  # repair Omgggg to oh my god
                line = re.sub(r'https?://[^ ]+','',line, flags=re.MULTILINE)  # remove link (http)
                line = re.sub(r'www.[^ ]+','',line, flags=re.MULTILINE)  # remove link (WWW)
                line = re.sub(r'[:]+', '', line, flags=re.MULTILINE)       
                line = list(filter(lambda x: x in string.printable, line)) # filter non-ascii characers
                new_line = ''
                line = ''.join(line)
                line=' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",line).split())
        
                for i in line.split():  # remove @ and #words, punctuataion
                    if not i.startswith('@') and not i.startswith('#') and i not in string.punctuation:
                        new_line += i + ' '
                line = new_line
        
       
                # # Do sentence correction
                Tline=''
                words =line.split()
                #print(stopwords)
                for r in words:
                  r=r.lower()
                  if not r in stop_words:
                    Tline=Tline+' '+r
                    #print(Tline)
                new_line = Tline
                #print(new_line)
                new_line = word_tokenize(new_line)
                sline=''
                for w in new_line:
                    sline=sline+' '+ps.stem(w)
                    #print(w + ":" + ps.stem(w))
                #print(sline)    
                WRITE_HANDLER.write(sline + '\n')
            return outputFile

DATA_FOLDER ='C:/Users/User1/Desktop/HeartAttack/Code/RCode/Proj05_22/Digital Libraries'
CLEANED_DATA = 'C:/Users/User1/Desktop/HeartAttack/Code/RCode/Proj05_22/Digital Libraries/HeartAttack_7000_Clean.txt'
clean(DATA_FOLDER, CLEANED_DATA)

