preprocessing.py

# -*- coding: utf-8 -*-
"""Preprocessing.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1070MYQzdGOBcouLpNZTROFGtLxdP1Qq9

# Data-Cleaning Code

# imports
"""

!pip install langdetect
!pip install -U textblob
!pip install -U textblob-fr
!python -m textblob.download_corpora

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
from langdetect import DetectorFactory, detect, detect_langs
from bs4 import BeautifulSoup
import re
import datetime as datetime
from textblob import TextBlob
from textblob_fr import PatternTagger, PatternAnalyzer

text = u"Quelle belle matinée"
blob = TextBlob(text, pos_tagger=PatternTagger(), analyzer=PatternAnalyzer())

df = pd.read_csv('/content/Sheet - Sheet5.csv')
#df=df.set_axis([*df.columns[:-1], 'Reply'], axis=1, inplace=False)

df.head()

# df['Corpus'] = df['Corpus'].replace({r'\s+$': '', r'^\s+': ''}, regex=True).replace(r'\n',  ' ', regex=True)
# df['Corpus'] = df['Corpus'].replace('\r',  ' ')



# import urllib.request
# from html.parser import HTMLParser
# import re
# page = urllib.request.urlopen('http://netherkingdom.netai.net/pycake.html')
# t = page.read()
# class MyHTMLParser(HTMLParser):
#     def handle_data(self, data):
#         print(data)
#         f = open('/Users/austinhitt/Desktop/Test.py', 'r')
#         t = f.read()
#         f = open('/Users/austinhitt/Desktop/Test.py', 'w')
#         f.write(t + '\n' + data)
#         f.close()
# parser = MyHTMLParser()
# t = t.decode()
# parser.feed(t)

df['Corpus'].head()

"""# Seperate code to show how I have seperated the comments for Ayush and Barkha"""

# df1= df.loc[df['Comment'].str.contains("Ayush", case=False)]

#df1.to_csv('Ayush_E6.csv',index=False)

#df2 = df.loc[df['Comment'].str.contains("Barkha", case=False)]

#df2.to_csv('Barkha_E6.csv', index=False)

#df.sample(5)

"""# preprocess"""

#df_copy = df.copy()
#df_copy['Comment'] = [BeautifulSoup(text).get_text() for text in df_copy['Comment']]#remove HTML tags
#df_copy['Comment'] = df_copy['Comment'].apply(lambda x: re.split(' "https:\/\/.*', str(x))[0])#remove HTTP

df["Corpus"] = df["Corpus"].str.lower()

"""##### Clean data creation """

# # lists for the data columns 
# Name=[]
# Comment=[]
# Time=[]
# Likes=[]
# Reply=[]
# Time=[]
# # list for checking error in detect Library
# done=[]
# Notedone=[]
# #loop for the new data
# for i in range(len(df_copy)):
#     try:
#         if detect(df_copy['Comment'][i]) == 'en':
#             Comment.append(df_copy['Comment'][i])
#             done.append(i)#check
#             #others appended
#             Name.append(df_copy['Name'][i])
#             Time.append(df_copy['Time'][i])
#             Likes.append(df_copy['Likes'][i])
#             Reply.append(df_copy['Reply'][i])
#             Time.append(df_copy['Time'][i])
#     except:
#         Notedone.append(i)#check

# print(len(Name),len(Comment),len(Time),len(Likes),len(Reply))

# print(len(Notedone),'text languages is not getting detected and gave and error')

# df_cl = pd.DataFrame(list(zip(Name,Comment,Time,Likes,Reply)),columns =['Names','Comment','Time','Likes','Reply_count'])
# #df_cl.to_csv('clean_data.csv',index=False)#save
# #df_cl

# temp= pd.DataFrame(list(Comment),columns=['Comment'])
# temp1= pd.DataFrame(list(Likes),columns=['Likes'])
# temp2= pd.DataFrame(list(Name),columns=['Name'])
# temp3= pd.DataFrame(list(Reply),columns=['Reply_Count'])
# temp4 = pd.DataFrame(list(Time),columns=['Time'])

# temp['Comment'] = temp.astype(str).apply(lambda x: x.str.encode('ascii', 'ignore').str.decode('ascii'))

#removing special characters from comments
spec_chars = ["!",'"',"#","%","&","'","(",")",
              "*","+",",","-",".","/",":",";","<",
              "=",">","?","@","[","\\","]","^","_",
              "`","{","|","}","~","–"]
for char in spec_chars:
    df['Corpus'] = df['Corpus'].str.replace(char, ' ')

# df['Pos answer'] = df['Pos answer'].str.split().str.join(" ")

df.head()

#removing emojis
temp2['Name'] = temp2.astype(str).apply(lambda x: x.str.encode('ascii', 'ignore').str.decode('ascii'))

# #removing special characters from name
# spec_chars = ["!",'"',"#","%","&","'","(",")",
#               "*","+",",","-",".","/",":",";","<",
#               "=",">","?","@","[","\\","]","^","_",
#               "`","{","|","}","~","–"]
# for char in spec_chars:
#     temp2['Name'] = temp2['Name'].str.replace(char, ' ')

"""### Concating data"""

from datetime import *

# #Extracting month, date and hour from time column
# date = []
# hour = []
# month = []
# for i in range(len(result)):
#     d1 = datetime.fromisoformat(result['Time'][i][:-1])
#     h = d1.strftime('%H')
#     d = d1.strftime('%d')
#     m = d1.strftime('%m')
#     date.append(d)
#     hour.append(h)
#     month.append(m)

# result['date']=date
# result['hour']=hour
# result['month']=month

# #lowering the strings
# df['Pos answer'] = df['Pos answer'].str.lower()

df.to_csv('clean_corpus.csv')

#result.head()

"""# Sentimental Scoring methods

# Importing libraries
"""

import nltk
nltk.download('punkt')

from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.stem.snowball import SnowballStemmer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
nltk.download('vader_lexicon')

df['Answer3'] = df['Answer3'].fillna('zero')
df['Answer4'] = df['Answer4'].fillna('zero')
df['Answer5'] = df['Answer5'].fillna('zero')
df['Answer 6'] = df['Answer 6'].fillna('zero')

df['Answer 6']= df['Answer 6'].replace(0, 'zero')

#tokenizing the comments
df['Answer 6']= df['Answer 6'].apply(word_tokenize)

#creating stem of comments
stemmer = SnowballStemmer("english")

df['Answer 1'] = df['Answer 1'].apply(lambda x: [stemmer.stem(y) for y in x])
df['Answer2'] = df['Answer2'].apply(lambda x: [stemmer.stem(y) for y in x])
df['Answer3'] = df['Answer3'].apply(lambda x: [stemmer.stem(y) for y in x])
df['Answer4'] = df['Answer4'].apply(lambda x: [stemmer.stem(y) for y in x])
df['Answer5'] = df['Answer5'].apply(lambda x: [stemmer.stem(y) for y in x])
df['Answer 6'] = df['Answer 6'].apply(lambda x: [stemmer.stem(y) for y in x])

df.head()

df.to_csv('imagescore.csv')

#Creating sentiment scores positive, negative, neutral and compound
analyzer = SentimentIntensityAnalyzer()
df['compound'] = [analyzer.polarity_scores(x)['compound'] for x in df['Answer 6']]
df['neg'] = [analyzer.polarity_scores(x)['neg'] for x in df['Answer 6']]
df['neu'] = [analyzer.polarity_scores(x)['neu'] for x in df['Answer 6']]
df['pos'] = [analyzer.polarity_scores(x)['pos'] for x in df['Answer 6']]

#Calculating polarity and subjectivity
df[['polarity_3', 'subjectivity_3']] = df['answer 3'].apply(lambda Text: pd.Series(TextBlob(Text).sentiment))

df.head()

"""### Empath score 

"""

!pip install empath

#function for creating dictionary
def returnSum(dict): 
    sum = 0
    for i in dict: 
        sum = sum + dict[i]  
    return sum

#calculating empath score
from empath import Empath
lexicon = Empath()
l = []
m = []
for i in range(len(df)):
  lex= lexicon.analyze(df['Answer 6'][i], normalize=True)
  emp = empath=lexicon.analyze(df['Answer 6'][i], normalize=True)
  empath_score= returnSum(empath) 
  l.append(empath_score)

l = pd.DataFrame(list(l))

l.info()

l.rename(columns = {0:'empath_score'}, inplace = True)

df = pd.concat([df, l], axis=1, join='inner')
display(df)

df.to_csv('last.csv')

def maximum(a, b, c):
  
    if (a >= b) and (a >= c):
        largest = 'neg'
  
    elif (b >= a) and (b >= c):
        largest = 'neu'
    else:
        largest = 'pos'
          
    return largest

l=[]
for i in range(len(result)):
    x=maximum(df['neg'][i],df['neu'][i],df['pos'][i])
    l.append(x)

df['sentiment_status']=l

#Final dataset cleaned and scored
df.to_csv("Barkha E3 score.csv", index=False)