import numpy as np
import pandas as pd
import seaborn as sns
sns.set_style("darkgrid")
import matplotlib.pyplot as plt
import nltk
from sklearn.preprocessing import LabelBinarizer
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from wordcloud import WordCloud,STOPWORDS
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize,sent_tokenize
from bs4 import BeautifulSoup
import re,string,unicodedata
from keras.preprocessing import text, sequence
from sklearn.metrics import classification_report,confusion_matrix,accuracy_score
from sklearn.model_selection import train_test_split
from string import punctuation
from nltk import pos_tag
from nltk.corpus import wordnet
import keras
from keras.models import Sequential
from keras.layers import Dense,Embedding,LSTM,Dropout
from keras.callbacks import ReduceLROnPlateau
import tensorflow as tf
import os
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
true = pd.read_csv("./True.csv")
true.head()
fake = pd.read_csv("./Fake.csv")
fake.head()
true['category'] = 1
fake['category'] = 0
df = pd.concat([true,fake])
df.head()
df.shape
df.describe(include="object")
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# Assuming 'category' is a column in the 'df' DataFrame
print(df['category'].value_counts())
# Convert the Series to a DataFrame
data_subset = df['category'].value_counts().to_frame()
# Reset the index to convert the 'category' values to a column
data_subset.reset_index(inplace=True)
# Rename the columns to have appropriate names
data_subset.columns = ['category', 'count']
# Set the seaborn style
sns.set_style("darkgrid")
# Create the count plot
plt.figure(figsize=(10, 10))
sns.barplot(x='category', y='count', data=data_subset)
# Show the plot
plt.show()
sns.countplot(x='subject',hue='category',data=df,)
plt.xticks(rotation=90)
plt.show()
df["text"] =df["title"]+df["text"]+df['subject']
df=df[["text","category"]]
!python -m spacy download en_core_web_sm
import spacy
nlp = spacy.load('en_core_web_sm')
list1 = nlp.Defaults.stop_words
list2 = stopwords.words('english')
punctuation = list(string.punctuation)
Stopwords = set((set(list1)|set(list2)|set(punctuation)))
len(Stopwords)
#creating instance
lemma=WordNetLemmatizer()
#text cleaning function
def clean_text(text):
"""
It takes text as an input and clean it by applying several methods
"""
string = ""
#lower casing
text=text.lower()
#simplifying text
text=re.sub(r"i'm","i am",text)
text=re.sub(r"he's","he is",text)
text=re.sub(r"she's","she is",text)
text=re.sub(r"that's","that is",text)
text=re.sub(r"what's","what is",text)
text=re.sub(r"where's","where is",text)
text=re.sub(r"\'ll"," will",text)
text=re.sub(r"\'ve"," have",text)
text=re.sub(r"\'re"," are",text)
text=re.sub(r"\'d"," would",text)
text=re.sub(r"won't","will not",text)
text=re.sub(r"can't","cannot",text)
#removing any special character
text=re.sub(r"[-()\"#!@$%^&*{}?.,:]"," ",text)
text=re.sub(r"\s+"," ",text)
text=re.sub('[^A-Za-z0-9]+',' ', text)
for word in text.split():
if word not in Stopwords:
string+=lemma.lemmatize(word)+" "
return string
import nltk
# Download the WordNet resource
nltk.download('wordnet')
# Assuming the NLTK data path issue has been resolved, and the necessary libraries are imported
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
# Download NLTK resources (if not already downloaded)
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
# Define the function for text cleaning
def clean_text(text):
Stopwords = set(stopwords.words('english'))
lemma = WordNetLemmatizer()
string = ""
for word in nltk.word_tokenize(text):
if word not in Stopwords:
string += lemma.lemmatize(word) + " "
return string
# Clean the "text" column in the DataFrame
df["text"] = df["text"].apply(clean_text)
import nltk
nltk.download('omw-1.4')
import pandas as pd
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
# Download NLTK resources (if you haven't done it already)
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
# Load the DataFrame
# (Assuming your DataFrame is named 'df' and contains a column named 'text')
# df = pd.read_csv('your_data.csv')
# Function to clean and tokenize the text
def clean_text(text):
Stopwords = set(stopwords.words('english'))
lemma = WordNetLemmatizer()
string = ""
for word in nltk.word_tokenize(text):
if word not in Stopwords:
string += lemma.lemmatize(word) + " "
return string
# Check Data Types
print(df.dtypes)
# Check for Null or NaN Values
print(df.isnull().sum())
# Clean the "text" column in the DataFrame
df["text"] = df["text"].astype(str).apply(clean_text)
# Drop any rows with null or NaN values in the "text" column
df = df.dropna(subset=["text"])
# Verify Tokenization
print(df["text"].head())
# Continue with word cloud generation, classifier training, and evaluation
plt.figure(figsize = (20,20))
wc = WordCloud(max_words = 2000 , width = 1000 , height = 500 , stopwords =
Stopwords).generate(" ".join(df[df.category == 0].text))
plt.axis("off")
plt.imshow(wc , interpolation = 'bilinear')
plt.show()
plt.figure(figsize = (20,20))
wc = WordCloud(max_words = 2000 , width = 1000 , height = 500 , stopwords =
Stopwords).generate(" ".join(df[df.category == 1].text))
plt.axis("off")
plt.imshow(wc , interpolation = 'bilinear')
plt.show()
plt.figure(figsize = (20,20))
wc = WordCloud(max_words = 2000 , width = 1000 , height = 500 , stopwords =
Stopwords,background_color='white').generate(" ".join(df.text))
plt.axis("off")
plt.imshow(wc , interpolation = 'bilinear')
plt.show()
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
#Split the dataset into Train And Test Dataset.
X=df["text"] #feature
y=df["category"] # traget
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
#pipe line will take the text and vectorise it , and then TF-IDF, then fitting the model
clf_text=Pipeline([("tfidf",TfidfVectorizer()),("clf",LogisticRegression())])
clf_text.fit(X_train,y_train)
#making prediction using the model
predictions=clf_text.predict(X_test)
print(metrics.classification_report(y_test,predictions))
#overall acuracy
print(metrics.accuracy_score(y_test,predictions))
print(metrics.f1_score(y_test,predictions))
#confusion matrix
print(metrics.confusion_matrix(y_test,predictions))
#pipe line will take the text and vectorise it , and then TF-IDF, then fitting the model
clf_text=Pipeline([("tfidf",TfidfVectorizer()),("clf",MultinomialNB(alpha=0.5))])
clf_text.fit(X_train,y_train)
#making prediction using the model
predictions=clf_text.predict(X_test)
print(metrics.classification_report(y_test,predictions))
#overall acuracy
print(metrics.accuracy_score(y_test,predictions))
print(metrics.f1_score(y_test,predictions))
#confusion matrix
print(metrics.confusion_matrix(y_test,predictions))
#pipe line will take the text and vectorise it , and then TF-IDF, then fitting the model
clf_text=Pipeline([("tfidf",TfidfVectorizer()),("clf",LinearSVC())])
clf_text.fit(X_train,y_train)
#making prediction using the model
predictions=clf_text.predict(X_test)
print(metrics.classification_report(y_test,predictions))
#overall acuracy
print(metrics.accuracy_score(y_test,predictions))
print(metrics.f1_score(y_test,predictions))
#confusion matrix
print(metrics.confusion_matrix(y_test,predictions))
#pipe line will take the text and vectorise it , and then TF-IDF, then fitting the model
clf_rf=Pipeline([("tfidf",TfidfVectorizer()),("clf",RandomForestClassifier(random_state=0))])
clf_rf.fit(X_train,y_train)
#making prediction using the model
predictions=clf_rf.predict(X_test)
print(metrics.classification_report(y_test,predictions))
#overall acuracy
print(metrics.accuracy_score(y_test,predictions))
print(metrics.f1_score(y_test,predictions))
#confusion matrix
print(metrics.confusion_matrix(y_test,predictions))
#pipe line will take the text and vectorise it , and then TF-IDF, then fitting the model
clf_dt=Pipeline([("tfidf",TfidfVectorizer()),("clf",DecisionTreeClassifier(random_state=2))])
clf_dt.fit(X_train,y_train)
#making prediction using the model
predictions=clf_dt.predict(X_test)
print(metrics.classification_report(y_test,predictions))
#overall acuracy
print(metrics.accuracy_score(y_test,predictions))
print(metrics.f1_score(y_test,predictions))
#confusion matrix
print(metrics.confusion_matrix(y_test,predictions))
