1.

Write a Python program for Data pre-processing and include Data

cleaning steps

import pandas as pd

import numpy as np

from sklearn.preprocessing import LabelEncoder, StandardScaler

from scipy.stats import zscore

# Sample dataset creation

data = {

'Name': ['Alice', 'Bob', 'Charlie', None, 'Eve', 'Frank', 'Alice', 'Grace'],

'Age': [25, np.nan, 30, 35, 29, 120, 25, 33],

'Gender': ['Female', 'Male', 'Male', 'Female', None, 'Male', 'Female', 'Female'],

'Income': [50000, 60000, np.nan, 80000, 75000, 300000, 50000, 72000],

'Location': ['New York', 'San Francisco', 'Chicago', None, 'New York', 'Chicago', 'New York',
'Chicago']

# Load data into a DataFrame

df = pd.DataFrame(data)

print("Original DataFrame:")

print(df)

# Step 1: Handling Missing Values

# Fill missing numerical values with the column median

df['Age'] = df['Age'].fillna(df['Age'].median())

df['Income'] = df['Income'].fillna(df['Income'].median())

# Fill missing categorical values with the mode

df['Name'] = df['Name'].fillna('Unknown')

df['Gender'] = df['Gender'].fillna(df['Gender'].mode()[0])
df['Location'] = df['Location'].fillna('Unknown')

# Step 2: Removing Duplicates

# Check for duplicates and drop them

df = df.drop_duplicates()

# Step 3: Handling Outliers

# Remove rows where numerical features have Z-scores > 3 (extreme outliers)

z_scores = zscore(df[['Age', 'Income']])

df = df[(np.abs(z_scores) < 3).all(axis=1)]

# Step 4: Encoding Categorical Variables

# Convert categorical columns to numerical using Label Encoding

label_encoder = LabelEncoder()

df['Gender'] = label_encoder.fit_transform(df['Gender']) # Female = 0, Male = 1

df['Location'] = label_encoder.fit_transform(df['Location']) # Encoding Location

# Step 5: Removing Irrelevant Features

# Drop the 'Name' column as it's non-informative for analysis

df = df.drop(columns=['Name'])

# Step 6: Scaling Features

# Normalize numerical features using StandardScaler

scaler = StandardScaler()

df[['Age', 'Income']] = scaler.fit_transform(df[['Age', 'Income']])

# Final Cleaned DataFrame

print("\nCleaned DataFrame:")

print(df)
 2. Write a Python program and include Data Integration steps and Data
cleaning steps
import pandas as pd

import numpy as np

from sklearn.preprocessing import LabelEncoder, StandardScaler

# Sample Datasets

data1 = {

'CustomerID': [1, 2, 3, 4],

'Name': ['Alice', 'Bob', 'Charlie', 'Diana'],

'Age': [25, np.nan, 30, 35],

'Gender': ['Female', 'Male', 'Male', 'Female']

data2 = {

'CustomerID': [3, 4, 5, 6],

'Income': [60000, 80000, np.nan, 75000],

'Location': ['New York', 'Chicago', None, 'San Francisco']

# Step 1: Load Data

df1 = pd.DataFrame(data1)

df2 = pd.DataFrame(data2)

print("Dataset 1:")

print(df1)

print("\nDataset 2:")

print(df2)

# Step 2: Data Integration (Merging Datasets)

# Merge datasets on CustomerID

df = pd.merge(df1, df2, on='CustomerID', how='outer')

print("\nIntegrated Dataset:")

print(df)

df10 = pd.merge(df1, df2, on='CustomerID', how='inner')

print("\nIntegrated Dataset:")

print(df10)

# Step 3: Handling Missing Values

# Fill missing numerical values with the median

df['Age'] = df['Age'].fillna(df['Age'].median())

df['Income'] = df['Income'].fillna(df['Income'].median())

# Fill missing categorical values with the mode

df['Gender'] = df['Gender'].fillna(df['Gender'].mode()[0])

df['Location'] = df['Location'].fillna('Unknown')

# Step 4: Removing Duplicates

df = df.drop_duplicates()

# Step 5: Encoding Categorical Variables

# Encode Gender and Location columns

label_encoder = LabelEncoder()

df['Gender'] = label_encoder.fit_transform(df['Gender'])

df['Location'] = label_encoder.fit_transform(df['Location'])

# Step 6: Scaling Numerical Features

scaler = StandardScaler()

df[['Age', 'Income']] = scaler.fit_transform(df[['Age', 'Income']])

# Final Processed Dataset

print("\nProcessed Dataset:")

print(df)

3. Write a Python Program and include data reduction functions

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.model_selection import train_test_split

# Sample Dataset
data = {
'Feature1': [2.5, 3.0, 2.8, 3.2, 2.7, 3.5, 2.9],
'Feature2': [1.2, 1.5, 1.3, 1.7, 1.4, 1.8, 1.6],
'Feature3': [0.8, 0.9, 1.0, 0.7, 1.1, 0.6, 1.2],
'Feature4': [10, 20, 15, 10, 20, 15, 10],
'Target': [0, 1, 1, 0, 1, 0, 1]
}

# Load data into a DataFrame

df = pd.DataFrame(data)
print("Original Dataset:")
print(df)

# Splitting features and target

X = df.drop(columns=['Target'])
y = df['Target']

# Step 1: Feature Scaling

# Scale features to normalize the data
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Step 2: Dimensionality Reduction using PCA

# Reduce dimensions to 2 principal components
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)
print("\nData after PCA (Dimensionality Reduction):")
print(pd.DataFrame(X_pca, columns=['PC1', 'PC2']))
 # Step 3: Feature Selection
# Select top 2 features based on statistical significance
selector = SelectKBest(score_func=f_classif, k=2)
X_selected = selector.fit_transform(X, y)
print("\nData after Feature Selection (Top 2 Features):")
print(pd.DataFrame(X_selected, columns=['Feature1', 'Feature2']))

# Step 4: Sampling (Data Size Reduction)

# Split data into a smaller sample for testing
X_train, X_sample, y_train, y_sample = train_test_split(X, y, test_size=0.4,
random_state=42)
print("\nSampled Data (40% of Original Dataset):")
print(pd.DataFrame(X_sample, columns=X.columns))

4. Write a Python Program and include data transformation

functions

import pandas as pd

import numpy as np

from sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder, OneHotEncoder

from scipy.stats import boxcox

# Sample Dataset

data = {

'Feature1': [10, 20, 30, 40, 50],

'Feature2': [1.2, 3.5, 5.1, 7.3, 9.0],

'Feature3': [1000, 1500, 2000, 2500, 3000],

'Category': ['A', 'B', 'A', 'C', 'B']

# Load data into a DataFrame

df = pd.DataFrame(data)

print("Original Dataset:")
print(df)

# Step 1: Normalization

# Normalize Feature2 to a range of [0, 1]

minmax_scaler = MinMaxScaler()

df['Feature2_Normalized'] = minmax_scaler.fit_transform(df[['Feature2']])

# Step 2: Standardization

# Standardize Feature1 to have a mean of 0 and standard deviation of 1

std_scaler = StandardScaler()

df['Feature1_Standardized'] = std_scaler.fit_transform(df[['Feature1']])

# Step 3: Log Transformation

# Apply log transformation to Feature3 to reduce skewness

df['Feature3_Log'] = np.log(df['Feature3'])

# Step 4: Power Transformation (Box-Cox Transformation)

# Apply Box-Cox transformation to Feature2 (requires positive values)

df['Feature2_BoxCox'], _ = boxcox(df['Feature2'])

# Step 5: Encoding Categorical Variables

# Label Encoding for Category

label_encoder = LabelEncoder()

df['Category_LabelEncoded'] = label_encoder.fit_transform(df['Category'])

# One-Hot Encoding for Category

onehot_encoder = OneHotEncoder(sparse=False)

onehot_encoded = onehot_encoder.fit_transform(df[['Category']])

onehot_encoded_df = pd.DataFrame(onehot_encoded,
columns=onehot_encoder.get_feature_names_out(['Category']))

df = pd.concat([df, onehot_encoded_df], axis=1)

# Display Transformed Dataset

print("\nTransformed Dataset:")

print(df)

5. Write a Python program for Feature Engineering concepts.

(include Titanic dataset)
import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler
from sklearn.impute import SimpleImputer

# Load Titanic dataset (replace 'train.csv' with the path to your dataset)
data = pd.read_csv('train.csv')

# Display the first few rows of the dataset

print("Initial Data Sample:")
print(data.head())

# Function for feature engineering

def feature_engineering(data):
# Make a copy of the dataset
df = data.copy()

# Handle missing values

imputer_age = SimpleImputer(strategy='median')
df['Age'] = imputer_age.fit_transform(df[['Age']])

imputer_embarked = SimpleImputer(strategy='most_frequent')
df['Embarked'] = imputer_embarked.fit_transform(df[['Embarked']])

# Drop columns that are not useful

df.drop(['Cabin', 'Ticket', 'Name'], axis=1, inplace=True)

# Encode categorical variables

label_encoder = LabelEncoder()
df['Sex'] = label_encoder.fit_transform(df['Sex'])

one_hot_encoder = OneHotEncoder(sparse=False, drop='first')

embarked_encoded = one_hot_encoder.fit_transform(df[['Embarked']])
embarked_encoded_df = pd.DataFrame(embarked_encoded,
columns=one_hot_encoder.get_feature_names_out(['Embarked']))
df = pd.concat([df, embarked_encoded_df], axis=1)
 df.drop(['Embarked'], axis=1, inplace=True)

# Create new features

df['FamilySize'] = df['SibSp'] + df['Parch']
df['IsAlone'] = (df['FamilySize'] == 0).astype(int)

# Scale numerical features

scaler = StandardScaler()
numerical_features = ['Age', 'Fare']
df[numerical_features] = scaler.fit_transform(df[numerical_features])

return df

# Perform feature engineering on the Titanic dataset

processed_data = feature_engineering(data)

# Display the processed data

print("\nProcessed Data Sample:")
print(processed_data.head())

# Save the processed data to a new CSV file

processed_data.to_csv('processed_titanic_data.csv', index=False)
print("\nProcessed data saved to 'processed_titanic_data.csv'.")