Pandas syntax revision for ML

Data Loading and Initial Exploration

Reading Data
# Basic data loading
df = pd.read_csv('file.csv')
df = pd.read_csv('file.csv', low_memory=False) # Prevents mixed data types
df = pd.read_csv('file.csv', index_col=0) # Set first column as index
df = pd.read_csv('file.csv', parse_dates=['date_col']) # Parse dates automatically

# Loading with specific parameters

df = pd.read_csv('file.csv',
sep=';', # Different separator
encoding='utf-8', # Handle special characters
nrows=1000) # Load only first 1000 rows

Basic Data Inspection

# Shape and basic info
df.shape # Returns (rows, columns)
df.info() # Data types, memory usage, non-null counts
df.describe() # Statistical summary for numerical columns
df.describe(include='all') # Summary for all columns including categorical

# Column information
df.columns # Column names
df.dtypes # Data types of each column
df.index # Index information
df.head(10) # First 10 rows
df.tail(5) # Last 5 rows
df.sample(3) # Random 3 rows

Data Selection and Indexing

Column Selection
# Single column
df['column_name'] # Returns Series
df[['column_name']] # Returns DataFrame

# Multiple columns
df[['col1', 'col2', 'col3']]

# Column selection by condition

numeric_cols = df.select_dtypes(include=[np.number]).columns
categorical_cols = df.select_dtypes(include=['object']).columns
df[numeric_cols] # Select only numeric columns

Row Selection
# By index position
df.iloc[^0] # First row
df.iloc[0:5] # First 5 rows
df.iloc[:, 0:3] # All rows, first 3 columns
df.iloc[0:5, 0:3] # First 5 rows, first 3 columns

# By label
df.loc[^0] # Row with index 0
df.loc[0:4] # Rows with index 0 to 4 (inclusive)
df.loc[:, 'col1':'col3'] # All rows, columns from col1 to col3
df.loc[df['column'] > 5] # Conditional selection

Boolean Indexing
# Single condition
df[df['age'] > 25]
df[df['category'] == 'A']
df[df['name'].str.contains('John')]

# Multiple conditions
df[(df['age'] > 25) & (df['salary'] < 50000)]
df[(df['category'] == 'A') | (df['category'] == 'B')]
df[df['column'].isin(['value1', 'value2', 'value3'])]

# Negation
df[~df['column'].isin(['unwanted_value'])]
df[df['column'] != 'unwanted_value']

Data Cleaning and Preprocessing

Handling Missing Values
# Detecting missing values
df.isnull().sum() # Count of missing values per column
df.isna().sum() # Same as isnull()
df.isnull().any() # Boolean - any missing values per column
df.info() # Shows non-null counts

# Removing missing values

df.dropna() # Drop rows with any missing values
df.dropna(subset=['col1']) # Drop rows where col1 is missing
df.dropna(axis=1) # Drop columns with any missing values
df.dropna(thresh=2) # Keep rows with at least 2 non-null values

# Filling missing values

df.fillna(0) # Fill with 0
df.fillna(method='ffill') # Forward fill
df.fillna(method='bfill') # Backward fill
df['column'].fillna(df['column'].mean()) # Fill with mean
df['column'].fillna(df['column'].mode()[^0]) # Fill with mode

Data Type Conversion

# Converting data types
df['column'] = df['column'].astype('int64')
df['column'] = df['column'].astype('float64')
df['column'] = df['column'].astype('category')
df['date_col'] = pd.to_datetime(df['date_col'])

# Converting multiple columns

cols_to_convert = ['col1', 'col2', 'col3']
df[cols_to_convert] = df[cols_to_convert].astype('float64')

Duplicate Handling
# Finding duplicates
df.duplicated().sum() # Count of duplicate rows
df.duplicated(subset=['col1']).sum() # Duplicates based on specific column

# Removing duplicates
df.drop_duplicates() # Remove duplicate rows
df.drop_duplicates(subset=['col1']) # Remove duplicates based on column
df.drop_duplicates(keep='last') # Keep last occurrence

Data Transformation

Creating New Columns

# Simple operations
df['new_col'] = df['col1'] + df['col2']
df['ratio'] = df['numerator'] / df['denominator']
df['log_price'] = np.log(df['price'])

# Conditional column creation

df['category'] = df['score'].apply(lambda x: 'High' if x > 80 else 'Low')
df['grade'] = np.where(df['score'] > 90, 'A',
np.where(df['score'] > 80, 'B', 'C'))

# Using map for categorical mapping

mapping = {'small': 1, 'medium': 2, 'large': 3}
df['size_numeric'] = df['size'].map(mapping)

String Operations
# Basic string operations
df['column'].str.lower() # Convert to lowercase
df['column'].str.upper() # Convert to uppercase
df['column'].str.strip() # Remove leading/trailing whitespace
df['column'].str.replace('old', 'new') # Replace text

# String splitting and extraction

df['column'].str.split('_') # Split by underscore
df['column'].str.split('_', expand=True) # Split into separate columns
df['column'].str[^0] # First character
df['column'].str[:3] # First 3 characters
df['column'].str.extract('(\d+)') # Extract numbers using regex

# String conditions
df['column'].str.contains('pattern')
df['column'].str.startswith('prefix')
df['column'].str.endswith('suffix')
df['column'].str.len() # Length of strings
Date and Time Operations
# Converting to datetime
df['date'] = pd.to_datetime(df['date'])

# Extracting date components (like in your notebook)

df['year'] = df['date'].dt.year
df['month'] = df['date'].dt.month
df['day'] = df['date'].dt.day
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['is_weekend'] = df['date'].dt.dayofweek >= 5

# Date arithmetic
df['days_ago'] = (pd.Timestamp.now() - df['date']).dt.days
df['date_plus_30'] = df['date'] + pd.Timedelta(days=30)

Grouping and Aggregation

Basic GroupBy Operations

# Single column grouping
df.groupby('category').mean()
df.groupby('category').sum()
df.groupby('category').count()
df.groupby('category').size() # Count including NaN

# Multiple column grouping

df.groupby(['category', 'subcategory']).mean()

# Specific column aggregation

df.groupby('category')['price'].mean()
df.groupby('category')['price'].agg(['mean', 'std', 'count'])

Advanced Aggregation
# Multiple aggregations
df.groupby('category').agg({
'price': ['mean', 'std'],
'quantity': 'sum',
'date': 'count'
})
# Custom aggregation functions
df.groupby('category')['price'].agg(lambda x: x.max() - x.min())

# Apply custom functions

def custom_function(group):
return group['price'].mean() / group['quantity'].sum()

df.groupby('category').apply(custom_function)

Transform and Filter

# Transform (returns same shape as original)
df['price_normalized'] = df.groupby('category')['price'].transform('mean')
df['price_zscore'] = df.groupby('category')['price'].transform(lambda x: (x - x.mean()) / x.std())

# Filter groups
df.groupby('category').filter(lambda x: len(x) > 10) # Groups with >10 records

Merging and Joining

Concatenation
# Vertical concatenation (stacking rows)
combined = pd.concat([df1, df2], axis=0, ignore_index=True)

# Horizontal concatenation (side by side)

combined = pd.concat([df1, df2], axis=1)

# With keys for identification

combined = pd.concat([df1, df2], keys=['train', 'test'])

Merging DataFrames
# Inner join (only matching records)
merged = pd.merge(df1, df2, on='key_column', how='inner')

# Left join (all records from left DataFrame)

merged = pd.merge(df1, df2, on='key_column', how='left')

# Outer join (all records from both DataFrames)

merged = pd.merge(df1, df2, on='key_column', how='outer')

# Multiple key columns

merged = pd.merge(df1, df2, on=['key1', 'key2'])

# Different column names

merged = pd.merge(df1, df2, left_on='col1', right_on='col2')

Reshaping Data

Pivot Operations
# Pivot table
pivot = df.pivot_table(values='sales',
index='date',
columns='product',
aggfunc='sum')

# Simple pivot
pivot = df.pivot(index='date', columns='product', values='sales')

# Melt (wide to long format)

melted = df.melt(id_vars=['id', 'date'],
value_vars=['sales1', 'sales2'],
var_name='product',
value_name='sales')

Stack and Unstack

# Stack (columns to rows)
stacked = df.stack()

# Unstack (rows to columns)

unstacked = df.unstack()

# Multi-level operations
df.unstack(level=0)
df.unstack(level='column_name')

Statistical Operations
Descriptive Statistics
# Basic statistics
df['column'].mean()
df['column'].median()
df['column'].std()
df['column'].var()
df['column'].min()
df['column'].max()
df['column'].quantile(0.25) # 25th percentile

# Multiple statistics at once

df['column'].describe()

# Correlation
df.corr() # Correlation matrix
df['col1'].corr(df['col2']) # Correlation between two columns

Value Counts and Frequencies

# Count unique values
df['column'].value_counts()
df['column'].value_counts(normalize=True) # As percentages
df['column'].value_counts(dropna=False) # Include NaN counts

# Unique values
df['column'].unique()
df['column'].nunique() # Number of unique values

Advanced Techniques for Kaggle

Feature Engineering Patterns

# Binning continuous variables
df['age_group'] = pd.cut(df['age'], bins=[0, 25, 50, 75, 100],
labels=['Young', 'Adult', 'Middle', 'Senior'])

# Quantile-based binning
df['price_quartile'] = pd.qcut(df['price'], q=4, labels=['Low', 'Med-Low', 'Med-High', 'High'])

# Rolling statistics (time series)

df['rolling_mean'] = df['value'].rolling(window=7).mean()
df['rolling_std'] = df['value'].rolling(window=7).std()

# Lag features
df['value_lag1'] = df['value'].shift(1)
df['value_lag7'] = df['value'].shift(7)

Memory Optimization
# Reduce memory usage
def reduce_mem_usage(df):
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
df[col] = df[col].astype(np.int8)
elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
df[col] = df[col].astype(np.int16)
return df

# Check memory usage

df.memory_usage(deep=True).sum()

Efficient Data Processing

# Using query for complex filtering (faster than boolean indexing)
df.query('age > 25 and salary < 50000')
df.query('category in ["A", "B", "C"]')

# Using eval for complex calculations

df.eval('new_col = col1 * col2 + col3')

# Categorical data for memory efficiency

df['category'] = df['category'].astype('category')

Common Kaggle Competition Patterns

Train-Test Split Preparation

# Combining train and test for consistent preprocessing (like in your notebook)
train_df = pd.read_csv('train.csv')
test_df = pd.read_csv('test.csv')

# Combine for preprocessing

combined = pd.concat([train_df, test_df], axis=0, ignore_index=True)

# After preprocessing, split back

train_processed = combined.iloc[:len(train_df), :].reset_index(drop=True)
test_processed = combined.iloc[len(train_df):, :].reset_index(drop=True)

Target Encoding and Feature Engineering

# Target encoding (mean encoding)
target_mean = train_df.groupby('category')['target'].mean()
train_df['category_encoded'] = train_df['category'].map(target_mean)

# Frequency encoding
freq_encoding = train_df['category'].value_counts()
train_df['category_freq'] = train_df['category'].map(freq_encoding)

# Interaction features
train_df['feature_interaction'] = train_df['feature1'] * train_df['feature2']

These pandas operations form the backbone of most data science workflows and Kaggle
competitions.