# Suppress all warnings to keep outputs clean
import warnings; warnings.filterwarnings("ignore")
import numpy as np
np.random.seed(42)

# Data setup (Titanic Dataset)
import pandas as pd
url = 'https://raw.githubusercontent.com/datasciencedojo/datasets/master/titanic.csv'
df = pd.read_csv(url)
print(df.shape)
print(df.head(3))

# Check for missing values in all columns
df.isnull().sum()

# Get the percentage of missing values per column
(df.isnull().sum() / len(df) * 100).round(2)

# See which rows are missing any data
df[df.isnull().any(axis=1)].head()

import seaborn as sns
import matplotlib.pyplot as plt
# Draw a heatmap of missing values
plt.figure(figsize=(10,4))
sns.heatmap(df.isnull(), cbar=False, yticklabels=False, cmap='viridis')
plt.title('Missing Data on Titanic')
plt.show()

# Count missing values in Cabin by passenger class
df.groupby('Pclass')['Cabin'].apply(lambda x: x.isnull().mean()).round(2)

# Drop rows where Age is missing
df_drop_age = df.dropna(subset=['Age'])
print('Original:', df.shape)
print('After dropping:', df_drop_age.shape)

# Fill missing Age values with the median age
median_age = df['Age'].median()
df['Age_filled'] = df['Age'].fillna(median_age)
print(f'Median Age: {median_age}')
print('Missing after fill:', df["Age_filled"].isnull().sum())

# Fill missing Embarked values with most common port
mode_embarked = df['Embarked'].mode()[0]
df['Embarked_filled'] = df['Embarked'].fillna(mode_embarked)
print(f'Most common port: {mode_embarked}')
print('Missing after fill:', df["Embarked_filled"].isnull().sum())

# Mark missing Cabin values with a flag
df['Cabin_missing'] = df['Cabin'].isnull().astype(int)
df[['Cabin', 'Cabin_missing']].head(5)

# Demonstrate forward fill and backward fill on small Age sample
sample = df[['Age']].head(10).copy()
sample.loc[[2, 5], 'Age'] = None
sample['ffill'] = sample['Age'].ffill()
sample['bfill'] = sample['Age'].bfill()
print(sample)

# Impute missing Age by median per Passenger Class
df['Age_group_median'] = df.groupby('Pclass')['Age'].transform(lambda x: x.fillna(x.median()))
print(df[['Pclass', 'Age', 'Age_group_median']].head(8))

# Drop columns with over 70 percent missing values
thresh = 0.7 * len(df)
df_reduced = df.dropna(axis=1, thresh=thresh)
print(f'Columns dropped: {set(df.columns) - set(df_reduced.columns)}')

# Replace placeholder like '?' and 'NA' with true missing values
df2 = df.copy()
df2.replace(['?', 'NA', '-', ''], pd.NA, inplace=True)
print('Done recoding placeholders.')

# Pipeline: fill Age by Pclass median, fill Embarked by mode, drop 'Cabin' and rows still missing
clean_df = df.copy()
clean_df['Age'] = clean_df.groupby('Pclass')['Age'].transform(lambda x: x.fillna(x.median()))
clean_df['Embarked'] = clean_df['Embarked'].fillna(clean_df['Embarked'].mode()[0])
clean_df = clean_df.drop(columns=['Cabin'])
clean_df = clean_df.dropna()
print(clean_df.isnull().sum().sum(), 'missing left')
print(clean_df.shape)

# Challenge: Identify if a record is missing more than one field
df['missing_count'] = df.isnull().sum(axis=1)
challenge_result = df[df['missing_count'] > 1]
print(challenge_result.shape)