# Basic setup
import warnings
warnings.filterwarnings("ignore")

import pandas as pd
import numpy as np

# Load Titanic dataset straight from a web address
url = "https://raw.githubusercontent.com/datasciencedojo/datasets/master/titanic.csv"
df = pd.read_csv(url)
print("Data shape:", df.shape)
df.head()

# Find out how many missing values are in each column
df.isnull().sum()

# Check the percentage of missing values in each column
(df.isnull().mean() * 100).round(2)

# View rows where at least one value is missing
df[df.isnull().any(axis=1)].head()

# Visualize missing values with a simple bar plot
import matplotlib.pyplot as plt
df.isnull().sum().plot(kind="bar")
plt.title("Missing values per column")
plt.ylabel("Count")
plt.show()

# Remove any rows with missing values
df_no_missing = df.dropna()
print("Original shape:", df.shape)
print("Shape after removing missing rows:", df_no_missing.shape)

# Remove any columns with too many missing values (more than 40%)
threshold = len(df) * 0.6
df_few_missing = df.dropna(axis=1, thresh=threshold)
print("Shape after dropping columns:", df_few_missing.shape)

# Fill missing ages with the average age (mean imputation)
mean_age = df["Age"].mean()
df["Age_filled"] = df["Age"].fillna(mean_age)
df[["Age", "Age_filled"]].head(8)

# Fill missing embarked place with the most common value (mode imputation)
mode_embarked = df["Embarked"].mode()[0]
df["Embarked_filled"] = df["Embarked"].fillna(mode_embarked)
df[["Embarked", "Embarked_filled"]].head(8)

# Forward fill: copy previous value to fill missing values
df["Cabin_ffill"] = df["Cabin"].fillna(method="ffill")
df[["Cabin", "Cabin_ffill"]].head(10)

# Interpolate: estimate missing values between numbers
df["Age_interp"] = df["Age"].interpolate()
df[["Age", "Age_interp"]].head(10)

# Advanced: Use SimpleImputer from scikit-learn for column imputation
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy="median")
df["Age_median"] = imputer.fit_transform(df[["Age"]])
df[["Age", "Age_median"]].head(10)

# Checking that there are no missing ages now
df["Age_median"].isnull().sum()

# Optional: ask the user how to fill a missing value
guess_age = input("Imagine you know nothing about a missing person's age. What would you use to fill it? (Type a number): ")
print("You picked:", guess_age)

# Mini project Part 1: Remove all rows with missing Embarked and fill missing Age with the median
df_clean = df.dropna(subset=["Embarked"]).copy()
median_age = df_clean["Age"].median()
df_clean["Age"] = df_clean["Age"].fillna(median_age)
print("Missing values after cleaning:")
print(df_clean.isnull().sum())

# Mini project Part 2: Compare the survival rate before and after cleaning
orig_survival = df["Survived"].mean()
clean_survival = df_clean["Survived"].mean()
print("Original survival rate:", round(orig_survival, 3))
print("After cleaning survival rate:", round(clean_survival, 3))

# Extra: flag which rows were imputed in Age column
df["Was_age_missing"] = df["Age"].isnull()
df["Age_filled_with_mean"] = df["Age"].fillna(mean_age)
df[["Age", "Age_filled_with_mean", "Was_age_missing"]].head(10)

# Challenge: Try filling Age with a constant value using input
custom_age = int(input("Type a number to use for all missing ages (try 30): "))
df["Age_filled_custom"] = df["Age"].fillna(custom_age)
df[["Age", "Age_filled_custom"]].head(10)