import warnings
from statsmodels.tools.sm_exceptions import ConvergenceWarning, ValueWarning
warnings.filterwarnings("ignore", category=ValueWarning)
warnings.filterwarnings("ignore", category=ConvergenceWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning)
# First we import pandas and matplotlib. These tools help us work with data easily.
import pandas as pd
import matplotlib.pyplot as plt

# Data setup
url = "https://raw.githubusercontent.com/jbrownlee/Datasets/master/shampoo.csv"
df = pd.read_csv(url)
# Create a sequence of monthly dates starting at January 1901 (one entry per row).
df['Month'] = pd.date_range(start='1901-01', periods=df.shape[0], freq='MS')
df['Month'] = pd.to_datetime(df['Month'])

print("Shape:", df.shape)
df.head()

# Let us see a simple line plot of the sales data.
plt.figure(figsize=(8, 4))
plt.plot(df["Sales"])
plt.title("Monthly shampoo sales over time")
plt.xlabel("Month (row number)")
plt.ylabel("Sales")
plt.show()

# Check column types
print(df.dtypes)

# Convert the 'Month' column to datetime if needed.
df["Month"] = pd.to_datetime(df["Month"], errors="coerce")
df.dtypes

# Show missing values in each column.
print(df.isnull().sum())

# Fill missing sales values with the average of the column.
mean_sales = df["Sales"].mean()
df["Sales"] = df["Sales"].fillna(mean_sales)
print(df.isnull().sum())

# Remove rows where 'Month' could not be converted to a date.
before = df.shape[0]
df = df.dropna(subset=["Month"])
after = df.shape[0]
print(f"Rows before: {before}, Rows after: {after}")

# Create a new column with sales in thousands.
df["Sales_K"] = df["Sales"] / 1000
df.head()

# Remove outliers above 10,000 in 'Sales'.
outliers = df[df["Sales"] > 10000]
df = df[df["Sales"] <= 10000]
print(f"Removed {outliers.shape[0]} outliers.")

# Reset index so numbers run smoothly after any row changes.
df = df.reset_index(drop=True)
df.head()

# Sort by date just to be safe.
df = df.sort_values("Month").reset_index(drop=True)
df.head()

# Use describe to quickly see summary statistics.
df.describe()

# Find months with above average sales.
above_avg = df[df["Sales"] > df["Sales"].mean()]
print(f"Months with above average sales: {above_avg.shape[0]}")
above_avg.head()

# Group by year and add up sales.
df["Year"] = df["Month"].dt.year
yearly_sales = df.groupby("Year")["Sales"].sum()
print(yearly_sales)

# Save the clean data to a new file.
df.to_csv("shampoo_cleaned.csv", index=False)
print("Cleaned data saved!")

# HANDS-ON: Practice reading user data and fixing missing sales
try:
    user_month = input("Enter a month (2001-01): ")
    user_sales = input("Enter sales for this month (number): ")
    if user_sales == "":
        user_sales = df["Sales"].mean()
        print(f"No sales entered. Using average: {user_sales}")
    else:
        user_sales = float(user_sales)
    new_row = {"Month": pd.to_datetime(user_month, errors="coerce"), "Sales": user_sales}
    import numpy as np
    # Fill missing columns if needed
    for col in df.columns:
        if col not in new_row:
            new_row[col] = np.nan
    new_df = pd.DataFrame([new_row])
    df = pd.concat([df, new_df], ignore_index=True)
    df.tail()
except Exception as e:
    print("An error occurred. Please try again. Error:", str(e))
    

# Mini-Project Part 2: Find missing months and fill them
all_months = pd.date_range(df["Month"].min(), df["Month"].max(), freq="MS")
df_full = df.set_index("Month").reindex(all_months).rename_axis("Month").reset_index()
missing = df_full[df_full["Sales"].isnull()]
print(f"Missing months: {missing.shape[0]}")
df_full["Sales"] = df_full["Sales"].fillna(df_full["Sales"].mean())
df_full.head()

# Best Practice: Always keep a copy of raw data.
df_raw = pd.read_csv(url)
print("Raw data shape:", df_raw.shape)

# Troubleshooting: What if .csv uses a different separator?
test_df = pd.read_csv(url, sep=",")
print(test_df.head())

# Extra tip: Quickly see unique values with .unique()
print(df["Year"].unique())