import openml
import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')

# Beginner Example 1: Quantitative survey data (customer satisfaction)
dataset = openml.datasets.get_dataset(42178)
df_quant = dataset.get_data(dataset_format='dataframe')[0]
print(df_quant.shape)
print(df_quant.head(3))

# Beginner Example 2: Qualitative open-ended feedback
df_qual = pd.DataFrame({'CustomerID':[1,2,3,4,5], 'Feedback':['Great service and friendly staff','Delivery was slow and packaging was poor','Excellent quality, will buy again','Customer support needs improvement','Good value for money']})
print(df_qual.shape)
print(df_qual.head(3))

# Beginner Example 3: Quantitative NPS survey (synthetic dataset)
np.random.seed(42)
nps_df = pd.DataFrame({
    'CustomerID': range(1, 21),
    'Age': np.random.randint(18, 70, 20),
    'Region': np.random.choice(['North', 'South', 'East', 'West'], 20),
    'NPS_Score': np.random.randint(0,11,20)
})
print(nps_df.head())

# Intermediate Example 1: Calculate mean NPS by region (quantitative segmentation)
nps_region = nps_df.groupby('Region')['NPS_Score'].mean().reset_index()
print(nps_region)

# Intermediate Example 2: Simple sentiment keyword count (qualitative analysis)
keywords = ['great','poor','excellent','improvement','good']
for kw in keywords:
    count = df_qual['Feedback'].str.lower().str.count(kw).sum()
    print(f"Keyword '{kw}' appears {count} times.")

# Intermediate Example 3: Cross-tabulationCustomer satisfaction by gender
df_quant_crosstab = pd.crosstab(df_quant['gender'], df_quant['Churn'])
print(df_quant_crosstab)

# Advanced Example 1: Quantitative and qualitative combinedsentiment by NPS segment
def nps_label(score):
    if score >= 9: return 'Promoter'
    elif score >= 7: return 'Passive'
    else: return 'Detractor'
nps_df['Segment'] = nps_df['NPS_Score'].apply(nps_label)
example_feedback = {
    'Detractor': ['Too slow','Bad packaging','Expensive','Unhelpful support','Rude staff'],
    'Passive': ['Okay service','Fine','Could be better','Average','Nothing special'],
    'Promoter': ['Excellent support','Love the product','Fast delivery','Great quality','Will recommend']
}
nps_df['Feedback'] = nps_df['Segment'].apply(lambda seg: np.random.choice(example_feedback[seg]))
sentiments = nps_df.groupby('Segment')['Feedback'].apply(lambda x: ', '.join(x.head(2))).reset_index()
print(sentiments)

# Advanced Example 2: Quantitative analysisfrequency of missing survey responses
missing_counts = df_quant.isnull().sum().sort_values(ascending=False)
missing_counts = missing_counts[missing_counts > 0]
print('Variables with missing values:')
print(missing_counts)

# Advanced Example 3: Market response to campaign (quantitative analysis over time)
dataset = openml.datasets.get_dataset(1461)
df_campaign = dataset.get_data(dataset_format='dataframe')[0]
df_campaign.columns = ['age','job','marital','education','default','balance','housing','loan','contact','day','month','duration','campaign','pdays','previous','poutcome','response']
monthly_counts = df_campaign.groupby('month')['response'].value_counts().unstack().fillna(0)
print(monthly_counts)

# Error Handling 1: Detect missing or ambiguous survey responses
sample_missing = df_quant[df_quant.isnull().any(axis=1)]
print('Rows with missing responses:')
print(sample_missing.head())

# Error Handling 2: Catch grouping errors (wrong columns in groupby)
try:
    df_quant.groupby('UnknownColumn').size()
except Exception as e:
    print('Grouping failed:', e)

# Error Handling 3: Interpreting NPS scores (Likert confusion)
print('NPS example scores:')
print(nps_df[['CustomerID','NPS_Score','Segment']].head())
print('Remember: Promoters = 9-10, Passives = 7-8, Detractors = 0-6.')

# Segmentation Example: Average monthly charges by contract type
avg_monthly = df_quant.groupby('Contract')['MonthlyCharges'].mean().reset_index()
print(avg_monthly)

# Cross-tabulation Pattern: Payment method vs. churn status
pay_churn_ct = pd.crosstab(df_quant['PaymentMethod'], df_quant['Churn'])
print(pay_churn_ct)

# Index Construction: Create a simple satisfaction index from multiple variables
satisfaction_vars = ['OnlineSecurity','TechSupport','StreamingTV','StreamingMovies']
index_map = { 'Yes': 2, 'No': 1, 'No internet service': 0 }
df_quant['SatisfactionIndex'] = df_quant[satisfaction_vars].replace(index_map).sum(axis=1)
print(df_quant[['SatisfactionIndex']].head())

# Trend Analysis: Monthly customer activity (cohort retention example)
np.random.seed(0)
dates = pd.date_range('2021-01-01', periods=24, freq='ME')
df_cohort = pd.DataFrame({
    'CustomerID': np.random.randint(1000,2000,len(dates)),
    'Signup_Month': dates,
    'Active_Users': np.random.randint(50,300,len(dates))
})
print(df_cohort.head())
print('Average Active Users per Month:', round(df_cohort['Active_Users'].mean(),2))

# End-to-End Example: What drives high NPS scores?
grouped = nps_df.groupby('Segment')['Age'].mean().reset_index()
print('Average age by NPS segment:')
print(grouped)
feedback_counts = nps_df.groupby('Segment')['Feedback'].value_counts().groupby(level=0).head(2)
print('Top feedback keywords by segment:')
print(feedback_counts)