import pandas as pd
import numpy as np
import openml
import scipy.stats as stats
import warnings
warnings.filterwarnings('ignore')

# Beginner Example 1: Load a customer satisfaction dataset from OpenML
dataset = openml.datasets.get_dataset(42178)
df, _, _, _ = dataset.get_data(dataset_format='dataframe')
print(df.shape)
print(df.head(3))

# Beginner Example 2: Check for missing values
missing = df.isnull().sum()
print(missing[missing > 0])

# Beginner Example 3: Compare Monthly Charges by Gender using t-test
male = df[df['gender'] == 'Male']['MonthlyCharges'].dropna()
female = df[df['gender'] == 'Female']['MonthlyCharges'].dropna()
t_stat, p_value = stats.ttest_ind(male, female, equal_var=False)
print('t-statistic:', t_stat)
print('p-value:', p_value)

# Beginner Example 4: Visualize Monthly Charges Distribution by Gender
import matplotlib.pyplot as plt
plt.hist(male, bins=30, alpha=0.6, label='Male')
plt.hist(female, bins=30, alpha=0.6, label='Female')
plt.xlabel('Monthly Charges')
plt.ylabel('Number of Customers')
plt.title('Distribution of Monthly Charges by Gender')
plt.legend()
plt.show()

# Beginner Example 5: Summarize mean and standard deviation for groups
print('Male monthly charges: Mean = {:.2f}, SD = {:.2f}'.format(male.mean(), male.std()))
print('Female monthly charges: Mean = {:.2f}, SD = {:.2f}'.format(female.mean(), female.std()))

# Intermediate Example 1: Load marketing campaign dataset and prepare group array
dataset2 = openml.datasets.get_dataset(1461)
df2, _, _, _ = dataset2.get_data(dataset_format='dataframe')
df2.columns = ['age','job','marital','education','default','balance','housing','loan','contact','day','month','duration','campaign','pdays','previous','poutcome','response']
print(df2[['age','duration','response']].head())

# Intermediate Example 2: Test if campaign responders spent longer on calls
yes = df2[df2['response'] == 'yes']['duration']
no = df2[df2['response'] == 'no']['duration']
t_stat2, p_value2 = stats.ttest_ind(yes, no, equal_var=False)
print('t-statistic:', t_stat2)
print('p-value:', p_value2)

# Intermediate Example 3: Calculate effect size (Cohen's d) for business impact
def cohens_d(a, b):
    return (a.mean() - b.mean()) / np.sqrt((a.std() ** 2 + b.std() ** 2) / 2)
effect_size = cohens_d(yes, no)
print('Cohen\'s d effect size:', effect_size)

# Intermediate Example 4: Compare age between responders and non-responders
age_yes = df2[df2['response'] == 'yes']['age']
age_no = df2[df2['response'] == 'no']['age']
t_stat_age, p_value_age = stats.ttest_ind(age_yes, age_no, equal_var=False)
print('Age t-statistic:', t_stat_age)
print('Age p-value:', p_value_age)

# Intermediate Example 5: Visualize campaign response rates by education
import seaborn as sns
sns.countplot(y='education', hue='response', data=df2)
plt.title('Campaign Response by Education Level')
plt.xlabel('Number of Customers')
plt.ylabel('Education Level')
plt.show()

# Intermediate Example 6: Analyze cross-tab of campaign response and job
cross = pd.crosstab(df2['job'], df2['response'])
print(cross)

# Advanced Example 1: Using NPS survey to compare regions (synthetic data)
np.random.seed(42)
df_nps = pd.DataFrame({
    'CustomerID': range(1,501),
    'Age': np.random.randint(18,70,500),
    'Region': np.random.choice(['North','South','East','West'],500),
    'NPS_Score': np.random.randint(0,11,500)
})
region1 = df_nps[df_nps['Region'] == 'North']['NPS_Score']
region2 = df_nps[df_nps['Region'] == 'South']['NPS_Score']
t_stat_nps, p_value_nps = stats.ttest_ind(region1, region2, equal_var=False)
print('NPS t-statistic:', t_stat_nps)
print('NPS p-value:', p_value_nps)

# Advanced Example 2: Compare NPS categories (Promoters vs. Detractors)
promoters = df_nps[df_nps['NPS_Score'] >= 9]
detractors = df_nps[df_nps['NPS_Score'] <= 6]
age_promoters = promoters['Age']
age_detractors = detractors['Age']
t_stat_agecat, p_value_agecat = stats.ttest_ind(age_promoters, age_detractors, equal_var=False)
print('Age t-statistic between Promoters and Detractors:', t_stat_agecat)
print('p-value:', p_value_agecat)

# Advanced Example 3: Handle multiple group t-tests with Bonferroni correction
regions = df_nps['Region'].unique()
results = []
for i in range(len(regions)):
    for j in range(i+1, len(regions)):
        group1 = df_nps[df_nps['Region'] == regions[i]]['NPS_Score']
        group2 = df_nps[df_nps['Region'] == regions[j]]['NPS_Score']
        t_stat, p_val = stats.ttest_ind(group1, group2, equal_var=False)
        results.append({'regionA': regions[i], 'regionB': regions[j], 'p_value': p_val})
# Bonferroni-corrected alpha for all pairwise tests
alpha = 0.05 / len(results)
sig_results = [r for r in results if r['p_value'] < alpha]
print('Significant differences after correction:', sig_results)

# Error Handling Example 1: What if there are missing NPS values?
df_nps_nan = df_nps.copy()
df_nps_nan.loc[0:9, 'NPS_Score'] = np.nan  # Set first 10 NPS_Score as missing
clean_nps = df_nps_nan['NPS_Score'].dropna()
print('Clean NPS shape after dropping NaN:', clean_nps.shape)

# Error Handling Example 2: What happens if groups are empty?
empty_group = df_nps[df_nps['Region'] == 'Central']['NPS_Score'] if 'Central' in df_nps['Region'].unique() else pd.Series([])
if empty_group.empty:
    print('No customers in this region (Central). Test cannot be performed.')

# Error Handling Example 3: Incorrect grouping (comparing continuous rather than categorical)
try:
    stats.ttest_ind(df_nps['NPS_Score'], df_nps['Age'])
except Exception as e:
    print('Error:', e)

# Error Handling Example 4: Misinterpreting Likert scales as interval
df_nps['Likert'] = np.random.choice(['Strongly Disagree', 'Disagree', 'Neutral', 'Agree', 'Strongly Agree'], 500)
try:
    stats.ttest_ind(df_nps[df_nps['Likert'] == 'Strongly Agree']['NPS_Score'],
                   df_nps[df_nps['Likert'] == 'Strongly Disagree']['NPS_Score'])
    print('T-test completed, but be cautious: Likert scale is ordinal, not interval.')
except Exception as e:
    print('Error:', e)

# Step 1: Prepare groups and clean numeric column
with_tech = df[df['TechSupport'] == 'Yes']['MonthlyCharges'].dropna()
without_tech = df[df['TechSupport'] == 'No']['MonthlyCharges'].dropna()
print('Sample sizes:', len(with_tech), len(without_tech))

# Step 2: Run the t-test and interpret result
t_stat_final, p_value_final = stats.ttest_ind(with_tech, without_tech, equal_var=False)
print('t-statistic:', t_stat_final)
print('p-value:', p_value_final)
if p_value_final < 0.05:
    print('Result: There is a significant difference in charges. Business may explore pricing strategies!')
else:
    print('Result: No significant difference detected. Keep tech support pricing model stable.')