import warnings
warnings.filterwarnings('ignore')
import pandas as pd
import numpy as np
import openml
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.stats import pearsonr, spearmanr

# Example 1: Load a simple NPS survey dataset
np.random.seed(42)
df_nps = pd.DataFrame({'CustomerID': range(1,501),
                      'Age': np.random.randint(18,70,500),
                      'Region': np.random.choice(['North','South','East','West'],500),
                      'NPS_Score': np.random.randint(0,11,500)})
print('NPS dataset shape:', df_nps.shape)
print(df_nps.head(3))

# Example 2: Calculate average NPS by region
avg_nps_region = df_nps.groupby('Region')['NPS_Score'].mean()
print('Average NPS by region:')
print(avg_nps_region)

# Example 3: Visualizing NPS Score Distribution
sns.histplot(df_nps['NPS_Score'], bins=11, kde=True, color='blue')
plt.title('Distribution of NPS Scores')
plt.xlabel('NPS Score')
plt.ylabel('Count')
plt.show()

# Example 4: Load real customer satisfaction survey data from OpenML
dataset = openml.datasets.get_dataset(42178)
df_cs, _, _, _ = dataset.get_data(dataset_format='dataframe')
print('Shape:', df_cs.shape)
df_cs.head(3)

# Example 5: Checking for missing customer feedback data
missing_values = df_cs.isnull().sum()
print(missing_values[missing_values > 0])

# Example: Calculate pairwise correlation safely after cleaning
num_columns = ['SeniorCitizen', 'tenure', 'MonthlyCharges', 'TotalCharges']
df_cs['TotalCharges'] = (
    df_cs['TotalCharges']
    .replace(' ', np.nan)
)
df_cs['TotalCharges'] = pd.to_numeric(
    df_cs['TotalCharges'],
    errors='coerce'
)
correlation_matrix = df_cs[num_columns].corr()
print('Correlation matrix:')
print(correlation_matrix)

# Example 7: Visualize correlation heatmap
plt.figure(figsize=(6,5))
sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt='.2f')
plt.title('Correlation Heatmap: Customer Satisfaction Variables')
plt.show()

# Example 8: Correlation between age and NPS score
corr_pearson, pval = pearsonr(df_nps['Age'], df_nps['NPS_Score'])
print(f'Pearson correlation between Age and NPS_Score: {corr_pearson:.2f}, p-value: {pval:.3f}')

# Example 9: Spearman correlation of TotalCharges and tenure (capture non-linear effects)
corr_spearman, pval_s = spearmanr(df_cs['TotalCharges'], df_cs['tenure'])
print(f'Spearman correlation between TotalCharges and Tenure: {corr_spearman:.2f}, p-value: {pval_s:.3f}')

# Example 10: Scatter plot of MonthlyCharges vs. TotalCharges to spot relationship shape
plt.scatter(df_cs['MonthlyCharges'], df_cs['TotalCharges'], alpha=0.3)
plt.title('Monthly Charges vs. Total Charges')
plt.xlabel('Monthly Charges')
plt.ylabel('Total Charges')
plt.show()

# Example 11: Segmenting NPS Scores by customer age group
df_nps['AgeGroup'] = pd.cut(df_nps['Age'], bins=[17,30,45,60,70], labels=['18-30','31-45','46-60','61-70'])
grouped_nps = df_nps.groupby('AgeGroup')['NPS_Score'].mean()
print('Average NPS by Age Group:')
print(grouped_nps)

# Example 12: Key Driver Analysis - simple OLS regression
import statsmodels.api as sm
X = df_cs[['MonthlyCharges', 'tenure', 'SeniorCitizen']].fillna(0)
y = df_cs['TotalCharges'].fillna(0)
X = sm.add_constant(X)
model = sm.OLS(y, X).fit()
print(model.summary())

# Multiple Linear Regression with Categorical Variables (Statsmodels)
import statsmodels.api as sm
X_cat = pd.get_dummies(
    df_cs[['gender', 'Contract', 'InternetService']].fillna('Unknown'),
    drop_first=True
)
X_full = pd.concat([X, X_cat], axis=1)
X_full, y = X_full.align(y, join='inner', axis=0)
X_full = X_full.fillna(0)
y = y.fillna(0)
X_full = X_full.astype(float)
y = y.astype(float)
X_full = sm.add_constant(X_full)
model_cat = sm.OLS(y, X_full).fit()
print(model_cat.summary())

# Example 14: Key Driver Analysis on NPS using logistic regression
from sklearn.linear_model import LogisticRegression
df_nps['Promoter'] = (df_nps['NPS_Score'] >= 9).astype(int)
X_demo = pd.get_dummies(df_nps[['AgeGroup', 'Region']], drop_first=True)
y_nps = df_nps['Promoter']
model_lr = LogisticRegression().fit(X_demo, y_nps)
coefs = pd.Series(model_lr.coef_[0], index=X_demo.columns)
print('Logistic Regression Coefficients (drivers of high NPS):')
print(coefs.sort_values(ascending=False))

# Example 15: Online Retail - correlation of quantity purchased and price
url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00502/online_retail_II.xlsx'
df_retail = pd.read_excel(url, sheet_name='Year 2010-2011')
corr_qty_price, _ = pearsonr(df_retail['Quantity'], df_retail['Price'])
print(f'Correlation between Quantity and Price: {corr_qty_price:.2f}')

# Example 16: Multi-variable correlation - Heatmap for select retail variables
retail_cols = ['Quantity','Price']
retail_corr = df_retail[retail_cols].corr()
sns.heatmap(retail_corr, annot=True, cmap='YlGnBu')
plt.title('Correlation in Retail Data')
plt.show()

# Example 17: Constructing a composite satisfaction index (normalize and average)
df_cs['Tenure_Norm'] = (df_cs['tenure'] - df_cs['tenure'].min()) / (df_cs['tenure'].max() - df_cs['tenure'].min())
df_cs['Monthly_Norm'] = (df_cs['MonthlyCharges'] - df_cs['MonthlyCharges'].min()) / (df_cs['MonthlyCharges'].max() - df_cs['MonthlyCharges'].min())
df_cs['SatisfactionIndex'] = (df_cs['Tenure_Norm'] + (1 - df_cs['Monthly_Norm'])) / 2
print(df_cs[['Tenure_Norm','Monthly_Norm','SatisfactionIndex']].head())

# Example 18: Trend analysis of NPS over simulated time periods
date_rng = pd.date_range('2023-01-01', periods=500, freq='D')
df_nps['ResponseDate'] = np.random.choice(date_rng, 500)
df_nps['Period'] = df_nps['ResponseDate'].dt.to_period('M')
trend = df_nps.groupby('Period')['NPS_Score'].mean()
trend.plot(marker='o')
plt.title('Monthly NPS Trend')
plt.ylabel('Average NPS')
plt.xlabel('Period')
plt.show()

# Example 19: Handling missing NPS survey responses
df_nps_missing = df_nps.copy()
df_nps_missing.loc[np.random.choice(df_nps_missing.index, 20, replace=False), 'NPS_Score'] = np.nan
missing_count = df_nps_missing['NPS_Score'].isnull().sum()
print(f'NPS responses missing: {missing_count}')
print('Imputing with mean NPS score:')
df_nps_missing['NPS_Score'] = df_nps_missing['NPS_Score'].fillna(df_nps_missing['NPS_Score'].mean())
print(f'Missing after imputation: {df_nps_missing['NPS_Score'].isnull().sum()}')

# Example 20: Debugging incorrect groupings (using wrong column type)
try:
    grouped = df_nps.groupby('NPS_Score')['Region'].mean()
except Exception as e:
    print('Error:', e)
    print('You cannot calculate the mean of a categorical variable like Region.')

# Example 21: Debugging Likert scale misinterpretation
likert_map = {'Strongly disagree':1, 'Disagree':2, 'Neutral':3, 'Agree':4, 'Strongly agree':5}
df_likert = pd.DataFrame({'Satisfaction': ['Agree','Neutral','Strongly agree','Disagree','Strongly disagree']})
df_likert['Satisfaction_Score'] = df_likert['Satisfaction'].map(likert_map)
print(df_likert)

# End-to-end: Identify the strongest driver of churn from real customer data
df_cs['TotalCharges'] = pd.to_numeric(df_cs['TotalCharges'], errors='coerce')
df_cs = df_cs.dropna(subset=['MonthlyCharges','tenure','TotalCharges','Churn'])
df_cs['Churn_Flag'] = df_cs['Churn'].map({'Yes':1, 'No':0})
X = df_cs[['MonthlyCharges','tenure','TotalCharges']]
y = df_cs['Churn_Flag']
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(max_iter=200).fit(X, y)
drivers = pd.Series(model.coef_[0], index=X.columns).sort_values(key=abs, ascending=False)
print('Churn Key Drivers:')
print(drivers)