import pandas as pd
import numpy as np
import openml
import warnings
warnings.filterwarnings('ignore')

# Beginner Example 1: Load a customer satisfaction survey dataset
dataset = openml.datasets.get_dataset(42178)
satisfaction_df, _, _, _ = dataset.get_data(dataset_format='dataframe')
print(satisfaction_df.shape)
print(satisfaction_df.head(3))

# Beginner Example 2: Calculate the overall churn rate (percentage of customers who left)
churn_rate = satisfaction_df['Churn'].value_counts(normalize=True).get('Yes', 0) * 100
print(f'Churn Rate: {churn_rate:.2f}%')

# Beginner Example 3: Count the most common contract type
common_contract = satisfaction_df['Contract'].value_counts().idxmax()
print(f'Most common contract type: {common_contract}')

# Beginner Example 4: Load a marketing campaign dataset
campaign_dataset = openml.datasets.get_dataset(1461)
campaign_df, _, _, _ = campaign_dataset.get_data(dataset_format='dataframe')
campaign_df.columns = ['age','job','marital','education','default','balance','housing','loan','contact','day','month','duration','campaign','pdays','previous','poutcome','response']
print(campaign_df.shape)
print(campaign_df.head(3))

# Beginner Example 5: Calculate the response rate for a marketing campaign
response_rate = campaign_df['response'].value_counts(normalize=True).get('2', 0) * 100
print(f'Response Rate: {response_rate:.2f}%')

# Intermediate Example 1: Segment churn rate by contract type
churn_by_contract = satisfaction_df.groupby('Contract')['Churn'].value_counts(normalize=True).unstack().get('Yes', 0) * 100
print('Churn Rate by Contract Type:')
print(churn_by_contract)

# Intermediate Example 2: Average monthly charges for customers by churn status
avg_charges = satisfaction_df.groupby('Churn')['MonthlyCharges'].mean()
print('Average Monthly Charges grouped by Churn:')
print(avg_charges)

# Intermediate Example 3: Creating a Net Promoter Score (NPS) analysis
np.random.seed(42)
nps_df = pd.DataFrame({
    'CustomerID': range(1, 501),
    'Age': np.random.randint(18, 70, 500),
    'Region': np.random.choice(['North', 'South', 'East', 'West'], 500),
    'NPS_Score': np.random.randint(0, 11, 500)
})
n_promoters = (nps_df['NPS_Score'] >= 9).sum()
n_detractors = (nps_df['NPS_Score'] <= 6).sum()
total_responses = nps_df.shape[0]
nps = ((n_promoters - n_detractors) / total_responses) * 100
print(f'Net Promoter Score: {nps:.1f}')

# Intermediate Example 4: Segment NPS by region
nps_by_region = nps_df.groupby('Region')['NPS_Score'].agg([
    lambda x: (x >= 9).sum(),
    lambda x: (x <= 6).sum(),
    'count'
])
nps_by_region.columns = ['Promoters', 'Detractors', 'Total']
nps_by_region['NPS'] = ((nps_by_region['Promoters'] - nps_by_region['Detractors']) / nps_by_region['Total']) * 100
print(nps_by_region[['NPS']])

# Intermediate Example 5: Analyze open-ended customer feedback with sentiment scoring
feedback_df = pd.DataFrame({'CustomerID':[1,2,3,4,5],
    'Feedback':[
        'Great service and friendly staff',
        'Delivery was slow and packaging was poor',
        'Excellent quality, will buy again',
        'Customer support needs improvement',
        'Good value for money'
    ]})
from textblob import TextBlob
feedback_df['Sentiment'] = feedback_df['Feedback'].apply(lambda x: TextBlob(x).sentiment.polarity)
print(feedback_df)

# Advanced Example 1: Segment campaign response rates by education
response_by_edu = campaign_df.groupby('education')['response'].value_counts(normalize=True).unstack().get('2', 0) * 100
print(response_by_edu.sort_values(ascending=False))

# Advanced Example 2: Analyze trends in customer retention using synthetic cohort data
np.random.seed(0)
dates = pd.date_range('2021-01-01', periods=24, freq='ME')
cohort_df = pd.DataFrame({
    'CustomerID': np.random.randint(1000, 2000, len(dates)),
    'Signup_Month': dates,
    'Active_Users': np.random.randint(50, 300, len(dates))
})
cohort_df['Month'] = cohort_df['Signup_Month'].dt.strftime('%Y-%m')
monthly_users = cohort_df.groupby('Month')['Active_Users'].sum()
monthly_users.plot(title='Monthly Active Users Trend')

# Advanced Example 3: Cross-tabulation of churn by contract and payment method
ct = pd.crosstab(satisfaction_df['Contract'], satisfaction_df['PaymentMethod'], values=(satisfaction_df['Churn']=='Yes'), aggfunc='mean').fillna(0)*100
print('Churn % by Contract and Payment Method:')
print(ct.round(1))

# Error Handling Example 1: Detecting missing survey responses
missing_counts = satisfaction_df.isnull().sum()
print('Missing values in each column:')
print(missing_counts[missing_counts > 0])

# Error Handling Example 2: Catching incorrect groupings
try:
    bogus_group = satisfaction_df.groupby('NonexistentColumn')['Churn'].mean()
except Exception as e:
    print('Grouping error:', e)

# Error Handling Example 3: Misinterpreting NPS scale
sample_nps = pd.Series([10,9,8,7,6,5,4,0])
promoters = (sample_nps >= 9).sum()
detractors = (sample_nps <= 6).sum()
nps_score = ((promoters - detractors) / len(sample_nps)) * 100
print('Correct NPS calculation:', nps_score)

# Best Practice: Create a segmentation index by combining multiple features
satisfaction_df['HighValue'] = (satisfaction_df['MonthlyCharges'] > 80) & (satisfaction_df['tenure'] > 24)
segment_churn = satisfaction_df.groupby('HighValue')['Churn'].value_counts(normalize=True).unstack().get('Yes', 0) * 100
print('Churn Rate by HighValue segment:')
print(segment_churn)

# Best Practice: Run a trend analysis of campaign response rate over calendar month
campaign_df['month_num'] = pd.to_datetime(campaign_df['month'].astype(str), format='%b').dt.month.fillna(0).astype(int)
monthly_resp = campaign_df.groupby('month_num')['response'].value_counts(normalize=True).unstack().get('2', 0) * 100
monthly_resp = monthly_resp[monthly_resp.index > 0]
monthly_resp.plot(title='Campaign Positive Response Rate by Month')

# End-to-end example: From raw NPS scores to an executive summary
np.random.seed(42)
summary_nps_df = pd.DataFrame({'Region': np.random.choice(['North', 'South', 'East', 'West'], 200),
                                'NPS_Score': np.random.randint(0, 11, 200)})
nps_summary = summary_nps_df.groupby('Region')['NPS_Score'].agg([
    lambda x: (x >= 9).sum(),
    lambda x: (x <= 6).sum(),
    'size'
])
nps_summary.columns = ['Promoters', 'Detractors', 'Total']
nps_summary['NPS'] = ((nps_summary['Promoters'] - nps_summary['Detractors']) / nps_summary['Total']) * 100
top_region = nps_summary['NPS'].idxmax()
bottom_region = nps_summary['NPS'].idxmin()
with open('market_research_summary.txt', 'w') as f:
    for region, row in nps_summary.iterrows():
        f.write(f'Region: {region}, NPS: {row.NPS:.1f}\n')
    f.write(f'Highest NPS: {top_region}\n')
    f.write(f'Lowest NPS: {bottom_region}\n')
print(nps_summary[['NPS']])