import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')

# Load the YouTube Analytics Dataset (synthetic fallback if needed)
np.random.seed(42)
n_videos = 300
views = np.random.randint(100, 500000, n_videos)
df = pd.DataFrame({
    'video_id': range(1, n_videos+1),
    'publish_date': pd.date_range('2022-01-01', periods=n_videos, freq='D'),
    'views': views,
    'watch_time': np.random.randint(1000, 500000, n_videos),
    'likes': (views * np.random.uniform(0.01, 0.08, n_videos)).astype(int),
    'comments': (views * np.random.uniform(0.001, 0.02, n_videos)).astype(int),
    'ctr': np.round(np.random.uniform(2, 10, n_videos), 2)
})
print(df.shape)
print(df.head(3))

# Beginner Example: List videos with the highest view counts
top_views = df.nlargest(5, 'views')[['video_id', 'views', 'watch_time']]
print(top_views)

# Beginner Example: List videos with the highest total watch time
top_watch_time = df.nlargest(5, 'watch_time')[['video_id', 'views', 'watch_time']]
print(top_watch_time)

# Beginner Example: Compute the average views and watch time across all videos
avg_views = df['views'].mean()
avg_watch_time = df['watch_time'].mean()
print(f'Average views per video: {int(avg_views):,}')
print(f'Average total watch time per video: {int(avg_watch_time):,} minutes')

# Beginner Example: Calculate average watch time PER VIEW (engagement depth)
df['avg_watch_time_per_view'] = df['watch_time'] / df['views']
print(df[['video_id', 'views', 'watch_time', 'avg_watch_time_per_view']].head(3))

# Beginner Example: Visualize the distribution of views
import matplotlib.pyplot as plt
plt.figure(figsize=(7,4))
plt.hist(df['views'], bins=30, color='skyblue', edgecolor='black')
plt.title('Distribution of Video Views')
plt.xlabel('Views')
plt.ylabel('Number of Videos')
plt.tight_layout()
plt.show()

# Intermediate Example: Visualize total watch time distribution
plt.figure(figsize=(7,4))
plt.hist(df['watch_time'], bins=30, color='coral', edgecolor='black')
plt.title('Distribution of Video Total Watch Time (minutes)')
plt.xlabel('Total Watch Time (minutes)')
plt.ylabel('Number of Videos')
plt.tight_layout()
plt.show()

# Intermediate Example: Find videos with above-average views and watch time
above_avg = df[(df['views'] > avg_views) & (df['watch_time'] > avg_watch_time)]
print(f'Number of videos above average in both views and watch time: {len(above_avg)}')
print(above_avg[['video_id', 'views', 'watch_time']].head(5))

# Intermediate Example: Analyze correlation between views and watch time
correlation = df['views'].corr(df['watch_time'])
print(f'Correlation between views and watch time: {correlation:.2f}')

# Intermediate Example: Flag videos with low watch time per view (shallow engagement)
low_engage = df[df['avg_watch_time_per_view'] < df['avg_watch_time_per_view'].quantile(0.25)]
print('Example of lower-engagement videos:')
print(low_engage[['video_id', 'views', 'watch_time', 'avg_watch_time_per_view']].head(5))

# Intermediate Example: Plot views vs. watch time as a scatter plot
plt.figure(figsize=(7,5))
plt.scatter(df['views'], df['watch_time'], alpha=0.6, edgecolors='w')
plt.xlabel('Views')
plt.ylabel('Total Watch Time (minutes)')
plt.title('Scatter Plot: Views vs. Total Watch Time')
plt.grid(alpha=0.3)
plt.tight_layout()
plt.show()

# Advanced Example: Calculate the top 5% of videos by average watch time per view
threshold = df['avg_watch_time_per_view'].quantile(0.95)
top_5pct = df[df['avg_watch_time_per_view'] >= threshold]
print(f'Top 5% high-engagement videos: {len(top_5pct)}')
print(top_5pct[['video_id', 'views', 'watch_time', 'avg_watch_time_per_view']].sort_values('avg_watch_time_per_view', ascending=False).head(5))

# Advanced Example: Compare average watch time per view across time periods
df['month'] = df['publish_date'].dt.to_period('M')
monthly_engagement = df.groupby('month')['avg_watch_time_per_view'].mean()
monthly_engagement.plot(kind='line', marker='o', figsize=(10,4), title='Avg Watch Time per View by Month')
plt.ylabel('Avg Watch Time per View (min)')
plt.xlabel('Month')
plt.tight_layout()
plt.show()

# Advanced Example: Benchmark videos by click-through rate vs. watch time
plt.figure(figsize=(8,5))
plt.scatter(df['ctr'], df['avg_watch_time_per_view'], c=df['watch_time'], cmap='viridis', s=40, alpha=0.7)
plt.colorbar(label='Total Watch Time (minutes)')
plt.xlabel('CTR (%)')
plt.ylabel('Avg Watch Time per View (minutes)')
plt.title('CTR vs. Avg Watch Time per View (colored by Watch Time)')
plt.grid(alpha=0.3)
plt.tight_layout()
plt.show()

# Error Handling: Handle missing values in views or watch_time
df.loc[np.random.choice(df.index, 5, replace=False), 'views'] = np.nan
df.loc[np.random.choice(df.index, 5, replace=False), 'watch_time'] = np.nan
missing = df[df['views'].isnull() | df['watch_time'].isnull()]
print(f'Rows with missing views or watch_time: {missing.shape[0]}')

# Error Handling: Fill missing values and recalculate average watch time per view
df['views'] = df['views'].fillna(0)
df['watch_time'] = df['watch_time'].fillna(0)
df['avg_watch_time_per_view'] = df.apply(
    lambda row: row['watch_time']/row['views'] if row['views'] > 0 else 0, axis=1)
print('Recalculated avg_watch_time_per_view with filled missing data:')
print(df[['video_id', 'views', 'watch_time', 'avg_watch_time_per_view']].head(3))

# Error Handling: Avoiding incorrect aggregation (should not sum ratios)
incorrect_total = df['avg_watch_time_per_view'].sum()
print(f'INCORRECT: Total of avg_watch_time_per_view (should not sum ratios): {incorrect_total:.2f}')
correct_mean = df['avg_watch_time_per_view'].mean()
print(f'CORRECT: Mean avg_watch_time_per_view: {correct_mean:.2f}')

# Error Handling: Misinterpreting CTR/engagement rates (division by zero protections already shown)
zero_row = pd.DataFrame({'views':[0], 'watch_time':[5000]})
zero_row['avg_watch_time_per_view'] = zero_row.apply(lambda row: row['watch_time']/row['views'] if row['views'] > 0 else 0, axis=1)
print(zero_row)

# Common Analytics Pattern: Identify top-performing videos by category (synthetic)
df['category'] = np.random.choice(['Education', 'Entertainment', 'HowTo', 'Gaming'], len(df))
category_grouped = df.groupby('category')['avg_watch_time_per_view'].mean().sort_values(ascending=False)
print('Average Watch Time per View by Category:')
print(category_grouped)

# Common Analytics Pattern: Track monthly growth in total watch time
monthly_growth = df.groupby('month')['watch_time'].sum()
monthly_growth.plot(kind='bar', figsize=(10,4), color='teal', title='Monthly Total Watch Time Growth')
plt.ylabel('Total Watch Time (minutes)')
plt.xlabel('Month')
plt.tight_layout()
plt.show()

# End-to-End Solution: Find top three videos and summarize patterns
top_videos = df.sort_values('avg_watch_time_per_view', ascending=False).head(3)
print('Top 3 videos (by avg watch time per view):')
print(top_videos[['video_id', 'category', 'month', 'avg_watch_time_per_view']])

categories = top_videos['category'].unique()
months = top_videos['month'].unique()
print(f'Pattern: Categories = {categories}, Months = {months}')
print('Recommendation: Focus more on these content categories in upcoming months for maximum viewer engagement.')