import pandas as pd
from itertools import combinations
from collections import Counter
clean = pd.read_csv('online_retail_clean.csv', parse_dates=['InvoiceDate'])
clean.shape

uk = clean[clean['Country'] == 'United Kingdom']
uk.shape[0]
uk['InvoiceNo'].nunique()

top_products = uk.groupby('Description')['InvoiceNo'].nunique().sort_values(ascending=False).head(40)
top_products.head(10)
top_set = set(top_products.index)

in_top = uk[uk['Description'].isin(top_set)]
baskets = in_top.groupby('InvoiceNo')['Description'].apply(set)
len(baskets)

baskets = baskets[baskets.apply(len) >= 2]
len(baskets)

n_invoices = uk['InvoiceNo'].nunique()
item_counts = Counter()
for basket in baskets:
    for item in basket:
        item_counts[item] += 1
support = {item: cnt / n_invoices for item, cnt in item_counts.items()}
sorted(support.items(), key=lambda x: -x[1])[:5]

pair_counts = Counter()
for basket in baskets:
    for a, b in combinations(sorted(basket), 2):
        pair_counts[(a, b)] += 1
len(pair_counts)

pair_support = {pair: cnt / n_invoices for pair, cnt in pair_counts.items()}
sorted(pair_support.items(), key=lambda x: -x[1])[:5]

def confidence(a, b):
    key = tuple(sorted((a, b)))
    return pair_counts[key] / item_counts[a]
example_a, example_b = sorted(pair_counts, key=lambda p: -pair_counts[p])[0]
round(confidence(example_a, example_b), 3), round(confidence(example_b, example_a), 3)

def lift(a, b):
    key = tuple(sorted((a, b)))
    return pair_support[key] / (support[a] * support[b])
round(lift(example_a, example_b), 2)

rows = []
for (a, b), cnt in pair_counts.items():
    rows.append((a, b, cnt, pair_support[(a, b)], confidence(a, b), confidence(b, a), lift(a, b)))
rules = pd.DataFrame(rows, columns=['A', 'B', 'co_count', 'support', 'conf_A_to_B', 'conf_B_to_A', 'lift'])
rules.shape[0]

rules.sort_values('lift', ascending=False).head(10)

rules.sort_values('conf_A_to_B', ascending=False).head(10)[['A', 'B', 'conf_A_to_B', 'lift']]

meaningful = rules[rules['co_count'] >= 20]
meaningful.shape[0]
meaningful.sort_values('lift', ascending=False).head(10)

top_rule = meaningful.sort_values('lift', ascending=False).iloc[0]
print(f"Customers who buy '{top_rule['A']}' are {round(top_rule['lift'], 1)}x more likely to also buy '{top_rule['B']}' than random chance would predict.")

rules['lift'].mean().round(2)
(rules['lift'] > 2).sum()

import matplotlib.pyplot as plt
top15 = meaningful.sort_values('lift', ascending=False).head(15)
labels = [f"{a[:15]}.. + {b[:15]}.." for a, b in zip(top15['A'], top15['B'])]
plt.figure(figsize=(9, 7))
plt.barh(labels[::-1], top15['lift'][::-1], color='darkorange')
plt.xlabel('Real Lift')
plt.title('Real Top 15 Real Product Associations by Lift')
plt.tight_layout()
plt.savefig('market_basket_top_lift.png', dpi=120)
plt.close()

plt.figure(figsize=(8, 6))
plt.scatter(rules['support'], rules['lift'], alpha=0.5, color='teal')
plt.xlabel('Real Support')
plt.ylabel('Real Lift')
plt.title('Real Support vs Real Lift Across All Candidate Pairs')
plt.tight_layout()
plt.savefig('market_basket_support_vs_lift.png', dpi=120)
plt.close()

strong = meaningful[meaningful['lift'] > 2]
appearance_counts = pd.concat([strong['A'], strong['B']]).value_counts()
appearance_counts.head(10)

most_connected = appearance_counts.index[0]
bundle_candidates = strong[(strong['A'] == most_connected) | (strong['B'] == most_connected)]
bundle_candidates[['A', 'B', 'lift']].sort_values('lift', ascending=False)

meaningful.sort_values('lift', ascending=False).to_csv('market_basket_rules.csv', index=False)
reloaded_rules = pd.read_csv('market_basket_rules.csv')
reloaded_rules.shape[0] == meaningful.shape[0]

basket_sizes = baskets.apply(len)
basket_sizes.value_counts().sort_index()
basket_sizes.mean().round(2)

from math import comb
possible_pairs = comb(len(top_set), 2)
len(pair_counts), possible_pairs
round(len(pair_counts) / possible_pairs * 100, 1)

manual_count = sum(1 for basket in baskets if example_a in basket and example_b in basket)
manual_count == pair_counts[tuple(sorted((example_a, example_b)))]

item_counts[example_a], item_counts[example_b]
print(f"'{example_a}' appears in {item_counts[example_a]} real baskets; '{example_b}' appears in {item_counts[example_b]}, which is exactly why the two real confidence values differ.")

plt.figure(figsize=(8, 5))
plt.hist(rules['lift'], bins=30, color='slateblue', edgecolor='white')
plt.axvline(1, color='red', linestyle='--', label='Real independence (lift = 1)')
plt.xlabel('Real Lift')
plt.ylabel('Real Number of Rules')
plt.title('Real Distribution of Lift Across All Candidate Pairs')
plt.legend()
plt.tight_layout()
plt.savefig('market_basket_lift_distribution.png', dpi=120)
plt.close()

support_df = pd.DataFrame(sorted(support.items(), key=lambda x: -x[1]), columns=['Product', 'Support'])
support_df.to_csv('market_basket_item_support.csv', index=False)
support_df.head(5)

baskets_with_strong_rule = sum(1 for basket in baskets if any(a in basket and b in basket for a, b in zip(strong['A'], strong['B'])))
round(baskets_with_strong_rule / len(baskets) * 100, 1)

for min_count in [5, 10, 20, 50, 100]:
    n_rules = (rules['co_count'] >= min_count).sum()
    print(f'min_count={min_count}: {n_rules} real rules survive')

(rules['support'] <= rules[['A']].join(pd.Series(support, name='sA'), on='A')['sA']).all()