# First, suppress warnings to keep our notebook tidy
import warnings; warnings.filterwarnings("ignore")
import numpy as np
np.random.seed(42)

# Data setup
import requests
from bs4 import BeautifulSoup
url = 'https://www.bbc.com/news'
html = requests.get(url).text
soup = BeautifulSoup(html, 'html.parser')
headlines = [h.get_text(strip=True) for h in soup.select('h2') if h.get_text(strip=True)]
print(headlines[:10])

# How many headlines did we scrape?
print("Total headlines scraped:", len(headlines))

# Let us preview the first five headlines
for i, h in enumerate(headlines[:5], 1):
    print(f"{i}. {h}")

# Let us put our headlines into a pandas DataFrame
import pandas as pd
df = pd.DataFrame({'headline': headlines})
print(df.shape)
df.head()

# Remove duplicate headlines if there are any
df = df.drop_duplicates('headline').reset_index(drop=True)
print("Unique headlines:", len(df))

# Check for any empty or very short headlines
short_headlines = df[df['headline'].str.len() < 8]
print(short_headlines.shape)
short_headlines.head()

# Drop all headlines less than 8 characters
df = df[df['headline'].str.len() >= 8].reset_index(drop=True)
print("Cleaned headline count:", len(df))

# Get word frequency from all headlines
from collections import Counter
all_words = ' '.join(df['headline']).lower().split()
common_words = Counter(all_words).most_common(10)
for word, count in common_words:
    print(f"{word}: {count}")

# Plot the word frequency as a bar chart
import matplotlib.pyplot as plt
words, counts = zip(*common_words)
plt.figure(figsize=(8,4))
plt.bar(words, counts, color='skyblue')
plt.title('Top 10 Words in BBC News Headlines')
plt.ylabel('Count')
plt.xlabel('Word')
plt.show()

# Are there any headlines with certain keywords?
key = input("Type a word to search for in the headlines: ").strip().lower()
matches = df[df['headline'].str.lower().str.contains(key)]
print(f'Headlines with "{key}":', len(matches))
if not matches.empty:
    print(matches.head(5))

# Let us count headlines by length for more insights
df['len'] = df['headline'].str.len()
print(df['len'].describe())

# Plot a histogram of headline lengths
import seaborn as sns
plt.figure(figsize=(7,3))
sns.histplot(df['len'], bins=12, color='purple')
plt.title('Distribution of Headline Lengths')
plt.xlabel('Number of characters')
plt.ylabel('Count')
plt.show()

# Find headlines with numbers (like years or stats)
import re
num_headlines = df[df['headline'].str.contains(r'\d')]
print("Total with numbers:", len(num_headlines))
num_headlines.head(5)

# Optional: Save collected headlines for future analysis
df[['headline']].to_csv('bbc_headlines.csv', index=False)
print('Saved to bbc_headlines.csv!')