# Suppress warnings for a cleaner output
import warnings; warnings.filterwarnings("ignore")
import numpy as np
np.random.seed(42)

# Data setup
import requests
from bs4 import BeautifulSoup
url = 'https://webscraper.io/test-sites/e-commerce/static/computers'
html = requests.get(url).text
soup = BeautifulSoup(html, 'html.parser')
cards = soup.select('.thumbnail')
records = [{'title': c.select_one('.title').get_text(strip=True), 'price': c.select_one('.price').get_text(strip=True)} for c in cards]
print(records[:5])

# Convert records to pandas DataFrame for easy analysis
import pandas as pd
df = pd.DataFrame(records)
print(df.shape)
print(df.head())

# Check column names and missing values
print(df.columns)
print(df.isnull().sum())

# Clean price column (remove currency signs and convert to float)
df['price'] = df['price'].str.replace('$', '').astype(float)

# Show the basic statistics for the price
print(df['price'].describe())

# Plot a simple price distribution
import matplotlib.pyplot as plt
plt.figure(figsize=(8, 4))
plt.hist(df['price'], bins=10, color='skyblue', edgecolor='black')
plt.title('Distribution of Scraped Property Prices')
plt.xlabel('Price')
plt.ylabel('Count')
plt.show()

# Display top 5 most expensive listings
print(df.sort_values('price', ascending=False).head())

# Search for a property keyword using input
keyword = input("Enter a keyword to search titles: ")
matches = df[df['title'].str.lower().str.contains(keyword.lower())]
print(matches)

# Count how many listings contain the word 'laptop'
laptop_count = df['title'].str.lower().str.contains('laptop').sum()
print(f"Number of listings with 'laptop': {laptop_count}")

# Create a bar plot of the top 5 most common words in titles
from collections import Counter
words = " ".join(df['title']).lower().split()
stopwords = set(['and','or','the','with','for','of','by','to','in'])
filtered = [w for w in words if w not in stopwords]
top5 = Counter(filtered).most_common(5)
labels, counts = zip(*top5)
plt.figure(figsize=(7,4))
plt.bar(labels, counts, color='orange')
plt.title('Top 5 Words in Property Titles')
plt.ylabel('Count')
plt.show()

# Export your cleaned data for later use
df.to_csv('real_estate_scraped.csv', index=False)
print('Saved as real_estate_scraped.csv')