import json

order = {"drink": "flat white", "size": "large", "price": 3.40, "extras": ["oat milk"]}
text = json.dumps(order)
print(type(text), text)

back = json.loads(text)
print(back["drink"], "with", back["extras"][0])

menu = [{"item": "Croissant", "price": 2.20}, {"item": "Flat white", "price": 3.40}]
for row in menu:
    print(f"{row['item']:<12}${row['price']:.2f}")
print("JSON is just text, so it travels well:", json.dumps(menu)[:40], "...")

request_text = '''GET /menu.html HTTP/1.1
Host: cornercafe.test
User-Agent: MyBrowser/1.0
Accept: text/html'''
print(request_text)

from urllib.parse import urlparse

url = urlparse("https://cornercafe.test:8080/api/cafes?city=Leeds&limit=3")
print("scheme:", url.scheme)
print("host:  ", url.hostname)
print("port:  ", url.port)
print("path:  ", url.path)
print("query: ", url.query)

from urllib.parse import urlencode
print("/api/cafes?" + urlencode({"city": "York", "open_late": "yes"}))
print("/search?" + urlencode({"q": "flat white & cake"}))

from http import HTTPStatus
for code in [200, 201, 301, 400, 401, 404, 500]:
    print(code, HTTPStatus(code).phrase)

def status_family(code):
    families = {2: "success", 3: "redirect",
                4: "your mistake (client error)", 5: "their mistake (server error)"}
    return families.get(code // 100, "unknown")
for code in [200, 302, 404, 503]:
    print(code, "->", status_family(code))

print(403, "->", status_family(403))
print("Meaning:", HTTPStatus(403).phrase)

from pathlib import Path

SITE = Path("lesson_files/site")
SITE.mkdir(parents=True, exist_ok=True)

(SITE / "index.html").write_text('''<html><head><title>Corner Cafe</title></head>
<body>
  <h1>Corner Cafe</h1>
  <p class="tagline">Good coffee, slow mornings.</p>
  <a href="menu.html">Menu</a> | <a href="about.html">About us</a> | <a href="jobs.html">Jobs</a>
</body></html>''', encoding="utf-8")
print(sorted(p.name for p in SITE.iterdir()))

menu_html = '''<html><head><title>Menu</title></head><body>
<h1>Our Menu</h1>
<table id="menu">
  <tr><th>Item</th><th>Type</th><th>Price</th></tr>
  <tr class="drink"><td>Flat white</td><td>drink</td><td>$3.40</td></tr>
  <tr class="drink"><td>Chai latte</td><td>drink</td><td>$3.60</td></tr>
  <tr class="food"><td>Cheese toastie</td><td>food</td><td>$5.20</td></tr>
  <tr class="food special"><td>Banana bread</td><td>food</td><td>$2.80</td></tr>
</table>
<p>Back to <a href="index.html">home</a></p>
</body></html>'''
(SITE / "menu.html").write_text(menu_html, encoding="utf-8")
print(len(menu_html), "characters written to menu.html")

(SITE / "about.html").write_text('''<html><head><title>About us</title></head><body>
<h1>About us</h1>
<p id="story">Two friends, one very old espresso machine.</p>
<img src="team.jpg" alt="The cafe team">
</body></html>''', encoding="utf-8")

(SITE / "robots.txt").write_text("User-agent: *\nDisallow: /private/\nCrawl-delay: 1\n")
print(sorted(p.name for p in SITE.iterdir()))

CAFES = [
    {"id": 1, "name": "Bean There", "city": "Leeds", "rating": 4.6, "wifi": True},
    {"id": 2, "name": "Grind House", "city": "York", "rating": 4.1, "wifi": False},
    {"id": 3, "name": "The Daily Cup", "city": "Leeds", "rating": 3.9, "wifi": True},
    {"id": 4, "name": "Steam and Leaf", "city": "Bristol", "rating": 4.8, "wifi": True},
    {"id": 5, "name": "Mug Life", "city": "York", "rating": 4.4, "wifi": True},
]
WEATHER = {"leeds": {"temp_c": 12, "sky": "drizzle"},
           "york": {"temp_c": 10, "sky": "cloudy"},
           "bristol": {"temp_c": 14, "sky": "sunny spells"}}
print(len(CAFES), "cafes and", len(WEATHER), "cities with weather")

def cafes_route(query, headers):
    results = CAFES
    if "city" in query:
        results = [c for c in results if c["city"].lower() == query["city"].lower()]
    if "min_rating" in query:
        results = [c for c in results if c["rating"] >= float(query["min_rating"])]
    return 200, {"count": len(results), "cafes": results}

def weather_route(query, headers):
    city = query.get("city", "").lower()
    if city not in WEATHER:
        return 404, {"error": f"No weather for '{city}'"}
    return 200, {"city": city.title(), **WEATHER[city]}

API_ROUTES = {"/api/cafes": cafes_route, "/api/weather": weather_route}
POST_ROUTES = {}
print("GET routes:", list(API_ROUTES))

print(cafes_route({"city": "york"}, {}))
print(weather_route({"city": "Mars"}, {}))

from http.server import SimpleHTTPRequestHandler

class QuietHandler(SimpleHTTPRequestHandler):
    extensions_map = {".html": "text/html; charset=utf-8", ".txt": "text/plain; charset=utf-8"}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, directory=str(SITE), **kwargs)

    def log_message(self, *args):
        pass  # don't print a line for every request

    def send_json(self, data, status=200):
        body = json.dumps(data).encode("utf-8")
        try:
            self.send_response(status)
            self.send_header("Content-Type", "application/json; charset=utf-8")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)
        except (BrokenPipeError, ConnectionResetError):
            pass  # the visitor gave up waiting and left

print("QuietHandler is ready")

from urllib.parse import parse_qs

class LessonHandler(QuietHandler):
    def do_GET(self):
        url = urlparse(self.path)
        route = API_ROUTES.get(url.path)
        if route is None:
            return super().do_GET()  # not an API path: serve a file
        query = {key: values[0] for key, values in parse_qs(url.query).items()}
        try:
            status, data = route(query, self.headers)
        except ValueError:
            status, data = 400, {"error": "Bad value in your query"}
        self.send_json(data, status)

    def do_POST(self):
        size = int(self.headers.get("Content-Length", 0))
        body = json.loads(self.rfile.read(size) or b"{}")
        route = POST_ROUTES.get(urlparse(self.path).path)
        status, data = route(body, self.headers) if route else (404, {"error": "Not found"})
        self.send_json(data, status)

print("LessonHandler can answer GET and POST")

import threading
from http.server import ThreadingHTTPServer
server = ThreadingHTTPServer(("127.0.0.1", 0), LessonHandler)  # port 0 = any free port
port = server.server_address[1]
BASE = f"http://127.0.0.1:{port}"

thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
print("Server running in the background:", thread.is_alive())

import requests
r = requests.get(BASE + "/index.html")
print(r.status_code, r.reason)
print("ok?", r.ok)
print(r.text[:90])

print(r.headers["Content-Type"])
print(r.headers["content-type"])   # header names ignore case
print("Body size:", len(r.content), "bytes")

missing = requests.get(BASE + "/pancakes.html")
print(missing.status_code, missing.reason)
print("ok?", missing.ok)

for page in ["/index.html", "/menu.html", "/about.html", "/jobs.html"]:
    r = requests.get(BASE + page)
    print(f"{page:<13} {r.status_code}  {status_family(r.status_code)}")

r = requests.get(BASE + "/api/cafes")
print(r.status_code, r.headers["Content-Type"])
data = r.json()          # JSON text -> Python dict
print(type(data), "with keys", list(data))
print("Cafes found:", data["count"])

for cafe in data["cafes"]:
    wifi = "wifi" if cafe["wifi"] else "no wifi"
    print(f"{cafe['name']:<16}{cafe['city']:<9}{cafe['rating']:>4}  {wifi}")

r = requests.get(BASE + "/api/cafes", params={"city": "Leeds"})
print(r.url.removeprefix(BASE))
for cafe in r.json()["cafes"]:
    print(" ", cafe["name"], cafe["rating"])

r = requests.get(BASE + "/api/cafes", params={"city": "York", "min_rating": 4.2})
print(r.url.removeprefix(BASE))
print([cafe["name"] for cafe in r.json()["cafes"]])

while True:
    city = input("City for weather (blank to stop): ").strip()
    if not city:
        break
    r = requests.get(BASE + "/api/weather", params={"city": city})
    if r.status_code == 200:
        w = r.json()
        print(f"  {w['city']}: {w['temp_c']}C and {w['sky']}")
    else:
        print(f"  {r.status_code}: {r.json()['error']}")

menu_data = requests.get(BASE + "/menu.html").json()

r = requests.get(BASE + "/menu.html")
if "application/json" in r.headers["Content-Type"]:
    print(r.json())
else:
    print("That's", r.headers["Content-Type"], "so read r.text instead")

import time

def slow_route(query, headers):
    time.sleep(2)  # pretend the server is very busy
    return 200, {"message": "Sorry for the wait!"}

API_ROUTES["/api/slow"] = slow_route
print("Routes now:", list(API_ROUTES))

try:
    r = requests.get(BASE + "/api/slow", timeout=0.5)
    print(r.json())
except requests.exceptions.Timeout:
    print("Gave up after 0.5 seconds. The server is too slow.")

r = requests.get(BASE + "/api/weather", params={"city": "Atlantis"})
try:
    r.raise_for_status()      # turns 4xx and 5xx into an exception
    print("All good")
except requests.exceptions.HTTPError as err:
    print("HTTP problem:", err.response.status_code, err.response.reason)

try:
    requests.get("http://127.0.0.1:1/", timeout=2)   # nothing listens on port 1
except requests.exceptions.ConnectionError:
    print("Could not connect. Is the server running?")

def safe_get(url, **kwargs):
    try:
        r = requests.get(url, timeout=3, **kwargs)
        r.raise_for_status()
        return r
    except requests.exceptions.RequestException as err:
        print(f"  Could not fetch {url.removeprefix(BASE)}: {type(err).__name__}")
        return None

for path in ["/menu.html", "/pancakes.html", "/api/weather?city=york"]:
    r = safe_get(BASE + path)
    print(path, "->", "got it" if r else "skipped")

r = requests.get("127.0.0.1/menu.html")

print(BASE.startswith("http://"))
r = requests.get(BASE + "/menu.html")
print(r.status_code, "Menu page fetched")

r = requests.get(BASE + "/api/weather", params={"city": "Atlantis"})
print(r.json()["temp_c"], "degrees")

r = requests.get(BASE + "/api/weather", params={"city": "Atlantis"})
if r.ok:
    print(r.json()["temp_c"], "degrees")
else:
    print("No luck:", r.status_code, r.json()["error"])

ORDERS = []

def new_order_route(body, headers):
    if "drink" not in body:
        return 400, {"error": "Every order needs a drink"}
    order = {"id": len(ORDERS) + 1, "drink": body["drink"], "name": body.get("name", "guest")}
    ORDERS.append(order)
    return 201, order

POST_ROUTES["/api/orders"] = new_order_route
print("POST routes:", list(POST_ROUTES))

r = requests.post(BASE + "/api/orders", json={"drink": "flat white", "name": "Ana"}, timeout=3)
print(r.status_code, r.reason)
print(r.json())

new_orders = [{"drink": "chai latte", "name": "Ben"},
              {"name": "Cleo"},
              {"drink": "mocha"}]
for item in new_orders:
    r = requests.post(BASE + "/api/orders", json=item, timeout=3)
    print(r.status_code, r.json())

def orders_route(query, headers):
    return 200, {"orders": ORDERS}

API_ROUTES["/api/orders"] = orders_route
for order in requests.get(BASE + "/api/orders", timeout=3).json()["orders"]:
    print(f"#{order['id']}  {order['name']:<6}{order['drink']}")

def whoami_route(query, headers):
    return 200, {"user_agent": headers.get("User-Agent")}

API_ROUTES["/api/whoami"] = whoami_route
r = requests.get(BASE + "/api/whoami", timeout=3)
print("Default:", r.json()["user_agent"].split("/")[0])

my_headers = {"User-Agent": "CornerCafeBot/1.0 (learning project)"}
r = requests.get(BASE + "/api/whoami", headers=my_headers, timeout=3)
print("Custom: ", r.json()["user_agent"])

def secret_route(query, headers):
    if headers.get("X-API-Key") != "latte-123":
        return 401, {"error": "Missing or wrong API key"}
    return 200, {"recipe": "honey, cinnamon and a pinch of salt"}

API_ROUTES["/api/secret-syrup"] = secret_route
r = requests.get(BASE + "/api/secret-syrup", timeout=3)
print(r.status_code, r.json())

r = requests.get(BASE + "/api/secret-syrup", headers={"X-API-Key": "latte-123"}, timeout=3)
print(r.status_code, r.json())

r = requests.post(BASE + "/api/orders", json={"drink": "espresso"}, timeout=3)
print(r.status_code, HTTPStatus(r.status_code).phrase)

def find_cafes(city, min_rating=0):
    r = safe_get(BASE + "/api/cafes", params={"city": city, "min_rating": min_rating})
    return r.json()["cafes"] if r else []

def get_weather(city):
    r = safe_get(BASE + "/api/weather", params={"city": city})
    return r.json() if r else None

print([c["name"] for c in find_cafes("Leeds", 4)], get_weather("York"))

def ask_rating():
    while True:
        answer = input("Minimum rating (0-5)? ")
        try:
            return float(answer)
        except ValueError:
            print(f"  '{answer}' isn't a number, try something like 4.2")

print("ask_rating is ready")

def cafe_finder():
    while True:
        city = input("Which city? ").strip()
        rating = ask_rating()
        cafes = sorted(find_cafes(city, rating), key=lambda c: c["rating"], reverse=True)
        print(f"  {len(cafes)} cafe(s) in {city.title()} rated {rating}+")
        for cafe in cafes:
            print(f"   * {cafe['name']:<15}{cafe['rating']}")
        weather = get_weather(city)
        if weather:
            print(f"  Weather: {weather['temp_c']}C, {weather['sky']}")
        if input("Search again? (y/n) ").strip().lower() != "y":
            break
    print("Enjoy your coffee!")

cafe_finder()

from bs4 import BeautifulSoup

playlist_html = '''<html><head><title>Sunday Playlist</title></head><body>
<h1>Sunday Playlist</h1>
<p class="intro">Slow songs for a slow morning.</p>
<ul id="songs">
  <li class="song" data-mins="4">River Song</li>
  <li class="song" data-mins="3">Golden Hour</li>
  <li class="song fav" data-mins="5">Paper Boats</li>
</ul>
<a href="/playlists/rainy-day" title="Next playlist">Rainy Day</a>
</body></html>'''

soup = BeautifulSoup(playlist_html, "html.parser")
print(soup.title.text)
print(soup.h1.text)

intro = soup.find("p", class_="intro")
print(intro)
print(intro.text)

songs = soup.find_all("li")
print(len(songs), "songs")
for number, song in enumerate(songs, start=1):
    print(number, song.text)

link = soup.find("a")
print(link["href"])
print(link.get("title"))
print(link.attrs)

total = 0
for song in soup.find_all("li", class_="song"):
    minutes = int(song["data-mins"])
    total += minutes
    print(f"{song.text:<12}{minutes} min")
print("Playlist length:", total, "minutes")

print(soup.find("p", class_="outro").text)

outro = soup.find("p", class_="outro")
if outro is None:
    print("No outro on this page, skipping it")
else:
    print(outro.text)

print([li.text for li in soup.select("li.fav")])
print(soup.select_one("#songs li").text)
print(len(soup.select("ul#songs > li")), "songs inside the list")

r = requests.get(BASE + "/menu.html", timeout=3)
menu_soup = BeautifulSoup(r.text, "html.parser")
print(menu_soup.title.text)
print(menu_soup.select_one("tr.drink td").text)

table = menu_soup.find("table", id="menu")
rows = []
for tr in table.find_all("tr"):
    cells = [cell.get_text(strip=True) for cell in tr.find_all(["th", "td"])]
    rows.append(cells)
for row in rows:
    print(row)

header, *body = rows
menu_items = [dict(zip(header, row)) for row in body]
for item in menu_items:
    item["Price"] = float(item["Price"].replace("$", ""))
print(menu_items[0])
print("Cheapest:", min(menu_items, key=lambda i: i["Price"])["Item"])

home = BeautifulSoup(requests.get(BASE + "/index.html", timeout=3).text, "html.parser")
print(home.select_one(".tagline").text)
links = [a["href"] for a in home.find_all("a")]
print(links)

from urllib.parse import urljoin
print(urljoin(BASE + "/index.html", "menu.html").removeprefix(BASE))
print(urljoin(BASE + "/shop/index.html", "/about.html").removeprefix(BASE))

for href in links:
    r = safe_get(urljoin(BASE + "/index.html", href))
    if r:
        page = BeautifulSoup(r.text, "html.parser")
        print(f"{href:<12} title: {page.title.text}")

print(requests.get(BASE + "/robots.txt", timeout=3).text)

from urllib.robotparser import RobotFileParser
robots = RobotFileParser()
robots.set_url(BASE + "/robots.txt")
robots.read()
for path in ["/menu.html", "/private/staff-rota.html"]:
    print(f"{path:<26} allowed: {robots.can_fetch('CornerCafeBot', BASE + path)}")
print("Crawl delay:", robots.crawl_delay("CornerCafeBot"), "second(s)")

page_cache = {}

def polite_get(url, delay=0.2):
    if url in page_cache:
        return page_cache[url]
    if not robots.can_fetch("CornerCafeBot", url):
        print("  robots.txt says no:", url.removeprefix(BASE))
        return None
    time.sleep(delay)  # be kind to the server
    r = safe_get(url, headers={"User-Agent": "CornerCafeBot/1.0"})
    page_cache[url] = r
    return r

print("polite_get is ready")

start = time.perf_counter()
for path in ["/menu.html", "/about.html", "/menu.html", "/private/staff-rota.html"]:
    r = polite_get(BASE + path)
    print(path, "->", r.status_code if r else "not fetched")
print(f"Pages in cache: {len(page_cache)}, took about {time.perf_counter() - start:.1f}s")

print("Because it came from page_cache, so no new request was sent.")
print("Cached pages:", [url.removeprefix(BASE) for url in page_cache])

TODAY = {
    "pour-over":  ("Ceramic pour-over dripper", 18.50, True),
    "grinder":    ("Burr coffee grinder", 64.99, True),
    "kettle":     ("Gooseneck kettle", 39.00, True),
    "jar":        ("Glass coffee jar", 12.75, True),
    "scale":      ("Digital coffee scale", 21.00, False),
    "travel-mug": ("Travel mug", 14.99, True),
}
YESTERDAY = {"pour-over": 22.00, "grinder": 79.99, "kettle": 39.00,
             "jar": 11.50, "scale": 24.50, "travel-mug": 14.99}
print(len(TODAY), "products in the shop")

def product_page(slug, name, price, in_stock):
    stock = "In stock" if in_stock else "Out of stock"
    return f'''<html><head><title>{name} | Brew Gear</title></head><body>
<div class="product" data-sku="{slug}">
  <h1 class="name">{name}</h1>
  <p>Now only <span class="price">${price:.2f}</span></p>
  <p class="stock">{stock}</p>
</div></body></html>'''

print(product_page("demo", "Demo mug", 9.5, True))

SHOP = SITE / "shop"
SHOP.mkdir(exist_ok=True)
links = []
for slug, (name, price, in_stock) in TODAY.items():
    (SHOP / f"{slug}.html").write_text(product_page(slug, name, price, in_stock), encoding="utf-8")
    links.append(f'<li><a class="product-link" href="{slug}.html">{name}</a></li>')

(SHOP / "index.html").write_text("<html><head><title>Brew Gear</title></head><body><ul>"
                                 + "\n".join(links) + "</ul></body></html>", encoding="utf-8")
print(sorted(p.name for p in SHOP.iterdir()))

def yesterday_route(query, headers):
    return 200, {"date": "2026-03-14", "currency": "GBP", "prices": YESTERDAY}

API_ROUTES["/api/prices/yesterday"] = yesterday_route
print(requests.get(BASE + "/api/prices/yesterday", timeout=3).json()["date"])

SHOP_URL = BASE + "/shop/index.html"

def product_links():
    r = polite_get(SHOP_URL)
    shop_soup = BeautifulSoup(r.text, "html.parser")
    return [urljoin(SHOP_URL, a["href"]) for a in shop_soup.select("a.product-link")]

product_urls = product_links()
print(len(product_urls), "product pages, for example", product_urls[0].removeprefix(BASE))

def scrape_product(url):
    r = polite_get(url)
    if r is None:
        return None
    page = BeautifulSoup(r.text, "html.parser")
    box = page.select_one("div.product")
    return {
        "sku": box["data-sku"],
        "name": box.select_one(".name").get_text(strip=True),
        "price": float(box.select_one(".price").text.replace("$", "")),
        "in_stock": box.select_one(".stock").text == "In stock",
    }

print(scrape_product(product_urls[1]))

start = time.perf_counter()
today = [scrape_product(url) for url in product_urls]
print(f"Scraped {len(today)} products in about {time.perf_counter() - start:.1f}s")
old = safe_get(BASE + "/api/prices/yesterday").json()
yesterday = old["prices"]
print("Yesterday's prices from", old["date"], "for", len(yesterday), "products")

def compare(product, old_prices):
    before = old_prices.get(product["sku"])
    change = round(product["price"] - before, 2)
    percent = change / before * 100
    return {**product, "was": before, "change": change, "percent": round(percent, 1)}

changes = [compare(p, yesterday) for p in today]
print(changes[0])

drops = sorted([c for c in changes if c["change"] < 0], key=lambda c: c["percent"])
rises = [c for c in changes if c["change"] > 0]
same = [c for c in changes if c["change"] == 0]
print(f"{len(drops)} drop(s), {len(rises)} rise(s), {len(same)} unchanged")

def print_report():
    print("BREW GEAR PRICE TRACKER")
    print("=" * 67)
    print(f"{'Product':<28}{'Was':>8}{'Now':>8}{'Change':>10}  Trend Stock")
    for c in drops + rises + same:
        arrow = "DOWN" if c["change"] < 0 else "UP" if c["change"] > 0 else "-"
        stock = "yes" if c["in_stock"] else "SOLD OUT"
        was, now = f"${c['was']:.2f}", f"${c['price']:.2f}"
        print(f"{c['name']:<28}{was:>8}{now:>8}{c['percent']:>+9.1f}%  {arrow:<5}{stock}")
    print("=" * 67)

print_report()

best = drops[0]
saving = -best["change"]
print(f"Biggest drop: {best['name']}, now ${best['price']:.2f} (save ${saving:.2f})")
buyable = [d["name"] for d in drops if d["in_stock"]]
print("Drops you can buy right now:", ", ".join(buyable))

def check_watch(name, target):
    match = [c for c in changes if c["name"].lower() == name.lower()]
    if not match:
        return f"No product called '{name}'"
    item = match[0]
    if item["price"] <= target:
        return f"Buy now! {item['name']} is ${item['price']:.2f}, under your ${target:.2f}"
    return f"Wait. {item['name']} is ${item['price']:.2f}, above your ${target:.2f}"

watch_name = input("Product to watch: ")
watch_price = float(input("Your target price: $"))
print(check_watch(watch_name, watch_price))

report = {"compared_with": old["date"], "drops": drops, "rises": rises}
report_path = Path("lesson_files/price_report.json")
report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
saved = json.loads(report_path.read_text(encoding="utf-8"))
print("Saved", report_path.name, "with", len(saved["drops"]), "drops")

server.shutdown()
server.server_close()
thread.join(timeout=2)
print("Server thread alive?", thread.is_alive())

try:
    requests.get(BASE + "/shop/index.html", timeout=2)
except requests.exceptions.ConnectionError:
    print("Shop is closed. Goodnight!")