Nicolas Fryder
260200e251
- apps/crawler/ : service Python complet, remplace les scripts locaux - FlareSolverr pour bypasser Cloudflare (cookies CF → session requests) - Crawl incrémental : /archives/band-list/by/created et /by/modified - Crawl complet Europe : pagination AJAX /browse/ajax-country/ - Enrichissement : pages individuelles de bands (themes, membres, label, hash) - Écriture directe en DB (upserts bulk, idempotents) - Scheduler intégré (schedule library) : incrémental 4h, enrich 2h, full le 1er du mois - Tracking via crawl_run et crawl_checkpoint (migration 004) - docker-compose.dev.yml : flaresolverr + crawler ajoutés Full crawl désactivé en dev (CRAWLER_SCHED_FULL_DAY=0) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
98 lines
3.2 KiB
Python
98 lines
3.2 KiB
Python
"""
|
|
Parser d'une page individuelle de band Metal Archives.
|
|
Adapté de C:\Users\nicol\Documents\crawlerbm\src\scrape_band_page.py
|
|
"""
|
|
import hashlib
|
|
import re
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
def parse_band_page(html: str) -> Dict[str, Any]:
|
|
soup = BeautifulSoup(html, "lxml")
|
|
out: Dict[str, Any] = {}
|
|
|
|
title = soup.find("title")
|
|
out["title"] = _text(title)
|
|
|
|
h1 = (
|
|
soup.find("h1", class_=re.compile(r"band_name", re.I))
|
|
or soup.find("h1")
|
|
)
|
|
out["name"] = _text(h1)
|
|
|
|
# Bloc #band_stats : dt/dd pairs
|
|
stats = soup.find("div", id="band_stats") or soup.find("div", id="band_info") or soup
|
|
kv: Dict[str, str] = {}
|
|
for dl in stats.find_all("dl"):
|
|
for dt in dl.find_all("dt"):
|
|
dd = dt.find_next_sibling("dd")
|
|
k = _text(dt).rstrip(":")
|
|
v = _text(dd)
|
|
if k and v:
|
|
kv[k] = v
|
|
|
|
out["info"] = kv
|
|
out["status"] = _pick(kv, "Status")
|
|
out["formed_in"] = _pick(kv, "Formed in", "Formed")
|
|
out["genre"] = _pick(kv, "Genre")
|
|
out["themes"] = _pick(kv, "Lyrical themes", "Lyrical Themes")
|
|
out["label"] = _pick(kv, "Current label", "Label")
|
|
out["years_active"] = _pick(kv, "Years active")
|
|
|
|
# Dates "Added on" / "Last modified" (souvent dans #auditTrail ou en bas de page)
|
|
audit = soup.find(id="auditTrail") or soup.find("div", class_=re.compile(r"audit", re.I))
|
|
if audit:
|
|
trail_text = _text(audit)
|
|
m_added = re.search(r"Added on:\s*([^\n,]+)", trail_text, re.I)
|
|
m_modified = re.search(r"Last modified on:\s*([^\n,]+)", trail_text, re.I)
|
|
out["ma_created_at"] = m_added.group(1).strip() if m_added else None
|
|
out["ma_modified_at"] = m_modified.group(1).strip() if m_modified else None
|
|
|
|
# Membres
|
|
members: List[Dict] = []
|
|
for table in soup.find_all("table"):
|
|
cls = " ".join(table.get("class") or [])
|
|
if "lineupTable" not in cls:
|
|
continue
|
|
section_el = table.find_previous(["h2", "h3"])
|
|
section = _text(section_el) if section_el else None
|
|
headers = [_text(th) for th in table.find_all("th")]
|
|
for tr in table.find_all("tr")[1:]:
|
|
tds = tr.find_all(["td", "th"])
|
|
if not tds:
|
|
continue
|
|
cells = [_text(td) for td in tds]
|
|
row = dict(zip(headers, cells)) if headers and len(headers) == len(cells) else {"cells": cells}
|
|
if section:
|
|
row["section"] = section
|
|
members.append(row)
|
|
if members:
|
|
out["members"] = members
|
|
|
|
# Liens externes
|
|
links_div = soup.find("div", id="band_links")
|
|
if links_div:
|
|
out["links"] = [
|
|
{"text": _text(a), "url": a["href"]}
|
|
for a in links_div.find_all("a", href=True)
|
|
]
|
|
|
|
return out
|
|
|
|
|
|
def page_hash(html: str) -> str:
|
|
"""MD5 du HTML pour détecter les changements."""
|
|
return hashlib.md5(html.encode()).hexdigest()
|
|
|
|
|
|
def _text(el) -> str:
|
|
return (el.get_text(" ", strip=True) if el else "").strip()
|
|
|
|
|
|
def _pick(kv: Dict, *keys: str) -> Optional[str]:
|
|
for k in keys:
|
|
if k in kv:
|
|
return kv[k]
|
|
return None
|