"""
Parseur des pages Metal Archives.
Module le plus exposé du crawler : c'est lui qui extrait genre, statut, thèmes,
line-up et dates. Il n'avait AUCUN test. Si Metal Archives change son HTML, il
renvoie silencieusement des champs vides et le crawler enregistre des fiches
creuses sans lever la moindre erreur — la panne la plus coûteuse possible,
parce qu'elle est invisible.
Les fixtures reproduisent la structure réelle des pages (dl/dt/dd pour les
stats, table.lineupTable pour le line-up, #auditTrail pour les dates).
"""
import hashlib
import pytest
from src.scraper_band import page_hash, parse_band_page
def page(stats="", lineup="", extra=""):
rows = "".join(f"{k}:{v}" for k, v in stats) if stats else ""
return f"""
{lineup}{extra}
"""
FULL_STATS = [
("Country of origin", "Norway"),
("Location", "Oslo"),
("Status", "Active"),
("Formed in", "1984"),
("Genre", "Black Metal"),
("Lyrical themes", "Death, Satanism, Darkness"),
("Current label", "Century Media"),
("Years active", "1984-1993, 1995-present"),
]
class TestChampsPrincipaux:
def test_extrait_le_nom(self):
assert parse_band_page(page(FULL_STATS))["name"] == "Mayhem"
@pytest.mark.parametrize("champ,attendu", [
("status", "Active"),
("genre", "Black Metal"),
("formed_in", "1984"),
("location", "Oslo"),
("years_active", "1984-1993, 1995-present"),
("label", "Century Media"),
])
def test_extrait_chaque_champ_de_stats(self, champ, attendu):
assert parse_band_page(page(FULL_STATS))[champ] == attendu
def test_themes_est_un_alias_de_lyrical_themes(self):
"""upsert_band_enriched() lit data['themes'] : l'alias doit suivre."""
out = parse_band_page(page(FULL_STATS))
assert out["themes"] == out["lyrical_themes"] == "Death, Satanism, Darkness"
# Metal Archives n'est pas constant sur ce libellé : les trois orthographes
# ont été observées. En rater une revient à perdre les thèmes en silence.
@pytest.mark.parametrize("libelle", ["Lyrical themes", "Lyrical Themes", "Themes"])
def test_accepte_les_variantes_du_libelle_themes(self, libelle):
out = parse_band_page(page([(libelle, "War")]))
assert out["themes"] == "War"
@pytest.mark.parametrize("libelle", ["Current label", "Last label", "Label"])
def test_accepte_les_variantes_du_libelle_label(self, libelle):
assert parse_band_page(page([(libelle, "Peaceville")]))["label"] == "Peaceville"
@pytest.mark.parametrize("libelle", ["Formed in", "Formed"])
def test_accepte_les_variantes_de_formed(self, libelle):
assert parse_band_page(page([(libelle, "1991")]))["formed_in"] == "1991"
def test_info_raw_conserve_tout_en_minuscules(self):
raw = parse_band_page(page(FULL_STATS))["info_raw"]
assert raw["country of origin"] == "Norway"
assert all(k == k.lower() for k in raw)
class TestRobustesse:
"""Le parseur ne doit jamais lever : une page inattendue donne des champs vides."""
@pytest.mark.parametrize("html", [
"", "", "",
"pas du html du tout", "fragment
",
"Sans stats
",
])
def test_ne_leve_jamais_sur_une_page_degradee(self, html):
out = parse_band_page(html)
assert isinstance(out, dict)
assert "name" in out and "lineup" in out
def test_champs_absents_valent_none(self):
out = parse_band_page(page([]))
for champ in ("status", "genre", "formed_in", "location", "label", "themes"):
assert out[champ] is None, champ
def test_page_sans_h1_ne_plante_pas(self):
assert parse_band_page("")["name"] == ""
def test_dt_sans_dd_est_ignore(self):
html = ""
assert parse_band_page(html)["genre"] is None
def test_retombe_sur_band_info_si_band_stats_absent(self):
html = """"""
assert parse_band_page(html)["genre"] == "Doom"
def test_la_structure_de_sortie_est_toujours_complete(self):
"""upsert_band_enriched lit ces clés sans les tester : elles doivent exister."""
out = parse_band_page("")
for cle in ("name", "info_raw", "status", "genre", "formed_in", "location",
"themes", "lyrical_themes", "label", "lineup", "links",
"discography_url"):
assert cle in out, cle
assert isinstance(out["lineup"], dict)
assert isinstance(out["links"], list)
LINEUP_HTML = """
Current lineup
Past members
"""
class TestLineup:
def test_regroupe_par_section(self):
lineup = parse_band_page(page(FULL_STATS, LINEUP_HTML))["lineup"]
assert [m["name"] for m in lineup["current"]] == ["Necrobutcher", "Hellhammer"]
assert [m["name"] for m in lineup["past"]] == ["Dead"]
def test_conserve_role_et_url(self):
m = parse_band_page(page(FULL_STATS, LINEUP_HTML))["lineup"]["current"][0]
assert m["role"] == "Bass"
assert m["url"] == "/artists/Necrobutcher/12"
def test_ignore_la_ligne_den_tete(self):
lineup = parse_band_page(page(FULL_STATS, LINEUP_HTML))["lineup"]
assert all(m["name"] not in ("Nom", "Rôle") for m in lineup["current"])
def test_ignore_les_lignes_sans_lien_artiste(self):
html = """Current
"""
assert parse_band_page(page(FULL_STATS, html))["lineup"]["current"] == []
def test_section_inconnue_retombe_sur_current(self):
html = """Quelque chose d'imprévu
"""
assert len(parse_band_page(page(FULL_STATS, html))["lineup"]["current"]) == 1
def test_les_quatre_sections_existent_meme_vides(self):
lineup = parse_band_page(page(FULL_STATS))["lineup"]
assert set(lineup) >= {"current", "past", "live", "session"}
class TestLiensEtDates:
def test_extrait_lurl_de_discographie(self):
extra = 'Complete discography'
assert parse_band_page(page(FULL_STATS, extra=extra))["discography_url"] == "/band/discography/id/67"
def test_discographie_absente_vaut_none(self):
assert parse_band_page(page(FULL_STATS))["discography_url"] is None
def test_extrait_les_liens_externes(self):
extra = ''
links = parse_band_page(page(FULL_STATS, extra=extra))["links"]
assert links == [{"text": "Site officiel", "url": "https://x.example"}]
def test_extrait_les_dates_de_laudit_trail(self):
extra = ('Added on: 2003-05-12 14:02 '
'Last modified on: 2026-08-01 09:31
')
out = parse_band_page(page(FULL_STATS, extra=extra))
assert out["ma_created_at"] == "2003-05-12 14:02"
assert out["ma_modified_at"] == "2026-08-01 09:31"
def test_audit_trail_absent_nintroduit_pas_les_cles(self):
out = parse_band_page(page(FULL_STATS))
assert out.get("ma_created_at") is None
class TestPageHash:
def test_stable_et_sensible(self):
assert page_hash("a") == page_hash("a")
assert page_hash("a") != page_hash("b")
def test_correspond_a_md5(self):
# upsert_band_enriched compare ce hash pour éviter les écritures inutiles :
# changer d'algorithme invaliderait tout le cache d'un coup.
assert page_hash("x") == hashlib.md5(b"x").hexdigest()
def test_accepte_lunicode(self):
assert len(page_hash("Mötley Crüe — ✝")) == 32