fix(geocoder): band_locations.ma_id en BIGINT — débloque 62k bands
bands.ma_id est BIGINT (crawler génère jusqu'à ~3.5e9) mais band_locations.ma_id avait été créé en INTEGER (max 2.1e9). Tout INSERT pour un band à ma_id élevé échouait en "integer out of range", erreur avalée silencieusement par l'enqueue. 62 239 bands (sur 96 143 localisables) n'entraient donc jamais dans le pipeline, d'où 0 llm_needed / 0 erreur trompeurs. - migration 008: ma_id BIGINT (installs neuves) - migration 009: ALTER COLUMN ma_id TYPE BIGINT (DB existante) - enqueue.py: compte les inserts échoués (failed=) au lieu de les avaler - worker.py: sync_bands_primary prend le lieu d'ORIGINE (step 0) et non la dernière localisation — garde les groupes européens en Europe - admin app.js: coût LLM robuste au NaN (NUMERIC accepte la valeur spéciale NaN) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
dba6b9f730
commit
f145188d1f
5 changed files with 31 additions and 8 deletions
|
|
@ -1064,7 +1064,8 @@ async function loadGeocoding() {
|
||||||
const locManual = locs.find((x) => x.status === "manual")?.n || 0;
|
const locManual = locs.find((x) => x.status === "manual")?.n || 0;
|
||||||
const locErr = locs.find((x) => x.status === "error")?.n || 0;
|
const locErr = locs.find((x) => x.status === "error")?.n || 0;
|
||||||
const locPct = locTotal ? Math.round((locDone / locTotal) * 100) : 0;
|
const locPct = locTotal ? Math.round((locDone / locTotal) * 100) : 0;
|
||||||
const llmCost = parseFloat(r.llm_cache?.total_cost_usd || 0).toFixed(4);
|
const rawCost = Number(r.llm_cache?.total_cost_usd);
|
||||||
|
const llmCost = (Number.isFinite(rawCost) ? rawCost : 0).toFixed(4);
|
||||||
|
|
||||||
const locStatsEl = document.getElementById("loc-stats");
|
const locStatsEl = document.getElementById("loc-stats");
|
||||||
if (locStatsEl) locStatsEl.innerHTML = `
|
if (locStatsEl) locStatsEl.innerHTML = `
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS band_locations (
|
CREATE TABLE IF NOT EXISTS band_locations (
|
||||||
id BIGSERIAL PRIMARY KEY,
|
id BIGSERIAL PRIMARY KEY,
|
||||||
ma_id INTEGER NOT NULL REFERENCES bands(ma_id) ON DELETE CASCADE,
|
ma_id BIGINT NOT NULL REFERENCES bands(ma_id) ON DELETE CASCADE,
|
||||||
step_order SMALLINT NOT NULL DEFAULT 0,
|
step_order SMALLINT NOT NULL DEFAULT 0,
|
||||||
step_label TEXT,
|
step_label TEXT,
|
||||||
location_raw TEXT NOT NULL,
|
location_raw TEXT NOT NULL,
|
||||||
|
|
|
||||||
9
apps/api/migrations/009_band_locations_bigint.sql
Normal file
9
apps/api/migrations/009_band_locations_bigint.sql
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
-- 009_band_locations_bigint.sql
|
||||||
|
-- bands.ma_id est BIGINT (valeurs jusqu'à ~3.5e9 côté crawler), mais
|
||||||
|
-- band_locations.ma_id avait été créé en INTEGER (008). Tout INSERT pour un
|
||||||
|
-- band avec ma_id > 2147483647 échouait en "integer out of range" — l'erreur
|
||||||
|
-- était avalée silencieusement par l'enqueue, laissant ~62k bands hors pipeline.
|
||||||
|
-- On aligne le type sur bands.ma_id.
|
||||||
|
|
||||||
|
ALTER TABLE band_locations
|
||||||
|
ALTER COLUMN ma_id TYPE BIGINT;
|
||||||
|
|
@ -43,7 +43,8 @@ def run_enqueue(cur) -> tuple[int, int, int]:
|
||||||
bands = cur.fetchall()
|
bands = cur.fetchall()
|
||||||
print(f"[enqueue] {len(bands)} bands à parser")
|
print(f"[enqueue] {len(bands)} bands à parser")
|
||||||
|
|
||||||
inserted = country_fixed = skipped = 0
|
inserted = country_fixed = skipped = failed = 0
|
||||||
|
err_samples: list[str] = []
|
||||||
|
|
||||||
for ma_id, country, location_text in bands:
|
for ma_id, country, location_text in bands:
|
||||||
steps = parse_location_text(location_text, country)
|
steps = parse_location_text(location_text, country)
|
||||||
|
|
@ -67,8 +68,14 @@ def run_enqueue(cur) -> tuple[int, int, int]:
|
||||||
if cur.rowcount > 0:
|
if cur.rowcount > 0:
|
||||||
inserted += 1
|
inserted += 1
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
failed += 1
|
||||||
|
if len(err_samples) < 5:
|
||||||
|
err_samples.append(f"ma_id={ma_id}: {exc}")
|
||||||
print(f"[enqueue] WARN insert ma_id={ma_id}: {exc}")
|
print(f"[enqueue] WARN insert ma_id={ma_id}: {exc}")
|
||||||
|
|
||||||
|
if failed:
|
||||||
|
print(f"[enqueue] {failed} inserts échoués — échantillon: {err_samples}")
|
||||||
|
|
||||||
# Résoudre immédiatement les entrées pays-seulement
|
# Résoudre immédiatement les entrées pays-seulement
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
"""
|
||||||
|
|
@ -101,7 +108,7 @@ def run_enqueue(cur) -> tuple[int, int, int]:
|
||||||
else:
|
else:
|
||||||
print(f"[enqueue] no centroid id={loc_id} raw='{location_raw}'")
|
print(f"[enqueue] no centroid id={loc_id} raw='{location_raw}'")
|
||||||
|
|
||||||
return inserted, country_fixed, skipped
|
return inserted, country_fixed, skipped, failed
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
@ -141,10 +148,11 @@ def main():
|
||||||
print(f"[enqueue] job #{job_id} démarré")
|
print(f"[enqueue] job #{job_id} démarré")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
inserted, country_fixed, skipped = run_enqueue(cur)
|
inserted, country_fixed, skipped, failed = run_enqueue(cur)
|
||||||
summary = (
|
summary = (
|
||||||
f"[geocoder_enqueue] job #{job_id} terminé — "
|
f"[geocoder_enqueue] job #{job_id} terminé — "
|
||||||
f"inserted={inserted} country_fixed={country_fixed} skipped={skipped}"
|
f"inserted={inserted} country_fixed={country_fixed} "
|
||||||
|
f"skipped={skipped} failed={failed}"
|
||||||
)
|
)
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"UPDATE job_triggers SET status='done', finished_at=now() WHERE id=%s",
|
"UPDATE job_triggers SET status='done', finished_at=now() WHERE id=%s",
|
||||||
|
|
|
||||||
|
|
@ -66,14 +66,19 @@ def geoapify_search(query: str) -> dict | None:
|
||||||
|
|
||||||
|
|
||||||
def sync_bands_primary(cur, ma_id: int):
|
def sync_bands_primary(cur, ma_id: int):
|
||||||
"""Met à jour bands.lat/lon avec le step le plus récent géocodé."""
|
"""Met à jour bands.lat/lon avec le lieu d'ORIGINE géocodé (step_order le plus bas).
|
||||||
|
|
||||||
|
Carte « metal from Europe » : on veut le point de formation du groupe, pas
|
||||||
|
sa dernière localisation (ex. un groupe de Thessaloniki parti à Boston doit
|
||||||
|
rester à Thessaloniki, en Europe).
|
||||||
|
"""
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
"""
|
||||||
SELECT lat, lon FROM band_locations
|
SELECT lat, lon FROM band_locations
|
||||||
WHERE ma_id = %s
|
WHERE ma_id = %s
|
||||||
AND geocode_status IN ('done', 'country_only')
|
AND geocode_status IN ('done', 'country_only')
|
||||||
AND lat IS NOT NULL AND lon IS NOT NULL
|
AND lat IS NOT NULL AND lon IS NOT NULL
|
||||||
ORDER BY step_order DESC, id DESC
|
ORDER BY step_order ASC, id ASC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
""",
|
""",
|
||||||
(ma_id,),
|
(ma_id,),
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue