fix(geocoder): band_locations.ma_id en BIGINT — débloque 62k bands
bands.ma_id est BIGINT (crawler génère jusqu'à ~3.5e9) mais band_locations.ma_id avait été créé en INTEGER (max 2.1e9). Tout INSERT pour un band à ma_id élevé échouait en "integer out of range", erreur avalée silencieusement par l'enqueue. 62 239 bands (sur 96 143 localisables) n'entraient donc jamais dans le pipeline, d'où 0 llm_needed / 0 erreur trompeurs. - migration 008: ma_id BIGINT (installs neuves) - migration 009: ALTER COLUMN ma_id TYPE BIGINT (DB existante) - enqueue.py: compte les inserts échoués (failed=) au lieu de les avaler - worker.py: sync_bands_primary prend le lieu d'ORIGINE (step 0) et non la dernière localisation — garde les groupes européens en Europe - admin app.js: coût LLM robuste au NaN (NUMERIC accepte la valeur spéciale NaN) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
dba6b9f730
commit
f145188d1f
5 changed files with 31 additions and 8 deletions
|
|
@ -1064,7 +1064,8 @@ async function loadGeocoding() {
|
|||
const locManual = locs.find((x) => x.status === "manual")?.n || 0;
|
||||
const locErr = locs.find((x) => x.status === "error")?.n || 0;
|
||||
const locPct = locTotal ? Math.round((locDone / locTotal) * 100) : 0;
|
||||
const llmCost = parseFloat(r.llm_cache?.total_cost_usd || 0).toFixed(4);
|
||||
const rawCost = Number(r.llm_cache?.total_cost_usd);
|
||||
const llmCost = (Number.isFinite(rawCost) ? rawCost : 0).toFixed(4);
|
||||
|
||||
const locStatsEl = document.getElementById("loc-stats");
|
||||
if (locStatsEl) locStatsEl.innerHTML = `
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
CREATE TABLE IF NOT EXISTS band_locations (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
ma_id INTEGER NOT NULL REFERENCES bands(ma_id) ON DELETE CASCADE,
|
||||
ma_id BIGINT NOT NULL REFERENCES bands(ma_id) ON DELETE CASCADE,
|
||||
step_order SMALLINT NOT NULL DEFAULT 0,
|
||||
step_label TEXT,
|
||||
location_raw TEXT NOT NULL,
|
||||
|
|
|
|||
9
apps/api/migrations/009_band_locations_bigint.sql
Normal file
9
apps/api/migrations/009_band_locations_bigint.sql
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
-- 009_band_locations_bigint.sql
|
||||
-- bands.ma_id est BIGINT (valeurs jusqu'à ~3.5e9 côté crawler), mais
|
||||
-- band_locations.ma_id avait été créé en INTEGER (008). Tout INSERT pour un
|
||||
-- band avec ma_id > 2147483647 échouait en "integer out of range" — l'erreur
|
||||
-- était avalée silencieusement par l'enqueue, laissant ~62k bands hors pipeline.
|
||||
-- On aligne le type sur bands.ma_id.
|
||||
|
||||
ALTER TABLE band_locations
|
||||
ALTER COLUMN ma_id TYPE BIGINT;
|
||||
|
|
@ -43,7 +43,8 @@ def run_enqueue(cur) -> tuple[int, int, int]:
|
|||
bands = cur.fetchall()
|
||||
print(f"[enqueue] {len(bands)} bands à parser")
|
||||
|
||||
inserted = country_fixed = skipped = 0
|
||||
inserted = country_fixed = skipped = failed = 0
|
||||
err_samples: list[str] = []
|
||||
|
||||
for ma_id, country, location_text in bands:
|
||||
steps = parse_location_text(location_text, country)
|
||||
|
|
@ -67,8 +68,14 @@ def run_enqueue(cur) -> tuple[int, int, int]:
|
|||
if cur.rowcount > 0:
|
||||
inserted += 1
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
if len(err_samples) < 5:
|
||||
err_samples.append(f"ma_id={ma_id}: {exc}")
|
||||
print(f"[enqueue] WARN insert ma_id={ma_id}: {exc}")
|
||||
|
||||
if failed:
|
||||
print(f"[enqueue] {failed} inserts échoués — échantillon: {err_samples}")
|
||||
|
||||
# Résoudre immédiatement les entrées pays-seulement
|
||||
cur.execute(
|
||||
"""
|
||||
|
|
@ -101,7 +108,7 @@ def run_enqueue(cur) -> tuple[int, int, int]:
|
|||
else:
|
||||
print(f"[enqueue] no centroid id={loc_id} raw='{location_raw}'")
|
||||
|
||||
return inserted, country_fixed, skipped
|
||||
return inserted, country_fixed, skipped, failed
|
||||
|
||||
|
||||
def main():
|
||||
|
|
@ -141,10 +148,11 @@ def main():
|
|||
print(f"[enqueue] job #{job_id} démarré")
|
||||
|
||||
try:
|
||||
inserted, country_fixed, skipped = run_enqueue(cur)
|
||||
inserted, country_fixed, skipped, failed = run_enqueue(cur)
|
||||
summary = (
|
||||
f"[geocoder_enqueue] job #{job_id} terminé — "
|
||||
f"inserted={inserted} country_fixed={country_fixed} skipped={skipped}"
|
||||
f"inserted={inserted} country_fixed={country_fixed} "
|
||||
f"skipped={skipped} failed={failed}"
|
||||
)
|
||||
cur.execute(
|
||||
"UPDATE job_triggers SET status='done', finished_at=now() WHERE id=%s",
|
||||
|
|
|
|||
|
|
@ -66,14 +66,19 @@ def geoapify_search(query: str) -> dict | None:
|
|||
|
||||
|
||||
def sync_bands_primary(cur, ma_id: int):
|
||||
"""Met à jour bands.lat/lon avec le step le plus récent géocodé."""
|
||||
"""Met à jour bands.lat/lon avec le lieu d'ORIGINE géocodé (step_order le plus bas).
|
||||
|
||||
Carte « metal from Europe » : on veut le point de formation du groupe, pas
|
||||
sa dernière localisation (ex. un groupe de Thessaloniki parti à Boston doit
|
||||
rester à Thessaloniki, en Europe).
|
||||
"""
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT lat, lon FROM band_locations
|
||||
WHERE ma_id = %s
|
||||
AND geocode_status IN ('done', 'country_only')
|
||||
AND lat IS NOT NULL AND lon IS NOT NULL
|
||||
ORDER BY step_order DESC, id DESC
|
||||
ORDER BY step_order ASC, id ASC
|
||||
LIMIT 1
|
||||
""",
|
||||
(ma_id,),
|
||||
|
|
|
|||
Loading…
Reference in a new issue