nico/metalfrom.eu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: geocoder Geoapify + fixes crawler + améliorations admin dashboard

Browse source 
Geocoder → Geoapify (bcb790b9007644d1a44ff2391118479c) :
- Remplace Nominatim par Geoapify dans apps/geocoder/src/worker.py
- Délai réduit à 0.22s (5 req/s vs 1 req/s Nominatim) → bien plus rapide
- Même logique de cache geocode_cache, même fallback backoff progressif
- Env vars : GEOAPIFY_API_KEY (obligatoire), GEOCODER_MIN_DELAY/JITTER

Fix crawler incremental_modified (Expecting value: line 1 column 1) :
- ma_http.get_json() retenait sans retry sur JSONDecodeError (corps vide =
  session Chrome morte). Désormais : refresh session + retry, comme pour
  les erreurs 403/429.

Admin dashboard :
- Bands : colonne Lieu, champ recherche lieu séparé (location_q), filtres
  "Géocodé oui/non" (has_lat) et "Lieu vide/renseigné" (has_location)
- Queue : bouton "Annuler runs bloqués >30min" (POST /admin/api/crawl-runs/
  cleanup), auto-refresh 15s, colonne Progression avec durée elapsed pour
  les runs actifs
- Dashboard : toutes les listes pays/genre/statut sans limite (scroll interne)
- Progression live : crawler écrit les stats dans crawl_run toutes les 50
  bands (update_crawl_run_progress), visible dans Queue en temps réel

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Nicolas Fryder 2026-06-30 22:44:28 +02:00
parent 10ab0055d5
commit 3dc483bc4f
8 changed files with 204 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

131
apps/admin/site/app.js
View file

@ -3,11 +3,12 @@
const state = { const state = {
username: null, username: null,
view: "dashboard", view: "dashboard",
bands: { page: 1, pageSize: 50, q: "", country: "", status: "", genre: "", enriched: "", sort: "ma_id", dir: "asc", total: 0 }, bands: { page: 1, pageSize: 50, q: "", location_q: "", country: "", status: "", genre: "", enriched: "", has_lat: "", has_location: "", sort: "ma_id", dir: "asc", total: 0 },
runs: { page: 1, pageSize: 50, run_type: "", status: "", total: 0 }, runs: { page: 1, pageSize: 50, run_type: "", status: "", total: 0 },
logs: { page: 1, pageSize: 100, level: "", autoRefresh: true }, logs: { page: 1, pageSize: 100, level: "", autoRefresh: true },
audit: { page: 1, pageSize: 50 }, audit: { page: 1, pageSize: 50 },
logsTimer: null, logsTimer: null,
queueTimer: null,
}; };
// ------------------------------------------------------------------ // ------------------------------------------------------------------
@ -37,6 +38,7 @@ function showLogin() {
document.getElementById("login-screen").classList.remove("hidden"); document.getElementById("login-screen").classList.remove("hidden");
document.getElementById("app").classList.add("hidden"); document.getElementById("app").classList.add("hidden");
if (state.logsTimer) { clearInterval(state.logsTimer); state.logsTimer = null; } if (state.logsTimer) { clearInterval(state.logsTimer); state.logsTimer = null; }
if (state.queueTimer) { clearInterval(state.queueTimer); state.queueTimer = null; }
} }
function showApp() { function showApp() {
@ -102,6 +104,7 @@ function router() {
a.classList.toggle("active", a.dataset.view === view); a.classList.toggle("active", a.dataset.view === view);
}); });
if (state.logsTimer) { clearInterval(state.logsTimer); state.logsTimer = null; } if (state.logsTimer) { clearInterval(state.logsTimer); state.logsTimer = null; }
if (state.queueTimer) { clearInterval(state.queueTimer); state.queueTimer = null; }
const renderers = { const renderers = {
dashboard: renderDashboard, dashboard: renderDashboard,
bands: renderBands, bands: renderBands,
@ -149,16 +152,16 @@ async function renderDashboard() {
${statCard("Stale (>30j)", t.stale, "warn")} ${statCard("Stale (>30j)", t.stale, "warn")}
</div> </div>
<div class="grid grid-2"> <div class="grid grid-2">
<div class="card"> <div class="card" style="max-height:420px;overflow-y:auto">
<h2>Par statut</h2> <h2>Par statut</h2>
${r.by_status.map((x) => barRow(x.status, x.total, maxStatus)).join("") || emptyRow()} ${r.by_status.map((x) => barRow(x.status, x.total, maxStatus)).join("") || emptyRow()}
</div> </div>
<div class="card"> <div class="card" style="max-height:420px;overflow-y:auto">
<h2>Par pays (top 20)</h2> <h2>Par pays (${r.by_country.length})</h2>
${r.by_country.map((x) => barRow(x.country, x.total, maxCountry)).join("") || emptyRow()} ${r.by_country.map((x) => barRow(x.country, x.total, maxCountry)).join("") || emptyRow()}
</div> </div>
<div class="card"> <div class="card" style="max-height:420px;overflow-y:auto">
<h2>Par genre (top 20)</h2> <h2>Par genre (${r.by_genre.length})</h2>
${r.by_genre.map((x) => barRow(x.genre, x.total, maxGenre)).join("") || emptyRow()} ${r.by_genre.map((x) => barRow(x.genre, x.total, maxGenre)).join("") || emptyRow()}
</div> </div>
</div> </div>
@ -181,39 +184,72 @@ function emptyRow() { return `<div class="empty">Aucune donnée</div>`; }
// Queue // Queue
// ------------------------------------------------------------------ // ------------------------------------------------------------------
async function renderQueue() { async function renderQueue() {
content().innerHTML = `<div class="loading">Chargement…</div>`; if (!document.getElementById("q-body")) {
try {
const r = await api("/admin/api/queue");
const b = r.breakdown;
content().innerHTML = ` content().innerHTML = `
<div class="grid grid-stats"> <div class="grid grid-stats" id="q-stats"></div>
${statCard("Nouveaux (priorité 1)", b.new_bands, "err")}
${statCard("Modifiés depuis enrich (priorité 2)", b.modified_since_enrich, "warn")}
${statCard("Legacy à ré-enrichir (priorité 3)", b.legacy_pending, "warn")}
${statCard("Stale >30j (priorité 4)", b.stale, "ok")}
</div>
<div class="card"> <div class="card">
<h2>Derniers runs d'enrichissement</h2> <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:14px">
<h2 style="margin:0">Derniers runs d'enrichissement</h2>
<div style="display:flex;gap:8px;align-items:center">
<span id="q-refresh-status" style="font-size:11px;color:var(--muted)">Auto-refresh 15s</span>
<button class="btn btn-mini" id="q-cleanup-btn">🧹 Annuler runs bloqués (>30min)</button>
</div>
</div>
<div class="table-wrap"><table> <div class="table-wrap"><table>
<thead><tr><th>ID</th><th>Statut</th><th>Début</th><th>Fin</th><th>Vus</th><th>Enrichis</th><th>Erreur</th></tr></thead> <thead><tr><th>ID</th><th>Statut</th><th>Début</th><th>Fin</th><th>Vus</th><th>Enrichis</th><th>Progression</th><th>Erreur</th></tr></thead>
<tbody> <tbody id="q-body"></tbody>
${r.recent_enrich_runs.map((x) => `
<tr>
<td>${x.id}</td>
<td>${statusBadge(x.status)}</td>
<td>${fmtDate(x.started_at)}</td>
<td>${fmtDate(x.finished_at)}</td>
<td>${x.bands_seen}</td>
<td>${x.bands_enriched}</td>
<td>${esc(x.error || "")}</td>
</tr>
`).join("") || `<tr><td colspan="7" class="empty">Aucun run</td></tr>`}
</tbody>
</table></div> </table></div>
</div> </div>
`; `;
document.getElementById("q-cleanup-btn").addEventListener("click", async () => {
const btn = document.getElementById("q-cleanup-btn");
btn.disabled = true;
try {
const r = await api("/admin/api/crawl-runs/cleanup", { method: "POST", body: JSON.stringify({ older_than_minutes: 30 }) });
alert(`${r.cleaned} run(s) annulé(s).`);
await loadQueue();
} catch (e) {
alert(`Erreur: ${e.message}`);
} finally { btn.disabled = false; }
});
}
await loadQueue();
if (state.queueTimer) clearInterval(state.queueTimer);
state.queueTimer = setInterval(loadQueue, 15000);
}
async function loadQueue() {
try {
const r = await api("/admin/api/queue");
const b = r.breakdown;
const statsEl = document.getElementById("q-stats");
if (statsEl) statsEl.innerHTML = `
${statCard("Nouveaux (priorité 1)", b.new_bands, "err")}
${statCard("Modifiés depuis enrich (p.2)", b.modified_since_enrich, "warn")}
${statCard("Legacy à ré-enrichir (p.3)", b.legacy_pending, "warn")}
${statCard("Stale >30j (p.4)", b.stale, "ok")}
`;
const tbody = document.getElementById("q-body");
if (tbody) tbody.innerHTML = r.recent_enrich_runs.map((x) => {
const isRunning = x.status === "running";
const elapsed = isRunning ? Math.round((Date.now() - new Date(x.started_at)) / 60000) : null;
const prog = isRunning && x.bands_seen > 0
? `<span style="color:var(--muted)">${x.bands_seen} vus, ${x.bands_enriched} enrichis${elapsed ? `, ${elapsed}min` : ""}</span>`
: isRunning ? `<span style="color:var(--warn)">en attente… (${elapsed}min)</span>` : "—";
return `<tr>
<td>${x.id}</td>
<td>${statusBadge(x.status)}</td>
<td>${fmtDate(x.started_at)}</td>
<td>${fmtDate(x.finished_at)}</td>
<td>${x.bands_seen}</td>
<td>${x.bands_enriched}</td>
<td>${prog}</td>
<td style="max-width:200px;overflow:hidden;text-overflow:ellipsis;color:var(--err)">${esc(x.error || "")}</td>
</tr>`;
}).join("") || `<tr><td colspan="8" class="empty">Aucun run</td></tr>`;
} catch (e) { } catch (e) {
content().innerHTML = `<div class="empty">Erreur : ${esc(e.message)}</div>`; const tbody = document.getElementById("q-body");
if (tbody) tbody.innerHTML = `<tr><td colspan="8" class="empty">Erreur : ${esc(e.message)}</td></tr>`;
} }
} }
@ -229,14 +265,25 @@ async function renderBands() {
const s = state.bands; const s = state.bands;
content().innerHTML = ` content().innerHTML = `
<div class="toolbar"> <div class="toolbar">
<input type="text" id="b-q" placeholder="Recherche (nom, genre, lieu)…" value="${esc(s.q)}" style="min-width:240px"> <input type="text" id="b-q" placeholder="Nom / genre…" value="${esc(s.q)}" style="min-width:180px">
<input type="text" id="b-country" placeholder="Pays (FR, DE…)" value="${esc(s.country)}" style="width:120px"> <input type="text" id="b-location_q" placeholder="Lieu…" value="${esc(s.location_q)}" style="width:160px">
<input type="text" id="b-status" placeholder="Statut" value="${esc(s.status)}" style="width:140px"> <input type="text" id="b-country" placeholder="Pays (FR, DE…)" value="${esc(s.country)}" style="width:110px">
<input type="text" id="b-genre" placeholder="Genre" value="${esc(s.genre)}" style="width:160px"> <input type="text" id="b-status" placeholder="Statut" value="${esc(s.status)}" style="width:130px">
<input type="text" id="b-genre" placeholder="Genre" value="${esc(s.genre)}" style="width:150px">
<select id="b-enriched"> <select id="b-enriched">
<option value="">Enrichi: tous</option> <option value="">Enrichi: tous</option>
<option value="true" ${s.enriched === "true" ? "selected" : ""}>Enrichi</option> <option value="true" ${s.enriched === "true" ? "selected" : ""}>Enrichi ✓</option>
<option value="false" ${s.enriched === "false" ? "selected" : ""}>Non enrichi</option> <option value="false" ${s.enriched === "false" ? "selected" : ""}>Non enrichi ✗</option>
</select>
<select id="b-has_lat">
<option value="">Géocodé: tous</option>
<option value="true" ${s.has_lat === "true" ? "selected" : ""}>Géocodé ✓</option>
<option value="false" ${s.has_lat === "false" ? "selected" : ""}>Sans coords ✗</option>
</select>
<select id="b-has_location">
<option value="">Lieu texte: tous</option>
<option value="true" ${s.has_location === "true" ? "selected" : ""}>Lieu renseigné ✓</option>
<option value="false" ${s.has_location === "false" ? "selected" : ""}>Lieu vide ✗</option>
</select> </select>
<button class="btn" id="b-apply">Filtrer</button> <button class="btn" id="b-apply">Filtrer</button>
</div> </div>
@ -245,10 +292,13 @@ async function renderBands() {
`; `;
document.getElementById("b-apply").addEventListener("click", () => { document.getElementById("b-apply").addEventListener("click", () => {
s.q = document.getElementById("b-q").value.trim(); s.q = document.getElementById("b-q").value.trim();
s.location_q = document.getElementById("b-location_q").value.trim();
s.country = document.getElementById("b-country").value.trim(); s.country = document.getElementById("b-country").value.trim();
s.status = document.getElementById("b-status").value.trim(); s.status = document.getElementById("b-status").value.trim();
s.genre = document.getElementById("b-genre").value.trim(); s.genre = document.getElementById("b-genre").value.trim();
s.enriched = document.getElementById("b-enriched").value; s.enriched = document.getElementById("b-enriched").value;
s.has_lat = document.getElementById("b-has_lat").value;
s.has_location = document.getElementById("b-has_location").value;
s.page = 1; s.page = 1;
loadBands(); loadBands();
}); });
@ -261,6 +311,7 @@ const BAND_COLUMNS = [
{ key: "country", label: "Pays" }, { key: "country", label: "Pays" },
{ key: "status", label: "Statut" }, { key: "status", label: "Statut" },
{ key: "genre", label: "Genre" }, { key: "genre", label: "Genre" },
{ key: "location_text", label: "Lieu" },
{ key: "formed_year", label: "Année" }, { key: "formed_year", label: "Année" },
{ key: "enriched", label: "Enrichi" }, { key: "enriched", label: "Enrichi" },
{ key: "crawled_at", label: "Crawlé le" }, { key: "crawled_at", label: "Crawlé le" },
@ -273,10 +324,13 @@ async function loadBands() {
page: s.page, pageSize: s.pageSize, sort: s.sort, dir: s.dir, page: s.page, pageSize: s.pageSize, sort: s.sort, dir: s.dir,
}); });
if (s.q) params.set("q", s.q); if (s.q) params.set("q", s.q);
if (s.location_q) params.set("location_q", s.location_q);
if (s.country) params.set("country", s.country); if (s.country) params.set("country", s.country);
if (s.status) params.set("status", s.status); if (s.status) params.set("status", s.status);
if (s.genre) params.set("genre", s.genre); if (s.genre) params.set("genre", s.genre);
if (s.enriched) params.set("enriched", s.enriched); if (s.enriched) params.set("enriched", s.enriched);
if (s.has_lat) params.set("has_lat", s.has_lat);
if (s.has_location) params.set("has_location", s.has_location);
try { try {
const r = await api(`/admin/api/bands?${params}`); const r = await api(`/admin/api/bands?${params}`);
@ -295,6 +349,7 @@ async function loadBands() {
<td>${esc(b.country || "")}</td> <td>${esc(b.country || "")}</td>
<td>${esc(b.status || "")}</td> <td>${esc(b.status || "")}</td>
<td>${esc(b.genre || "")}</td> <td>${esc(b.genre || "")}</td>
<td style="max-width:160px;overflow:hidden;text-overflow:ellipsis" title="${esc(b.location_text || "")}">${esc(b.location_text || "")}</td>
<td>${b.formed_year || ""}</td> <td>${b.formed_year || ""}</td>
<td>${b.enriched ? '<span class="badge ok">oui</span>' : '<span class="badge warn">non</span>'}</td> <td>${b.enriched ? '<span class="badge ok">oui</span>' : '<span class="badge warn">non</span>'}</td>
<td>${fmtDate(b.crawled_at)}</td> <td>${fmtDate(b.crawled_at)}</td>

45
apps/api/src/adminRoutes.js
View file

@ -45,16 +45,16 @@ export default async function adminRoutes(fastify, opts) {
`), `),
pool.query(` pool.query(`
SELECT COALESCE(NULLIF(trim(status), ''), 'Unknown') AS status, count(*)::int AS total SELECT COALESCE(NULLIF(trim(status), ''), 'Unknown') AS status, count(*)::int AS total
FROM bands GROUP BY 1 ORDER BY total DESC LIMIT 20 FROM bands GROUP BY 1 ORDER BY total DESC
`), `),
pool.query(` pool.query(`
SELECT COALESCE(country, '??') AS country, count(*)::int AS total SELECT COALESCE(country, '??') AS country, count(*)::int AS total
FROM bands GROUP BY 1 ORDER BY total DESC LIMIT 20 FROM bands GROUP BY 1 ORDER BY total DESC
`), `),
pool.query(` pool.query(`
SELECT genre, count(*)::int AS total SELECT genre, count(*)::int AS total
FROM bands WHERE genre IS NOT NULL AND genre != '' FROM bands WHERE genre IS NOT NULL AND genre != ''
GROUP BY 1 ORDER BY total DESC LIMIT 20 GROUP BY 1 ORDER BY total DESC
`), `),
]); ]);
return { return {
@ -110,7 +110,7 @@ export default async function adminRoutes(fastify, opts) {
// ------------------------------------------------------------------ // ------------------------------------------------------------------
fastify.get("/admin/api/bands", async (req, reply) => { fastify.get("/admin/api/bands", async (req, reply) => {
try { try {
const { q, country, status, genre, enriched, sort, dir } = req.query || {}; const { q, location_q, country, status, genre, enriched, has_lat, has_location, sort, dir } = req.query || {};
const { page, pageSize, offset } = pagination(req.query || {}); const { page, pageSize, offset } = pagination(req.query || {});
const where = []; const where = [];
@ -120,11 +120,19 @@ export default async function adminRoutes(fastify, opts) {
if (q) { if (q) {
const query = String(q).trim().slice(0, 100); const query = String(q).trim().slice(0, 100);
if (query.length >= 2) { if (query.length >= 2) {
where.push(`(name ILIKE $${i} OR genre ILIKE $${i} OR location_text ILIKE $${i})`); where.push(`(name ILIKE $${i} OR genre ILIKE $${i})`);
vals.push(`%${query}%`); vals.push(`%${query}%`);
i++; i++;
} }
} }
if (location_q) {
const lq = String(location_q).trim().slice(0, 100);
if (lq.length >= 1) {
where.push(`location_text ILIKE $${i}`);
vals.push(`%${lq}%`);
i++;
}
}
if (country) { if (country) {
where.push(`country = $${i}`); where.push(`country = $${i}`);
vals.push(String(country).toUpperCase()); vals.push(String(country).toUpperCase());
@ -142,6 +150,10 @@ export default async function adminRoutes(fastify, opts) {
} }
if (enriched === "true" || enriched === "1") where.push(`enriched = true`); if (enriched === "true" || enriched === "1") where.push(`enriched = true`);
if (enriched === "false" || enriched === "0") where.push(`enriched = false`); if (enriched === "false" || enriched === "0") where.push(`enriched = false`);
if (has_lat === "true" || has_lat === "1") where.push(`lat IS NOT NULL`);
if (has_lat === "false" || has_lat === "0") where.push(`lat IS NULL`);
if (has_location === "true" || has_location === "1") where.push(`location_text IS NOT NULL AND location_text != ''`);
if (has_location === "false" || has_location === "0") where.push(`(location_text IS NULL OR location_text = '')`);
const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : ""; const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : "";
const sortCol = BAND_SORT_COLUMNS.has(sort) ? sort : "ma_id"; const sortCol = BAND_SORT_COLUMNS.has(sort) ? sort : "ma_id";
@ -324,6 +336,29 @@ export default async function adminRoutes(fastify, opts) {
} }
}); });
// ------------------------------------------------------------------
// Cleanup des runs bloqués (status=running depuis trop longtemps)
// ------------------------------------------------------------------
fastify.post("/admin/api/crawl-runs/cleanup", async (req, reply) => {
try {
const olderThanMinutes = Math.max(1, Number(req.body?.older_than_minutes) || 30);
const r = await pool.query(`
UPDATE crawl_run
SET status = 'error',
finished_at = now(),
error = 'annulé manuellement (run bloqué)'
WHERE status = 'running'
AND started_at < now() - make_interval(mins => $1)
RETURNING id, run_type, started_at
`, [olderThanMinutes]);
await writeAuditLog(pool, req.adminUsername, "cleanup_stuck_runs", "crawl_run", null, null, { cleaned: r.rows });
return { ok: true, cleaned: r.rows.length, runs: r.rows };
} catch (err) {
fastify.log.error(err);
return reply.code(500).send({ ok: false, error: "Erreur cleanup" });
}
});
// ------------------------------------------------------------------ // ------------------------------------------------------------------
// Journal d'audit (actions admin) // Journal d'audit (actions admin)
// ------------------------------------------------------------------ // ------------------------------------------------------------------

16
apps/crawler/src/db.py
View file

@ -196,6 +196,22 @@ def log_event(level: str, message: str, run_id: Optional[int] = None, ma_id: Opt
# Crawl run tracking # Crawl run tracking
# ------------------------------------------------------------------ # ------------------------------------------------------------------
def update_crawl_run_progress(run_id: int, stats: Dict[str, int]):
"""Mise à jour des compteurs d'un run en cours (progression live)."""
try:
with get_conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""UPDATE crawl_run SET
bands_seen=%s, bands_new=%s, bands_updated=%s, bands_enriched=%s
WHERE id=%s AND status='running'""",
(stats.get("seen", 0), stats.get("new", 0),
stats.get("updated", 0), stats.get("enriched", 0), run_id),
)
except Exception as e:
log.warning(f"[db] update_crawl_run_progress failed: {e}")
def start_crawl_run(run_type: str, countries: Optional[List[str]] = None) -> int: def start_crawl_run(run_type: str, countries: Optional[List[str]] = None) -> int:
sql = "INSERT INTO crawl_run (run_type, countries) VALUES (%s, %s) RETURNING id" sql = "INSERT INTO crawl_run (run_type, countries) VALUES (%s, %s) RETURNING id"
with get_conn() as conn: with get_conn() as conn:

4
apps/crawler/src/jobs.py
View file

@ -14,6 +14,7 @@ from .db import (
upsert_bands, upsert_band_enriched, upsert_bands, upsert_band_enriched,
get_bands_to_enrich, get_bands_to_enrich,
start_crawl_run, finish_crawl_run, start_crawl_run, finish_crawl_run,
update_crawl_run_progress,
get_checkpoint, set_checkpoint, get_checkpoint, set_checkpoint,
log_event, log_event,
) )
@ -162,6 +163,9 @@ def run_enrich(session: MASession, limit: int = 100, country: Optional[str] = No
if COOLDOWN_EVERY and (i + 1) % COOLDOWN_EVERY == 0: if COOLDOWN_EVERY and (i + 1) % COOLDOWN_EVERY == 0:
cooldown(COOLDOWN_MIN, COOLDOWN_MAX) cooldown(COOLDOWN_MIN, COOLDOWN_MAX)
if (i + 1) % 50 == 0:
update_crawl_run_progress(run_id, stats)
log.info(f"[enrich] done: {stats}") log.info(f"[enrich] done: {stats}")
log_event("info", f"enrich done: {stats}", run_id=run_id) log_event("info", f"enrich done: {stats}", run_id=run_id)
except Exception as e: except Exception as e:

10
apps/crawler/src/ma_http.py
View file

@ -62,9 +62,13 @@ class MASession:
raise RuntimeError(f"HTTP {status} on {full_url}") raise RuntimeError(f"HTTP {status} on {full_url}")
try: try:
return _extract_json(body) return _extract_json(body)
except (json.JSONDecodeError, ValueError): except (json.JSONDecodeError, ValueError) as e:
log.warning(f"[ma_http] non-JSON response from {url}") if attempt < retries:
raise log.warning(f"[ma_http] JSON invalide/vide depuis {url} (tentative {attempt+1}), refresh session…")
self.ensure_session(force_refresh=True)
sleep_range(LIST_MIN_DELAY * 2, LIST_MAX_DELAY * 2)
continue
raise RuntimeError(f"JSON parse error après {retries} tentatives sur {url}: {e}")
raise RuntimeError(f"Failed after {retries} retries: {url}") raise RuntimeError(f"Failed after {retries} retries: {url}")
def get_html(self, url: str, retries: int = 2) -> str: def get_html(self, url: str, retries: int = 2) -> str:

96
apps/geocoder/src/worker.py
View file

@ -5,13 +5,12 @@ import json
import requests import requests
import psycopg2 import psycopg2
NOMINATIM_BASE = os.environ.get("NOMINATIM_BASE", "https://nominatim.openstreetmap.org").rstrip("/") GEOAPIFY_API_KEY = os.environ.get("GEOAPIFY_API_KEY", "").strip()
NOMINATIM_EMAIL = os.environ.get("NOMINATIM_EMAIL", "").strip() GEOAPIFY_BASE = "https://api.geoapify.com/v1/geocode/search"
USER_AGENT = os.environ.get("NOMINATIM_USER_AGENT", "bm-geocoder/0.1 (contact: you@example.com)").strip()
MIN_DELAY = float(os.environ.get("NOMINATIM_MIN_DELAY", "1.05")) # >= 1s MIN_DELAY = float(os.environ.get("GEOCODER_MIN_DELAY", "0.22")) # 5 req/s free tier
JITTER = float(os.environ.get("NOMINATIM_JITTER", "0.35")) JITTER = float(os.environ.get("GEOCODER_JITTER", "0.10"))
MAX_PER_RUN = int(os.environ.get("GEOCODE_MAX_PER_RUN", "100000")) MAX_PER_RUN = int(os.environ.get("GEOCODE_MAX_PER_RUN", "100000"))
_last = 0.0 _last = 0.0
@ -22,28 +21,28 @@ def polite_sleep():
time.sleep(wait) time.sleep(wait)
_last = time.monotonic() _last = time.monotonic()
def nominatim_search(query: str) -> dict | None: def geoapify_search(query: str) -> dict | None:
# doc: /search + email conseillé pour gros volume :contentReference[oaicite:1]{index=1} if not GEOAPIFY_API_KEY:
params = { raise RuntimeError("GEOAPIFY_API_KEY not set")
"q": query, params = {"text": query, "apiKey": GEOAPIFY_API_KEY, "limit": 1}
"format": "jsonv2",
"limit": 1,
"addressdetails": 1,
}
if NOMINATIM_EMAIL:
params["email"] = NOMINATIM_EMAIL
headers = {"User-Agent": USER_AGENT} # requis par policy :contentReference[oaicite:2]{index=2}
polite_sleep() polite_sleep()
r = requests.get(f"{NOMINATIM_BASE}/search", params=params, headers=headers, timeout=30) r = requests.get(GEOAPIFY_BASE, params=params, timeout=30)
if r.status_code in (403, 429, 503): if r.status_code == 429:
raise RuntimeError(f"nominatim_throttle status={r.status_code} body={r.text[:200]}") raise RuntimeError(f"geoapify_throttle status=429")
r.raise_for_status() r.raise_for_status()
arr = r.json() data = r.json()
if not arr: features = data.get("features") or []
if not features:
return None return None
return arr[0] props = features[0].get("properties", {})
coords = features[0].get("geometry", {}).get("coordinates", [])
if len(coords) < 2:
return None
return {
"lat": props.get("lat") or coords[1],
"lon": props.get("lon") or coords[0],
"raw": data,
}
def main(): def main():
dsn = os.environ["DATABASE_URL"] dsn = os.environ["DATABASE_URL"]
@ -53,7 +52,6 @@ def main():
processed = 0 processed = 0
with conn.cursor() as cur: with conn.cursor() as cur:
while processed < MAX_PER_RUN: while processed < MAX_PER_RUN:
# prend 1 job à faire
cur.execute( cur.execute(
""" """
SELECT ma_id, query, country, tries SELECT ma_id, query, country, tries
@ -73,17 +71,12 @@ def main():
ma_id, query, country, tries = row ma_id, query, country, tries = row
# marque processing
cur.execute( cur.execute(
""" "UPDATE geocode_queue SET status='processing', updated_at=now() WHERE ma_id=%s",
UPDATE geocode_queue
SET status='processing', updated_at=now()
WHERE ma_id=%s
""",
(ma_id,), (ma_id,),
) )
# 1) cache ? # cache hit ?
cur.execute("SELECT lat, lon, raw FROM geocode_cache WHERE query=%s", (query,)) cur.execute("SELECT lat, lon, raw FROM geocode_cache WHERE query=%s", (query,))
cached = cur.fetchone() cached = cur.fetchone()
if cached and cached[0] is not None and cached[1] is not None: if cached and cached[0] is not None and cached[1] is not None:
@ -94,7 +87,7 @@ def main():
SET lat=%s, lon=%s, SET lat=%s, lon=%s,
geom=ST_SetSRID(ST_MakePoint(%s,%s),4326)::geography, geom=ST_SetSRID(ST_MakePoint(%s,%s),4326)::geography,
geocoded_at=now(), geocoded_at=now(),
geocode_provider='nominatim-cache', geocode_provider='geoapify-cache',
geocode_query=%s, geocode_query=%s,
geocode_raw=%s, geocode_raw=%s,
geocode_error=NULL, geocode_error=NULL,
@ -108,24 +101,19 @@ def main():
print(f"[worker] ma_id={ma_id} cache HIT -> done") print(f"[worker] ma_id={ma_id} cache HIT -> done")
continue continue
# 2) requête nominatim # requête Geoapify
try: try:
res = nominatim_search(query) res = geoapify_search(query)
if not res: if not res:
cur.execute( cur.execute(
""" "UPDATE bands SET geocode_error=%s, geocode_error_at=now() WHERE ma_id=%s",
UPDATE bands
SET geocode_error=%s, geocode_error_at=now()
WHERE ma_id=%s
""",
("no_result", ma_id), ("no_result", ma_id),
) )
cur.execute( cur.execute(
""" """
UPDATE geocode_queue UPDATE geocode_queue
SET status='error', tries=tries+1, last_error=%s, SET status='error', tries=tries+1, last_error=%s,
next_run_at=now() + interval '7 days', next_run_at=now() + interval '7 days', updated_at=now()
updated_at=now()
WHERE ma_id=%s WHERE ma_id=%s
""", """,
("no_result", ma_id), ("no_result", ma_id),
@ -137,33 +125,31 @@ def main():
lat = float(res["lat"]) lat = float(res["lat"])
lon = float(res["lon"]) lon = float(res["lon"])
# écrit cache
cur.execute( cur.execute(
""" """
INSERT INTO geocode_cache(query, provider, lat, lon, geom, raw, updated_at) INSERT INTO geocode_cache(query, provider, lat, lon, geom, raw, updated_at)
VALUES (%s,'nominatim',%s,%s,ST_SetSRID(ST_MakePoint(%s,%s),4326)::geography,%s,now()) VALUES (%s,'geoapify',%s,%s,ST_SetSRID(ST_MakePoint(%s,%s),4326)::geography,%s,now())
ON CONFLICT (query) DO UPDATE ON CONFLICT (query) DO UPDATE
SET lat=EXCLUDED.lat, lon=EXCLUDED.lon, geom=EXCLUDED.geom, SET lat=EXCLUDED.lat, lon=EXCLUDED.lon, geom=EXCLUDED.geom,
raw=EXCLUDED.raw, updated_at=now() raw=EXCLUDED.raw, provider=EXCLUDED.provider, updated_at=now()
""", """,
(query, lat, lon, lon, lat, json.dumps(res)), (query, lat, lon, lon, lat, json.dumps(res["raw"])),
) )
# écrit bands
cur.execute( cur.execute(
""" """
UPDATE bands UPDATE bands
SET lat=%s, lon=%s, SET lat=%s, lon=%s,
geom=ST_SetSRID(ST_MakePoint(%s,%s),4326)::geography, geom=ST_SetSRID(ST_MakePoint(%s,%s),4326)::geography,
geocoded_at=now(), geocoded_at=now(),
geocode_provider='nominatim', geocode_provider='geoapify',
geocode_query=%s, geocode_query=%s,
geocode_raw=%s, geocode_raw=%s,
geocode_error=NULL, geocode_error=NULL,
geocode_error_at=NULL geocode_error_at=NULL
WHERE ma_id=%s WHERE ma_id=%s
""", """,
(lat, lon, lon, lat, query, json.dumps(res), ma_id), (lat, lon, lon, lat, query, json.dumps(res["raw"]), ma_id),
) )
cur.execute("UPDATE geocode_queue SET status='done', updated_at=now() WHERE ma_id=%s", (ma_id,)) cur.execute("UPDATE geocode_queue SET status='done', updated_at=now() WHERE ma_id=%s", (ma_id,))
@ -171,22 +157,16 @@ def main():
print(f"[worker] ma_id={ma_id} OK lat={lat} lon={lon}") print(f"[worker] ma_id={ma_id} OK lat={lat} lon={lon}")
except Exception as e: except Exception as e:
# backoff progressif
backoff_minutes = min(60, 5 * (tries + 1)) backoff_minutes = min(60, 5 * (tries + 1))
cur.execute( cur.execute(
""" "UPDATE bands SET geocode_error=%s, geocode_error_at=now() WHERE ma_id=%s",
UPDATE bands
SET geocode_error=%s, geocode_error_at=now()
WHERE ma_id=%s
""",
(str(e)[:400], ma_id), (str(e)[:400], ma_id),
) )
cur.execute( cur.execute(
""" """
UPDATE geocode_queue UPDATE geocode_queue
SET status='error', tries=tries+1, last_error=%s, SET status='error', tries=tries+1, last_error=%s,
next_run_at=now() + (%s || ' minutes')::interval, next_run_at=now() + (%s || ' minutes')::interval, updated_at=now()
updated_at=now()
WHERE ma_id=%s WHERE ma_id=%s
""", """,
(str(e)[:400], backoff_minutes, ma_id), (str(e)[:400], backoff_minutes, ma_id),

8
docker-compose.dev.yml
View file

@ -41,11 +41,9 @@ services:
working_dir: /app working_dir: /app
environment: environment:
DATABASE_URL: ${DATABASE_URL} DATABASE_URL: ${DATABASE_URL}
NOMINATIM_BASE: "https://nominatim.openstreetmap.org" GEOAPIFY_API_KEY: ${GEOAPIFY_API_KEY}
NOMINATIM_EMAIL: ${NOMINATIM_EMAIL} GEOCODER_MIN_DELAY: "0.22"
NOMINATIM_USER_AGENT: ${NOMINATIM_USER_AGENT} GEOCODER_JITTER: "0.10"
NOMINATIM_MIN_DELAY: "1.05"
NOMINATIM_JITTER: "0.35"
GEOCODE_MAX_PER_RUN: "5000" GEOCODE_MAX_PER_RUN: "5000"
command: ["python", "src/worker.py"] command: ["python", "src/worker.py"]
networks: networks:

8
docker-compose.yml
View file

@ -13,11 +13,9 @@ services:
working_dir: /app working_dir: /app
environment: environment:
DATABASE_URL: ${DATABASE_URL} DATABASE_URL: ${DATABASE_URL}
NOMINATIM_BASE: "https://nominatim.openstreetmap.org" GEOAPIFY_API_KEY: ${GEOAPIFY_API_KEY}
NOMINATIM_EMAIL: ${NOMINATIM_EMAIL} GEOCODER_MIN_DELAY: "0.22"
NOMINATIM_USER_AGENT: ${NOMINATIM_USER_AGENT} GEOCODER_JITTER: "0.10"
NOMINATIM_MIN_DELAY: "1.05"
NOMINATIM_JITTER: "0.35"
GEOCODE_MAX_PER_RUN: "5000" GEOCODE_MAX_PER_RUN: "5000"
command: ["python", "src/worker.py"] command: ["python", "src/worker.py"]
networks: networks: