feat: locked_fields, genres splittés, conflits, géocodage, jobs dashboard
Migration 007 :
- bands.locked_fields : champs édités manuellement = ne jamais réécrire
- bands.crawler_pending : valeurs MA différentes des valeurs verrouillées
- job_triggers : déclenchement de jobs depuis l'admin sans docker socket
- DELETE non-EU bands (6852 bands US/BR/CA/… entrés via crawl incrémental)
Crawler :
- upsert_band_enriched respecte locked_fields et stocke crawler_pending
- run_incremental filtre les bands hors Europe (EU-only désormais confirmé)
- update_crawl_run_progress toutes les 10 bands (granularité améliorée)
- main loop consomme job_triggers (enrich / incremental / full_crawl)
- retry sur JSONDecodeError dans get_json (FlareSolverr corps vide)
Admin PATCH bands :
- les champs édités sont automatiquement ajoutés à locked_fields
Dashboard :
- Genres découpés en mots-clés (Doom/Death Metal → Doom, Death Metal…)
- Listes pays/statut/genre sans limite, avec scroll interne
Bands admin :
- Colonne Thèmes + champ recherche thèmes_q
- Indicateur 🔒 sur les bands avec champs verrouillés
Nouvelles pages admin :
- Conflits : résolution champ par champ (garder ma valeur / accepter MA)
- Géocodage : stats queue, barre de progression, 20 derniers géocodages
- Jobs : boutons pour déclencher enrich/incremental/full_crawl manuellement
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
3dc483bc4f
commit
c90b5063f1
7 changed files with 514 additions and 38 deletions
|
|
@ -94,7 +94,7 @@ document.getElementById("logout-btn").addEventListener("click", async () => {
|
|||
// ------------------------------------------------------------------
|
||||
// Router
|
||||
// ------------------------------------------------------------------
|
||||
const VIEWS = ["dashboard", "bands", "queue", "runs", "logs", "checkpoints", "audit"];
|
||||
const VIEWS = ["dashboard", "bands", "queue", "runs", "logs", "geocoding", "conflicts", "jobs", "checkpoints", "audit"];
|
||||
|
||||
function router() {
|
||||
const hash = (location.hash || "#/dashboard").replace("#/", "");
|
||||
|
|
@ -108,9 +108,12 @@ function router() {
|
|||
const renderers = {
|
||||
dashboard: renderDashboard,
|
||||
bands: renderBands,
|
||||
conflicts: renderConflicts,
|
||||
queue: renderQueue,
|
||||
runs: renderRuns,
|
||||
logs: renderLogs,
|
||||
geocoding: renderGeocoding,
|
||||
jobs: renderJobs,
|
||||
checkpoints: renderCheckpoints,
|
||||
audit: renderAudit,
|
||||
};
|
||||
|
|
@ -131,6 +134,26 @@ const content = () => document.getElementById("content");
|
|||
// ------------------------------------------------------------------
|
||||
// Dashboard
|
||||
// ------------------------------------------------------------------
|
||||
// Découpe un genre complexe ("Doom/Death Metal; Gothic/Progressive") en mots-clés
|
||||
function splitGenreElements(genreRows) {
|
||||
const counts = new Map();
|
||||
for (const { genre, total } of genreRows) {
|
||||
if (!genre) continue;
|
||||
// Retire les parenthèses et leur contenu (ex: "(early)", "(later)")
|
||||
const cleaned = genre.replace(/\([^)]*\)/g, "");
|
||||
// Découpe sur / ; ,
|
||||
const tokens = cleaned.split(/[\/;,]+/);
|
||||
for (const tok of tokens) {
|
||||
const kw = tok.trim();
|
||||
if (kw.length < 3) continue;
|
||||
counts.set(kw, (counts.get(kw) || 0) + total);
|
||||
}
|
||||
}
|
||||
return [...counts.entries()]
|
||||
.map(([kw, n]) => ({ kw, n }))
|
||||
.sort((a, b) => b.n - a.n);
|
||||
}
|
||||
|
||||
async function renderDashboard() {
|
||||
content().innerHTML = `<div class="loading">Chargement…</div>`;
|
||||
try {
|
||||
|
|
@ -138,7 +161,8 @@ async function renderDashboard() {
|
|||
const t = r.totals;
|
||||
const maxStatus = Math.max(1, ...r.by_status.map((x) => x.total));
|
||||
const maxCountry = Math.max(1, ...r.by_country.map((x) => x.total));
|
||||
const maxGenre = Math.max(1, ...r.by_genre.map((x) => x.total));
|
||||
const elements = splitGenreElements(r.by_genre);
|
||||
const maxEl = Math.max(1, ...elements.map((x) => x.n));
|
||||
|
||||
content().innerHTML = `
|
||||
<div class="grid grid-stats">
|
||||
|
|
@ -160,9 +184,10 @@ async function renderDashboard() {
|
|||
<h2>Par pays (${r.by_country.length})</h2>
|
||||
${r.by_country.map((x) => barRow(x.country, x.total, maxCountry)).join("") || emptyRow()}
|
||||
</div>
|
||||
<div class="card" style="max-height:420px;overflow-y:auto">
|
||||
<h2>Par genre (${r.by_genre.length})</h2>
|
||||
${r.by_genre.map((x) => barRow(x.genre, x.total, maxGenre)).join("") || emptyRow()}
|
||||
<div class="card" style="max-height:500px;overflow-y:auto">
|
||||
<h2>Éléments de genre (${elements.length} mots-clés)</h2>
|
||||
<p style="font-size:11px;color:var(--muted);margin:0 0 10px">Chaque genre complexe (ex: "Doom/Death Metal; Gothic") est découpé en mots-clés.</p>
|
||||
${elements.map((x) => barRow(x.kw, x.n, maxEl)).join("") || emptyRow()}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
|
@ -280,6 +305,7 @@ async function renderBands() {
|
|||
<option value="true" ${s.has_lat === "true" ? "selected" : ""}>Géocodé ✓</option>
|
||||
<option value="false" ${s.has_lat === "false" ? "selected" : ""}>Sans coords ✗</option>
|
||||
</select>
|
||||
<input type="text" id="b-themes_q" placeholder="Thèmes…" value="${esc(s.themes_q||"")}" style="width:150px">
|
||||
<select id="b-has_location">
|
||||
<option value="">Lieu texte: tous</option>
|
||||
<option value="true" ${s.has_location === "true" ? "selected" : ""}>Lieu renseigné ✓</option>
|
||||
|
|
@ -297,6 +323,7 @@ async function renderBands() {
|
|||
s.status = document.getElementById("b-status").value.trim();
|
||||
s.genre = document.getElementById("b-genre").value.trim();
|
||||
s.enriched = document.getElementById("b-enriched").value;
|
||||
s.themes_q = document.getElementById("b-themes_q").value.trim();
|
||||
s.has_lat = document.getElementById("b-has_lat").value;
|
||||
s.has_location = document.getElementById("b-has_location").value;
|
||||
s.page = 1;
|
||||
|
|
@ -311,6 +338,7 @@ const BAND_COLUMNS = [
|
|||
{ key: "country", label: "Pays" },
|
||||
{ key: "status", label: "Statut" },
|
||||
{ key: "genre", label: "Genre" },
|
||||
{ key: "themes", label: "Thèmes" },
|
||||
{ key: "location_text", label: "Lieu" },
|
||||
{ key: "formed_year", label: "Année" },
|
||||
{ key: "enriched", label: "Enrichi" },
|
||||
|
|
@ -329,6 +357,7 @@ async function loadBands() {
|
|||
if (s.status) params.set("status", s.status);
|
||||
if (s.genre) params.set("genre", s.genre);
|
||||
if (s.enriched) params.set("enriched", s.enriched);
|
||||
if (s.themes_q) params.set("themes_q", s.themes_q);
|
||||
if (s.has_lat) params.set("has_lat", s.has_lat);
|
||||
if (s.has_location) params.set("has_location", s.has_location);
|
||||
|
||||
|
|
@ -349,9 +378,10 @@ async function loadBands() {
|
|||
<td>${esc(b.country || "")}</td>
|
||||
<td>${esc(b.status || "")}</td>
|
||||
<td>${esc(b.genre || "")}</td>
|
||||
<td style="max-width:160px;overflow:hidden;text-overflow:ellipsis" title="${esc(b.location_text || "")}">${esc(b.location_text || "")}</td>
|
||||
<td style="max-width:140px;overflow:hidden;text-overflow:ellipsis" title="${esc(b.themes || "")}">${esc(b.themes || "")}</td>
|
||||
<td style="max-width:140px;overflow:hidden;text-overflow:ellipsis" title="${esc(b.location_text || "")}">${esc(b.location_text || "")}</td>
|
||||
<td>${b.formed_year || ""}</td>
|
||||
<td>${b.enriched ? '<span class="badge ok">oui</span>' : '<span class="badge warn">non</span>'}</td>
|
||||
<td>${b.enriched ? '<span class="badge ok">oui</span>' : '<span class="badge warn">non</span>'}${Object.keys(b.locked_fields||{}).length ? ' <span title="Champs verrouillés manuellement" style="color:var(--blood)">🔒</span>' : ""}</td>
|
||||
<td>${fmtDate(b.crawled_at)}</td>
|
||||
<td>${fmtDate(b.updated_at)}</td>
|
||||
</tr>
|
||||
|
|
@ -578,6 +608,184 @@ async function loadLogs() {
|
|||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Conflits (champs verrouillés ≠ valeur crawler)
|
||||
// ------------------------------------------------------------------
|
||||
async function renderConflicts() {
|
||||
content().innerHTML = `<div class="loading">Chargement…</div>`;
|
||||
try {
|
||||
const r = await api("/admin/api/conflicts?page=1&pageSize=100");
|
||||
if (!r.items.length) {
|
||||
content().innerHTML = `<div class="card"><div class="empty">Aucun conflit — toutes les éditions manuelles sont cohérentes avec les données Metal Archives.</div></div>`;
|
||||
return;
|
||||
}
|
||||
content().innerHTML = `
|
||||
<p style="font-size:12px;color:var(--muted);margin:0 0 14px">
|
||||
Ces bands ont des champs édités manuellement (🔒) pour lesquels le crawler a trouvé une valeur différente.
|
||||
Choisissez champ par champ ce que vous voulez conserver.
|
||||
</p>
|
||||
<div id="conflict-list">${r.items.map(conflictCard).join("")}</div>
|
||||
`;
|
||||
document.getElementById("conflict-list").addEventListener("click", async (e) => {
|
||||
const btn = e.target.closest("[data-action]");
|
||||
if (!btn) return;
|
||||
const { action, maId, field } = btn.dataset;
|
||||
btn.disabled = true;
|
||||
try {
|
||||
await api(`/admin/api/bands/${maId}/resolve-conflict`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ field, action }),
|
||||
});
|
||||
renderConflicts();
|
||||
} catch (err) { alert(err.message); btn.disabled = false; }
|
||||
});
|
||||
} catch (e) {
|
||||
content().innerHTML = `<div class="empty">Erreur : ${esc(e.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function conflictCard(b) {
|
||||
const pending = b.crawler_pending || {};
|
||||
const locked = b.locked_fields || {};
|
||||
const rows = Object.entries(pending).map(([field, crawlerVal]) => {
|
||||
const myVal = b[field] ?? "—";
|
||||
return `<tr>
|
||||
<td><strong>${esc(field)}</strong></td>
|
||||
<td style="color:var(--bone)">${esc(String(myVal))}</td>
|
||||
<td style="color:var(--warn)">${esc(String(crawlerVal))}</td>
|
||||
<td>
|
||||
<button class="btn btn-mini" data-action="keep_mine" data-ma-id="${b.ma_id}" data-field="${field}">Garder ma valeur</button>
|
||||
<button class="btn btn-mini" style="border-color:rgba(198,26,26,0.4)" data-action="accept_crawler" data-ma-id="${b.ma_id}" data-field="${field}">Accepter MA</button>
|
||||
</td>
|
||||
</tr>`;
|
||||
}).join("");
|
||||
return `<div class="card" style="margin-bottom:14px">
|
||||
<h2 style="margin-bottom:8px">${esc(b.name)} <span style="color:var(--muted);font-weight:400">#${b.ma_id} · ${esc(b.country||"")}</span></h2>
|
||||
<div class="table-wrap"><table>
|
||||
<thead><tr><th>Champ</th><th>Ma valeur (verrouillée)</th><th>Valeur Metal Archives</th><th>Action</th></tr></thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table></div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Géocodage
|
||||
// ------------------------------------------------------------------
|
||||
async function renderGeocoding() {
|
||||
content().innerHTML = `<div class="loading">Chargement…</div>`;
|
||||
try {
|
||||
const r = await api("/admin/api/geocoding");
|
||||
const total = r.queue.reduce((s, x) => s + x.n, 0);
|
||||
const done = r.queue.find((x) => x.status === "done")?.n || 0;
|
||||
const queued = r.queue.find((x) => x.status === "queued")?.n || 0;
|
||||
const errored = r.queue.find((x) => x.status === "error")?.n || 0;
|
||||
const pct = total ? Math.round((done / total) * 100) : 0;
|
||||
|
||||
content().innerHTML = `
|
||||
<div class="grid grid-stats">
|
||||
${statCard("Total queue", total)}
|
||||
${statCard("Géocodés ✓", done, "ok")}
|
||||
${statCard("En attente", queued, "warn")}
|
||||
${statCard("Erreurs", errored, "err")}
|
||||
${statCard("Cache Geoapify", r.cache_size)}
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:16px">
|
||||
<h2>Progression</h2>
|
||||
<div style="background:rgba(255,255,255,0.06);border-radius:8px;height:18px;overflow:hidden;margin-bottom:8px">
|
||||
<div style="width:${pct}%;height:100%;background:linear-gradient(90deg,var(--blood),rgba(198,26,26,0.6))"></div>
|
||||
</div>
|
||||
<div style="font-size:12px;color:var(--muted)">${pct}% — ${done.toLocaleString("fr-FR")} / ${total.toLocaleString("fr-FR")}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>20 derniers géocodages</h2>
|
||||
<div class="table-wrap"><table>
|
||||
<thead><tr><th>MA ID</th><th>Nom</th><th>Pays</th><th>Provider</th><th>Query</th><th>Géocodé le</th><th>Erreur</th></tr></thead>
|
||||
<tbody>
|
||||
${r.recent.map((x) => `<tr>
|
||||
<td>${x.ma_id}</td>
|
||||
<td>${esc(x.name)}</td>
|
||||
<td>${esc(x.country||"")}</td>
|
||||
<td>${esc(x.geocode_provider||"")}</td>
|
||||
<td style="max-width:200px;overflow:hidden;text-overflow:ellipsis">${esc(x.geocode_query||"")}</td>
|
||||
<td>${fmtDate(x.geocoded_at)}</td>
|
||||
<td style="color:var(--err)">${esc(x.geocode_error||"")}</td>
|
||||
</tr>`).join("") || `<tr><td colspan="7" class="empty">Aucun</td></tr>`}
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
`;
|
||||
} catch (e) {
|
||||
content().innerHTML = `<div class="empty">Erreur : ${esc(e.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Jobs (déclenchement manuel des workers)
|
||||
// ------------------------------------------------------------------
|
||||
async function renderJobs() {
|
||||
content().innerHTML = `
|
||||
<div class="card" style="margin-bottom:16px">
|
||||
<h2>Déclencher un job manuellement</h2>
|
||||
<p style="font-size:12px;color:var(--muted);margin:0 0 14px">
|
||||
Le crawler consomme ces demandes dans sa prochaine itération (~1 min d'attente max).
|
||||
</p>
|
||||
<div style="display:flex;gap:10px;flex-wrap:wrap">
|
||||
<button class="btn" data-job="enrich">▶ Enrich (500 bands)</button>
|
||||
<button class="btn" data-job="incremental">▶ Crawl incrémental (créations + modifs)</button>
|
||||
<button class="btn" data-job="full_crawl">▶ Crawl complet Europe</button>
|
||||
</div>
|
||||
<div id="job-feedback" style="margin-top:12px;font-size:12px;color:var(--muted)"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>Historique des jobs déclenchés</h2>
|
||||
<div id="jobs-list"><div class="loading">Chargement…</div></div>
|
||||
</div>
|
||||
`;
|
||||
document.querySelectorAll("[data-job]").forEach((btn) => {
|
||||
btn.addEventListener("click", async () => {
|
||||
const job_type = btn.dataset.job;
|
||||
btn.disabled = true;
|
||||
const fb = document.getElementById("job-feedback");
|
||||
fb.textContent = "Envoi…";
|
||||
try {
|
||||
const r = await api("/admin/api/job-triggers", { method: "POST", body: JSON.stringify({ job_type }) });
|
||||
fb.textContent = `✓ Job #${r.id} créé — le crawler le consommera sous ~1 min.`;
|
||||
fb.style.color = "var(--ok)";
|
||||
await loadJobsList();
|
||||
} catch (e) {
|
||||
fb.textContent = `Erreur: ${e.message}`;
|
||||
fb.style.color = "var(--err)";
|
||||
} finally { btn.disabled = false; }
|
||||
});
|
||||
});
|
||||
await loadJobsList();
|
||||
}
|
||||
|
||||
async function loadJobsList() {
|
||||
const el = document.getElementById("jobs-list");
|
||||
if (!el) return;
|
||||
try {
|
||||
const r = await api("/admin/api/job-triggers");
|
||||
el.innerHTML = `<div class="table-wrap"><table>
|
||||
<thead><tr><th>ID</th><th>Type</th><th>Statut</th><th>Demandé par</th><th>Créé le</th><th>Démarré</th><th>Terminé</th><th>Erreur</th></tr></thead>
|
||||
<tbody>
|
||||
${r.items.map((x) => `<tr>
|
||||
<td>${x.id}</td>
|
||||
<td>${esc(x.job_type)}</td>
|
||||
<td>${statusBadge(x.status)}</td>
|
||||
<td>${esc(x.requested_by||"—")}</td>
|
||||
<td>${fmtDate(x.created_at)}</td>
|
||||
<td>${fmtDate(x.started_at)}</td>
|
||||
<td>${fmtDate(x.finished_at)}</td>
|
||||
<td style="color:var(--err)">${esc(x.error||"")}</td>
|
||||
</tr>`).join("") || `<tr><td colspan="8" class="empty">Aucun job</td></tr>`}
|
||||
</tbody>
|
||||
</table></div>`;
|
||||
} catch (e) {
|
||||
el.innerHTML = `<div class="empty">Erreur : ${esc(e.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Checkpoints
|
||||
// ------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -32,9 +32,12 @@
|
|||
<nav id="nav" class="nav">
|
||||
<a href="#/dashboard" data-view="dashboard">Dashboard</a>
|
||||
<a href="#/bands" data-view="bands">Bands</a>
|
||||
<a href="#/conflicts" data-view="conflicts">Conflits</a>
|
||||
<a href="#/queue" data-view="queue">Queue</a>
|
||||
<a href="#/runs" data-view="runs">Crawl runs</a>
|
||||
<a href="#/logs" data-view="logs">Logs</a>
|
||||
<a href="#/geocoding" data-view="geocoding">Géocodage</a>
|
||||
<a href="#/jobs" data-view="jobs">Jobs</a>
|
||||
<a href="#/checkpoints" data-view="checkpoints">Checkpoints</a>
|
||||
<a href="#/audit" data-view="audit">Audit</a>
|
||||
</nav>
|
||||
|
|
|
|||
33
apps/api/migrations/007_locked_fields_and_jobs.sql
Normal file
33
apps/api/migrations/007_locked_fields_and_jobs.sql
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
-- 007: champs verrouillés (éditions manuelles), conflits crawler, déclencheurs de jobs,
|
||||
-- et nettoyage des 6852 bands non-européens entrés via le crawl incrémental global.
|
||||
|
||||
-- 1. Protection des éditions manuelles
|
||||
ALTER TABLE bands
|
||||
ADD COLUMN IF NOT EXISTS locked_fields JSONB NOT NULL DEFAULT '{}',
|
||||
ADD COLUMN IF NOT EXISTS crawler_pending JSONB;
|
||||
|
||||
-- 2. Table de déclencheurs de jobs (depuis le dashboard admin sans socket Docker)
|
||||
CREATE TABLE IF NOT EXISTS job_triggers (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_type TEXT NOT NULL, -- 'enrich'|'incremental'|'full_crawl'|'geocode'
|
||||
status TEXT NOT NULL DEFAULT 'pending', -- pending|running|done|error
|
||||
requested_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
started_at TIMESTAMPTZ,
|
||||
finished_at TIMESTAMPTZ,
|
||||
error TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_job_triggers_pending ON job_triggers (status, created_at)
|
||||
WHERE status = 'pending';
|
||||
|
||||
-- 3. Suppression des bands hors Europe (entrés via crawl incrémental global)
|
||||
-- Liste officielle des codes pays européens utilisés par Metal Archives.
|
||||
DELETE FROM bands
|
||||
WHERE country IS NOT NULL
|
||||
AND country NOT IN (
|
||||
'AD','AL','AT','BA','BE','BG','BY','CH','CY','CZ',
|
||||
'DE','DK','EE','ES','FI','FR','GB','GE','GR','HR',
|
||||
'HU','IE','IS','IT','LI','LT','LU','LV','MC','MD',
|
||||
'ME','MK','MT','NL','NO','PL','PT','RO','RS','RU',
|
||||
'SE','SI','SK','SM','TR','UA','VA','XK'
|
||||
);
|
||||
|
|
@ -110,7 +110,7 @@ export default async function adminRoutes(fastify, opts) {
|
|||
// ------------------------------------------------------------------
|
||||
fastify.get("/admin/api/bands", async (req, reply) => {
|
||||
try {
|
||||
const { q, location_q, country, status, genre, enriched, has_lat, has_location, sort, dir } = req.query || {};
|
||||
const { q, location_q, country, status, genre, themes_q, enriched, has_lat, has_location, sort, dir } = req.query || {};
|
||||
const { page, pageSize, offset } = pagination(req.query || {});
|
||||
|
||||
const where = [];
|
||||
|
|
@ -154,6 +154,10 @@ export default async function adminRoutes(fastify, opts) {
|
|||
if (has_lat === "false" || has_lat === "0") where.push(`lat IS NULL`);
|
||||
if (has_location === "true" || has_location === "1") where.push(`location_text IS NOT NULL AND location_text != ''`);
|
||||
if (has_location === "false" || has_location === "0") where.push(`(location_text IS NULL OR location_text = '')`);
|
||||
if (themes_q) {
|
||||
const tq = String(themes_q).trim().slice(0, 100);
|
||||
if (tq.length >= 1) { where.push(`themes ILIKE $${i}`); vals.push(`%${tq}%`); i++; }
|
||||
}
|
||||
|
||||
const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : "";
|
||||
const sortCol = BAND_SORT_COLUMNS.has(sort) ? sort : "ma_id";
|
||||
|
|
@ -161,8 +165,8 @@ export default async function adminRoutes(fastify, opts) {
|
|||
|
||||
const countSql = `SELECT count(*)::int AS total FROM bands ${whereSql}`;
|
||||
const dataSql = `
|
||||
SELECT ma_id, name, country, status, genre, location_text, formed_year,
|
||||
themes, enriched, lat, lon, crawled_at, updated_at, first_seen_at
|
||||
SELECT ma_id, name, country, status, genre, location_text, themes, formed_year,
|
||||
enriched, lat, lon, crawled_at, updated_at, first_seen_at, locked_fields
|
||||
FROM bands
|
||||
${whereSql}
|
||||
ORDER BY ${sortCol} ${sortDir} NULLS LAST
|
||||
|
|
@ -245,12 +249,22 @@ export default async function adminRoutes(fastify, opts) {
|
|||
if (!before.rows.length) return reply.code(404).send({ ok: false, error: "not found" });
|
||||
|
||||
const setCols = Object.keys(updates);
|
||||
// Marque les champs édités comme verrouillés (ne seront plus écrasés par le crawler)
|
||||
const newLocked = Object.fromEntries(setCols.map((c) => [c, true]));
|
||||
const setSql = setCols.map((c, idx) => `${c} = $${idx + 2}`).join(", ");
|
||||
const vals = [id, ...setCols.map((c) => updates[c])];
|
||||
|
||||
const after = await pool.query(
|
||||
`UPDATE bands SET ${setSql} WHERE ma_id = $1 RETURNING *`,
|
||||
vals
|
||||
`UPDATE bands SET ${setSql},
|
||||
locked_fields = locked_fields || $${vals.length + 1}::jsonb,
|
||||
crawler_pending = (
|
||||
SELECT jsonb_strip_nulls(jsonb_build_object(${
|
||||
setCols.map((c) => `'${c}', crawler_pending->'${c}'`).join(", ")
|
||||
}))
|
||||
FROM bands WHERE ma_id = $1
|
||||
)
|
||||
WHERE ma_id = $1 RETURNING *`,
|
||||
[...vals, JSON.stringify(newLocked)]
|
||||
);
|
||||
|
||||
await writeAuditLog(
|
||||
|
|
@ -359,6 +373,123 @@ export default async function adminRoutes(fastify, opts) {
|
|||
}
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Conflits : champs verrouillés avec valeur différente côté crawler
|
||||
// ------------------------------------------------------------------
|
||||
fastify.get("/admin/api/conflicts", async (req, reply) => {
|
||||
try {
|
||||
const { page, pageSize, offset } = pagination(req.query || {});
|
||||
const r = await pool.query(`
|
||||
SELECT ma_id, name, country, status, genre, themes, formed_year,
|
||||
locked_fields, crawler_pending
|
||||
FROM bands
|
||||
WHERE crawler_pending IS NOT NULL AND crawler_pending != '{}'::jsonb
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT $1 OFFSET $2
|
||||
`, [pageSize, offset]);
|
||||
const count = await pool.query(`SELECT count(*)::int AS n FROM bands WHERE crawler_pending IS NOT NULL AND crawler_pending != '{}'::jsonb`);
|
||||
return { ok: true, items: r.rows, total: count.rows[0].n, page, pageSize };
|
||||
} catch (err) {
|
||||
fastify.log.error(err);
|
||||
return reply.code(500).send({ ok: false, error: "Erreur conflicts" });
|
||||
}
|
||||
});
|
||||
|
||||
fastify.post("/admin/api/bands/:ma_id/resolve-conflict", async (req, reply) => {
|
||||
try {
|
||||
const id = Number(req.params.ma_id);
|
||||
if (!Number.isFinite(id)) return reply.code(400).send({ ok: false, error: "bad ma_id" });
|
||||
const { field, action } = req.body || {}; // action: 'keep_mine' | 'accept_crawler'
|
||||
if (!field || !["keep_mine", "accept_crawler"].includes(action)) {
|
||||
return reply.code(400).send({ ok: false, error: "field et action requis" });
|
||||
}
|
||||
const before = await pool.query(`SELECT * FROM bands WHERE ma_id = $1`, [id]);
|
||||
if (!before.rows.length) return reply.code(404).send({ ok: false, error: "not found" });
|
||||
|
||||
let sql, vals;
|
||||
if (action === "keep_mine") {
|
||||
// Garde la valeur actuelle, efface juste le pending pour ce champ
|
||||
sql = `UPDATE bands SET crawler_pending = crawler_pending - $2 WHERE ma_id = $1`;
|
||||
vals = [id, field];
|
||||
} else {
|
||||
// Applique la valeur du crawler, déverrouille le champ
|
||||
const crawlerVal = before.rows[0].crawler_pending?.[field];
|
||||
sql = `UPDATE bands SET ${field} = $2::text, locked_fields = locked_fields - $3, crawler_pending = crawler_pending - $3 WHERE ma_id = $1`;
|
||||
vals = [id, crawlerVal, field];
|
||||
}
|
||||
const after = await pool.query(sql + " RETURNING *", vals);
|
||||
await writeAuditLog(pool, req.adminUsername, `resolve_conflict:${action}`, "bands", id, before.rows[0], after.rows[0]);
|
||||
return { ok: true, item: after.rows[0] };
|
||||
} catch (err) {
|
||||
fastify.log.error(err);
|
||||
return reply.code(500).send({ ok: false, error: "Erreur résolution conflit" });
|
||||
}
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Géocodage : stats + déclenchement
|
||||
// ------------------------------------------------------------------
|
||||
fastify.get("/admin/api/geocoding", async (req, reply) => {
|
||||
try {
|
||||
const [queue, cache, recent] = await Promise.all([
|
||||
pool.query(`
|
||||
SELECT status, count(*)::int AS n FROM geocode_queue GROUP BY status ORDER BY n DESC
|
||||
`),
|
||||
pool.query(`SELECT count(*)::int AS n FROM geocode_cache`),
|
||||
pool.query(`
|
||||
SELECT b.ma_id, b.name, b.country, b.geocode_provider, b.geocoded_at,
|
||||
b.geocode_query, b.geocode_error
|
||||
FROM bands b
|
||||
WHERE b.geocoded_at IS NOT NULL
|
||||
ORDER BY b.geocoded_at DESC LIMIT 20
|
||||
`),
|
||||
]);
|
||||
return {
|
||||
ok: true,
|
||||
queue: queue.rows,
|
||||
cache_size: cache.rows[0].n,
|
||||
recent: recent.rows,
|
||||
};
|
||||
} catch (err) {
|
||||
fastify.log.error(err);
|
||||
return reply.code(500).send({ ok: false, error: "Erreur geocoding stats" });
|
||||
}
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Job triggers (déclenche un job dans le crawler sans redéployer)
|
||||
// ------------------------------------------------------------------
|
||||
fastify.get("/admin/api/job-triggers", async (req, reply) => {
|
||||
try {
|
||||
const r = await pool.query(`
|
||||
SELECT id, job_type, status, requested_by, created_at, started_at, finished_at, error
|
||||
FROM job_triggers ORDER BY created_at DESC LIMIT 30
|
||||
`);
|
||||
return { ok: true, items: r.rows };
|
||||
} catch (err) {
|
||||
fastify.log.error(err);
|
||||
return reply.code(500).send({ ok: false, error: "Erreur job triggers" });
|
||||
}
|
||||
});
|
||||
|
||||
fastify.post("/admin/api/job-triggers", async (req, reply) => {
|
||||
try {
|
||||
const { job_type } = req.body || {};
|
||||
const allowed = ["enrich", "incremental", "full_crawl"];
|
||||
if (!allowed.includes(job_type)) {
|
||||
return reply.code(400).send({ ok: false, error: `job_type doit être: ${allowed.join(", ")}` });
|
||||
}
|
||||
const r = await pool.query(
|
||||
`INSERT INTO job_triggers (job_type, requested_by) VALUES ($1, $2) RETURNING id`,
|
||||
[job_type, req.adminUsername]
|
||||
);
|
||||
return { ok: true, id: r.rows[0].id };
|
||||
} catch (err) {
|
||||
fastify.log.error(err);
|
||||
return reply.code(500).send({ ok: false, error: "Erreur création job" });
|
||||
}
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Journal d'audit (actions admin)
|
||||
// ------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -100,36 +100,60 @@ def upsert_bands(bands: List[Dict[str, Any]]) -> Dict[str, int]:
|
|||
|
||||
|
||||
def upsert_band_enriched(ma_id: int, data: Dict[str, Any], html_hash: str) -> bool:
|
||||
"""Met à jour les champs d'enrichissement d'un band."""
|
||||
sql = """
|
||||
UPDATE bands SET
|
||||
status = COALESCE(%s, status),
|
||||
genre = COALESCE(%s, genre),
|
||||
themes = COALESCE(%s, themes),
|
||||
formed_year = COALESCE(%s, formed_year),
|
||||
data = data || %s::jsonb,
|
||||
enriched = true,
|
||||
crawled_at = %s,
|
||||
crawled_hash = %s,
|
||||
ma_created_at = COALESCE(%s, ma_created_at),
|
||||
ma_modified_at= COALESCE(%s, ma_modified_at)
|
||||
WHERE ma_id = %s
|
||||
"""
|
||||
Met à jour les champs d'enrichissement d'un band.
|
||||
Respecte locked_fields : les champs édités manuellement dans l'admin ne sont pas
|
||||
écrasés. Si le crawler a une valeur différente pour un champ verrouillé, elle est
|
||||
stockée dans crawler_pending pour résolution manuelle.
|
||||
"""
|
||||
ts = now_utc()
|
||||
new_vals: Dict[str, Any] = {
|
||||
"status": data.get("status"),
|
||||
"genre": data.get("genre"),
|
||||
"themes": data.get("themes"),
|
||||
"formed_year": _parse_year(data.get("formed_in")),
|
||||
}
|
||||
band_data = json.dumps({"band_page": data, "enriched_at": ts.isoformat()})
|
||||
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (
|
||||
data.get("status"),
|
||||
data.get("genre"),
|
||||
data.get("themes"),
|
||||
_parse_year(data.get("formed_in")),
|
||||
json.dumps({"band_page": data, "enriched_at": ts.isoformat()}),
|
||||
ts,
|
||||
html_hash,
|
||||
data.get("ma_created_at"),
|
||||
data.get("ma_modified_at"),
|
||||
cur.execute(
|
||||
"SELECT status, genre, themes, formed_year, locked_fields FROM bands WHERE ma_id = %s",
|
||||
(ma_id,)
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return False
|
||||
cur_vals = {"status": row[0], "genre": row[1], "themes": row[2], "formed_year": row[3]}
|
||||
locked: set = set((row[4] or {}).keys())
|
||||
|
||||
# Construire les clauses SET + détecter les conflits
|
||||
set_parts, params, pending = [], [], {}
|
||||
for field in ("status", "genre", "themes", "formed_year"):
|
||||
val = new_vals[field]
|
||||
if field in locked:
|
||||
if val is not None and val != cur_vals[field]:
|
||||
pending[field] = val
|
||||
else:
|
||||
set_parts.append(f"{field} = COALESCE(%s, {field})")
|
||||
params.append(val)
|
||||
|
||||
set_parts += [
|
||||
"data = data || %s::jsonb",
|
||||
"enriched = true",
|
||||
"crawled_at = %s",
|
||||
"crawled_hash = %s",
|
||||
"ma_created_at = COALESCE(%s, ma_created_at)",
|
||||
"ma_modified_at = COALESCE(%s, ma_modified_at)",
|
||||
"crawler_pending = %s::jsonb",
|
||||
]
|
||||
params += [
|
||||
band_data, ts, html_hash,
|
||||
data.get("ma_created_at"), data.get("ma_modified_at"),
|
||||
json.dumps(pending) if pending else None,
|
||||
ma_id,
|
||||
))
|
||||
]
|
||||
cur.execute(f"UPDATE bands SET {', '.join(set_parts)} WHERE ma_id = %s", params)
|
||||
return cur.rowcount > 0
|
||||
|
||||
|
||||
|
|
@ -262,6 +286,49 @@ def set_checkpoint(key: str, value: str):
|
|||
cur.execute(sql, (key, value))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Job triggers (déclenchement depuis le dashboard admin)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def claim_job_trigger(job_type: Optional[str] = None) -> Optional[int]:
|
||||
"""Récupère et verrouille un job trigger en attente. Retourne son id ou None."""
|
||||
where = "status = 'pending'"
|
||||
params: list = []
|
||||
if job_type:
|
||||
where += " AND job_type = %s"
|
||||
params.append(job_type)
|
||||
params.append(1)
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""UPDATE job_triggers SET status='running', started_at=now()
|
||||
WHERE id = (
|
||||
SELECT id FROM job_triggers WHERE {where}
|
||||
ORDER BY created_at ASC LIMIT %s FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING id, job_type""",
|
||||
params,
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return (row[0], row[1]) if row else None
|
||||
except Exception as e:
|
||||
log.warning(f"[db] claim_job_trigger failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def finish_job_trigger(trigger_id: int, error: Optional[str] = None):
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""UPDATE job_triggers SET status=%s, finished_at=now(), error=%s WHERE id=%s""",
|
||||
("error" if error else "done", error, trigger_id),
|
||||
)
|
||||
except Exception as e:
|
||||
log.warning(f"[db] finish_job_trigger failed: {e}")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ from .db import (
|
|||
log_event,
|
||||
)
|
||||
from .europe_codes import EUROPE_COUNTRY_CODES
|
||||
|
||||
_EU_SET = set(EUROPE_COUNTRY_CODES)
|
||||
from .europe_codes import EUROPE_COUNTRY_CODES
|
||||
from .ma_http import MASession
|
||||
from .polite import sleep_range, cooldown
|
||||
from .scraper_band import parse_band_page, page_hash
|
||||
|
|
@ -94,6 +97,9 @@ def run_incremental(session: MASession, order_by: str):
|
|||
if date_str and (latest_date is None or date_str > latest_date):
|
||||
latest_date = date_str
|
||||
|
||||
if band.get("country") not in _EU_SET:
|
||||
continue # on reste europe uniquement
|
||||
|
||||
band["data"] = {"url": band.pop("url", None)}
|
||||
buf.append(band)
|
||||
if len(buf) >= CHUNK:
|
||||
|
|
@ -163,7 +169,7 @@ def run_enrich(session: MASession, limit: int = 100, country: Optional[str] = No
|
|||
if COOLDOWN_EVERY and (i + 1) % COOLDOWN_EVERY == 0:
|
||||
cooldown(COOLDOWN_MIN, COOLDOWN_MAX)
|
||||
|
||||
if (i + 1) % 50 == 0:
|
||||
if (i + 1) % 10 == 0:
|
||||
update_crawl_run_progress(run_id, stats)
|
||||
|
||||
log.info(f"[enrich] done: {stats}")
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from .config import (
|
|||
ENRICH_LIMIT,
|
||||
)
|
||||
from .flaresolverr import FlareSolverr, FlareSolverrError
|
||||
from .db import claim_job_trigger, finish_job_trigger
|
||||
from .jobs import run_full_crawl, run_incremental, run_enrich
|
||||
from .ma_http import MASession
|
||||
|
||||
|
|
@ -100,8 +101,35 @@ def main():
|
|||
log.info("[main] entering scheduler loop")
|
||||
while True:
|
||||
schedule.run_pending()
|
||||
_check_job_triggers(ma)
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
def _check_job_triggers(ma: "MASession"):
|
||||
"""Consomme les job_triggers en attente créés depuis l'admin."""
|
||||
result = claim_job_trigger()
|
||||
if not result:
|
||||
return
|
||||
trigger_id, job_type = result
|
||||
log.info(f"[main] job trigger #{trigger_id}: {job_type}")
|
||||
error = None
|
||||
try:
|
||||
if job_type == "enrich":
|
||||
run_enrich(ma, limit=ENRICH_LIMIT)
|
||||
elif job_type == "incremental":
|
||||
run_incremental(ma, "created")
|
||||
run_incremental(ma, "modified")
|
||||
elif job_type == "full_crawl":
|
||||
run_full_crawl(ma)
|
||||
else:
|
||||
error = f"job_type inconnu: {job_type}"
|
||||
log.warning(f"[main] {error}")
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
log.error(f"[main] job trigger #{trigger_id} error: {e}", exc_info=True)
|
||||
finally:
|
||||
finish_job_trigger(trigger_id, error)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue