metalfrom.eu/apps/api/test/adminRoutesCoverage.test.js
Nicolas FRYDER 2043705fc7 fix(robustesse): session Chrome abandonnée, requêtes vides re-payées, nettoyage trop zélé
Le crawler n'avait aucun arrêt propre. Sa session FlareSolverr n'est détruite
qu'au moment d'en ouvrir une neuve : chaque redeploy abandonnait donc une
instance Chrome persistante côté FlareSolverr. Il gère désormais SIGTERM/SIGINT,
dort par tranches d'une seconde pour ne pas faire attendre une minute à l'arrêt,
et rend sa session dans un finally.

Le worker de géocodage ne mémorisait pas les résultats VIDES. Le commentaire
annonçait « toujours mettre en cache pour éviter de rappeler », mais l'insertion
était à l'intérieur du `if res:` : une requête sans résultat était re-payée à
chaque tentative — jusqu'à MAX_GEO_TRIES passages, multipliés par les requêtes
de repli, puis de nouveau après chaque aller-retour LLM qui remet les compteurs
à zéro. Elles sont désormais mises en cache avec lat/lon NULL, et la lecture
distingue « absent du cache » de « connu sans résultat ».

crawl-runs/cleanup utilisait 30 minutes par défaut, plus court qu'un crawl
complet Europe qui dure des heures. Le déclencher pendant un crawl légitime le
marquait en erreur alors qu'il tournait toujours, et update_crawl_run_progress
(filtré sur status='running') cessait silencieusement de publier : l'affichage
restait figé jusqu'à la fin. Le défaut passe à 24 h, et les runs dont
l'annulation est déjà demandée sont laissés au chemin coopératif.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-22 14:28:58 +02:00

480 lines
20 KiB
JavaScript

import { describe, it, expect, beforeAll, afterAll } from "vitest";
process.env.ADMIN_JWT_SECRET = "secret-de-test-suffisamment-long-pour-passer-32";
const { signAdminSession, ADMIN_COOKIE_NAME } = await import("../src/adminAuth.js");
const { rows } = await import("./helpers/fakePool.js");
const { createAdminApp } = await import("./helpers/testApp.js");
let auth, app, pool;
// Une seule construction d'app pour tout le fichier ; chaque test reprogramme
// le pool via buildApp(handlers).
beforeAll(async () => {
auth = { cookie: `${ADMIN_COOKIE_NAME}=${signAdminSession("nico")}` };
({ app, pool } = await createAdminApp());
});
afterAll(async () => { await app.close(); });
/** Reprogramme le pool partagé et renvoie l'app. */
function buildApp(handlers) {
pool.reset(Array.isArray(handlers) ? handlers : handlers ?? []);
return app;
}
/** Pool permissif : n'importe quelle requête renvoie une ligne plausible. */
function anyPool(extra = []) {
return [...extra, { match: /./, result: rows({ n: 1, total: 1, id: 1, status: "done", count: 1 }) }];
}
// ------------------------------------------------------------------
// Routes de lecture : forme de la réponse + protection
// ------------------------------------------------------------------
describe("routes de lecture", () => {
const READ_ROUTES = [
"/admin/api/stats",
"/admin/api/queue",
"/admin/api/crawl-checkpoints",
"/admin/api/logs",
"/admin/api/geocoding",
"/admin/api/llm",
"/admin/api/job-triggers",
"/admin/api/live",
"/admin/api/activity",
"/admin/api/bands",
];
it.each(READ_ROUTES)("%s exige une session", async (url) => {
const handlers = ([]);
const app = buildApp(handlers);
expect((await app.inject({ method: "GET", url })).statusCode).toBe(401);
expect(pool.calls).toHaveLength(0);
});
it.each(READ_ROUTES)("%s répond 200 et ok:true avec une session", async (url) => {
const app = buildApp(anyPool());
const res = await app.inject({ method: "GET", url, headers: auth });
expect(res.statusCode).toBe(200);
expect(res.json().ok).toBe(true);
});
it.each(READ_ROUTES)("%s renvoie 500 sans divulguer l'erreur si la base tombe", async (url) => {
const handlers = ([{ match: /./, throws: new Error("FATAL: password authentication failed for user bm") }]);
const app = buildApp(handlers);
const res = await app.inject({ method: "GET", url, headers: auth });
expect(res.statusCode).toBe(500);
expect(res.body).not.toMatch(/password|user bm/);
});
});
describe("GET /admin/api/stats", () => {
it("agrège totaux, statuts, pays et genres", async () => {
const handlers = ([
{ match: "count(*)::int AS total,", result: rows({ total: 10, enriched: 4, geocoded: 6 }) },
{ match: "AS status, count", result: rows({ status: "Active", total: 7 }) },
{ match: "AS country, count", result: rows({ country: "NO", total: 3 }) },
{ match: "SELECT genre, count", result: rows({ genre: "Black Metal", total: 2 }) },
]);
const app = buildApp(handlers);
const body = (await app.inject({ method: "GET", url: "/admin/api/stats", headers: auth })).json();
expect(body.totals.total).toBe(10);
expect(body.by_status[0].status).toBe("Active");
expect(body.by_country[0].country).toBe("NO");
expect(body.by_genre[0].genre).toBe("Black Metal");
});
it("les quatre agrégats sont lancés en parallèle, pas en cascade", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/stats", headers: auth });
expect(pool.calls.length).toBe(4);
});
});
describe("GET /admin/api/bands/:ma_id", () => {
const detailPool = () =>
([
{ match: "SELECT * FROM bands WHERE ma_id", result: rows({ ma_id: 1, name: "Mayhem" }) },
{ match: "FROM band_locations", result: rows({ id: 9, step_order: 0, location_raw: "Oslo" }) },
{ match: "FROM llm_cache", result: rows({ id: 3, model: "llama-3.3-70b-versatile" }) },
]);
it("renvoie le groupe, ses localisations et ses appels LLM", async () => {
const app = buildApp(detailPool());
const body = (await app.inject({ method: "GET", url: "/admin/api/bands/1", headers: auth })).json();
expect(body.item.name).toBe("Mayhem");
expect(body.locations).toHaveLength(1);
expect(body.llm).toHaveLength(1);
});
it("répond 400 pour un ma_id invalide", async () => {
const app = buildApp(detailPool());
for (const bad of ["abc", "-1"]) {
expect((await app.inject({ method: "GET", url: `/admin/api/bands/${bad}`, headers: auth })).statusCode).toBe(400);
}
});
it("répond 404 quand le groupe n'existe pas", async () => {
const handlers = ([{ match: "SELECT * FROM bands WHERE ma_id", result: rows() }]);
const app = buildApp(handlers);
expect((await app.inject({ method: "GET", url: "/admin/api/bands/1", headers: auth })).statusCode).toBe(404);
});
// Les tables du nouveau pipeline peuvent ne pas exister sur une base ancienne :
// le détail du groupe doit rester consultable malgré tout.
it("dégrade proprement si band_locations ou llm_cache sont absentes", async () => {
const handlers = ([
{ match: "SELECT * FROM bands WHERE ma_id", result: rows({ ma_id: 1, name: "Mayhem" }) },
{ match: "FROM band_locations", throws: new Error('relation "band_locations" does not exist') },
{ match: "FROM llm_cache", throws: new Error('relation "llm_cache" does not exist') },
]);
const app = buildApp(handlers);
const res = await app.inject({ method: "GET", url: "/admin/api/bands/1", headers: auth });
expect(res.statusCode).toBe(200);
expect(res.json().locations).toEqual([]);
expect(res.json().llm).toEqual([]);
});
});
describe("GET /admin/api/logs — filtres", () => {
it("filtre par niveau, run et curseur, en paramètres liés", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({
method: "GET",
url: "/admin/api/logs?level=error&run_id=12&min_id=100&pageSize=25",
headers: auth,
});
const call = pool.find("FROM crawl_log");
expect(call.sql).toContain("level = $1");
expect(call.sql).toContain("run_id = $2");
expect(call.sql).toContain("id > $3");
expect(call.values).toEqual(["error", 12, 100, 25, 0]);
});
it("sans filtre, aucune clause WHERE", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/logs", headers: auth });
expect(pool.find("FROM crawl_log").sql).not.toContain("WHERE");
});
// Régression : page/pageSize étaient calculés mais jamais renvoyés, le client
// ne pouvait donc pas savoir s'il restait des logs à charger.
it("renvoie les informations de pagination", async () => {
const app = buildApp(anyPool());
const body = (await app.inject({ method: "GET", url: "/admin/api/logs?page=3", headers: auth })).json();
expect(body.page).toBe(3);
expect(body.pageSize).toBe(50);
});
});
describe("GET /admin/api/llm — filtres", () => {
it("filtre par modèle, nullité et texte", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({
method: "GET",
url: "/admin/api/llm?model=llama-3.1-8b-instant&only_null=1&q=oslo",
headers: auth,
});
const call = pool.find("FROM llm_cache lc");
expect(call.sql).toContain("model = $1");
expect(call.sql).toContain("is_null = true");
expect(call.values).toContain("llama-3.1-8b-instant");
expect(call.values).toContain("%oslo%");
});
it("le count applique le même filtre que la page", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/llm?model=x", headers: auth });
const count = pool.find("count(*)::int AS total FROM llm_cache");
expect(count.sql).toContain("model = $1");
expect(count.values).toEqual(["x"]);
});
it("tronque une recherche exagérément longue", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: `/admin/api/llm?q=${"a".repeat(500)}`, headers: auth });
const bound = pool.find("FROM llm_cache lc").values.find((v) => String(v).startsWith("%aaa"));
expect(bound).toHaveLength(102); // 100 caractères + les deux %
});
});
describe("GET /admin/api/activity — liste unifiée", () => {
it("réunit crawl_run et admin_audit_log", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/activity", headers: auth });
const call = pool.find("UNION ALL");
expect(call.sql).toContain("FROM crawl_run");
expect(call.sql).toContain("FROM admin_audit_log");
});
it("filtre par type et statut en paramètres liés", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/activity?type=run&status=running", headers: auth });
const data = pool.find(/SELECT \* FROM \([\s\S]*UNION ALL/);
expect(data.values).toEqual(["run", "running", 50, 0]);
});
it("trie par date décroissante", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/activity", headers: auth });
expect(pool.find(/SELECT \* FROM \(/).sql).toContain("ORDER BY ts DESC");
});
});
// ------------------------------------------------------------------
// Actions de maintenance du géocodage
// ------------------------------------------------------------------
describe("actions sur band_locations", () => {
const ACTIONS = [
["/admin/api/locations/reset-errors", "geocode_status = 'error'"],
["/admin/api/locations/reset-llm", "geocode_status IN ('llm_needed', 'manual')"],
["/admin/api/locations/reset-all", "is_country_only = FALSE"],
["/admin/api/locations/requeue-all", "geocode_status = ANY($1::text[])"],
];
it.each(ACTIONS.map(([url]) => url))("%s exige une session", async (url) => {
const handlers = ([]);
const app = buildApp(handlers);
expect((await app.inject({ method: "POST", url, payload: {} })).statusCode).toBe(401);
expect(pool.calls).toHaveLength(0);
});
it.each(ACTIONS)("%s cible les bonnes lignes", async (url, expectedClause) => {
const handlers = ([
{ match: "UPDATE band_locations", result: { rows: [], rowCount: 12 } },
{ match: "INSERT INTO crawl_log", result: rows() },
]);
const app = buildApp(handlers);
const res = await app.inject({ method: "POST", url, headers: auth, payload: {} });
expect(res.statusCode).toBe(200);
expect(res.json().count).toBe(12);
expect(pool.find("UPDATE band_locations").sql).toContain(expectedClause);
});
it.each(ACTIONS.map(([url]) => url))("%s remet les lignes en queue", async (url) => {
const handlers = ([
{ match: "UPDATE band_locations", result: { rows: [], rowCount: 1 } },
{ match: "INSERT INTO crawl_log", result: rows() },
]);
const app = buildApp(handlers);
await app.inject({ method: "POST", url, headers: auth, payload: {} });
expect(pool.find("UPDATE band_locations").sql).toContain("geocode_status='queued'");
});
it.each(ACTIONS.map(([url]) => url))("%s trace l'action dans les logs avec l'auteur", async (url) => {
const handlers = ([
{ match: "UPDATE band_locations", result: { rows: [], rowCount: 1 } },
{ match: "INSERT INTO crawl_log", result: rows() },
]);
const app = buildApp(handlers);
await app.inject({ method: "POST", url, headers: auth, payload: {} });
expect(pool.find("INSERT INTO crawl_log").values[1]).toContain("admin:nico");
});
// reset-all efface les coordonnées (le worker va tout re-géocoder) alors que
// reset-errors ne remet en file que ce qui avait échoué.
it("reset-all efface les coordonnées, reset-errors non", async () => {
const mk = async (url) => {
const handlers = ([
{ match: "UPDATE band_locations", result: { rows: [], rowCount: 1 } },
{ match: "INSERT INTO crawl_log", result: rows() },
]);
const app = buildApp(handlers);
await app.inject({ method: "POST", url, headers: auth, payload: {} });
return pool.find("UPDATE band_locations").sql;
};
expect(await mk("/admin/api/locations/reset-all")).toContain("lat=NULL");
expect(await mk("/admin/api/locations/reset-errors")).not.toContain("lat=NULL");
});
it("requeue-all inclut 'done' seulement sur demande explicite", async () => {
const mk = async (payload) => {
const handlers = ([
{ match: "UPDATE band_locations", result: { rows: [], rowCount: 1 } },
{ match: "INSERT INTO crawl_log", result: rows() },
]);
const app = buildApp(handlers);
await app.inject({ method: "POST", url: "/admin/api/locations/requeue-all", headers: auth, payload });
return pool.find("UPDATE band_locations").values[0];
};
expect(await mk({})).toEqual(["error", "llm_needed"]);
expect(await mk({ include_done: true })).toEqual(["error", "llm_needed", "manual", "done"]);
});
it("requeue-all épargne toujours les localisations pays-seul", async () => {
const handlers = ([
{ match: "UPDATE band_locations", result: { rows: [], rowCount: 1 } },
{ match: "INSERT INTO crawl_log", result: rows() },
]);
const app = buildApp(handlers);
await app.inject({
method: "POST", url: "/admin/api/locations/requeue-all",
headers: auth, payload: { include_done: true },
});
expect(pool.find("UPDATE band_locations").sql).toContain("is_country_only = FALSE");
});
it("une action qui échoue renvoie 500 sans écrire de log de succès", async () => {
const handlers = ([{ match: "UPDATE band_locations", throws: new Error("deadlock") }]);
const app = buildApp(handlers);
const res = await app.inject({
method: "POST", url: "/admin/api/locations/reset-errors", headers: auth, payload: {},
});
expect(res.statusCode).toBe(500);
expect(pool.find("INSERT INTO crawl_log")).toBeUndefined();
});
});
describe("POST /admin/api/geocode-cache/purge-nominatim", () => {
it("ne supprime que les entrées de l'ancien pipeline", async () => {
const handlers = ([
{ match: "DELETE FROM geocode_cache", result: { rows: [], rowCount: 4 } },
{ match: "INSERT INTO crawl_log", result: rows() },
]);
const app = buildApp(handlers);
const res = await app.inject({
method: "POST", url: "/admin/api/geocode-cache/purge-nominatim", headers: auth, payload: {},
});
expect(res.json().count).toBe(4);
// Le marqueur place_rank distingue Nominatim de Geoapify : sans ce WHERE,
// la purge viderait aussi le cache Geoapify (appels payants).
expect(pool.find("DELETE FROM geocode_cache").sql).toContain("place_rank");
});
it("exige une session", async () => {
const handlers = ([]);
const app = buildApp(handlers);
expect((await app.inject({
method: "POST", url: "/admin/api/geocode-cache/purge-nominatim", payload: {},
})).statusCode).toBe(401);
expect(pool.calls).toHaveLength(0);
});
});
describe("POST /admin/api/crawl-runs/cleanup", () => {
const cleanupPool = () =>
([
{ match: "UPDATE crawl_run", result: rows({ id: 1, run_type: "enrich", started_at: "2026-01-01" }) },
{ match: "INSERT INTO admin_audit_log", result: rows() },
]);
it("le seuil par défaut dépasse la durée d'un crawl complet", async () => {
// 30 minutes était plus court qu'un crawl complet Europe, qui dure des
// heures : le nettoyage marquait alors en erreur un run parfaitement vivant,
// et update_crawl_run_progress (filtré sur status='running') cessait de
// publier — l'affichage restait figé jusqu'à la fin du run.
const handlers = cleanupPool();
const app = buildApp(handlers);
await app.inject({ method: "POST", url: "/admin/api/crawl-runs/cleanup", headers: auth, payload: {} });
expect(pool.find("UPDATE crawl_run").values).toEqual([24 * 60]);
});
it("laisse tranquille un run dont l'annulation est déjà demandée", async () => {
// L'annulation est coopérative : le crawler écrira lui-même 'cancelled'.
const handlers = cleanupPool();
const app = buildApp(handlers);
await app.inject({ method: "POST", url: "/admin/api/crawl-runs/cleanup", headers: auth, payload: {} });
expect(pool.find("UPDATE crawl_run").sql).toMatch(/cancel_requested = FALSE/);
});
it("ne touche que les runs marqués running", async () => {
const handlers = cleanupPool();
const app = buildApp(handlers);
await app.inject({ method: "POST", url: "/admin/api/crawl-runs/cleanup", headers: auth, payload: {} });
expect(pool.find("UPDATE crawl_run").sql).toContain("status = 'running'");
});
it("renvoie le nombre de runs nettoyés", async () => {
const app = buildApp(cleanupPool());
const res = await app.inject({
method: "POST", url: "/admin/api/crawl-runs/cleanup", headers: auth, payload: { older_than_minutes: 60 },
});
expect(res.json().cleaned).toBe(1);
});
});
describe("GET /admin/api/geocoding", () => {
it("dégrade proprement quand les tables du nouveau pipeline manquent", async () => {
const handlers = ([
{ match: "FROM geocode_cache", result: rows({ n: 5 }) },
{ match: "FROM bands b", result: rows() },
{ match: "FROM band_locations", throws: new Error("relation does not exist") },
{ match: "FROM llm_cache", throws: new Error("relation does not exist") },
]);
const app = buildApp(handlers);
const res = await app.inject({ method: "GET", url: "/admin/api/geocoding", headers: auth });
expect(res.statusCode).toBe(200);
expect(res.json().cache_size).toBe(5);
expect(res.json().locations).toEqual([]);
});
});
describe("GET /admin/api/live", () => {
it("remonte runs actifs, jobs en attente et file de géocodage", async () => {
const handlers = ([
{ match: "WHERE status = 'running'", result: rows({ id: 1, run_type: "enrich" }) },
{ match: "WHERE status = 'pending'", result: rows({ id: 2, job_type: "enrich" }) },
{ match: "GROUP BY geocode_status", result: rows({ status: "queued", n: 42 }) },
{ match: "geocode_status = 'processing'", result: rows() },
{ match: "FROM crawl_checkpoint", result: rows({ key: "last_full_crawl_at" }) },
]);
const app = buildApp(handlers);
const body = (await app.inject({ method: "GET", url: "/admin/api/live", headers: auth })).json();
expect(body.active_runs).toHaveLength(1);
expect(body.pending_jobs).toHaveLength(1);
expect(body.geo_queue[0].n).toBe(42);
expect(body.checkpoints).toHaveLength(1);
});
it("expose la santé des services de fond", async () => {
const handlers = ([
{ match: "FROM service_health", result: rows(
{ service: "crawler", ok: true, checks: { database: true }, error: null, stale: false },
{ service: "geocoder", ok: false, checks: { geoapify_joignable: false }, error: "HTTP 503", stale: false }) },
{ match: /./, result: rows() },
]);
const app = buildApp(handlers);
const body = (await app.inject({ method: "GET", url: "/admin/api/live", headers: auth })).json();
expect(body.services).toHaveLength(2);
expect(body.services[1].error).toBe("HTTP 503");
});
// Un service arrêté cesse simplement d'écrire : sans ce calcul, son dernier
// contrôle réussi le ferait passer pour sain indéfiniment.
it("marque comme silencieux un service qui n'écrit plus", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/live", headers: auth });
expect(pool.find("FROM service_health").sql).toContain("checked_at < now() - interval '2 hours'");
});
// La table est arrivée en migration 015 : une base plus ancienne ne doit pas
// faire tomber tout le tableau de bord.
it("dégrade proprement si service_health n'existe pas encore", async () => {
const handlers = ([
{ match: "FROM service_health", throws: new Error('relation "service_health" does not exist') },
{ match: /./, result: rows() },
]);
const app = buildApp(handlers);
const res = await app.inject({ method: "GET", url: "/admin/api/live", headers: auth });
expect(res.statusCode).toBe(200);
expect(res.json().services).toEqual([]);
});
it("limite l'aperçu des localisations en cours de traitement", async () => {
const handlers = anyPool();
const app = buildApp(handlers);
await app.inject({ method: "GET", url: "/admin/api/live", headers: auth });
expect(pool.find("geocode_status = 'processing'").sql).toContain("LIMIT 3");
});
});