Deux angles morts fermés. 1. Syntaxe SQL sans conteneur (apps/api/test/sqlSyntax.test.js) Le faux pool vérifiait la FORME du SQL mais ne l'exécutait jamais : une requête syntaxiquement invalide passait tous les tests et n'échouait qu'en production — c'est précisément ce qui s'était produit avec crawler_pending. Chaque requête réellement émise par les 28 routes est désormais parsée avec la grammaire PostgreSQL (node-sql-parser), y compris les SET dynamiques du PATCH et les casts par type de resolve-conflict. 48 tests, aucun conteneur. Limite déclarée explicitement : la sémantique n'est pas validée, et deux requêtes bâties sur jsonb_build_object ne sont pas parsables — le test échoue si une route cesse d'avoir la moindre requête vérifiable, pour éviter qu'il passe au vert à vide. 2. Supervision des services de fond (migration 015) Le crawler et les workers ne sont pas exposés par Traefik : aucune sonde HTTP ne peut les atteindre. Un crawler dont FlareSolverr était injoignable, ou un worker à court de quota Geoapify, restait muet — le seul symptôme était l'absence de données nouvelles, qu'il fallait remarquer soi-même. Chaque service écrit un battement de cœur horaire dans service_health : - crawler : base, FlareSolverr joignable, dernier run terminé < 12 h - geocoder : base, clé Geoapify présente, API joignable, progression < 6 h Une ligne par service, écrasée à chaque contrôle. L'API calcule `stale` en SQL (> 2 h sans écriture) : un service arrêté cesse d'écrire, et son dernier contrôle réussi le ferait sinon passer pour sain indéfiniment. Le Pilotage affiche une carte « Services » et remonte chaque service dégradé ou silencieux en alerte actionnable. Règle appliquée aux sondes : aucune ne peut interrompre le service qu'elle surveille. Toute exception devient un échec de sonde, l'écriture du résultat et la journalisation échouent en silence. Un contrôle de santé qui fait tomber le crawler serait pire que pas de contrôle. Tests : 580 JS (+51), 63 Python (+20), 85 Playwright (+3) Co-Authored-By: Claude <noreply@anthropic.com>
145 lines
8.2 KiB
JavaScript
145 lines
8.2 KiB
JavaScript
/**
|
|
* Jeu de données du serveur e2e.
|
|
*
|
|
* Le faux pool associe un fragment de SQL à un résultat : l'ordre compte, le
|
|
* premier fragment qui correspond gagne. Les entrées les plus spécifiques
|
|
* doivent donc précéder les plus générales.
|
|
*/
|
|
import bcrypt from "bcryptjs";
|
|
import { rows } from "../../../api/test/helpers/fakePool.js";
|
|
|
|
export const E2E_USER = "nico";
|
|
export const E2E_PASSWORD = "motdepasse-e2e";
|
|
// Coût 4 : suffisant pour exercer le vrai chemin bcrypt sans les ~330 ms du
|
|
// coût 12 utilisé en production.
|
|
const PASSWORD_HASH = bcrypt.hashSync(E2E_PASSWORD, 4);
|
|
|
|
const BANDS = [
|
|
{ ma_id: 1, name: "Mayhem", country: "NO", status: "Active", genre: "Black Metal",
|
|
location_text: "Oslo", themes: "Death, Satanism", formed_year: 1984, enriched: true,
|
|
lat: 59.91, lon: 10.75, crawled_at: "2026-08-01T10:00:00Z", updated_at: "2026-08-10T10:00:00Z",
|
|
first_seen_at: "2026-01-01T00:00:00Z", locked_fields: {} },
|
|
{ ma_id: 2, name: "Darkthrone", country: "NO", status: "Active", genre: "Black Metal",
|
|
location_text: "Kolbotn", themes: "Cold, Winter", formed_year: 1986, enriched: true,
|
|
lat: null, lon: null, crawled_at: "2026-08-02T10:00:00Z", updated_at: "2026-08-11T10:00:00Z",
|
|
first_seen_at: "2026-01-01T00:00:00Z", locked_fields: { name: true } },
|
|
{ ma_id: 3, name: "Gojira", country: "FR", status: "Active", genre: "Progressive Death Metal",
|
|
location_text: "Bayonne", themes: "Nature", formed_year: 1996, enriched: false,
|
|
lat: 43.49, lon: -1.47, crawled_at: null, updated_at: "2026-08-12T10:00:00Z",
|
|
first_seen_at: "2026-02-01T00:00:00Z", locked_fields: {} },
|
|
];
|
|
|
|
const LOCATIONS = [
|
|
{ id: 11, ma_id: 2, band_name: "Darkthrone", country: "NO", step_order: 0, step_label: null,
|
|
location_raw: "Kolbotn", is_country_only: false, lat: null, lon: null,
|
|
geocode_status: "error", geocode_provider: null, geocode_confidence: null,
|
|
geocode_granularity: null, geocode_query: "Kolbotn, Norway",
|
|
geocode_error: "Aucun résultat au-dessus du seuil de confiance",
|
|
geocode_tries_geo: 3, geocode_tries_llm: 0,
|
|
geocode_next_at: "2026-08-12T09:00:00Z", updated_at: "2026-08-12T09:00:00Z" },
|
|
{ id: 12, ma_id: 3, band_name: "Gojira", country: "FR", step_order: 0, step_label: "early",
|
|
location_raw: "Ondres / Bayonne", is_country_only: false, lat: null, lon: null,
|
|
geocode_status: "llm_needed", geocode_provider: null, geocode_confidence: null,
|
|
geocode_granularity: null, geocode_query: null, geocode_error: null,
|
|
geocode_tries_geo: 3, geocode_tries_llm: 1,
|
|
geocode_next_at: "2026-08-12T08:00:00Z", updated_at: "2026-08-12T08:00:00Z" },
|
|
];
|
|
|
|
/**
|
|
* @param {object} [over] surcharges par scénario (ex. aucun run actif)
|
|
*/
|
|
export function seedHandlers(over = {}) {
|
|
const {
|
|
activeRuns = [],
|
|
pendingJobs = [],
|
|
geoStatuses = [
|
|
{ status: "done", n: 1840 },
|
|
{ status: "error", n: 12 },
|
|
{ status: "llm_needed", n: 5 },
|
|
{ status: "queued", n: 3 },
|
|
],
|
|
bands = BANDS,
|
|
locations = LOCATIONS,
|
|
neverEnriched = 1,
|
|
} = over;
|
|
|
|
return [
|
|
// ---- Authentification ----
|
|
// Fragment volontairement précis : « SELECT count(*)::int AS n » seul
|
|
// capturait aussi le comptage du cache de géocodage.
|
|
{ match: "FROM admin_login_attempts\n WHERE success = false", result: rows({ n: 0 }) },
|
|
{ match: "FROM admin_users", result: rows({ password_hash: PASSWORD_HASH }) },
|
|
{ match: "INSERT INTO admin_login_attempts", result: rows() },
|
|
{ match: "UPDATE admin_users", result: rows() },
|
|
|
|
// ---- Écritures ----
|
|
{ match: "INSERT INTO admin_audit_log", result: rows() },
|
|
{ match: "INSERT INTO crawl_log", result: rows() },
|
|
{ match: "INSERT INTO job_triggers", result: rows({ id: 77 }) },
|
|
{ match: "UPDATE crawl_run", result: rows({ id: 5, run_type: "enrich" }) },
|
|
{ match: "UPDATE job_triggers", result: rows({ id: 7, job_type: "enrich" }) },
|
|
|
|
// ---- Détail d'un groupe (avant les listes : fragment plus spécifique) ----
|
|
{ match: "SELECT * FROM bands WHERE ma_id", result: (_s, v) => rows(bands.find((b) => b.ma_id === Number(v[0])) || bands[0]) },
|
|
{ match: "UPDATE bands SET", result: (_s, v) => rows({ ...bands[0], ma_id: Number(v[0]), name: v[1] ?? bands[0].name }) },
|
|
|
|
// ---- Localisations ----
|
|
{ match: "SELECT bl.id, bl.ma_id", result: rows(...locations) },
|
|
{ match: "count(*)::int AS total\n FROM band_locations", result: rows({ total: locations.length }) },
|
|
{ match: "SELECT * FROM band_locations", result: rows(locations[0]) },
|
|
{ match: "UPDATE band_locations", result: rows({ ...locations[0], lat: 59.79, lon: 10.8, geocode_status: "done" }) },
|
|
{ match: "FROM band_locations\n WHERE ma_id", result: rows(locations[0]) },
|
|
|
|
// ---- Statistiques ----
|
|
{ match: "never_enriched", result: rows({
|
|
total: 1950, enriched: 1200, not_enriched: 750, geocoded: 1840, not_geocoded: 110,
|
|
crawled_by_current_system: 1900, never_enriched: neverEnriched, stale: 4 }) },
|
|
{ match: "new_bands", result: rows({ new_bands: 3, modified_since_enrich: 1, legacy_pending: 0, stale: 4 }) },
|
|
{ match: "run_type = 'enrich'", result: rows() },
|
|
{ match: "AS status, count(*)::int AS total", result: rows({ status: "Active", total: 1500 }, { status: "Split-up", total: 450 }) },
|
|
{ match: "AS country, count", result: rows({ country: "NO", total: 900 }, { country: "FR", total: 400 }) },
|
|
{ match: "SELECT genre, count", result: rows({ genre: "Black Metal", total: 800 }) },
|
|
|
|
// ---- Supervision ----
|
|
{ match: "WHERE status = 'running'", result: rows(...activeRuns) },
|
|
{ match: "WHERE status = 'pending'", result: rows(...pendingJobs) },
|
|
{ match: "geocode_status = 'processing'", result: rows() },
|
|
{ match: "GROUP BY geocode_status", result: rows(...geoStatuses) },
|
|
{ match: "FROM service_health", result: rows(
|
|
{ service: "crawler", ok: true, checks: { database: true, flaresolverr: true }, error: null,
|
|
checked_at: "2026-08-18T14:00:00Z", stale: false },
|
|
{ service: "geocoder", ok: true, checks: { database: true, geoapify_joignable: true }, error: null,
|
|
checked_at: "2026-08-18T14:00:00Z", stale: false }) },
|
|
{ match: "FROM crawl_checkpoint", result: rows({ key: "last_full_crawl_at", value: "2026-08-01", updated_at: "2026-08-01T00:00:00Z" }) },
|
|
|
|
// ---- Géocodage / LLM ----
|
|
{ match: "FROM geocode_cache", result: rows({ n: 1500 }) },
|
|
{ match: "b.geocoded_at IS NOT NULL", result: rows() },
|
|
{ match: "sum(cost_usd)", result: rows({ n: 320, total_cost_usd: "0.041200", n_null: 18 }) },
|
|
{ match: "COALESCE(geocode_provider", result: rows({ provider: "geoapify", n: 1800, avg_conf: "0.91" }) },
|
|
{ match: "SELECT model,", result: rows({ model: "llama-3.3-70b-versatile", n: 320, n_null: 18, cost: "0.041200" }) },
|
|
{ match: "FROM llm_cache lc", result: rows({
|
|
id: 3, ma_id: 3, band_name: "Gojira", model: "llama-3.3-70b-versatile",
|
|
location_raw: "Ondres / Bayonne", country: "FR", parsed_city: "Bayonne",
|
|
parsed_country: "FR", is_null: false, tokens_in: 180, tokens_out: 12,
|
|
cost_usd: "0.000090", prompt: "Extract the city…", response: '{"city":"Bayonne"}',
|
|
created_at: "2026-08-12T08:00:00Z" }) },
|
|
|
|
// ---- Journal ----
|
|
{ match: "UNION ALL", result: (sql) => sql.includes("count(*)")
|
|
? rows({ total: 2 })
|
|
: rows(
|
|
{ type: "run", id: 5, subtype: "enrich", status: "done", actor: null,
|
|
ts: "2026-08-12T07:00:00Z", finished_at: "2026-08-12T07:30:00Z",
|
|
summary: { bands_seen: 500, bands_new: 12, bands_updated: 30, bands_enriched: 480, error: null, countries: null } },
|
|
{ type: "admin_action", id: 2, subtype: "update", status: "done", actor: "nico",
|
|
ts: "2026-08-12T06:00:00Z", finished_at: "2026-08-12T06:00:00Z",
|
|
summary: { target_table: "bands", target_id: "1", before: { name: "Mayhem" }, after: { name: "Mayhem" } } }) },
|
|
{ match: "FROM crawl_log", result: rows({ id: 1, run_id: 5, level: "info", message: "enrich done", ma_id: null, created_at: "2026-08-12T07:30:00Z" }) },
|
|
|
|
// ---- Liste des groupes (fragments génériques : en dernier) ----
|
|
{ match: "SELECT ma_id, name, country", result: rows(...bands) },
|
|
{ match: "count(*)::int AS total FROM bands", result: rows({ total: bands.length }) },
|
|
{ match: "FROM llm_cache", result: rows() },
|
|
];
|
|
}
|