From d2fcafc6be1d0e48a3b93f8f70294d60b8c93031 Mon Sep 17 00:00:00 2001 From: Nicolas Fryder Date: Sat, 27 Jun 2026 15:09:35 +0200 Subject: [PATCH] =?UTF-8?q?feat(db):=20syst=C3=A8me=20de=20migrations=20SQ?= =?UTF-8?q?L=20+=20refonte=20sch=C3=A9ma?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - apps/api/migrations/ : 4 migrations numérotées idempotentes 001 : schéma initial (bands, trigger geom, indexes) 002 : colonnes manquantes (enriched, themes, geocode_*) + fix trigger → updated_at mis à jour sur toute UPDATE (pas seulement lat/lon) 003 : geocode_queue et geocode_cache (formalisées) 004 : tracking crawl (crawled_at, crawled_hash, ma_created_at, ma_modified_at, first_seen_at) + crawl_run + crawl_checkpoint - apps/api/src/migrate.js : runner qui s applique au démarrage de l api - apps/api/Dockerfile : CMD lance migrate.js avant server.js - infra/init.sql : remplacé par notice de redirection Co-Authored-By: Claude Sonnet 4.6 --- apps/api/Dockerfile | 4 +- apps/api/migrations/001_initial_schema.sql | 41 +++++++++++++ .../migrations/002_add_missing_columns.sql | 37 ++++++++++++ apps/api/migrations/003_geocode_tables.sql | 26 ++++++++ apps/api/migrations/004_crawl_tracking.sql | 43 +++++++++++++ apps/api/src/migrate.js | 60 +++++++++++++++++++ infra/init.sql | 49 +++------------ 7 files changed, 217 insertions(+), 43 deletions(-) create mode 100644 apps/api/migrations/001_initial_schema.sql create mode 100644 apps/api/migrations/002_add_missing_columns.sql create mode 100644 apps/api/migrations/003_geocode_tables.sql create mode 100644 apps/api/migrations/004_crawl_tracking.sql create mode 100644 apps/api/src/migrate.js diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index f254fef..c774a5e 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -5,5 +5,7 @@ COPY package.json package-lock.json* ./ RUN npm install --omit=dev COPY src ./src +COPY migrations ./migrations + ENV PORT=3000 -CMD ["node", "src/server.js"] +CMD ["sh", "-c", "node src/migrate.js && node src/server.js"] diff --git a/apps/api/migrations/001_initial_schema.sql b/apps/api/migrations/001_initial_schema.sql new file mode 100644 index 0000000..f3a9a1c --- /dev/null +++ b/apps/api/migrations/001_initial_schema.sql @@ -0,0 +1,41 @@ +CREATE EXTENSION IF NOT EXISTS postgis; + +CREATE TABLE IF NOT EXISTS bands ( + id BIGSERIAL PRIMARY KEY, + ma_id BIGINT UNIQUE NOT NULL, + name TEXT NOT NULL, + country TEXT, + location_text TEXT, + status TEXT, + genre TEXT, + formed_year INT, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + geom GEOGRAPHY(Point, 4326), + data JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE OR REPLACE FUNCTION bands_set_geom() RETURNS trigger AS $$ +BEGIN + IF NEW.lat IS NOT NULL AND NEW.lon IS NOT NULL THEN + NEW.geom := ST_SetSRID(ST_MakePoint(NEW.lon, NEW.lat), 4326)::geography; + ELSE + NEW.geom := NULL; + END IF; + NEW.updated_at := now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_bands_set_geom ON bands; +CREATE TRIGGER trg_bands_set_geom +BEFORE INSERT OR UPDATE OF lat, lon ON bands +FOR EACH ROW EXECUTE FUNCTION bands_set_geom(); + +CREATE INDEX IF NOT EXISTS idx_bands_country ON bands (country); +CREATE INDEX IF NOT EXISTS idx_bands_status ON bands (status); +CREATE INDEX IF NOT EXISTS idx_bands_genre ON bands (genre); +CREATE INDEX IF NOT EXISTS idx_bands_geom ON bands USING GIST (geom); +CREATE INDEX IF NOT EXISTS idx_bands_data_gin ON bands USING GIN (data); diff --git a/apps/api/migrations/002_add_missing_columns.sql b/apps/api/migrations/002_add_missing_columns.sql new file mode 100644 index 0000000..4d99026 --- /dev/null +++ b/apps/api/migrations/002_add_missing_columns.sql @@ -0,0 +1,37 @@ +-- Colonnes utilisées par l'API et le geocoder mais absentes du schéma initial +ALTER TABLE bands ADD COLUMN IF NOT EXISTS enriched BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE bands ADD COLUMN IF NOT EXISTS themes TEXT; +ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocoded_at TIMESTAMPTZ; +ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_provider TEXT; +ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_query TEXT; +ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_raw JSONB; +ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_error TEXT; +ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_error_at TIMESTAMPTZ; + +-- Fix: le trigger précédent ne mettait updated_at à jour que sur changement de lat/lon. +-- Le remplacer pour qu'il se déclenche sur n'importe quelle UPDATE. +CREATE OR REPLACE FUNCTION bands_set_geom() RETURNS trigger AS $$ +BEGIN + -- Ne recalculer geom que si lat ou lon a vraiment changé + IF TG_OP = 'INSERT' + OR OLD.lat IS DISTINCT FROM NEW.lat + OR OLD.lon IS DISTINCT FROM NEW.lon + THEN + IF NEW.lat IS NOT NULL AND NEW.lon IS NOT NULL THEN + NEW.geom := ST_SetSRID(ST_MakePoint(NEW.lon, NEW.lat), 4326)::geography; + ELSE + NEW.geom := NULL; + END IF; + END IF; + NEW.updated_at := now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_bands_set_geom ON bands; +CREATE TRIGGER trg_bands_set_geom +BEFORE INSERT OR UPDATE ON bands +FOR EACH ROW EXECUTE FUNCTION bands_set_geom(); + +CREATE INDEX IF NOT EXISTS idx_bands_enriched ON bands (enriched); +CREATE INDEX IF NOT EXISTS idx_bands_geocoded_at ON bands (geocoded_at); diff --git a/apps/api/migrations/003_geocode_tables.sql b/apps/api/migrations/003_geocode_tables.sql new file mode 100644 index 0000000..3d23c47 --- /dev/null +++ b/apps/api/migrations/003_geocode_tables.sql @@ -0,0 +1,26 @@ +CREATE TABLE IF NOT EXISTS geocode_queue ( + ma_id BIGINT PRIMARY KEY REFERENCES bands(ma_id) ON DELETE CASCADE, + query TEXT NOT NULL, + country TEXT, + status TEXT NOT NULL DEFAULT 'queued', -- queued | processing | done | error + tries INT NOT NULL DEFAULT 0, + next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_geocode_queue_status_next + ON geocode_queue (status, next_run_at) + WHERE status IN ('queued', 'error'); + +CREATE TABLE IF NOT EXISTS geocode_cache ( + query TEXT PRIMARY KEY, + provider TEXT, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + geom GEOGRAPHY(Point, 4326), + raw JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/apps/api/migrations/004_crawl_tracking.sql b/apps/api/migrations/004_crawl_tracking.sql new file mode 100644 index 0000000..5d7fea0 --- /dev/null +++ b/apps/api/migrations/004_crawl_tracking.sql @@ -0,0 +1,43 @@ +-- Nouvelles colonnes sur bands pour tracker les crawls +ALTER TABLE bands ADD COLUMN IF NOT EXISTS first_seen_at TIMESTAMPTZ; -- quand on a découvert ce band pour la 1ère fois +ALTER TABLE bands ADD COLUMN IF NOT EXISTS crawled_at TIMESTAMPTZ; -- dernier crawl de la band page +ALTER TABLE bands ADD COLUMN IF NOT EXISTS crawled_hash TEXT; -- md5 du HTML de la band page (détection de changements) +ALTER TABLE bands ADD COLUMN IF NOT EXISTS ma_created_at TEXT; -- "Added" affiché par MA (texte brut, format variable) +ALTER TABLE bands ADD COLUMN IF NOT EXISTS ma_modified_at TEXT; -- "Last modified" affiché par MA + +-- Backfill first_seen_at pour les bands existantes +UPDATE bands SET first_seen_at = created_at WHERE first_seen_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_bands_crawled_at ON bands (crawled_at); +CREATE INDEX IF NOT EXISTS idx_bands_first_seen ON bands (first_seen_at); + +-- Historique des runs de crawl +CREATE TABLE IF NOT EXISTS crawl_run ( + id BIGSERIAL PRIMARY KEY, + run_type TEXT NOT NULL, -- full_europe | incremental_additions | incremental_modified | single_band + countries TEXT[], -- codes pays ciblés (null = tous) + status TEXT NOT NULL DEFAULT 'running', -- running | done | error | cancelled + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + bands_seen INT NOT NULL DEFAULT 0, + bands_new INT NOT NULL DEFAULT 0, + bands_updated INT NOT NULL DEFAULT 0, + bands_enriched INT NOT NULL DEFAULT 0, + error TEXT +); + +CREATE INDEX IF NOT EXISTS idx_crawl_run_type_started ON crawl_run (run_type, started_at DESC); + +-- État persistant des crawls (dernière date/page vérifiée, etc.) +CREATE TABLE IF NOT EXISTS crawl_checkpoint ( + key TEXT PRIMARY KEY, + value TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Valeurs initiales +INSERT INTO crawl_checkpoint (key, value) VALUES + ('last_additions_check', NULL), -- timestamp du dernier check de /archives/band-list/by/created + ('last_modified_check', NULL), -- timestamp du dernier check de /archives/band-list/by/modified + ('last_full_crawl_at', NULL) -- timestamp du dernier crawl complet Europe +ON CONFLICT (key) DO NOTHING; diff --git a/apps/api/src/migrate.js b/apps/api/src/migrate.js new file mode 100644 index 0000000..4119472 --- /dev/null +++ b/apps/api/src/migrate.js @@ -0,0 +1,60 @@ +import pg from 'pg'; +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const MIGRATIONS_DIR = path.join(__dirname, '../migrations'); + +const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); + +async function run() { + await client.connect(); + + await client.query(` + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `); + + const { rows } = await client.query( + 'SELECT version FROM schema_migrations ORDER BY version' + ); + const applied = new Set(rows.map(r => r.version)); + + const files = fs.readdirSync(MIGRATIONS_DIR) + .filter(f => f.endsWith('.sql')) + .sort(); + + let count = 0; + for (const file of files) { + if (applied.has(file)) continue; + + console.log(`[migrate] applying ${file}`); + const sql = fs.readFileSync(path.join(MIGRATIONS_DIR, file), 'utf8'); + + await client.query('BEGIN'); + try { + await client.query(sql); + await client.query( + 'INSERT INTO schema_migrations (version) VALUES ($1)', [file] + ); + await client.query('COMMIT'); + console.log(`[migrate] ok ${file}`); + count++; + } catch (err) { + await client.query('ROLLBACK'); + console.error(`[migrate] FAILED ${file}: ${err.message}`); + throw err; + } + } + + if (count === 0) console.log('[migrate] nothing to apply'); + await client.end(); +} + +run().catch(err => { + console.error('[migrate] fatal:', err.message); + process.exit(1); +}); diff --git a/infra/init.sql b/infra/init.sql index 15a49c2..8aeb37b 100644 --- a/infra/init.sql +++ b/infra/init.sql @@ -1,42 +1,7 @@ -CREATE EXTENSION IF NOT EXISTS postgis; - -CREATE TABLE IF NOT EXISTS bands ( - id BIGSERIAL PRIMARY KEY, - ma_id BIGINT UNIQUE NOT NULL, - name TEXT NOT NULL, - country TEXT, - location_text TEXT, - status TEXT, - genre TEXT, - formed_year INT, - lat DOUBLE PRECISION, - lon DOUBLE PRECISION, - geom GEOGRAPHY(Point, 4326), - data JSONB NOT NULL DEFAULT '{}'::jsonb, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - --- Keep geom in sync if lat/lon set -CREATE OR REPLACE FUNCTION bands_set_geom() RETURNS trigger AS $$ -BEGIN - IF NEW.lat IS NOT NULL AND NEW.lon IS NOT NULL THEN - NEW.geom := ST_SetSRID(ST_MakePoint(NEW.lon, NEW.lat), 4326)::geography; - ELSE - NEW.geom := NULL; - END IF; - NEW.updated_at := now(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS trg_bands_set_geom ON bands; -CREATE TRIGGER trg_bands_set_geom -BEFORE INSERT OR UPDATE OF lat, lon ON bands -FOR EACH ROW EXECUTE FUNCTION bands_set_geom(); - -CREATE INDEX IF NOT EXISTS idx_bands_country ON bands (country); -CREATE INDEX IF NOT EXISTS idx_bands_status ON bands (status); -CREATE INDEX IF NOT EXISTS idx_bands_genre ON bands (genre); -CREATE INDEX IF NOT EXISTS idx_bands_geom ON bands USING GIST (geom); -CREATE INDEX IF NOT EXISTS idx_bands_data_gin ON bands USING GIN (data); +-- Le schéma est désormais géré par les migrations numérotées dans apps/api/migrations/. +-- Ce fichier n'est plus utilisé. Les migrations sont appliquées automatiquement au +-- démarrage du service api (node src/migrate.js). +-- +-- Pour ajouter une migration : +-- 1. Créer apps/api/migrations/NNN_description.sql +-- 2. Commiter et pousser → le prochain déploiement l'applique automatiquement