feat(db): système de migrations SQL + refonte schéma
- apps/api/migrations/ : 4 migrations numérotées idempotentes
001 : schéma initial (bands, trigger geom, indexes)
002 : colonnes manquantes (enriched, themes, geocode_*) + fix trigger
→ updated_at mis à jour sur toute UPDATE (pas seulement lat/lon)
003 : geocode_queue et geocode_cache (formalisées)
004 : tracking crawl (crawled_at, crawled_hash, ma_created_at,
ma_modified_at, first_seen_at) + crawl_run + crawl_checkpoint
- apps/api/src/migrate.js : runner qui s applique au démarrage de l api
- apps/api/Dockerfile : CMD lance migrate.js avant server.js
- infra/init.sql : remplacé par notice de redirection
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
91166668a6
commit
d2fcafc6be
7 changed files with 217 additions and 43 deletions
|
|
@ -5,5 +5,7 @@ COPY package.json package-lock.json* ./
|
|||
RUN npm install --omit=dev
|
||||
|
||||
COPY src ./src
|
||||
COPY migrations ./migrations
|
||||
|
||||
ENV PORT=3000
|
||||
CMD ["node", "src/server.js"]
|
||||
CMD ["sh", "-c", "node src/migrate.js && node src/server.js"]
|
||||
|
|
|
|||
41
apps/api/migrations/001_initial_schema.sql
Normal file
41
apps/api/migrations/001_initial_schema.sql
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
CREATE EXTENSION IF NOT EXISTS postgis;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bands (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
ma_id BIGINT UNIQUE NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
country TEXT,
|
||||
location_text TEXT,
|
||||
status TEXT,
|
||||
genre TEXT,
|
||||
formed_year INT,
|
||||
lat DOUBLE PRECISION,
|
||||
lon DOUBLE PRECISION,
|
||||
geom GEOGRAPHY(Point, 4326),
|
||||
data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION bands_set_geom() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
IF NEW.lat IS NOT NULL AND NEW.lon IS NOT NULL THEN
|
||||
NEW.geom := ST_SetSRID(ST_MakePoint(NEW.lon, NEW.lat), 4326)::geography;
|
||||
ELSE
|
||||
NEW.geom := NULL;
|
||||
END IF;
|
||||
NEW.updated_at := now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_bands_set_geom ON bands;
|
||||
CREATE TRIGGER trg_bands_set_geom
|
||||
BEFORE INSERT OR UPDATE OF lat, lon ON bands
|
||||
FOR EACH ROW EXECUTE FUNCTION bands_set_geom();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_country ON bands (country);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_status ON bands (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_genre ON bands (genre);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_geom ON bands USING GIST (geom);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_data_gin ON bands USING GIN (data);
|
||||
37
apps/api/migrations/002_add_missing_columns.sql
Normal file
37
apps/api/migrations/002_add_missing_columns.sql
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
-- Colonnes utilisées par l'API et le geocoder mais absentes du schéma initial
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS enriched BOOLEAN NOT NULL DEFAULT false;
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS themes TEXT;
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocoded_at TIMESTAMPTZ;
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_provider TEXT;
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_query TEXT;
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_raw JSONB;
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_error TEXT;
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS geocode_error_at TIMESTAMPTZ;
|
||||
|
||||
-- Fix: le trigger précédent ne mettait updated_at à jour que sur changement de lat/lon.
|
||||
-- Le remplacer pour qu'il se déclenche sur n'importe quelle UPDATE.
|
||||
CREATE OR REPLACE FUNCTION bands_set_geom() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
-- Ne recalculer geom que si lat ou lon a vraiment changé
|
||||
IF TG_OP = 'INSERT'
|
||||
OR OLD.lat IS DISTINCT FROM NEW.lat
|
||||
OR OLD.lon IS DISTINCT FROM NEW.lon
|
||||
THEN
|
||||
IF NEW.lat IS NOT NULL AND NEW.lon IS NOT NULL THEN
|
||||
NEW.geom := ST_SetSRID(ST_MakePoint(NEW.lon, NEW.lat), 4326)::geography;
|
||||
ELSE
|
||||
NEW.geom := NULL;
|
||||
END IF;
|
||||
END IF;
|
||||
NEW.updated_at := now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_bands_set_geom ON bands;
|
||||
CREATE TRIGGER trg_bands_set_geom
|
||||
BEFORE INSERT OR UPDATE ON bands
|
||||
FOR EACH ROW EXECUTE FUNCTION bands_set_geom();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_enriched ON bands (enriched);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_geocoded_at ON bands (geocoded_at);
|
||||
26
apps/api/migrations/003_geocode_tables.sql
Normal file
26
apps/api/migrations/003_geocode_tables.sql
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
CREATE TABLE IF NOT EXISTS geocode_queue (
|
||||
ma_id BIGINT PRIMARY KEY REFERENCES bands(ma_id) ON DELETE CASCADE,
|
||||
query TEXT NOT NULL,
|
||||
country TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'queued', -- queued | processing | done | error
|
||||
tries INT NOT NULL DEFAULT 0,
|
||||
next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_geocode_queue_status_next
|
||||
ON geocode_queue (status, next_run_at)
|
||||
WHERE status IN ('queued', 'error');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS geocode_cache (
|
||||
query TEXT PRIMARY KEY,
|
||||
provider TEXT,
|
||||
lat DOUBLE PRECISION,
|
||||
lon DOUBLE PRECISION,
|
||||
geom GEOGRAPHY(Point, 4326),
|
||||
raw JSONB,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
43
apps/api/migrations/004_crawl_tracking.sql
Normal file
43
apps/api/migrations/004_crawl_tracking.sql
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
-- Nouvelles colonnes sur bands pour tracker les crawls
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS first_seen_at TIMESTAMPTZ; -- quand on a découvert ce band pour la 1ère fois
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS crawled_at TIMESTAMPTZ; -- dernier crawl de la band page
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS crawled_hash TEXT; -- md5 du HTML de la band page (détection de changements)
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS ma_created_at TEXT; -- "Added" affiché par MA (texte brut, format variable)
|
||||
ALTER TABLE bands ADD COLUMN IF NOT EXISTS ma_modified_at TEXT; -- "Last modified" affiché par MA
|
||||
|
||||
-- Backfill first_seen_at pour les bands existantes
|
||||
UPDATE bands SET first_seen_at = created_at WHERE first_seen_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_crawled_at ON bands (crawled_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_first_seen ON bands (first_seen_at);
|
||||
|
||||
-- Historique des runs de crawl
|
||||
CREATE TABLE IF NOT EXISTS crawl_run (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
run_type TEXT NOT NULL, -- full_europe | incremental_additions | incremental_modified | single_band
|
||||
countries TEXT[], -- codes pays ciblés (null = tous)
|
||||
status TEXT NOT NULL DEFAULT 'running', -- running | done | error | cancelled
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
finished_at TIMESTAMPTZ,
|
||||
bands_seen INT NOT NULL DEFAULT 0,
|
||||
bands_new INT NOT NULL DEFAULT 0,
|
||||
bands_updated INT NOT NULL DEFAULT 0,
|
||||
bands_enriched INT NOT NULL DEFAULT 0,
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_run_type_started ON crawl_run (run_type, started_at DESC);
|
||||
|
||||
-- État persistant des crawls (dernière date/page vérifiée, etc.)
|
||||
CREATE TABLE IF NOT EXISTS crawl_checkpoint (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Valeurs initiales
|
||||
INSERT INTO crawl_checkpoint (key, value) VALUES
|
||||
('last_additions_check', NULL), -- timestamp du dernier check de /archives/band-list/by/created
|
||||
('last_modified_check', NULL), -- timestamp du dernier check de /archives/band-list/by/modified
|
||||
('last_full_crawl_at', NULL) -- timestamp du dernier crawl complet Europe
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
60
apps/api/src/migrate.js
Normal file
60
apps/api/src/migrate.js
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import pg from 'pg';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const MIGRATIONS_DIR = path.join(__dirname, '../migrations');
|
||||
|
||||
const client = new pg.Client({ connectionString: process.env.DATABASE_URL });
|
||||
|
||||
async function run() {
|
||||
await client.connect();
|
||||
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
`);
|
||||
|
||||
const { rows } = await client.query(
|
||||
'SELECT version FROM schema_migrations ORDER BY version'
|
||||
);
|
||||
const applied = new Set(rows.map(r => r.version));
|
||||
|
||||
const files = fs.readdirSync(MIGRATIONS_DIR)
|
||||
.filter(f => f.endsWith('.sql'))
|
||||
.sort();
|
||||
|
||||
let count = 0;
|
||||
for (const file of files) {
|
||||
if (applied.has(file)) continue;
|
||||
|
||||
console.log(`[migrate] applying ${file}`);
|
||||
const sql = fs.readFileSync(path.join(MIGRATIONS_DIR, file), 'utf8');
|
||||
|
||||
await client.query('BEGIN');
|
||||
try {
|
||||
await client.query(sql);
|
||||
await client.query(
|
||||
'INSERT INTO schema_migrations (version) VALUES ($1)', [file]
|
||||
);
|
||||
await client.query('COMMIT');
|
||||
console.log(`[migrate] ok ${file}`);
|
||||
count++;
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
console.error(`[migrate] FAILED ${file}: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
if (count === 0) console.log('[migrate] nothing to apply');
|
||||
await client.end();
|
||||
}
|
||||
|
||||
run().catch(err => {
|
||||
console.error('[migrate] fatal:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,42 +1,7 @@
|
|||
CREATE EXTENSION IF NOT EXISTS postgis;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bands (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
ma_id BIGINT UNIQUE NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
country TEXT,
|
||||
location_text TEXT,
|
||||
status TEXT,
|
||||
genre TEXT,
|
||||
formed_year INT,
|
||||
lat DOUBLE PRECISION,
|
||||
lon DOUBLE PRECISION,
|
||||
geom GEOGRAPHY(Point, 4326),
|
||||
data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Keep geom in sync if lat/lon set
|
||||
CREATE OR REPLACE FUNCTION bands_set_geom() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
IF NEW.lat IS NOT NULL AND NEW.lon IS NOT NULL THEN
|
||||
NEW.geom := ST_SetSRID(ST_MakePoint(NEW.lon, NEW.lat), 4326)::geography;
|
||||
ELSE
|
||||
NEW.geom := NULL;
|
||||
END IF;
|
||||
NEW.updated_at := now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_bands_set_geom ON bands;
|
||||
CREATE TRIGGER trg_bands_set_geom
|
||||
BEFORE INSERT OR UPDATE OF lat, lon ON bands
|
||||
FOR EACH ROW EXECUTE FUNCTION bands_set_geom();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_country ON bands (country);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_status ON bands (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_genre ON bands (genre);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_geom ON bands USING GIST (geom);
|
||||
CREATE INDEX IF NOT EXISTS idx_bands_data_gin ON bands USING GIN (data);
|
||||
-- Le schéma est désormais géré par les migrations numérotées dans apps/api/migrations/.
|
||||
-- Ce fichier n'est plus utilisé. Les migrations sont appliquées automatiquement au
|
||||
-- démarrage du service api (node src/migrate.js).
|
||||
--
|
||||
-- Pour ajouter une migration :
|
||||
-- 1. Créer apps/api/migrations/NNN_description.sql
|
||||
-- 2. Commiter et pousser → le prochain déploiement l'applique automatiquement
|
||||
|
|
|
|||
Loading…
Reference in a new issue