metalfrom.eu/docker-compose.dev.yml
Nicolas Fryder 1ae5aa4b57 feat(geocoder): seuil de confiance strict, dedup, purge ancien pipeline
Étape 0 de la refonte pipeline crawl/géocodage.

Auto-requeue du géocodage
- migration 013: trigger trg_bands_geocode_dirty (AFTER UPDATE OF location_text
  ON bands) supprime les band_locations d'un band dès que sa localisation
  change réellement (IS DISTINCT FROM) — plus de points fantômes après un
  déménagement détecté par le crawler incrémental.
- enqueue.py: en plus du déclenchement manuel, un scan automatique tourne
  toutes les ENQUEUE_AUTO_INTERVAL_MIN minutes (défaut 60) et rattrape les
  bands rendus "dirty" par le trigger (idempotent, ON CONFLICT DO NOTHING —
  pas de canal de notification supplémentaire nécessaire). Chaque exécution
  (manuelle ou auto) crée une ligne crawl_run + logs rattachés, en miroir de
  crawler/src/db.py, pour apparaître dans l'activité admin à venir.

Crawl complet en calcul glissant
- remplace CRAWLER_SCHED_FULL_DAY (jour fixe du mois) par
  CRAWLER_FULL_CRAWL_INTERVAL_DAYS (défaut 60) basé sur le checkpoint
  last_full_crawl_at — robuste au calendrier et aux redémarrages (vérifié
  aussi au démarrage, pas seulement le tick quotidien 03:00 UTC).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-02 21:07:16 +02:00

171 lines
4.6 KiB
YAML

services:
redis:
image: redis:7-alpine
command: ["redis-server", "--appendonly", "yes"]
volumes:
- bm_dev_redis_data:/data
networks:
- coolify
flaresolverr:
image: ghcr.io/flaresolverr/flaresolverr:latest
environment:
LOG_LEVEL: info
CAPTCHA_SOLVER: none
networks:
- coolify
labels:
- traefik.enable=false
restart: unless-stopped
crawler:
build:
context: apps/crawler
environment:
DATABASE_URL: ${DATABASE_URL}
FLARESOLVERR_URL: http://flaresolverr:8191
CRAWLER_SCHED_INCREMENTAL_H: "4"
CRAWLER_SCHED_ENRICH_H: "2"
CRAWLER_FULL_CRAWL_INTERVAL_DAYS: "0"
depends_on:
- flaresolverr
networks:
- coolify
labels:
- traefik.enable=false
restart: unless-stopped
geocoder-enqueue:
build:
context: apps/geocoder
working_dir: /app
environment:
DATABASE_URL: ${DATABASE_URL}
GEOCODE_ENQUEUE_BATCH: "100000"
GEOCODE_ENQUEUE_POLL: "15"
ENQUEUE_AUTO_INTERVAL_MIN: "60"
command: ["python", "src/enqueue.py"]
networks:
- coolify
labels:
- traefik.enable=false
restart: unless-stopped
geocoder-worker:
build:
context: apps/geocoder
working_dir: /app
environment:
DATABASE_URL: ${DATABASE_URL}
GEOAPIFY_API_KEY: ${GEOAPIFY_API_KEY}
GEOCODER_MIN_DELAY: "0.22"
GEOCODER_JITTER: "0.10"
GEOCODE_MAX_PER_RUN: "100000"
GEOCODE_MAX_GEO_TRIES: "3"
GEOCODE_MIN_CONFIDENCE: "0.7"
command: ["python", "src/worker.py"]
networks:
- coolify
labels:
- traefik.enable=false
restart: unless-stopped
groq-worker:
build:
context: apps/geocoder
working_dir: /app
environment:
DATABASE_URL: ${DATABASE_URL}
GROQ_API_KEY: ${GROQ_API_KEY}
GROQ_MAX_TRIES: "5"
GROQ_CALL_DELAY: "2.5"
command: ["python", "src/groq_worker.py"]
networks:
- coolify
labels:
- traefik.enable=false
restart: unless-stopped
pgadmin:
image: dpage/pgadmin4:8
environment:
PGADMIN_DEFAULT_EMAIL: ${PGADMIN_EMAIL}
PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_PASSWORD}
PGADMIN_CONFIG_SERVER_MODE: "True"
volumes:
- bm_dev_pgadmin_data:/var/lib/pgadmin
networks:
- coolify
labels:
- traefik.enable=true
- traefik.docker.network=coolify
- traefik.http.routers.dev-pgadmin.rule=Host(`dev-pgadmin.metalfrom.eu`)
- traefik.http.routers.dev-pgadmin.entrypoints=https
- traefik.http.routers.dev-pgadmin.tls=true
- traefik.http.routers.dev-pgadmin.tls.certresolver=letsencrypt
- traefik.http.services.dev-pgadmin.loadbalancer.server.port=80
api:
build:
context: apps/api
environment:
PORT: "3000"
DATABASE_URL: ${DATABASE_URL}
BM_IMPORT_TOKEN: ${BM_IMPORT_TOKEN}
CORS_ORIGINS: ${CORS_ORIGINS:-https://dev.metalfrom.eu}
ADMIN_JWT_SECRET: ${ADMIN_JWT_SECRET}
ADMIN_SEED_USERNAME: ${ADMIN_SEED_USERNAME}
ADMIN_SEED_PASSWORD_HASH: ${ADMIN_SEED_PASSWORD_HASH}
networks:
coolify:
aliases:
- dev-api-internal
labels:
- traefik.enable=true
- traefik.docker.network=coolify
- traefik.http.routers.dev-api.rule=Host(`dev-api.metalfrom.eu`)
- traefik.http.routers.dev-api.entrypoints=https
- traefik.http.routers.dev-api.tls=true
- traefik.http.routers.dev-api.tls.certresolver=letsencrypt
- traefik.http.services.dev-api.loadbalancer.server.port=3000
admin:
build:
context: apps/admin
args:
API_UPSTREAM: http://dev-api-internal:3000
networks:
- coolify
labels:
- traefik.enable=true
- traefik.docker.network=coolify
- traefik.http.routers.dev-admin.rule=Host(`admin.dev.metalfrom.eu`)
- traefik.http.routers.dev-admin.entrypoints=https
- traefik.http.routers.dev-admin.tls=true
- traefik.http.routers.dev-admin.tls.certresolver=letsencrypt
- traefik.http.services.dev-admin.loadbalancer.server.port=80
web:
build:
context: apps/web
args:
API_BASE: https://dev-api.metalfrom.eu
networks:
- coolify
labels:
- traefik.enable=true
- traefik.docker.network=coolify
- traefik.http.routers.dev-web.rule=Host(`dev.metalfrom.eu`)
- traefik.http.routers.dev-web.entrypoints=https
- traefik.http.routers.dev-web.tls=true
- traefik.http.routers.dev-web.tls.certresolver=letsencrypt
- traefik.http.services.dev-web.loadbalancer.server.port=80
networks:
coolify:
external: true
name: coolify
volumes:
bm_dev_redis_data:
bm_dev_pgadmin_data: