Nicolas Fryder
79daa30a8a
- Backend NestJS orchestrant CloudCompare CLI (stats, decimation x2 sur 5 etapes, volume, assemblage .bin) - Scripts Python (laspy, scipy, matplotlib) pour la conversion LAS/LAZ et la generation d'images - Frontend web simple (Tailwind CDN) avec historique des runs - Image Docker (debian:trixie-slim + cloudcompare apt + xvfb) prete pour deploiement Coolify - Lint/typecheck/tests unitaires integres comme portes de qualite au build Docker - CLAUDE.md documentant les comportements CloudCompare CLI verifies empiriquement
447 lines
16 KiB
TypeScript
447 lines
16 KiB
TypeScript
import { Injectable, Logger } from '@nestjs/common';
|
|
import * as fs from 'fs/promises';
|
|
import * as path from 'path';
|
|
import { config } from '../config';
|
|
import { CcRunnerService } from '../cloudcompare/cc-runner.service';
|
|
import { PyHelperService } from '../pyhelper/pyhelper.service';
|
|
import { LevelResult, RunParamsInput, RunReport } from './types';
|
|
|
|
export function sanitizeBaseName(filename: string): string {
|
|
const base = filename.replace(/\.[^./\\]+$/, '');
|
|
return base.replace(/[^a-zA-Z0-9_-]+/g, '_').slice(0, 80) || 'nuage';
|
|
}
|
|
|
|
/**
|
|
* Orchestration complete d'un run :
|
|
* 1) statistiques (pas spatial initial, plan de reference Z)
|
|
* 2) niveau 0 = nuage complet -> volume + image
|
|
* 3) niveaux 1..N = decimation spatiale progressive (facteur 2, chainee depuis le niveau precedent)
|
|
* 4) assemblage des nuages exploitables dans un seul .bin
|
|
* 5) rapport CSV/JSON + graphiques de synthese
|
|
*/
|
|
@Injectable()
|
|
export class PipelineService {
|
|
private readonly logger = new Logger(PipelineService.name);
|
|
|
|
constructor(
|
|
private readonly cc: CcRunnerService,
|
|
private readonly py: PyHelperService,
|
|
) {}
|
|
|
|
async execute(
|
|
runId: string,
|
|
inputPath: string,
|
|
originalFilename: string,
|
|
workDir: string,
|
|
params: RunParamsInput,
|
|
onProgress: (msg: string) => void,
|
|
): Promise<RunReport> {
|
|
const levelsDir = path.join(workDir, 'levels');
|
|
const imagesDir = path.join(workDir, 'images');
|
|
const rawDir = path.join(workDir, 'raw');
|
|
await fs.mkdir(levelsDir, { recursive: true });
|
|
await fs.mkdir(imagesDir, { recursive: true });
|
|
await fs.mkdir(rawDir, { recursive: true });
|
|
|
|
// ---------- 0) Normalisation du format d'entree ----------
|
|
// Le paquet apt CloudCompare (Debian, utilise dans l'image Docker) ne fournit pas le plugin
|
|
// LAS/LAZ (seulement "Core I/O"). On convertit donc LAS/LAZ/COPC en ASCII XYZ via laspy avant
|
|
// de transmettre le nuage a CloudCompare, qui lit nativement l'ASCII sans plugin.
|
|
let normalizedInputPath = inputPath;
|
|
const lowerInput = inputPath.toLowerCase();
|
|
if (lowerInput.endsWith('.las') || lowerInput.endsWith('.laz')) {
|
|
onProgress('Conversion du nuage source (LAS/LAZ/COPC) en ASCII (CloudCompare ne lit pas le LAS nativement dans ce conteneur)...');
|
|
const xyzPath = path.join(workDir, 'input_converted.xyz');
|
|
await this.py.convertLasToXyz(inputPath, xyzPath);
|
|
normalizedInputPath = xyzPath;
|
|
}
|
|
|
|
// ---------- 1) Statistiques ----------
|
|
onProgress('Estimation du pas spatial initial (echantillonnage + KD-tree)...');
|
|
const t0 = Date.now();
|
|
const sampleRes = await this.cc.run(
|
|
['-O', normalizedInputPath, '-SS', 'RANDOM', String(config.statsSampleCap), '-C_EXPORT_FMT', 'ASC', '-PREC', '6', '-SAVE_CLOUDS', 'FILE', 'sample.xyz'],
|
|
workDir,
|
|
);
|
|
const totalPointCount = this.cc.parseLoadedPointCount(sampleRes.stdout);
|
|
if (totalPointCount === null) {
|
|
throw new Error(`Impossible de lire le nuage d'entree. Sortie CloudCompare:\n${sampleRes.stdout.slice(-2000)}`);
|
|
}
|
|
const samplePath = path.join(workDir, 'sample.xyz');
|
|
await this.assertExists(samplePath, 'echantillonnage du nuage (sample.xyz)', sampleRes.stdout);
|
|
const stats = await this.py.computeStats(samplePath);
|
|
|
|
const initialStep = params.initialStepOverride ?? stats.median_nn;
|
|
const zref = params.zrefOverride ?? stats.bbox.zmin;
|
|
const gridStep = initialStep * config.gridStepMultiplier;
|
|
|
|
onProgress(
|
|
`Pas initial=${initialStep.toFixed(5)}m (${params.initialStepOverride ? 'override' : 'auto, mediane NN'}), ` +
|
|
`grille volume=${gridStep.toFixed(5)}m, Zref=${zref.toFixed(4)} (${params.zrefOverride ? 'override' : 'auto, Zmin echantillon'}). ` +
|
|
`[${Date.now() - t0}ms]`,
|
|
);
|
|
|
|
const levels: LevelResult[] = [];
|
|
|
|
// ---------- 2) Niveau 0 : nuage complet ----------
|
|
onProgress('Niveau 0 (nuage complet) : conversion .bin + calcul de volume...');
|
|
const level0 = await this.computeLevel({
|
|
level: 0,
|
|
spatialStep: null,
|
|
sourceBinOrOriginal: normalizedInputPath,
|
|
isFirstConversion: true,
|
|
levelsDir,
|
|
imagesDir,
|
|
rawDir,
|
|
workDir,
|
|
gridStep,
|
|
zref,
|
|
refPointCount: totalPointCount,
|
|
onProgress,
|
|
});
|
|
levels.push(level0);
|
|
|
|
// ---------- 3) Niveaux 1..N : decimation progressive ----------
|
|
let prevBinAbs = level0.binFile ? path.join(workDir, level0.binFile) : null;
|
|
let prevPointCount = level0.pointCount;
|
|
let stopped = level0.status !== 'ok';
|
|
let stopReason = level0.status !== 'ok' ? 'le niveau 0 a echoue' : '';
|
|
|
|
for (let i = 1; i <= config.decimationSteps; i++) {
|
|
if (stopped || !prevBinAbs) {
|
|
levels.push(this.skippedLevel(i, stopReason));
|
|
continue;
|
|
}
|
|
if (prevPointCount < config.minPointsToContinue) {
|
|
stopReason = `nuage trop clairseme apres le niveau ${i - 1} (${prevPointCount} points < seuil ${config.minPointsToContinue})`;
|
|
levels.push(this.skippedLevel(i, stopReason));
|
|
stopped = true;
|
|
continue;
|
|
}
|
|
|
|
const spatialStep = initialStep * Math.pow(config.decimationFactor, i);
|
|
onProgress(`Niveau ${i} : decimation spatiale (pas=${spatialStep.toFixed(5)}m) depuis le niveau ${i - 1}...`);
|
|
|
|
try {
|
|
const lvl = await this.computeLevel({
|
|
level: i,
|
|
spatialStep,
|
|
sourceBinOrOriginal: prevBinAbs,
|
|
isFirstConversion: false,
|
|
levelsDir,
|
|
imagesDir,
|
|
rawDir,
|
|
workDir,
|
|
gridStep,
|
|
zref,
|
|
refPointCount: totalPointCount,
|
|
onProgress,
|
|
});
|
|
levels.push(lvl);
|
|
if (lvl.status === 'ok') {
|
|
prevBinAbs = lvl.binFile ? path.join(workDir, lvl.binFile) : null;
|
|
prevPointCount = lvl.pointCount;
|
|
} else {
|
|
stopped = true;
|
|
stopReason = lvl.message ?? `echec au niveau ${i}`;
|
|
}
|
|
} catch (err: any) {
|
|
this.logger.error(`Niveau ${i} en erreur: ${err.message}`);
|
|
levels.push({
|
|
level: i,
|
|
spatialStep,
|
|
pointCount: 0,
|
|
pointRatio: 0,
|
|
volume: null,
|
|
surface: null,
|
|
addedVolume: null,
|
|
removedVolume: null,
|
|
matchingCellsPct: null,
|
|
groundNonMatchingPct: null,
|
|
ceilNonMatchingPct: null,
|
|
warn: true,
|
|
status: 'error',
|
|
message: err.message,
|
|
binFile: null,
|
|
imageFile: null,
|
|
computeTimeMs: 0,
|
|
});
|
|
stopped = true;
|
|
stopReason = err.message;
|
|
}
|
|
}
|
|
|
|
// ---------- 4) Assemblage ----------
|
|
onProgress('Assemblage des nuages exploitables dans un seul fichier .bin...');
|
|
const okLevels = levels.filter((l) => l.status === 'ok' && l.binFile);
|
|
let mergedBinFile: string | null = null;
|
|
if (okLevels.length > 0) {
|
|
const baseName = sanitizeBaseName(originalFilename);
|
|
const mergedName = `${baseName}_all_levels.bin`;
|
|
const args: string[] = [];
|
|
for (const lvl of okLevels) args.push('-O', lvl.binFile!);
|
|
args.push('-C_EXPORT_FMT', 'BIN', '-SAVE_CLOUDS', 'ALL_AT_ONCE', 'FILE', mergedName);
|
|
const mergeRes = await this.cc.run(args, workDir);
|
|
const mergedAbs = path.join(workDir, mergedName);
|
|
await this.assertExists(mergedAbs, 'fusion des nuages', mergeRes.stdout);
|
|
mergedBinFile = mergedName;
|
|
}
|
|
|
|
const report: RunReport = {
|
|
runId,
|
|
originalFilename,
|
|
createdAt: new Date().toISOString(),
|
|
params: {
|
|
zref,
|
|
zrefSource: params.zrefOverride !== undefined ? 'override' : 'auto',
|
|
initialStep,
|
|
initialStepSource: params.initialStepOverride !== undefined ? 'override' : 'auto',
|
|
gridStep,
|
|
factor: config.decimationFactor,
|
|
steps: config.decimationSteps,
|
|
sampleStats: {
|
|
sampleCount: stats.count,
|
|
meanNn: stats.mean_nn,
|
|
medianNn: stats.median_nn,
|
|
bboxApprox: stats.bbox,
|
|
},
|
|
},
|
|
levels,
|
|
mergedBinFile,
|
|
};
|
|
|
|
await fs.writeFile(path.join(workDir, 'report.json'), JSON.stringify(report, null, 2), 'utf-8');
|
|
await fs.writeFile(path.join(workDir, 'report.csv'), this.toCsv(report), 'utf-8');
|
|
|
|
// ---------- 5) Graphiques de synthese ----------
|
|
onProgress('Generation des graphiques de synthese...');
|
|
try {
|
|
await this.py.renderSummary(path.join(workDir, 'report.json'), imagesDir);
|
|
} catch (err: any) {
|
|
this.logger.warn(`Graphiques de synthese non generes: ${err.message}`);
|
|
}
|
|
|
|
return report;
|
|
}
|
|
|
|
private skippedLevel(level: number, reason: string): LevelResult {
|
|
return {
|
|
level,
|
|
spatialStep: null,
|
|
pointCount: 0,
|
|
pointRatio: 0,
|
|
volume: null,
|
|
surface: null,
|
|
addedVolume: null,
|
|
removedVolume: null,
|
|
matchingCellsPct: null,
|
|
groundNonMatchingPct: null,
|
|
ceilNonMatchingPct: null,
|
|
warn: true,
|
|
status: 'skipped',
|
|
message: reason,
|
|
binFile: null,
|
|
imageFile: null,
|
|
computeTimeMs: 0,
|
|
};
|
|
}
|
|
|
|
private async assertExists(p: string, what: string, ccLog: string) {
|
|
try {
|
|
await fs.access(p);
|
|
} catch {
|
|
throw new Error(`Etape "${what}" : fichier attendu introuvable (${p}). Sortie CloudCompare:\n${ccLog.slice(-2000)}`);
|
|
}
|
|
}
|
|
|
|
private async computeLevel(opts: {
|
|
level: number;
|
|
spatialStep: number | null;
|
|
sourceBinOrOriginal: string;
|
|
isFirstConversion: boolean;
|
|
levelsDir: string;
|
|
imagesDir: string;
|
|
rawDir: string;
|
|
workDir: string;
|
|
gridStep: number;
|
|
zref: number;
|
|
refPointCount: number;
|
|
onProgress: (msg: string) => void;
|
|
}): Promise<LevelResult> {
|
|
const { level, spatialStep, workDir, gridStep, zref, refPointCount } = opts;
|
|
const t0 = Date.now();
|
|
let binRelPath: string;
|
|
let pointCount: number;
|
|
|
|
if (opts.isFirstConversion) {
|
|
// Niveau 0 : simple conversion du fichier d'entree en .bin canonique
|
|
binRelPath = path.posix.join('levels', 'L0_full.bin');
|
|
const res = await this.cc.run(['-O', opts.sourceBinOrOriginal, '-C_EXPORT_FMT', 'BIN', '-SAVE_CLOUDS', 'FILE', binRelPath], workDir);
|
|
const n = this.cc.parseLoadedPointCount(res.stdout);
|
|
if (n === null) {
|
|
return this.errorLevel(level, spatialStep, `Conversion niveau 0 echouee. Sortie:\n${res.stdout.slice(-1500)}`, t0);
|
|
}
|
|
pointCount = n;
|
|
await this.assertExists(path.join(workDir, binRelPath), 'conversion niveau 0', res.stdout);
|
|
} else {
|
|
// Niveaux 1..N : decimation spatiale depuis le niveau precedent
|
|
const stepStr = spatialStep!.toFixed(6);
|
|
binRelPath = path.posix.join('levels', `L${level}_step${stepStr}.bin`);
|
|
const res = await this.cc.run(
|
|
['-O', opts.sourceBinOrOriginal, '-SS', 'SPATIAL', stepStr, '-C_EXPORT_FMT', 'BIN', '-SAVE_CLOUDS', 'FILE', binRelPath],
|
|
workDir,
|
|
);
|
|
const n = this.cc.parseSubsampleResult(res.stdout);
|
|
if (n === null || n === 0) {
|
|
return {
|
|
level,
|
|
spatialStep,
|
|
pointCount: 0,
|
|
pointRatio: 0,
|
|
volume: null,
|
|
surface: null,
|
|
addedVolume: null,
|
|
removedVolume: null,
|
|
matchingCellsPct: null,
|
|
groundNonMatchingPct: null,
|
|
ceilNonMatchingPct: null,
|
|
warn: true,
|
|
status: 'skipped',
|
|
message: `decimation spatiale n'a produit aucun point exploitable (pas=${stepStr}m)`,
|
|
binFile: null,
|
|
imageFile: null,
|
|
computeTimeMs: Date.now() - t0,
|
|
};
|
|
}
|
|
pointCount = n;
|
|
await this.assertExists(path.join(workDir, binRelPath), `decimation niveau ${level}`, res.stdout);
|
|
}
|
|
|
|
// Volume (necessite AUTO_SAVE ON, sinon CloudCompare ne genere pas le rapport texte - voir cc-runner.service.ts)
|
|
const beforeVolume = Date.now();
|
|
const volRes = await this.cc.run(
|
|
['-O', binRelPath, '-VOLUME', '-GRID_STEP', String(gridStep), '-CONST_HEIGHT', String(zref)],
|
|
workDir,
|
|
{ autoSave: true },
|
|
);
|
|
const reportFile = await this.cc.findLatestFile(workDir, { prefix: 'VolumeCalculationReport', suffix: '.txt', after: beforeVolume });
|
|
if (!reportFile) {
|
|
return this.errorLevel(level, spatialStep, `Rapport de volume introuvable au niveau ${level}. Sortie:\n${volRes.stdout.slice(-1500)}`, t0, binRelPath);
|
|
}
|
|
const vol = await this.cc.parseVolumeReportFile(reportFile);
|
|
const rawReportDest = path.join(opts.rawDir, `L${level}_volume_report.txt`);
|
|
await fs.rename(reportFile, rawReportDest);
|
|
|
|
// AUTO_SAVE ON sauvegarde aussi une grille "*_HEIGHT_DIFFERENCE_*.bin" qu'on ne veut pas garder
|
|
const strayGrid = await this.cc.findLatestFile(workDir, { suffix: '.bin', after: beforeVolume });
|
|
if (strayGrid && strayGrid !== path.join(workDir, binRelPath)) {
|
|
await fs.unlink(strayGrid).catch(() => undefined);
|
|
}
|
|
|
|
// Carte de hauteur : export ASCII du nuage + binning numpy cote Python.
|
|
// (Le paquet apt CloudCompare, utilise dans l'image Docker, plante sur l'export GeoTIFF natif
|
|
// -RASTERIZE -OUTPUT_RASTER_Z faute de support GDAL compile - on reconstruit donc la grille
|
|
// nous-memes a partir d'un export ASCII, qui lui fonctionne sans plugin.)
|
|
let imageRelPath: string | null = null;
|
|
try {
|
|
const xyzExportRel = path.posix.join('raw', `L${level}_points.xyz`);
|
|
await this.cc.run(['-O', binRelPath, '-C_EXPORT_FMT', 'ASC', '-PREC', '6', '-SAVE_CLOUDS', 'FILE', xyzExportRel], workDir);
|
|
const xyzAbs = path.join(workDir, xyzExportRel);
|
|
await this.assertExists(xyzAbs, `export ASCII niveau ${level}`, '');
|
|
const pngName = `L${level}_heightmap.png`;
|
|
const pngAbs = path.join(opts.imagesDir, pngName);
|
|
const title = spatialStep === null ? `Niveau 0 (complet, ${pointCount} pts)` : `Niveau ${level} (pas=${spatialStep.toFixed(4)}m, ${pointCount} pts)`;
|
|
await this.py.renderHeightmap(xyzAbs, gridStep, pngAbs, title);
|
|
imageRelPath = path.posix.join('images', pngName);
|
|
} catch (err: any) {
|
|
this.logger.warn(`Image niveau ${level} non generee: ${err.message}`);
|
|
}
|
|
|
|
const warn = vol.matchingCellsPct < config.matchingCellsWarnThreshold;
|
|
|
|
return {
|
|
level,
|
|
spatialStep,
|
|
pointCount,
|
|
pointRatio: refPointCount > 0 ? pointCount / refPointCount : 0,
|
|
volume: vol.volume,
|
|
surface: vol.surface,
|
|
addedVolume: vol.addedVolume,
|
|
removedVolume: vol.removedVolume,
|
|
matchingCellsPct: vol.matchingCellsPct,
|
|
groundNonMatchingPct: vol.groundNonMatchingPct,
|
|
ceilNonMatchingPct: vol.ceilNonMatchingPct,
|
|
warn,
|
|
status: 'ok',
|
|
binFile: binRelPath,
|
|
imageFile: imageRelPath,
|
|
computeTimeMs: Date.now() - t0,
|
|
};
|
|
}
|
|
|
|
private errorLevel(level: number, spatialStep: number | null, message: string, t0: number, binFile: string | null = null): LevelResult {
|
|
return {
|
|
level,
|
|
spatialStep,
|
|
pointCount: 0,
|
|
pointRatio: 0,
|
|
volume: null,
|
|
surface: null,
|
|
addedVolume: null,
|
|
removedVolume: null,
|
|
matchingCellsPct: null,
|
|
groundNonMatchingPct: null,
|
|
ceilNonMatchingPct: null,
|
|
warn: true,
|
|
status: 'error',
|
|
message,
|
|
binFile,
|
|
imageFile: null,
|
|
computeTimeMs: Date.now() - t0,
|
|
};
|
|
}
|
|
|
|
private toCsv(report: RunReport): string {
|
|
return reportToCsv(report);
|
|
}
|
|
}
|
|
|
|
export function reportToCsv(report: RunReport): string {
|
|
const headers = [
|
|
'level',
|
|
'spatialStep',
|
|
'pointCount',
|
|
'pointRatio',
|
|
'volume',
|
|
'surface',
|
|
'addedVolume',
|
|
'removedVolume',
|
|
'matchingCellsPct',
|
|
'groundNonMatchingPct',
|
|
'ceilNonMatchingPct',
|
|
'status',
|
|
'warn',
|
|
'computeTimeMs',
|
|
'message',
|
|
];
|
|
const rows = report.levels.map((l) =>
|
|
[
|
|
l.level,
|
|
l.spatialStep ?? '',
|
|
l.pointCount,
|
|
l.pointRatio,
|
|
l.volume ?? '',
|
|
l.surface ?? '',
|
|
l.addedVolume ?? '',
|
|
l.removedVolume ?? '',
|
|
l.matchingCellsPct ?? '',
|
|
l.groundNonMatchingPct ?? '',
|
|
l.ceilNonMatchingPct ?? '',
|
|
l.status,
|
|
l.warn,
|
|
l.computeTimeMs,
|
|
(l.message ?? '').replace(/[\r\n,]+/g, ' '),
|
|
].join(','),
|
|
);
|
|
return [headers.join(','), ...rows].join('\n') + '\n';
|
|
}
|