import crypto from 'node:crypto';
|
|
export const MANIFEST_COLUMNS = ['case_id', 'run_id', 'batch_id', 'artifact_path', 'sha256', 'bytes', 'mtime'];
|
export const PRIMARY_HEADER_TEXT = '"case_id","run_id","batch_id","artifact_path","sha256","bytes","mtime"';
|
|
const sha256 = value => crypto.createHash('sha256').update(value).digest('hex');
|
|
function splitPhysicalRecords(buffer) {
|
if (!Buffer.isBuffer(buffer)) throw new TypeError('MANIFEST_BUFFER_REQUIRED');
|
if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
|
return { records: [], errors: ['MANIFEST_BOM_FORBIDDEN'] };
|
}
|
const text = buffer.toString('utf8');
|
if (!Buffer.from(text, 'utf8').equals(buffer)) return { records: [], errors: ['MANIFEST_UTF8_INVALID'] };
|
const records = [];
|
let start = 0;
|
let inQuote = false;
|
let physicalLine = 1;
|
for (let i = 0; i < text.length; i++) {
|
const ch = text[i];
|
if (ch === '"') {
|
if (inQuote && text[i + 1] === '"') i++;
|
else inQuote = !inQuote;
|
} else if (ch === '\n' && !inQuote) {
|
const raw = text.slice(start, i + 1);
|
records.push({ physical_line: physicalLine, raw, body: raw.endsWith('\r\n') ? raw.slice(0, -2) : raw.slice(0, -1) });
|
start = i + 1;
|
physicalLine++;
|
}
|
}
|
const errors = [];
|
if (inQuote) errors.push('MANIFEST_UNCLOSED_QUOTE');
|
if (start !== text.length) {
|
records.push({ physical_line: physicalLine, raw: text.slice(start), body: text.slice(start) });
|
errors.push('MANIFEST_TERMINAL_NEWLINE_REQUIRED');
|
}
|
return { records, errors };
|
}
|
|
function parseRecord(body) {
|
const values = [];
|
let cell = '';
|
let quoted = false;
|
let closed = false;
|
for (let i = 0; i < body.length; i++) {
|
const ch = body[i];
|
if (!quoted && cell === '' && ch === '"') {
|
quoted = true;
|
closed = false;
|
} else if (quoted && ch === '"') {
|
if (body[i + 1] === '"') {
|
cell += '"';
|
i++;
|
} else {
|
quoted = false;
|
closed = true;
|
}
|
} else if (!quoted && ch === ',') {
|
values.push(cell);
|
cell = '';
|
closed = false;
|
} else if (!quoted && closed) {
|
return { values: [], error: 'CHARACTER_AFTER_CLOSING_QUOTE' };
|
} else {
|
cell += ch;
|
}
|
}
|
if (quoted) return { values: [], error: 'UNCLOSED_QUOTE' };
|
values.push(cell);
|
return { values, error: null };
|
}
|
|
export function parseAuthorityCsv(text) {
|
const { records, errors } = splitPhysicalRecords(Buffer.from(text, 'utf8'));
|
if (errors.length) return { header: [], rows: [], errors };
|
if (!records.length) return { header: [], rows: [], errors: ['CSV_EMPTY'] };
|
const parsed = records.map(record => parseRecord(record.body));
|
const parseErrors = parsed.flatMap((row, index) => row.error ? [`CSV_RECORD_PARSE:${index + 1}:${row.error}`] : []);
|
if (parseErrors.length) return { header: [], rows: [], errors: parseErrors };
|
const header = parsed[0].values;
|
const rows = parsed.slice(1).filter(row => row.values.some(value => value !== '')).map(row =>
|
Object.fromEntries(header.map((column, index) => [column, row.values[index] ?? '']))
|
);
|
return { header, rows, errors: [] };
|
}
|
|
export function serializeCsv(rows, columns) {
|
return `${columns.map(csvCell).join(',')}\n${rows.map(row => columns.map(column => csvCell(row[column])).join(',')).join('\n')}${rows.length ? '\n' : ''}`;
|
}
|
|
export function serializeManifestDataRows(rows) {
|
return rows.map(row => MANIFEST_COLUMNS.map(column => csvCell(row[column])).join(',')).join('\n') + (rows.length ? '\n' : '');
|
}
|
|
function csvCell(value) {
|
return `"${String(value ?? '').replace(/"/g, '""')}"`;
|
}
|
|
function authorityMap(authorityRows) {
|
const errors = [];
|
const map = new Map();
|
for (const row of authorityRows) {
|
const line = Number(row.physical_line);
|
if (!Number.isInteger(line) || line < 2) errors.push(`ANOMALY_AUTHORITY_LINE:${row.anomaly_id}`);
|
if (map.has(line)) errors.push(`ANOMALY_AUTHORITY_DUPLICATE_LINE:${line}`);
|
map.set(line, row);
|
}
|
return { map, errors };
|
}
|
|
function validManifestRow(values, physicalLine) {
|
const errors = [];
|
if (values.length !== MANIFEST_COLUMNS.length) return [`MANIFEST_FIELD_COUNT:${physicalLine}:${values.length}`];
|
const row = Object.fromEntries(MANIFEST_COLUMNS.map((column, index) => [column, values[index]]));
|
if (!row.case_id || !row.run_id || !row.batch_id || !row.artifact_path) errors.push(`MANIFEST_REQUIRED_FIELD:${physicalLine}`);
|
if (!/^[0-9a-f]{64}$/.test(row.sha256)) errors.push(`MANIFEST_SHA256:${physicalLine}`);
|
if (!/^(0|[1-9][0-9]*)$/.test(row.bytes)) errors.push(`MANIFEST_BYTES:${physicalLine}`);
|
const supportedTimestamp = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d{1,7})? ?(?:Z|[+-]\d{2}:\d{2})$/;
|
if (!supportedTimestamp.test(row.mtime) || !Number.isFinite(Date.parse(row.mtime))) {
|
errors.push(`MANIFEST_MTIME:${physicalLine}`);
|
}
|
if (row.artifact_path.includes('\\') || row.artifact_path.startsWith('/') || /^[A-Za-z]:/.test(row.artifact_path)) {
|
errors.push(`MANIFEST_PATH_NOT_RELATIVE_POSIX:${physicalLine}`);
|
}
|
return errors.length ? errors : row;
|
}
|
|
export function validateAppendStableManifest(buffer, {
|
anomalyRows,
|
frozenPrefix,
|
originalRunId,
|
originalRunExpectedRows = 11,
|
expectedRunRows = [],
|
expectedRunId = '',
|
expectedPhysicalDelta = null,
|
filesystemIdentity = null,
|
} = {}) {
|
const errors = [];
|
if (!Array.isArray(anomalyRows)) return { errors: ['ANOMALY_AUTHORITY_REQUIRED'], logicalRows: [], records: [], anomalies: [] };
|
if (frozenPrefix) {
|
if (buffer.length < frozenPrefix.bytes) errors.push('MANIFEST_PREFIX_TRUNCATED');
|
else if (sha256(buffer.subarray(0, frozenPrefix.bytes)) !== frozenPrefix.sha256) errors.push('MANIFEST_PREFIX_DRIFT');
|
}
|
const split = splitPhysicalRecords(buffer);
|
errors.push(...split.errors);
|
if (!split.records.length) return { errors: [...new Set([...errors, 'MANIFEST_EMPTY'])], logicalRows: [], records: [], anomalies: [] };
|
const header = parseRecord(split.records[0].body);
|
if (header.error || header.values.join('|') !== MANIFEST_COLUMNS.join('|')) errors.push('PRIMARY_HEADER_MISMATCH');
|
if (split.records[0].body !== PRIMARY_HEADER_TEXT) errors.push('PRIMARY_HEADER_SERIALIZATION_MISMATCH');
|
|
const auth = authorityMap(anomalyRows);
|
errors.push(...auth.errors);
|
const anomalies = [];
|
const logicalRows = [];
|
const seenAuthorityLines = new Set();
|
let headerShapeCount = 1;
|
for (const record of split.records.slice(1)) {
|
const parsed = parseRecord(record.body);
|
if (parsed.error) {
|
errors.push(`MANIFEST_RECORD_PARSE:${record.physical_line}:${parsed.error}`);
|
continue;
|
}
|
const isHeaderShape = parsed.values.join('|') === MANIFEST_COLUMNS.join('|');
|
if (isHeaderShape) headerShapeCount++;
|
const authority = auth.map.get(record.physical_line);
|
if (authority) {
|
seenAuthorityLines.add(record.physical_line);
|
const rawBytes = Buffer.byteLength(record.raw, 'utf8');
|
const rawSha = sha256(Buffer.from(record.raw, 'utf8'));
|
if (authority.disposition !== 'LEGACY_DUPLICATE_HEADER_NON_DATA_ROW') errors.push(`ANOMALY_DISPOSITION:${record.physical_line}`);
|
if (record.body !== PRIMARY_HEADER_TEXT || !isHeaderShape) errors.push(`ANOMALY_HEADER_SHAPE:${record.physical_line}`);
|
if (String(rawBytes) !== String(authority.raw_record_bytes)) errors.push(`ANOMALY_BYTES:${record.physical_line}`);
|
if (rawSha !== authority.raw_record_sha256) errors.push(`ANOMALY_SHA256:${record.physical_line}`);
|
if (authority.raw_record_terminator !== (record.raw.endsWith('\r\n') ? 'CRLF' : record.raw.endsWith('\n') ? 'LF' : 'NONE')) {
|
errors.push(`ANOMALY_TERMINATOR:${record.physical_line}`);
|
}
|
anomalies.push({ ...authority, observed_raw_record_sha256: rawSha });
|
continue;
|
}
|
if (isHeaderShape) {
|
errors.push(`UNAUTHORIZED_HEADER_SHAPED_ROW:${record.physical_line}`);
|
continue;
|
}
|
const valid = validManifestRow(parsed.values, record.physical_line);
|
if (Array.isArray(valid)) errors.push(...valid);
|
else logicalRows.push({ ...valid, physical_line: String(record.physical_line) });
|
}
|
for (const line of auth.map.keys()) if (!seenAuthorityLines.has(line)) errors.push(`ANOMALY_AUTHORITY_NOT_FOUND:${line}`);
|
if (anomalies.length !== anomalyRows.length) errors.push(`ANOMALY_COUNT:${anomalies.length}:${anomalyRows.length}`);
|
if (headerShapeCount !== 1 + anomalyRows.length) errors.push(`HEADER_SHAPE_COUNT:${headerShapeCount}:${1 + anomalyRows.length}`);
|
if (originalRunId) {
|
const rows = logicalRows.filter(row => row.run_id === originalRunId);
|
if (rows.length !== originalRunExpectedRows || new Set(rows.map(row => row.artifact_path)).size !== originalRunExpectedRows) {
|
errors.push(`ORIGINAL_RUN_ROWS:${rows.length}:${new Set(rows.map(row => row.artifact_path)).size}`);
|
}
|
}
|
if (expectedRunId) {
|
const actual = logicalRows.filter(row => row.run_id === expectedRunId);
|
if (actual.length !== expectedRunRows.length || new Set(actual.map(row => row.artifact_path)).size !== expectedRunRows.length) {
|
errors.push(`EXPECTED_RUN_ROWS:${actual.length}:${expectedRunRows.length}`);
|
}
|
for (let i = 0; i < expectedRunRows.length; i++) {
|
const expected = expectedRunRows[i];
|
const observed = actual[i];
|
if (!observed || MANIFEST_COLUMNS.some(column => observed[column] !== String(expected[column] ?? ''))) {
|
errors.push(`EXPECTED_RUN_FIELD:${i + 1}`);
|
break;
|
}
|
}
|
}
|
if (frozenPrefix && expectedPhysicalDelta !== null) {
|
const prefixSplit = splitPhysicalRecords(buffer.subarray(0, frozenPrefix.bytes));
|
const delta = split.records.length - prefixSplit.records.length;
|
if (delta !== expectedPhysicalDelta) errors.push(`PHYSICAL_RECORD_DELTA:${delta}:${expectedPhysicalDelta}`);
|
}
|
if (filesystemIdentity) {
|
for (const row of logicalRows.filter(row => filesystemIdentity.runIds?.includes(row.run_id))) {
|
const identity = filesystemIdentity.get(row.artifact_path);
|
if (!identity) errors.push(`FILESYSTEM_PATH_MISSING:${row.artifact_path}`);
|
else if (row.sha256 !== identity.sha256 || row.bytes !== String(identity.bytes) || Date.parse(row.mtime) !== identity.mtimeMs) {
|
errors.push(`FILESYSTEM_IDENTITY:${row.artifact_path}`);
|
}
|
}
|
}
|
return {
|
errors: [...new Set(errors)],
|
logicalRows,
|
records: split.records,
|
anomalies,
|
primaryHeaderCount: 1,
|
headerShapeCount,
|
logicalDataRows: logicalRows.length,
|
physicalRecords: split.records.length,
|
};
|
}
|
|
export function hashBuffer(buffer) {
|
return sha256(buffer);
|
}
|
