From efbfc27e0c813c31b570d72fded40eb91d07f2db Mon Sep 17 00:00:00 2001
From: Ariver <shanghai3168@gmail.com>
Date: Thu, 28 May 2026 21:01:19 +0800
Subject: [PATCH] Add AppKit tag navigation reordering
---
Research/SmartStart/Scripts/build-ultimate-default-catalog.mjs | 1208 ++++++++++++++++++++++++++++++++++++++++-----------------
1 files changed, 840 insertions(+), 368 deletions(-)
diff --git a/Research/SmartStart/Scripts/build-ultimate-default-catalog.mjs b/Research/SmartStart/Scripts/build-ultimate-default-catalog.mjs
index 87a58f3..c8d46de 100644
--- a/Research/SmartStart/Scripts/build-ultimate-default-catalog.mjs
+++ b/Research/SmartStart/Scripts/build-ultimate-default-catalog.mjs
@@ -2,6 +2,8 @@
import fs from "node:fs";
import path from "node:path";
+import { spawnSync } from "node:child_process";
+import crypto from "node:crypto";
import { fileURLToPath } from "node:url";
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
@@ -10,10 +12,7 @@
const outputDir = path.join(researchDir, "UltimateDefaultCatalog");
const sources = {
- master5000: path.join(researchDir, "MacCommonApps_CandidateTop5000_Master.csv"),
- persona: path.join(researchDir, "PersonaTopApps", "MacPersonaTopApps_Unique.csv"),
- game: path.join(researchDir, "game.csv"),
- top1000: path.join(researchDir, "AppDefaultTags_Review.csv"),
+ curatedCatalog: path.join(outputDir, "SmartStart_UltimateDefaultCatalog.csv"),
appleNotes: path.join(projectDir, "Apptag", "AppleDefaultAppNotes.swift"),
localizationDir: path.join(projectDir, "Apptag", "Localization"),
};
@@ -21,19 +20,40 @@
const outputs = {
reviewCSV: path.join(outputDir, "SmartStart_UltimateDefaultCatalog.csv"),
runtimeJSON: path.join(outputDir, "SmartStart_UltimateDefaultCatalog.json"),
+ runtimeBaseJSON: path.join(outputDir, "SmartStart_UltimateDefaultCatalog.base.json"),
+ runtimeManifestJSON: path.join(outputDir, "SmartStart_UltimateDefaultCatalog.manifest.json"),
report: path.join(outputDir, "SmartStart_UltimateDefaultCatalog_Report.md"),
duplicateReview: path.join(outputDir, "SmartStart_UltimateDefaultCatalog_DuplicateReview.csv"),
translationQA: path.join(outputDir, "SmartStart_UltimateDefaultCatalog_TranslationQA.md"),
+ sourceNoteTranslationCache: path.join(outputDir, "SmartStart_UltimateDefaultCatalog_SourceNoteTranslationCache.json"),
};
+
+const resourceFormatVersion = 1;
+const catalogContentVersion = 2;
+const notesVersion = 1;
+const requiredFallbackLanguages = ["en", "zh-Hans", "zh-Hant"];
const stableTagOrder = [
"browser",
"communication",
- "productivity",
+ "GTD",
+ "Notes",
+ "Meeting",
+ "office",
+ "PDF",
"file-management",
"transfer",
- "development",
- "design",
+ "ai-tools",
+ "api-tools",
+ "database-tools",
+ "devops",
+ "ide",
+ "runtime-sdk",
+ "terminal-tools",
+ "Font",
+ "ui-prototyping",
+ "3d-cad",
+ "diagramming",
"writing",
"media",
"video",
@@ -41,110 +61,31 @@
"picture-photo",
"utilities",
"system",
- "system-enhancement",
+ "system-maintenance",
+ "window-management",
+ "device-management",
+ "input-tools",
+ "Automation",
+ "network-tools",
"entertainment",
"game",
"finance",
"education",
- "ai-tools",
"security",
"other",
];
const stableTags = new Set(stableTagOrder);
const noteLimit = 80;
-
-const sourcePriority = {
- apple: 100,
- persona: 90,
- top1000: 80,
- game: 70,
- master5000: 60,
-};
-
-const tagOverridesByNormalizedName = new Map(Object.entries({
- "android-ndk": ["development", "utilities"],
- "codeql": ["development", "security"],
- "prince": ["writing", "development", "utilities"],
- "qlstephen": ["utilities", "system-enhancement"],
- "kotlin-lsp": ["development"],
- "ghdl": ["development"],
- "android-sdk-platform-tools": ["development", "utilities"],
- "android-sdk-command-line-tools": ["development", "utilities"],
-}));
-
-const appleCatalogMetadata = new Map(Object.entries({
- "com.apple.activitymonitor": { name: "Activity Monitor", tags: ["system", "utilities", "system-enhancement"] },
- "com.apple.airport.airportutility": { name: "AirPort Utility", tags: ["system", "utilities"] },
- "com.apple.appstore": { name: "App Store", tags: ["system", "utilities"] },
- "com.apple.apps.launcher": { name: "Apps", tags: ["system", "utilities"] },
- "com.apple.audio.audiomidisetup": { name: "Audio MIDI Setup", tags: ["audio", "system", "utilities"] },
- "com.apple.automator": { name: "Automator", tags: ["system-enhancement", "utilities"] },
- "com.apple.bluetoothfileexchange": { name: "Bluetooth File Exchange", tags: ["transfer", "system", "utilities"] },
- "com.apple.ibooksx": { name: "Books", tags: ["writing", "education", "entertainment"] },
- "com.apple.bootcampassistant": { name: "Boot Camp Assistant", tags: ["system", "utilities"] },
- "com.apple.calculator": { name: "Calculator", tags: ["utilities"] },
- "com.apple.ical": { name: "Calendar", tags: ["productivity"] },
- "com.apple.chess": { name: "Chess", tags: ["game", "entertainment"] },
- "com.apple.clock": { name: "Clock", tags: ["productivity", "utilities"] },
- "com.apple.colorsyncutility": { name: "ColorSync Utility", tags: ["design", "picture-photo", "system", "utilities"] },
- "com.apple.console": { name: "Console", tags: ["system", "utilities"] },
- "com.apple.addressbook": { name: "Contacts", tags: ["communication", "productivity"] },
- "com.apple.dictionary": { name: "Dictionary", tags: ["writing", "education", "utilities"] },
- "com.apple.digitalcolormeter": { name: "Digital Color Meter", tags: ["design", "picture-photo", "utilities"] },
- "com.apple.diskutility": { name: "Disk Utility", tags: ["system", "utilities"] },
- "com.apple.facetime": { name: "FaceTime", tags: ["communication", "video"] },
- "com.apple.findmy": { name: "Find My", tags: ["system", "utilities"] },
- "com.apple.fontbook": { name: "Font Book", tags: ["design", "utilities"] },
- "com.apple.freeform": { name: "Freeform", tags: ["productivity", "design"] },
- "com.apple.games": { name: "Games", tags: ["game", "entertainment"] },
- "com.apple.grapher": { name: "Grapher", tags: ["education", "utilities"] },
- "com.apple.home": { name: "Home", tags: ["system", "utilities"] },
- "com.apple.image_capture": { name: "Image Capture", tags: ["picture-photo", "transfer", "utilities"] },
- "com.apple.generativeplaygroundapp": { name: "Image Playground", tags: ["ai-tools", "picture-photo", "design"] },
- "com.apple.imovieapp": { name: "iMovie", tags: ["video", "media"] },
- "com.apple.screencontinuity": { name: "iPhone Mirroring", tags: ["system-enhancement", "utilities"] },
- "com.apple.journal": { name: "Journal", tags: ["writing", "productivity"] },
- "com.apple.iwork.keynote": { name: "Keynote", tags: ["productivity", "writing"] },
- "com.apple.magnifier": { name: "Magnifier", tags: ["system", "utilities"] },
- "com.apple.mail": { name: "Mail", tags: ["communication"] },
- "com.apple.maps": { name: "Maps", tags: ["productivity", "utilities"] },
- "com.apple.mobilesms": { name: "Messages", tags: ["communication"] },
- "com.apple.migrateassistant": { name: "Migration Assistant", tags: ["transfer", "system", "utilities"] },
- "com.apple.exposelauncher": { name: "Mission Control", tags: ["system", "system-enhancement"] },
- "com.apple.music": { name: "Music", tags: ["audio", "media", "entertainment"] },
- "com.apple.news": { name: "News", tags: ["media", "writing"] },
- "com.apple.notes": { name: "Notes", tags: ["writing", "productivity"] },
- "com.apple.iwork.numbers": { name: "Numbers", tags: ["productivity"] },
- "com.apple.iwork.pages": { name: "Pages", tags: ["writing"] },
- "com.apple.passwords": { name: "Passwords", tags: ["security", "utilities"] },
- "com.apple.mobilephone": { name: "Phone", tags: ["communication"] },
- "com.apple.photobooth": { name: "Photo Booth", tags: ["picture-photo", "media", "entertainment"] },
- "com.apple.photos": { name: "Photos", tags: ["picture-photo", "media"] },
- "com.apple.podcasts": { name: "Podcasts", tags: ["audio", "media", "entertainment"] },
- "com.apple.preview": { name: "Preview", tags: ["picture-photo", "writing", "utilities"] },
- "com.apple.printcenter": { name: "Print Center", tags: ["system", "utilities"] },
- "com.apple.quicktimeplayerx": { name: "QuickTime Player", tags: ["video", "audio", "media"] },
- "com.apple.reminders": { name: "Reminders", tags: ["productivity"] },
- "com.apple.safari": { name: "Safari", tags: ["browser"] },
- "com.apple.screensharing": { name: "Screen Sharing", tags: ["system", "communication", "utilities"] },
- "com.apple.screenshot.launcher": { name: "Screenshot", tags: ["picture-photo", "video", "utilities"] },
- "com.apple.scripteditor2": { name: "Script Editor", tags: ["development", "system-enhancement", "utilities"] },
- "com.apple.sfsymbols": { name: "SF Symbols", tags: ["design", "development"] },
- "com.apple.shortcuts": { name: "Shortcuts", tags: ["system-enhancement", "productivity", "utilities"] },
- "com.apple.siri.launcher": { name: "Siri", tags: ["system", "utilities"] },
- "com.apple.stickies": { name: "Stickies", tags: ["writing", "productivity"] },
- "com.apple.stocks": { name: "Stocks", tags: ["finance"] },
- "com.apple.systemprofiler": { name: "System Information", tags: ["system", "utilities"] },
- "com.apple.systempreferences": { name: "System Settings", tags: ["system", "utilities"] },
- "com.apple.terminal": { name: "Terminal", tags: ["development", "system", "utilities"] },
- "com.apple.textedit": { name: "TextEdit", tags: ["writing"] },
- "com.apple.backup.launcher": { name: "Time Machine", tags: ["system", "utilities"] },
- "com.apple.helpviewer": { name: "Tips", tags: ["education", "utilities"] },
- "com.apple.tv": { name: "TV", tags: ["video", "media", "entertainment"] },
- "com.apple.voicememos": { name: "Voice Memos", tags: ["audio", "media"] },
- "com.apple.voiceoverutility": { name: "VoiceOver Utility", tags: ["system", "utilities"] },
- "com.apple.weather": { name: "Weather", tags: ["utilities"] },
- "com.apple.dt.xcode": { name: "Xcode", tags: ["development"] },
+const translationProvider = "google-translate-gtx-source-notes-20260520";
+const translationBackend = process.env.SMARTSTART_TRANSLATION_BACKEND || "google-gtx";
+const lingvaBaseURL = process.env.SMARTSTART_LINGVA_BASE_URL || "https://lingva.ml";
+const translationBatchSize = Number(process.env.SMARTSTART_TRANSLATION_BATCH_SIZE || 50);
+const translationPauseMS = Number(process.env.SMARTSTART_TRANSLATION_PAUSE_MS || 150);
+const translationRetryLimit = Number(process.env.SMARTSTART_TRANSLATION_RETRY_LIMIT || 4);
+const equivalentTargetLanguages = new Map(Object.entries({
+ "ar-Najdi": ["ar"],
+ "nn": ["nb", "no"],
+ "no": ["nb", "nn"],
}));
function parseCSV(text) {
@@ -210,7 +151,9 @@
function readCSVObjects(filePath) {
const rows = parseCSV(fs.readFileSync(filePath, "utf8"));
- const header = rows[0] ?? [];
+ const header = (rows[0] ?? []).map((column, index) =>
+ index === 0 ? String(column ?? "").replace(/^\uFEFF/, "") : column,
+ );
return rows.slice(1).map((row) =>
Object.fromEntries(header.map((column, index) => [column, row[index] ?? ""])),
);
@@ -232,8 +175,7 @@
cleaned.push(tag);
}
const meaningful = cleaned.filter((tag) => tag !== "other");
- const finalTags = meaningful.length > 0 ? meaningful : cleaned;
- return stableTagOrder.filter((tag) => finalTags.includes(tag));
+ return meaningful.length > 0 ? meaningful : cleaned;
}
function normalizeName(value) {
@@ -244,6 +186,52 @@
.replace(/&/g, " and ")
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "") || String(value ?? "").trim().toLowerCase();
+}
+
+function sha256(value) {
+ return crypto.createHash("sha256").update(value).digest("hex");
+}
+
+function smartStartNotesPath(languageCode) {
+ return path.join(outputDir, `SmartStart_UltimateDefaultCatalog.notes.${languageCode}.json`);
+}
+
+function entryIDFor(row) {
+ const baseSource = row.bundleIdentifier || row.normalizedName || row.name || "entry";
+ const base = normalizeName(baseSource) || "entry";
+ const identity = [
+ row.bundleIdentifier ?? "",
+ row.normalizedName ?? "",
+ row.name ?? "",
+ ].join("\u0000");
+ return `${base}-${sha256(identity).slice(0, 12)}`;
+}
+
+function loadRuntimeSmartCategoryIDs() {
+ const filePath = path.join(projectDir, "Apptag", "SmartCategorization", "SmartCategory.swift");
+ const source = fs.readFileSync(filePath, "utf8");
+ const enumBody = source.match(/enum SmartCategoryID:[\s\S]*?\n}/)?.[0] ?? "";
+ const ids = new Set();
+ const casePattern = /^\s*case\s+([A-Za-z_][A-Za-z0-9_]*)(?:\s*=\s*"([^"]+)")?/gm;
+ let match;
+ while ((match = casePattern.exec(enumBody)) !== null) {
+ ids.add(match[2] ?? match[1]);
+ }
+ return ids;
+}
+
+function normalizedForNoteComparison(value) {
+ return String(value ?? "")
+ .normalize("NFKD")
+ .replace(/[\u0300-\u036f]/g, "")
+ .toLowerCase()
+ .replace(/[^a-z0-9\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af]+/g, "");
+}
+
+function noteRepeatsAppName(note, name) {
+ const normalizedNameValue = normalizedForNoteComparison(name);
+ if (normalizedNameValue.length < 3) return false;
+ return normalizedForNoteComparison(note).includes(normalizedNameValue);
}
function normalizeBundle(value) {
@@ -258,60 +246,472 @@
return Array.from(String(value ?? "")).length;
}
+function stripTerminalPunctuation(value) {
+ return String(value ?? "")
+ .replace(/\s+/g, " ")
+ .trim()
+ .replace(/[。．\.!！\?？;；:：,，、]+$/u, "")
+ .trim();
+}
+
+function stripLeadingPunctuation(value) {
+ const clean = String(value ?? "")
+ .replace(/\s+/g, " ")
+ .trim();
+
+ if (clean.startsWith(".NET")) {
+ return `Microsoft .NET${clean.slice(".NET".length)}`.trim();
+ }
+
+ return clean
+ .replace(/^[\p{P}\s]+/u, "")
+ .trim();
+}
+
function truncateNote(value) {
- const clean = String(value ?? "").replace(/\s+/g, " ").trim();
+ const clean = stripTerminalPunctuation(stripLeadingPunctuation(value));
if (charLength(clean) <= noteLimit) return clean;
let sliced = Array.from(clean).slice(0, noteLimit).join("");
- sliced = sliced.replace(/[，、；：,.!！?？]$/, "");
- if (!/[。.!！?？]$/.test(sliced)) {
- sliced = `${Array.from(sliced).slice(0, noteLimit - 1).join("")}。`;
- }
- return sliced;
+ return stripTerminalPunctuation(sliced);
}
-function loadCategoryTranslations() {
- const translations = new Map();
+function normalizeTranslatedNote(value) {
+ return truncateNote(value);
+}
+
+function loadSupportedLanguageCodes() {
+ const codes = [];
for (const fileName of fs.readdirSync(sources.localizationDir).sort()) {
if (!fileName.endsWith(".json")) continue;
- const code = fileName.replace(/\.json$/, "");
- const raw = JSON.parse(fs.readFileSync(path.join(sources.localizationDir, fileName), "utf8"));
- const categories = {};
- for (const tag of stableTagOrder) {
- categories[tag] = raw[`smart.category.${tag}`] ?? tag;
- }
- translations.set(code, categories);
+ codes.push(fileName.replace(/\.json$/, ""));
}
- return translations;
+ return codes.sort();
}
-function localizedGeneratedNote(name, tags, code, translations) {
- const categoryMap = translations.get(code) ?? translations.get("en") ?? {};
- const meaningfulTags = tags.filter((tag) => tag !== "other").slice(0, 2);
- const shownTags = meaningfulTags.length > 0 ? meaningfulTags : tags.slice(0, 1);
- const labels = shownTags.map((tag) => categoryMap[tag] ?? tag);
- const joiner = ["ar", "ar-Najdi"].includes(code) ? "، " : ["zh-Hans", "zh-Hant", "ja"].includes(code) ? "、" : ", ";
- const separator = ["zh-Hans", "zh-Hant", "ja"].includes(code) ? "：" : ": ";
- const end = ["zh-Hans", "zh-Hant", "ja"].includes(code) ? "。" : ".";
- const full = `${name}${separator}${labels.join(joiner)}${end}`;
- if (charLength(full) <= noteLimit) return full;
-
- const compact = `${name}${separator}${labels[0] ?? "App"}${end}`;
- if (charLength(compact) <= noteLimit) return compact;
-
- return truncateNote(compact);
+function googleLanguageCode(code) {
+ switch (code) {
+ case "zh-Hans": return "zh-CN";
+ case "zh-Hant": return "zh-TW";
+ case "pt-BR": return "pt";
+ case "sr-Cyrl": return "sr";
+ case "ar-Najdi": return "ar";
+ case "nb":
+ case "nn":
+ case "no":
+ return "no";
+ default:
+ return code;
+ }
}
-function buildLocalizedNotes(name, tags, noteZH, translations) {
- if (!noteZH) return {};
+function sourceNotesFingerprint(sourceNotes) {
+ return crypto
+ .createHash("sha256")
+ .update(JSON.stringify([...sourceNotes].sort()))
+ .digest("hex");
+}
+
+function translationCacheKey(targetLanguage, sourceNote) {
+ return ["zh-Hans", targetLanguage, sourceNote].join("\u0001");
+}
+
+function loadSourceNoteTranslationCache(sourceFingerprint) {
+ if (!fs.existsSync(outputs.sourceNoteTranslationCache)) {
+ return {
+ version: 1,
+ provider: translationProvider,
+ sourceFingerprint,
+ entries: {},
+ ignoredExistingCache: false,
+ };
+ }
+
+ const cache = JSON.parse(fs.readFileSync(outputs.sourceNoteTranslationCache, "utf8"));
+ if (
+ cache.provider !== translationProvider ||
+ cache.sourceFingerprint !== sourceFingerprint ||
+ cache.version !== 1
+ ) {
+ return {
+ version: 1,
+ provider: translationProvider,
+ sourceFingerprint,
+ entries: {},
+ ignoredExistingCache: true,
+ };
+ }
+
+ return {
+ version: 1,
+ provider: translationProvider,
+ sourceFingerprint,
+ entries: cache.entries ?? {},
+ ignoredExistingCache: false,
+ };
+}
+
+function saveSourceNoteTranslationCache(cache) {
+ const sortedEntries = Object.fromEntries(
+ Object.entries(cache.entries).sort(([left], [right]) => left.localeCompare(right)),
+ );
+ fs.writeFileSync(outputs.sourceNoteTranslationCache, `${JSON.stringify({
+ version: 1,
+ provider: translationProvider,
+ sourceFingerprint: cache.sourceFingerprint,
+ updatedAt: new Date().toISOString(),
+ entries: sortedEntries,
+ }, null, 2)}\n`);
+}
+
+function sleep(ms) {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function parseGoogleTranslation(payload) {
+ return (payload?.[0] ?? [])
+ .map((segment) => segment?.[0] ?? "")
+ .join("");
+}
+
+async function requestGoogleTranslationPayload(params, attempt = 0) {
+ const result = spawnSync("curl", [
+ "-sS",
+ "--fail",
+ "--compressed",
+ "--max-time",
+ "45",
+ "--retry",
+ "2",
+ "--retry-delay",
+ "1",
+ "-A",
+ "TagLauncher-CatalogOps/1.0",
+ "-H",
+ "content-type: application/x-www-form-urlencoded;charset=UTF-8",
+ "--data",
+ params.toString(),
+ "https://translate.googleapis.com/translate_a/single",
+ ], {
+ encoding: "utf8",
+ maxBuffer: 16 * 1024 * 1024,
+ });
+
+ if (result.status === 0) {
+ try {
+ return JSON.parse(result.stdout);
+ } catch (error) {
+ if (attempt < translationRetryLimit) {
+ await sleep(2500 * (attempt + 1));
+ return requestGoogleTranslationPayload(params, attempt + 1);
+ }
+ throw new Error(`translation response was not JSON: ${error.message}`);
+ }
+ }
+
+ if (attempt < translationRetryLimit) {
+ await sleep(2500 * (attempt + 1));
+ return requestGoogleTranslationPayload(params, attempt + 1);
+ }
+
+ throw new Error(`translation request failed: ${result.stderr || result.stdout || result.error?.message}`);
+}
+
+async function requestLingvaTranslationBatch(notes, targetLanguage, attempt = 0) {
+ const targetCode = googleLanguageCode(targetLanguage);
+ const query = encodeURIComponent(notes.map((note) => note.replace(/\//g, "／")).join("\n"));
+ const result = spawnSync("curl", [
+ "-sS",
+ "--fail",
+ "--compressed",
+ "--max-time",
+ "60",
+ "--retry",
+ "4",
+ "--retry-all-errors",
+ "--retry-connrefused",
+ "--retry-delay",
+ "2",
+ "-A",
+ "TagLauncher-CatalogOps/1.0",
+ `${lingvaBaseURL.replace(/\/+$/, "")}/api/v1/zh/${targetCode}/${query}`,
+ ], {
+ encoding: "utf8",
+ maxBuffer: 16 * 1024 * 1024,
+ });
+
+ if (result.status === 0) {
+ try {
+ const payload = JSON.parse(result.stdout);
+ if (payload.error) throw new Error(payload.error);
+ const translated = String(payload.translation ?? "")
+ .replace(/\r\n/g, "\n")
+ .replace(/\r/g, "\n");
+ const parts = translated.split("\n").map(normalizeTranslatedNote);
+ if (parts.length !== notes.length || parts.some((part) => !part)) {
+ throw new Error(`translation batch line mismatch: expected ${notes.length}, got ${parts.length}`);
+ }
+ return parts;
+ } catch (error) {
+ if (attempt < translationRetryLimit && isRemoteThrottleError(error)) {
+ await sleep(2500 * (attempt + 1));
+ return requestLingvaTranslationBatch(notes, targetLanguage, attempt + 1);
+ }
+ throw error;
+ }
+ }
+
+ if (attempt < translationRetryLimit) {
+ await sleep(2500 * (attempt + 1));
+ return requestLingvaTranslationBatch(notes, targetLanguage, attempt + 1);
+ }
+
+ throw new Error(`lingva translation request failed: ${result.stderr || result.stdout || result.error?.message}`);
+}
+
+async function requestOnlineTranslationBatch(notes, targetLanguage) {
+ if (translationBackend === "lingva") {
+ return requestLingvaTranslationBatch(notes, targetLanguage);
+ }
+
+ const params = new URLSearchParams({
+ client: "gtx",
+ sl: googleLanguageCode("zh-Hans"),
+ tl: googleLanguageCode(targetLanguage),
+ dt: "t",
+ q: notes.join("\n"),
+ });
+ const translated = parseGoogleTranslation(await requestGoogleTranslationPayload(params))
+ .replace(/\r\n/g, "\n")
+ .replace(/\r/g, "\n");
+ const parts = translated.split("\n").map(normalizeTranslatedNote);
+ if (parts.length !== notes.length || parts.some((part) => !part)) {
+ throw new Error(`translation batch line mismatch: expected ${notes.length}, got ${parts.length}`);
+ }
+ return parts;
+}
+
+function isRemoteThrottleError(error) {
+ const message = String(error?.message ?? "");
+ return message.includes("not JSON") || message.includes("translation request failed");
+}
+
+async function translateNotesWithSplit(notes, targetLanguage) {
+ try {
+ return await requestOnlineTranslationBatch(notes, targetLanguage);
+ } catch (error) {
+ if (isRemoteThrottleError(error) || notes.length <= 1) {
+ throw error;
+ }
+ const middle = Math.ceil(notes.length / 2);
+ const left = await translateNotesWithSplit(notes.slice(0, middle), targetLanguage);
+ const right = await translateNotesWithSplit(notes.slice(middle), targetLanguage);
+ return [...left, ...right];
+ }
+}
+
+function reusableTranslation(cache, targetLanguage, sourceNote) {
+ const directKey = translationCacheKey(targetLanguage, sourceNote);
+ if (cache.entries[directKey]) return cache.entries[directKey];
+
+ for (const equivalentLanguage of equivalentTargetLanguages.get(targetLanguage) ?? []) {
+ const equivalentKey = translationCacheKey(equivalentLanguage, sourceNote);
+ if (cache.entries[equivalentKey]) return cache.entries[equivalentKey];
+ }
+ return null;
+}
+
+async function translateMissingNotes(uniqueSourceNotes, supportedLanguages, cache) {
+ let translatedCount = 0;
+ let failedCount = 0;
+ let reusedEquivalentCount = 0;
+ let cacheHits = 0;
+
+ for (const sourceNote of uniqueSourceNotes) {
+ cache.entries[translationCacheKey("zh-Hans", sourceNote)] = normalizeTranslatedNote(sourceNote);
+ }
+
+ for (const targetLanguage of supportedLanguages) {
+ if (targetLanguage === "zh-Hans" || targetLanguage === "zh-Hant") continue;
+
+ const missing = [];
+ for (const sourceNote of uniqueSourceNotes) {
+ const key = translationCacheKey(targetLanguage, sourceNote);
+ if (cache.entries[key]) {
+ cacheHits += 1;
+ continue;
+ }
+ const reusable = reusableTranslation(cache, targetLanguage, sourceNote);
+ if (reusable) {
+ cache.entries[key] = reusable;
+ reusedEquivalentCount += 1;
+ continue;
+ }
+ missing.push({ sourceNote, key });
+ }
+
+ if (missing.length > 0) {
+ console.warn(`Translating ${missing.length} source notes to ${targetLanguage}...`);
+ }
+
+ for (let index = 0; index < missing.length; index += translationBatchSize) {
+ const batch = missing.slice(index, index + translationBatchSize);
+ try {
+ const translations = await translateNotesWithSplit(
+ batch.map((item) => item.sourceNote),
+ targetLanguage,
+ );
+ translations.forEach((translation, translationIndex) => {
+ cache.entries[batch[translationIndex].key] = translation;
+ translatedCount += 1;
+ });
+ } catch (error) {
+ failedCount += batch.length;
+ saveSourceNoteTranslationCache(cache);
+ throw new Error(`Batch failed for ${targetLanguage}; cached progress saved.\n${error.message}`);
+ }
+ saveSourceNoteTranslationCache(cache);
+ await sleep(translationPauseMS);
+ }
+ }
+
+ saveSourceNoteTranslationCache(cache);
+ return { translatedCount, failedCount, reusedEquivalentCount, cacheHits };
+}
+
+function convertSimplifiedNotesToTraditional(sourceNotes) {
+ const notes = sourceNotes.map(normalizeTranslatedNote).filter(Boolean);
+ if (notes.length === 0) return new Map();
+
+ const swiftCode = `
+import Foundation
+
+let inputData = FileHandle.standardInput.readDataToEndOfFile()
+let notes = try JSONDecoder().decode([String].self, from: inputData)
+let converted = notes.map { note -> String in
+ let mutable = NSMutableString(string: note)
+ CFStringTransform(mutable, nil, "Simplified-Traditional" as CFString, false)
+ return mutable as String
+}
+let outputData = try JSONEncoder().encode(converted)
+FileHandle.standardOutput.write(outputData)
+`;
+ const result = spawnSync("/usr/bin/swift", ["-e", swiftCode], {
+ input: JSON.stringify(notes),
+ encoding: "utf8",
+ maxBuffer: 64 * 1024 * 1024,
+ });
+
+ if (result.error) {
+ throw result.error;
+ }
+ if (result.status !== 0) {
+ throw new Error(`zh-Hant conversion failed:\n${result.stderr}`);
+ }
+
+ const converted = JSON.parse(result.stdout);
+ if (!Array.isArray(converted) || converted.length !== notes.length) {
+ throw new Error("zh-Hant conversion returned an unexpected result shape");
+ }
+
+ return new Map(notes.map((note, index) => [note, normalizeTranslatedNote(converted[index])]));
+}
+
+function buildLocalizedNotes(sourceNote, supportedLanguages, cache, traditionalNote) {
+ const cleanSourceNote = normalizeTranslatedNote(sourceNote);
+ if (!cleanSourceNote) return {};
+
const notes = {};
- for (const code of translations.keys()) {
- if (code === "zh-Hans") {
- notes[code] = truncateNote(noteZH);
- } else {
- notes[code] = localizedGeneratedNote(name, tags, code, translations);
+ for (const languageCode of supportedLanguages) {
+ if (languageCode === "zh-Hans") {
+ notes[languageCode] = cleanSourceNote;
+ continue;
}
+ if (languageCode === "zh-Hant") {
+ const cleanTraditionalNote = normalizeTranslatedNote(traditionalNote);
+ if (cleanTraditionalNote) notes[languageCode] = cleanTraditionalNote;
+ continue;
+ }
+ const translatedNote = normalizeTranslatedNote(cache.entries[translationCacheKey(languageCode, cleanSourceNote)]);
+ if (translatedNote) notes[languageCode] = translatedNote;
}
- return notes;
+ return Object.fromEntries(
+ Object.entries(notes).sort(([left], [right]) => left.localeCompare(right)),
+ );
+}
+
+function writeJSON(filePath, value) {
+ const text = `${JSON.stringify(value, null, 2)}\n`;
+ fs.writeFileSync(filePath, text);
+ return {
+ file: path.basename(filePath),
+ sha256: sha256(text),
+ bytes: Buffer.byteLength(text),
+ };
+}
+
+function buildSplitResources({ finalRows, supportedLanguages, generatedAt, noteLimit }) {
+ const baseResource = {
+ resourceFormatVersion,
+ catalogContentVersion,
+ generatedAt,
+ noteLimit,
+ supportedLanguages,
+ fallbackLanguages: requiredFallbackLanguages,
+ entries: finalRows.map((row) => ({
+ entryID: row.entryID,
+ rank: row.rank,
+ name: row.name,
+ normalizedName: row.normalizedName,
+ bundleIdentifier: row.bundleIdentifier,
+ defaultTag: row.tags,
+ sourceEvidence: row.sourceEvidence,
+ })),
+ };
+ const baseInfo = writeJSON(outputs.runtimeBaseJSON, baseResource);
+
+ const notesResources = {};
+ for (const languageCode of supportedLanguages) {
+ const notesResource = {
+ resourceFormatVersion,
+ catalogContentVersion,
+ notesVersion,
+ generatedAt,
+ language: languageCode,
+ entries: finalRows
+ .map((row) => ({
+ entryID: row.entryID,
+ note: row.notes[languageCode],
+ }))
+ .filter((entry) => entry.note),
+ };
+ const notesPath = smartStartNotesPath(languageCode);
+ notesResources[languageCode] = {
+ ...writeJSON(notesPath, notesResource),
+ count: notesResource.entries.length,
+ };
+ }
+
+ const manifest = {
+ resourceFormatVersion,
+ catalogContentVersion,
+ notesVersion,
+ generatedAt,
+ noteLimit,
+ supportedLanguages,
+ fallbackLanguages: requiredFallbackLanguages,
+ baseResource: {
+ ...baseInfo,
+ count: baseResource.entries.length,
+ },
+ notesResources,
+ };
+ const manifestInfo = writeJSON(outputs.runtimeManifestJSON, manifest);
+
+ return {
+ baseInfo,
+ manifestInfo,
+ notesResources,
+ };
}
function parseAppleNotes() {
@@ -391,236 +791,230 @@
stats.mergedRows += 1;
}
-function build() {
- fs.mkdirSync(outputDir, { recursive: true });
- const translations = loadCategoryTranslations();
- const appleNotes = parseAppleNotes();
- const entries = [];
- const byBundle = new Map();
- const byName = new Map();
- const ambiguous = [];
- const mergedGroups = [];
- const stats = {
- sourceRows: {},
- mergedRows: 0,
- appleNotesAttached: 0,
- appleNotesSwiftOnly: 0,
+function mapCuratedTags(value) {
+ const tags = [];
+ const unknownTokens = [];
+ for (const token of splitTags(value)) {
+ if (!stableTags.has(token)) {
+ unknownTokens.push(token);
+ continue;
+ }
+ tags.push(token);
+ }
+
+ return {
+ tags: orderedTags(tags),
+ unknownTokens,
};
+}
- function addEntry(incoming) {
- if (!incoming.name || incoming.tags.length === 0) return;
+const duplicateAliasSuffixes = ["-app", "-desktop", "-mac", "-macos"];
- const bundleKey = incoming.bundleIdentifier?.toLowerCase() ?? null;
- let existing = bundleKey ? byBundle.get(bundleKey) : null;
+function duplicateAliasBase(normalizedName) {
+ const clean = String(normalizedName ?? "").trim();
+ for (const suffix of duplicateAliasSuffixes) {
+ if (clean.endsWith(suffix)) {
+ return clean.slice(0, -suffix.length);
+ }
+ }
+ return clean;
+}
- if (!existing) {
- const normalizedExisting = byName.get(incoming.normalizedName);
- if (normalizedExisting) {
- const existingBundle = normalizedExisting.bundleIdentifier?.toLowerCase() ?? null;
- if (bundleKey && existingBundle && bundleKey !== existingBundle) {
- ambiguous.push({
- kind: "same_normalized_different_bundle",
- identity: incoming.normalizedName,
- names: [normalizedExisting.name, incoming.name].join(" | "),
- bundles: [normalizedExisting.bundleIdentifier, incoming.bundleIdentifier].join(" | "),
- sources: [[...normalizedExisting.sources].join("|"), incoming.source].join(" | "),
- decision: "kept_separate_for_review",
- });
- } else {
- existing = normalizedExisting;
- }
+function duplicateComparable(row) {
+ return JSON.stringify({
+ defaultTag: String(row.defaultTag ?? "").trim(),
+ tager: String(row.tager ?? "").trim(),
+ bundleIdentifier: normalizeBundle(row.bundleIdentifier),
+ noteZH: truncateNote(row["defaultNote-ZH"]),
+ });
+}
+
+function dedupeCuratedRows(rows, stats) {
+ const byNormalized = new Map();
+ for (const row of rows) {
+ const normalizedName = String(row.normalizedName ?? "").trim() || normalizeName(row.Name ?? "");
+ if (normalizedName && !byNormalized.has(normalizedName)) {
+ byNormalized.set(normalizedName, row);
+ }
+ }
+
+ const filtered = [];
+ for (const row of rows) {
+ const normalizedName = String(row.normalizedName ?? "").trim() || normalizeName(row.Name ?? "");
+ const aliasBase = duplicateAliasBase(normalizedName);
+ const baseRow = aliasBase === normalizedName ? null : byNormalized.get(aliasBase);
+ if (baseRow && duplicateComparable(row) === duplicateComparable(baseRow)) {
+ stats.aliasDuplicatesRemoved += 1;
+ continue;
+ }
+ filtered.push(row);
+ }
+ return filtered;
+}
+
+async function build() {
+ fs.mkdirSync(outputDir, { recursive: true });
+ const supportedLanguages = loadSupportedLanguageCodes();
+ const runtimeSmartCategoryIDs = loadRuntimeSmartCategoryIDs();
+ const missingRuntimeCategories = stableTagOrder.filter((tag) => !runtimeSmartCategoryIDs.has(tag));
+ if (missingRuntimeCategories.length > 0) {
+ throw new Error(
+ `SmartCategoryID is missing generated catalog tags: ${missingRuntimeCategories.join(", ")}`,
+ );
+ }
+ for (const language of requiredFallbackLanguages) {
+ if (!supportedLanguages.includes(language)) {
+ throw new Error(`required fallback language is missing from Localization: ${language}`);
+ }
+ }
+ const appleNotes = parseAppleNotes();
+ const previousRuntimeEntries = fs.existsSync(outputs.runtimeJSON)
+ ? new Map(
+ (JSON.parse(fs.readFileSync(outputs.runtimeJSON, "utf8")).entries ?? []).map((entry) => [entry.normalizedName, entry]),
+ )
+ : new Map();
+ const rawCuratedRows = readCSVObjects(sources.curatedCatalog);
+ const unknownTagerTokens = new Set();
+ const stats = {
+ curatedRows: rawCuratedRows.length,
+ aliasDuplicatesRemoved: 0,
+ rowsWithSourceNotes: 0,
+ appleNotesAttached: 0,
+ tagChangedVsPrevious: 0,
+ zhNoteChangedVsPrevious: 0,
+ bundleChangedVsPrevious: 0,
+ missingLocalizedNotes: 0,
+ };
+ const curatedRows = rawCuratedRows;
+ const preparedRows = curatedRows.map((row, index) => {
+ const name = String(row.Name ?? "").trim();
+ const normalizedName = String(row.normalizedName ?? "").trim() || normalizeName(name);
+ const bundleIdentifier = normalizeBundle(row.bundleIdentifier);
+ const mapping = mapCuratedTags(row.tager);
+ for (const token of mapping.unknownTokens) unknownTagerTokens.add(token);
+
+ let noteZH = truncateNote(row["defaultNote-ZH"]);
+ let usedAppleFallback = false;
+ if (!noteZH) {
+ const bundleNote = bundleIdentifier ? appleNotes.byBundle.get(bundleIdentifier.toLowerCase()) : null;
+ const nameNote = appleNotes.byName.get(name);
+ if (bundleNote || nameNote) {
+ noteZH = truncateNote(bundleNote ?? nameNote);
+ usedAppleFallback = true;
+ stats.appleNotesAttached += 1;
+ }
+ } else {
+ stats.rowsWithSourceNotes += 1;
+ }
+
+ const previous = previousRuntimeEntries.get(normalizedName);
+ if (previous) {
+ if (JSON.stringify(previous.defaultTag ?? []) !== JSON.stringify(mapping.tags)) {
+ stats.tagChangedVsPrevious += 1;
+ }
+ if (((previous.notes ?? {})["zh-Hans"] ?? "") !== (noteZH ?? "")) {
+ stats.zhNoteChangedVsPrevious += 1;
+ }
+ if ((previous.bundleIdentifier ?? null) !== bundleIdentifier) {
+ stats.bundleChangedVsPrevious += 1;
}
}
- if (!existing) {
- const entry = {
- name: incoming.name,
- normalizedName: incoming.normalizedName,
- bundleIdentifier: incoming.bundleIdentifier,
- tags: incoming.tags,
- noteZH: incoming.noteZH,
- rank: incoming.rank,
- sourceEvidence: new Set([...incoming.evidence, incoming.source].filter(Boolean)),
- sources: new Set([incoming.source]),
- namePriority: incoming.priority,
- notePriority: incoming.noteZH ? incoming.priority : -1,
- };
- entries.push(entry);
- byName.set(entry.normalizedName, entry);
- if (bundleKey) byBundle.set(bundleKey, entry);
- return;
- }
-
- const beforeSources = new Set(existing.sources);
- mergeInto(existing, incoming, stats);
- if (incoming.bundleIdentifier) byBundle.set(incoming.bundleIdentifier.toLowerCase(), existing);
- byName.set(existing.normalizedName, existing);
- mergedGroups.push({
- identity: bundleKey ? `bundle:${bundleKey}` : `name:${incoming.normalizedName}`,
- sourcesBefore: [...beforeSources].join("|"),
- mergedSource: incoming.source,
- finalName: existing.name,
- });
- }
-
- const masterRows = readCSVObjects(sources.master5000);
- stats.sourceRows.master5000 = masterRows.length;
- for (const row of masterRows) {
- addEntry(rawEntry({
- source: "master5000",
- name: row.name,
- normalizedName: row.normalizedName,
- bundleIdentifier: row.bundleIdentifier,
- tags: splitTags(row.defaultTagIDs),
- rank: row.rank,
- evidence: splitTags(row.sourceEvidence),
- }));
- }
-
- const topRows = readCSVObjects(sources.top1000);
- stats.sourceRows.top1000 = topRows.length;
- for (const row of topRows) {
- addEntry(rawEntry({
- source: "top1000",
- name: row.name,
- normalizedName: row.normalizedName,
- bundleIdentifier: row.bundleIdentifier,
- tags: splitTags(row.defaultTagIDs),
- rank: row.rank,
- evidence: splitTags(row.sourceEvidence),
- }));
- }
-
- const gameRows = readCSVObjects(sources.game);
- stats.sourceRows.game = gameRows.length;
- for (const row of gameRows) {
- addEntry(rawEntry({
- source: "game",
- name: row.name,
- normalizedName: normalizeName(row.name),
- bundleIdentifier: row.bundleIdentifier,
- tags: orderedTags([...splitTags(row.defaultTagIDs), "entertainment"]),
- rank: row.rank,
- evidence: ["game_seed"],
- }));
- }
-
- const personaRows = readCSVObjects(sources.persona);
- stats.sourceRows.persona = personaRows.length;
- for (const row of personaRows) {
- addEntry(rawEntry({
- source: "persona",
- name: row.appName,
- normalizedName: row.normalizedName,
- bundleIdentifier: row.bundleIdentifier,
- tags: splitTags(row.defaultTagIDs),
- noteZH: row.defaultNoteZH,
- rank: row.masterRank || row.bestPersonaRank,
- evidence: ["persona_top_apps", ...(row.personaIDs ? [`personas:${row.personaIDs}`] : [])],
- }));
- }
-
- stats.sourceRows.apple = appleNotes.byBundle.size;
- for (const [bundleIdentifier, note] of appleNotes.byBundle.entries()) {
- const metadata = appleCatalogMetadata.get(bundleIdentifier);
- if (!metadata) continue;
- addEntry(rawEntry({
- source: "apple",
- name: metadata.name,
- normalizedName: normalizeName(metadata.name),
+ return {
+ rank: index + 1,
+ name,
+ normalizedName,
bundleIdentifier,
- tags: metadata.tags,
- noteZH: note,
- rank: 50,
- evidence: ["apple_default_notes"],
- }));
+ tags: mapping.tags,
+ noteZH,
+ sourceEvidence: usedAppleFallback
+ ? ["curated_tager_catalog", "apple_default_notes"]
+ : ["curated_tager_catalog"],
+ };
+ });
+
+ const uniqueSourceNotes = [
+ ...new Set(preparedRows.map((row) => normalizeTranslatedNote(row.noteZH)).filter(Boolean)),
+ ].sort((left, right) => left.localeCompare(right));
+ const sourceFingerprint = sourceNotesFingerprint(uniqueSourceNotes);
+ const translationCache = loadSourceNoteTranslationCache(sourceFingerprint);
+ const traditionalNotesBySource = convertSimplifiedNotesToTraditional(uniqueSourceNotes);
+ for (const [sourceNote, traditionalNote] of traditionalNotesBySource.entries()) {
+ translationCache.entries[translationCacheKey("zh-Hant", sourceNote)] = traditionalNote;
+ }
+ const translationStats = await translateMissingNotes(uniqueSourceNotes, supportedLanguages, translationCache);
+ if (translationStats.failedCount > 0) {
+ throw new Error(`translation failed for ${translationStats.failedCount} source-note/language pairs; JSON generation blocked`);
}
- for (const entry of entries) {
- const bundleNote = entry.bundleIdentifier
- ? appleNotes.byBundle.get(entry.bundleIdentifier.toLowerCase())
- : null;
- const nameNote = appleNotes.byName.get(entry.name);
- const note = bundleNote ?? nameNote;
- if (note) {
- entry.noteZH = truncateNote(note);
- entry.notePriority = sourcePriority.apple;
- entry.sources.add("apple");
- entry.sourceEvidence.add("apple_default_notes");
- stats.appleNotesAttached += 1;
- }
- }
-
- for (const bundle of appleNotes.byBundle.keys()) {
- if (!byBundle.has(bundle)) stats.appleNotesSwiftOnly += 1;
- }
-
- const finalRows = entries
- .map((entry) => {
- entry.tags = orderedTags(entry.tags);
- if (entry.tags.length === 1 && entry.tags[0] === "other") {
- const overrideTags = tagOverridesByNormalizedName.get(entry.normalizedName);
- if (overrideTags) {
- entry.tags = orderedTags(overrideTags);
- entry.sourceEvidence.add("manual_other_reduction");
- }
- }
- const notes = buildLocalizedNotes(entry.name, entry.tags, entry.noteZH, translations);
- return {
- name: entry.name,
- normalizedName: entry.normalizedName || normalizeName(entry.name),
- bundleIdentifier: entry.bundleIdentifier,
- tags: entry.tags,
- noteZH: entry.noteZH,
- notes,
- rank: entry.rank,
- sourceEvidence: [...entry.sourceEvidence].sort(),
- sources: [...entry.sources].sort(),
- };
- })
- .sort((left, right) => {
- const rankDiff = left.rank - right.rank;
- if (rankDiff !== 0) return rankDiff;
- return left.name.localeCompare(right.name);
- });
+ const finalRows = preparedRows.map((row) => ({
+ ...row,
+ entryID: entryIDFor(row),
+ notes: buildLocalizedNotes(
+ row.noteZH,
+ supportedLanguages,
+ translationCache,
+ traditionalNotesBySource.get(normalizeTranslatedNote(row.noteZH)),
+ ),
+ }));
const invalidTagRows = finalRows.filter((row) => row.tags.some((tag) => !stableTags.has(tag)));
+ const runtimeInvalidTagRows = finalRows.filter((row) => row.tags.some((tag) => !runtimeSmartCategoryIDs.has(tag)));
+ const entryIDCounts = new Map();
+ for (const row of finalRows) {
+ entryIDCounts.set(row.entryID, (entryIDCounts.get(row.entryID) ?? 0) + 1);
+ }
+ const duplicateEntryIDs = [...entryIDCounts.entries()].filter(([, count]) => count > 1);
const emptyNormalizedRows = finalRows.filter((row) => !row.normalizedName);
const exactOtherRows = finalRows.filter((row) => row.tags.length === 1 && row.tags[0] === "other");
const mixedOtherRows = finalRows.filter((row) => row.tags.length > 1 && row.tags.includes("other"));
const noteRows = finalRows.filter((row) => row.noteZH);
- const localizedNoteIssues = [];
+ const actualLocalizedNoteCount = noteRows.reduce(
+ (total, row) => total + Object.keys(row.notes).filter((code) => supportedLanguages.includes(code)).length,
+ 0,
+ );
+ const missingLocalizedNotes = noteRows.length * supportedLanguages.length - actualLocalizedNoteCount;
+ const missingLocalizedNoteRows = noteRows.filter(
+ (row) => supportedLanguages.some((code) => !row.notes[code]),
+ );
+ if (runtimeInvalidTagRows.length > 0) {
+ throw new Error(
+ `runtime SmartCategoryID cannot decode ${runtimeInvalidTagRows.length} catalog rows`,
+ );
+ }
+ if (duplicateEntryIDs.length > 0) {
+ throw new Error(
+ `duplicate Smart Start entryID values: ${duplicateEntryIDs.map(([id]) => id).slice(0, 10).join(", ")}`,
+ );
+ }
+ const noteQualityIssues = [];
for (const row of noteRows) {
- for (const code of translations.keys()) {
- const note = row.notes[code];
- if (!note) {
- localizedNoteIssues.push({ name: row.name, code, issue: "missing" });
- } else if (charLength(note) > noteLimit) {
- localizedNoteIssues.push({ name: row.name, code, issue: `over_limit:${charLength(note)}` });
+ for (const code of supportedLanguages) {
+ if (!row.notes[code]) {
+ noteQualityIssues.push({ name: row.name || row.normalizedName, code, issue: "missing_translation" });
+ }
+ }
+ for (const [code, note] of Object.entries(row.notes)) {
+ if (charLength(note) > noteLimit) {
+ noteQualityIssues.push({ name: row.name || row.normalizedName, code, issue: `over_limit:${charLength(note)}` });
+ } else if (stripTerminalPunctuation(note) != note) {
+ noteQualityIssues.push({ name: row.name || row.normalizedName, code, issue: "trailing_punctuation" });
+ } else if (stripLeadingPunctuation(note) != note) {
+ noteQualityIssues.push({ name: row.name || row.normalizedName, code, issue: "leading_punctuation" });
+ } else if (noteRepeatsAppName(note, row.name)) {
+ noteQualityIssues.push({ name: row.name || row.normalizedName, code, issue: "repeats_app_name" });
}
}
}
-
- const reviewHeader = ["Name", "normalizedName", "defaultTag", "bundleIdentifier", "defaultNote-ZH"];
- const reviewRows = [
- reviewHeader,
- ...finalRows.map((row) => [
- row.name,
- row.normalizedName,
- row.tags.join("|"),
- row.bundleIdentifier ?? "null",
- row.noteZH ?? "",
- ]),
- ];
- fs.writeFileSync(outputs.reviewCSV, `${stringifyCSV(reviewRows)}\n`);
const runtime = {
version: 2,
generatedAt: new Date().toISOString(),
noteLimit,
- supportedLanguages: [...translations.keys()].sort(),
- entries: finalRows.map((row, index) => ({
- rank: index + 1,
+ supportedLanguages,
+ entries: finalRows.map((row) => ({
+ entryID: row.entryID,
+ rank: row.rank,
name: row.name,
normalizedName: row.normalizedName,
bundleIdentifier: row.bundleIdentifier,
@@ -630,16 +1024,37 @@
})),
};
fs.writeFileSync(outputs.runtimeJSON, `${JSON.stringify(runtime, null, 2)}\n`);
+ const splitResources = buildSplitResources({
+ finalRows,
+ supportedLanguages,
+ generatedAt: runtime.generatedAt,
+ noteLimit,
+ });
const duplicateRows = [
- ["kind", "identity", "names", "bundles", "sources", "decision"],
- ...ambiguous.map((row) => [row.kind, row.identity, row.names, row.bundles, row.sources, row.decision]),
+ ["name", "normalizedName", "previousRuntimeDefaultTag", "runtimeDefaultTag", "bundleIdentifier", "defaultNote-ZH"],
+ ...finalRows
+ .filter((row) => {
+ const previous = previousRuntimeEntries.get(row.normalizedName);
+ return previous && JSON.stringify(previous.defaultTag ?? []) !== JSON.stringify(row.tags);
+ })
+ .map((row) => [
+ row.name,
+ row.normalizedName,
+ (previousRuntimeEntries.get(row.normalizedName)?.defaultTag ?? []).join("|"),
+ row.tags.join("|"),
+ row.bundleIdentifier ?? "null",
+ row.noteZH ?? "",
+ ]),
];
fs.writeFileSync(outputs.duplicateReview, `${stringifyCSV(duplicateRows)}\n`);
- const sourceLines = Object.entries(stats.sourceRows)
- .map(([source, count]) => `- ${source}: ${count}`)
- .join("\n");
+ const sourceLines = [
+ "- curated CSV: `Research/SmartStart/UltimateDefaultCatalog/SmartStart_UltimateDefaultCatalog.csv`",
+ `- curated rows: ${stats.curatedRows}`,
+ `- curated rows used for JSON: ${curatedRows.length}`,
+ `- unknown tager tokens: ${unknownTagerTokens.size}`,
+ ].join("\n");
const topTagLines = stableTagOrder
.map((tag) => [tag, finalRows.filter((row) => row.tags.includes(tag)).length])
.filter(([, count]) => count > 0)
@@ -657,22 +1072,45 @@
- Review CSV: \`Research/SmartStart/UltimateDefaultCatalog/SmartStart_UltimateDefaultCatalog.csv\`
- Runtime JSON: \`Research/SmartStart/UltimateDefaultCatalog/SmartStart_UltimateDefaultCatalog.json\`
+- Split base JSON: \`Research/SmartStart/UltimateDefaultCatalog/${splitResources.baseInfo.file}\`
+- Split manifest JSON: \`Research/SmartStart/UltimateDefaultCatalog/${splitResources.manifestInfo.file}\`
+- Split notes JSON files: ${Object.keys(splitResources.notesResources).length}
- Duplicate review: \`Research/SmartStart/UltimateDefaultCatalog/SmartStart_UltimateDefaultCatalog_DuplicateReview.csv\`
- Translation QA: \`Research/SmartStart/UltimateDefaultCatalog/SmartStart_UltimateDefaultCatalog_TranslationQA.md\`
+- Source-note translation cache: \`Research/SmartStart/UltimateDefaultCatalog/SmartStart_UltimateDefaultCatalog_SourceNoteTranslationCache.json\`
## Summary
- Final rows: ${finalRows.length}
- Rows with Chinese default notes: ${noteRows.length}
-- Merged source rows: ${stats.mergedRows}
-- Ambiguous duplicate candidates: ${ambiguous.length}
+- Rows with source Chinese notes from curated CSV: ${stats.rowsWithSourceNotes}
+- Rows with Apple note fallback attached: ${stats.appleNotesAttached}
+- Supported languages: ${supportedLanguages.length}
+- Unique source notes: ${uniqueSourceNotes.length}
+- Expected localized notes: ${noteRows.length * supportedLanguages.length}
+- Actual localized notes: ${actualLocalizedNoteCount}
+- Missing localized notes: ${missingLocalizedNotes}
+- Rows missing at least one localized note: ${missingLocalizedNoteRows.length}
+- Source note fingerprint: ${sourceFingerprint}
+- Existing source-note cache ignored: ${translationCache.ignoredExistingCache ? "yes" : "no"}
+- Translation provider: ${translationProvider}
+- Translation backend: ${translationBackend}
+- Translation cache hits before run: ${translationStats.cacheHits}
+- Equivalent-language cache reuses: ${translationStats.reusedEquivalentCount}
+- New non-Chinese translations generated: ${translationStats.translatedCount}
+- Translation failures: ${translationStats.failedCount}
+- zh-Hant generation: macOS CFStringTransform Simplified-Traditional from zh-Hans source notes
- Invalid tag rows: ${invalidTagRows.length}
+- Runtime-invalid tag rows: ${runtimeInvalidTagRows.length}
+- Duplicate entryID values: ${duplicateEntryIDs.length}
- Empty normalizedName rows: ${emptyNormalizedRows.length}
- Exact \`other\` rows: ${exactOtherRows.length}
- Mixed \`other\` rows after cleanup: ${mixedOtherRows.length}
-- Apple notes attached to catalog rows: ${stats.appleNotesAttached}
-- Apple notes still handled by Swift-only fallback: ${stats.appleNotesSwiftOnly}
-- Localized note issues: ${localizedNoteIssues.length}
+- Tag changes vs previous runtime JSON: ${stats.tagChangedVsPrevious}
+- Chinese note changes vs previous runtime JSON: ${stats.zhNoteChangedVsPrevious}
+- Bundle identifier changes vs previous runtime JSON: ${stats.bundleChangedVsPrevious}
+- Unknown tager tokens: ${unknownTagerTokens.size}
+- Note quality issues: ${noteQualityIssues.length}
## Tag Distribution
@@ -680,9 +1118,10 @@
## Notes
-- \`other\` is removed whenever a row has at least one meaningful tag.
-- Runtime notes are generated for all supported localization files when a Chinese source note exists.
-- Non-Chinese localized notes currently use localized tag-summary copy generated from existing category translations; high-traffic languages can be manually polished later.
+- Runtime JSON is generated strictly from the curated CSV. The generator does not rewrite the source CSV.
+- Runtime tags are copied directly from the curated CSV \`tager\` column.
+- Runtime notes are generated only from curated CSV \`defaultNote-ZH\`: \`zh-Hans\` is the source note, \`zh-Hant\` is converted from it, and all other supported languages are translated from the same source note.
+- The source-note translation cache is only a fingerprint-validated acceleration artifact. Localized tag-summary notes are not valid sources for runtime notes.
`;
fs.writeFileSync(outputs.report, report);
@@ -694,27 +1133,60 @@
- Supported languages: ${runtime.supportedLanguages.length}
- Rows with source Chinese notes: ${noteRows.length}
-- Required localized notes: ${noteRows.length * runtime.supportedLanguages.length}
-- Issues: ${localizedNoteIssues.length}
+- Source notes: ${noteRows.length}
+- Required zh-Hant conversions: ${noteRows.length}
+- Required non-Chinese machine/reviewed translations: ${noteRows.length * (runtime.supportedLanguages.length - 2)}
+- Unique source notes: ${uniqueSourceNotes.length}
+- Expected localized notes including source language: ${noteRows.length * runtime.supportedLanguages.length}
+- Actual localized notes: ${actualLocalizedNoteCount}
+- Missing localized notes: ${missingLocalizedNotes}
+- Rows missing at least one localized note: ${missingLocalizedNoteRows.length}
+- Source note fingerprint: ${sourceFingerprint}
+- Existing source-note cache ignored: ${translationCache.ignoredExistingCache ? "yes" : "no"}
+- Translation provider: ${translationProvider}
+- Translation backend: ${translationBackend}
+- Translation cache hits before run: ${translationStats.cacheHits}
+- Equivalent-language cache reuses: ${translationStats.reusedEquivalentCount}
+- New non-Chinese translations generated: ${translationStats.translatedCount}
+- Translation failures: ${translationStats.failedCount}
+- zh-Hant generation: macOS CFStringTransform Simplified-Traditional from zh-Hans source notes
+- Generated placeholder translations: 0
+- Note quality issues: ${noteQualityIssues.length}
- Note limit: ${noteLimit}
## Issue Sample
-${localizedNoteIssues.slice(0, 50).map((issue) => `- ${issue.name} / ${issue.code}: ${issue.issue}`).join("\n") || "- None"}
+${noteQualityIssues.slice(0, 50).map((issue) => `- ${issue.name} / ${issue.code}: ${issue.issue}`).join("\n") || "- None"}
`;
fs.writeFileSync(outputs.translationQA, translationReport);
return {
finalRows: finalRows.length,
noteRows: noteRows.length,
- ambiguousDuplicates: ambiguous.length,
+ splitBaseBytes: splitResources.baseInfo.bytes,
+ splitNotesFiles: Object.keys(splitResources.notesResources).length,
+ changedVsPrevious: stats.tagChangedVsPrevious,
invalidTagRows: invalidTagRows.length,
+ runtimeInvalidTagRows: runtimeInvalidTagRows.length,
+ duplicateEntryIDs: duplicateEntryIDs.length,
emptyNormalizedRows: emptyNormalizedRows.length,
exactOtherRows: exactOtherRows.length,
mixedOtherRows: mixedOtherRows.length,
- localizedNoteIssues: localizedNoteIssues.length,
+ supportedLanguages: supportedLanguages.length,
+ uniqueSourceNotes: uniqueSourceNotes.length,
+ expectedLocalizedNotes: noteRows.length * supportedLanguages.length,
+ actualLocalizedNotes: actualLocalizedNoteCount,
+ missingLocalizedNotes,
+ sourceFingerprint,
+ sourceNoteCacheIgnoredExisting: translationCache.ignoredExistingCache,
+ translationCacheHitsBeforeRun: translationStats.cacheHits,
+ equivalentLanguageCacheReuses: translationStats.reusedEquivalentCount,
+ newNonChineseTranslationsGenerated: translationStats.translatedCount,
+ translationFailures: translationStats.failedCount,
+ noteQualityIssues: noteQualityIssues.length,
+ unknownTagerTokens: [...unknownTagerTokens].sort(),
outputDir,
};
}
-console.log(JSON.stringify(build(), null, 2));
+console.log(JSON.stringify(await build(), null, 2));
--
Gitblit v1.9.3