#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Knomo 月度归档去重工具 (memoblock dedupe) ======================================== 扫描 X1.Knomo/Memos-YYYY-MM.md,删除"同一天标题下、时间戳+正文完全一致"的 重复 memoblock(Knomo 增量追加归档时中断重试会产生的重复)。 设计: - 纯标准库,可用 /usr/bin/python3 运行,与 WorkBuddy 无关。 - 默认 dry-run(只预览,不写文件);--apply 才真正删除。 - 仅删"整行逐字节相同"的重复(含其续行),不同正文/不同时间戳都保留, 因此你在 memo 里手加的 #成链 等修改不会被误删。 - 无重复时不改写文件(保留 mtime)。 用法: python3 knomo_dedupe.py --vault "" # dry-run 预览 python3 knomo_dedupe.py --vault "" --apply # 真正删除重复 """ import re import os import argparse from collections import defaultdict def find_memos_files(vault): k = os.path.join(vault, "X1.Knomo") if not os.path.isdir(k): return [] return sorted( os.path.join(k, f) for f in os.listdir(k) if re.match(r"^Memos-\d{4}-\d{2}\.md$", f) ) def process_file(fp, apply): """返回 (removed_count, changed_bool)。apply=False 时不写文件。""" lines = open(fp, encoding="utf-8").read().split("\n") out = [] removed = [] date = None seen = defaultdict(set) # date -> {(ts, body)} cur_entry = None # 当前正在累积的 memo 条目(含续行) cur_is_memo = False def flush_entry(): nonlocal cur_entry, cur_is_memo if cur_entry is None: return if cur_is_memo and date: head = cur_entry[0] bm = re.match(r"^(\s*-\s*)(\d{2}:\d{2}:\d{2})(\s+)(.*)$", head) if bm: ts = bm.group(2) body = bm.group(4).strip() key = (ts, body) if key in seen[date]: removed.append((date, ts, body[:40])) # 丢弃整条 entry(不写入 out),含其续行 else: seen[date].add(key) out.extend(cur_entry) else: out.extend(cur_entry) else: out.extend(cur_entry) cur_entry = None cur_is_memo = False for line in lines: m = re.match(r"^##\s*\[\[(\d{4}-\d{2}-\d{2})\]\]", line) if m: flush_entry() date = m.group(1) seen[date] = set() out.append(line) continue b = re.match(r"^(\s*-\s*)(\d{2}:\d{2}:\d{2})(\s+)(.*)$", line) if b: flush_entry() cur_entry = [line] cur_is_memo = True continue # 其它行(续行 / 空行 / 其它标题) if cur_entry is not None: cur_entry.append(line) else: out.append(line) flush_entry() changed = len(removed) > 0 if apply and changed: with open(fp, "w", encoding="utf-8") as f: f.write("\n".join(out)) return len(removed), changed def main(): ap = argparse.ArgumentParser(description="Knomo 月度归档 memoblock 去重") ap.add_argument("--vault", required=True, help="vault 根目录") ap.add_argument("--apply", action="store_true", help="真正删除重复(默认 dry-run)") args = ap.parse_args() vault = os.path.abspath(args.vault) files = find_memos_files(vault) if not files: print("未找到任何 Memos-*.md 月归档文件。") return print(f"vault : {vault}") print(f"mode : {'APPLY (删除重复)' if args.apply else 'DRY-RUN (仅预览)'}") print("=" * 60) total = 0 for fp in files: n, changed = process_file(fp, args.apply) total += n rel = os.path.relpath(fp, vault) if n: print(f"[{'删除' if args.apply else '将删'} {n} 条] {rel}") else: print(f"[无重复] {rel}") print("=" * 60) if args.apply: print(f"完成:共删除 {total} 条重复 memoblock。") else: print(f"预览:发现 {total} 条重复。加 --apply 才真正删除。") if __name__ == "__main__": main()