diff --git a/docs/maintenance/FREENODE.md b/docs/maintenance/FREENODE.md index b87206cf..ca5d81d1 100644 --- a/docs/maintenance/FREENODE.md +++ b/docs/maintenance/FREENODE.md @@ -217,4 +217,15 @@ Nodehub仍运行`507cbf7`，已在今天00:15正常生成20261022四站10份文 恢复发布批次`20261022045639-8d73d22a4e86`（北京时间12:56:39准备），URI 2592、Clash 2559、Sing-box 2373，16份本地产物。随后重放既有Nodehub同步，四站均成功，10份当日订阅落盘与当前批次哈希匹配（Clash仅既有trim差异），Nodehub服务器侧10个精确HTTPS地址全部200、响应哈希与文件一致。已完成恢复和服务器侧验收，未做线上页面验收或节点联网测速。回滚产物可用原current批次`20261021160648-829d2ad82938`，代码回退点`21ab4b84307056716664d8a0bc1e95d5daea448d`；代码回滚不自动回滚Nodehub文件。 -本次仅按已证实的满盘故障清理三个可再生下载缓存，没有修改共享Gitea的缓存保留策略。未来缓存仍可能增长，阶段日志会记录剩余空间，但不等于已建立磁盘容量治理或自动清理服务。 +初次恢复仅清理三个可再生缓存。用户随后明确授权为这些缓存加自动清理，实施规则见下一节。 + + +### Gitea下载缓存自动清理 + +源码与单元测试在`tools/maintenance/gitea_archive_cache.py`、`test_gitea_archive_cache.py`，systemd unit同目录。仅处理`/var/lib/gitea/data/repo-archive/<数字仓库ID>/<哈希前两位>/<40位哈希>.zip|.tar.gz|.bundle`。每5分钟执行：删除超过1小时缓存；仍超过512MiB时从旧到新淘汰。保护正在被进程打开的inode和生成不足10分钟的文件；不跟随符号链接，未知文件保留并计数，因此存在受保护文件时512MiB是清理目标，不是硬配额。只记录计数/字节，不输出仓库内容；执行互斥，重检文件身份和修改时间后删除，目录异常或无法检查打开文件时失败。不会接触Git对象库、数据库、附件、备份或订阅产物。 + +安装目标：`/usr/local/sbin/gitea-archive-cache-cleaner`，units为`/etc/systemd/system/gitea-archive-cache-cleaner.{service,timer}`。独立任务无需重启Gitea；原每5分钟`log-budget-cleaner`保持原样。日志用`journalctl -u gitea-archive-cache-cleaner.service`；手工预览为`/usr/local/sbin/gitea-archive-cache-cleaner --dry-run`（需root，运行目录由service创建）。容量超标但只剩保护文件时返回失败并记录，不为达配额删除正在使用的文件。 + +回退：先`systemctl disable --now gitea-archive-cache-cleaner.timer`，停止service，再按部署备份恢复/移除本次独立脚本和两个unit并`systemctl daemon-reload`；无需回退免费节点业务代码。已删除缓存可按需重新生成。部署前检查生产HEAD、并行状态及旧文件，保留安装前状态和验收记录到服务器私有备份目录；本地5项测试覆盖过期/容量淘汰、打开文件/新文件/未知文件保护、符号链接、预览与重复执行。 + +当前状态：本地测试通过，尚待Git部署与线上真实清理和timer验收。 diff --git a/tools/maintenance/gitea-archive-cache-cleaner.service b/tools/maintenance/gitea-archive-cache-cleaner.service new file mode 100644 index 00000000..4cea2195 --- /dev/null +++ b/tools/maintenance/gitea-archive-cache-cleaner.service @@ -0,0 +1,20 @@ +[Unit] +Description=Bound Gitea regenerable download archive cache +After=local-fs.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/gitea-archive-cache-cleaner +User=root +UMask=0077 +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=90 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +RuntimeDirectory=gitea-archive-cache-cleaner +RuntimeDirectoryMode=0700 +RuntimeDirectoryPreserve=yes +ReadWritePaths=/var/lib/gitea/data/repo-archive /run/gitea-archive-cache-cleaner +PrivateTmp=true diff --git a/tools/maintenance/gitea-archive-cache-cleaner.timer b/tools/maintenance/gitea-archive-cache-cleaner.timer new file mode 100644 index 00000000..f2f2cd02 --- /dev/null +++ b/tools/maintenance/gitea-archive-cache-cleaner.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check Gitea download archive cache every five minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +AccuracySec=15s +Unit=gitea-archive-cache-cleaner.service + +[Install] +WantedBy=timers.target diff --git a/tools/maintenance/gitea_archive_cache.py b/tools/maintenance/gitea_archive_cache.py new file mode 100644 index 00000000..21e11ab9 --- /dev/null +++ b/tools/maintenance/gitea_archive_cache.py @@ -0,0 +1,101 @@ +#!/usr/bin/python3 +"""Bound only Gitea's regenerable repository download archives, never repositories.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import re +import stat +import time + +ROOT = Path('/var/lib/gitea/data/repo-archive') +BUDGET = 512 * 1024 * 1024 +TTL = 3600 +GRACE = 600 +ARCHIVE = re.compile(r'([0-9a-f]{40})\.(?:zip|tar\.gz|bundle)\Z') + + +def open_inodes(): + result = set() + for proc in Path('/proc').iterdir(): + if not proc.name.isdigit(): + continue + try: + for fd in (proc / 'fd').iterdir(): + try: + s = fd.stat() + if stat.S_ISREG(s.st_mode): + result.add((s.st_dev, s.st_ino)) + except (FileNotFoundError, ProcessLookupError): + pass + except (FileNotFoundError, ProcessLookupError): + pass + return result + + +def clean(root, now, opened, dry_run=False, budget=BUDGET, ttl=TTL, grace=GRACE): + if root.is_symlink() or root.resolve() != root or not root.is_dir(): + raise RuntimeError('cache_root_missing_or_symlink') + files = [] + unknown = 0 + for base, dirs, names in os.walk(root, followlinks=False): + dirs[:] = [d for d in dirs if not Path(base, d).is_symlink()] + for name in names: + p = Path(base, name) + try: + s = p.lstat() + except FileNotFoundError: + continue + if not stat.S_ISREG(s.st_mode): + continue + parts = p.relative_to(root).parts + match = ARCHIVE.fullmatch(name) + known = len(parts) == 3 and parts[0].isdigit() and match and parts[1] == match[1][:2] + if not known: + unknown += 1 + files.append((p, s, bool(known))) + total = sum(s.st_size for _, s, _ in files) + before = total + removed = 0 + protected = 0 + for p, s, known in sorted(files, key=lambda row: row[1].st_mtime): + age = now - s.st_mtime + if not known or (s.st_dev, s.st_ino) in opened or age < grace: + protected += 1 + continue + if age < ttl and total <= budget: + continue + try: + current = p.lstat() + except FileNotFoundError: + continue + if (current.st_dev, current.st_ino, current.st_size, current.st_mtime_ns) != (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns): + continue + if not dry_run: + p.unlink() + total -= s.st_size + removed += 1 + return dict(dry_run=dry_run, before_bytes=before, after_bytes=total, + removed_files=removed, protected_files=protected, unknown_files=unknown, + budget_bytes=budget, over_budget=total > budget) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + with open('/run/gitea-archive-cache-cleaner/cleanup.lock', 'w') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return + result = clean(ROOT, time.time(), open_inodes(), args.dry_run) + result['free_bytes'] = os.statvfs(ROOT).f_bavail * os.statvfs(ROOT).f_frsize + print(json.dumps(result), flush=True) + if result['over_budget']: + raise SystemExit('Cache remains over budget: fresh/open/unknown files are protected') + + +if __name__ == '__main__': + main() diff --git a/tools/maintenance/test_gitea_archive_cache.py b/tools/maintenance/test_gitea_archive_cache.py new file mode 100644 index 00000000..f4c572b5 --- /dev/null +++ b/tools/maintenance/test_gitea_archive_cache.py @@ -0,0 +1,55 @@ +import importlib.util +import os +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('cache', Path(__file__).with_name('gitea_archive_cache.py')) +cache = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cache) + +class CleanerTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name).resolve() + self.now = 10000 + def tearDown(self): + self.temp.cleanup() + def file(self, number, age, size=20): + sha = f'{number:040x}' + p = self.root / '1' / sha[:2] / (sha + '.zip') + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b'x' * size) + os.utime(p, (self.now-age, self.now-age)) + return p + def test_expiry_preserves_open_recent_unknown_and_symlink_targets(self): + old = self.file(1, 4000) + active = self.file(2, 4000) + fresh = self.file(3, 100) + unknown = self.root / 'repository.git'; unknown.write_bytes(b'protected') + (self.root/'outside').symlink_to('/tmp', target_is_directory=True) + s = active.stat() + r = cache.clean(self.root, self.now, {(s.st_dev, s.st_ino)}, budget=1000) + self.assertFalse(old.exists()) + for p in (active, fresh, unknown): self.assertTrue(p.exists()) + self.assertEqual(1, r['removed_files']) + def test_capacity_evicts_oldest_then_stops(self): + old = self.file(1, 1200); newer = self.file(2, 1000); newest = self.file(3, 700) + r = cache.clean(self.root, self.now, set(), budget=40) + self.assertFalse(old.exists()); self.assertTrue(newer.exists()); self.assertTrue(newest.exists()) + self.assertEqual(40, r['after_bytes']) + def test_dry_run_does_not_delete_and_repeated_cleanup_is_idempotent(self): + p = self.file(1, 4000) + r = cache.clean(self.root, self.now, set(), dry_run=True) + self.assertTrue(p.exists()); self.assertEqual(1, r['removed_files']) + cache.clean(self.root, self.now, set()) + self.assertEqual(0, cache.clean(self.root, self.now, set())['removed_files']) + def test_fresh_files_may_exceed_budget_without_deletion(self): + p = self.file(1, 100) + r = cache.clean(self.root, self.now, set(), budget=1) + self.assertTrue(p.exists()); self.assertTrue(r['over_budget']) + def test_symlink_root_is_rejected(self): + link = self.root/'link'; link.symlink_to(self.root, target_is_directory=True) + with self.assertRaises(RuntimeError): cache.clean(link, self.now, set()) + +if __name__ == '__main__': unittest.main()