From a2bcd4267cc7051c7f070a94fd3862cb615ae6db Mon Sep 17 00:00:00 2001 From: hellcat Date: Thu, 22 Oct 2026 12:55:49 +0800 Subject: [PATCH] Record cron stage outcomes and available disk space --- app/Services/Cron.php | 23 ++++++----- app/Services/Cron/Stage.php | 45 ++++++++++++++++++++++ docs/maintenance/FREENODE.md | 4 +- tests/Unit/Freenode/CronStageTest.php | 55 +++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 app/Services/Cron/Stage.php create mode 100644 tests/Unit/Freenode/CronStageTest.php diff --git a/app/Services/Cron.php b/app/Services/Cron.php index fae1f662..394a363f 100755 --- a/app/Services/Cron.php +++ b/app/Services/Cron.php @@ -21,6 +21,7 @@ use App\Services\Cron\NasaTestFnSubfile; use App\Services\Cron\ShadowRocketCrawl; use App\Services\Cron\ShadowRocketPublish; use App\Services\Cron\ReportUmami; +use App\Services\Cron\Stage as CronStage; final class Cron { @@ -46,34 +47,36 @@ final class Cron echo "cron开始执行 \n"; if ($h === 8 && $i === 3) { - (new ReportUmami())->handle(); + CronStage::run('umami', fn () => (new ReportUmami())->handle()); } if ($h % 8 === 0 && $i === 3) { - CronBook::randbox_notify(); // note提醒 + CronStage::run('book', fn () => CronBook::randbox_notify()); // note提醒 } if ($h % 4 === 0 && $i === 3) { - ShadowRocketCrawl::run(); // 爬取小火箭id - ShadowRocketPublish::run(); // 小火箭id发布到site + CronStage::run('shadowrocket_crawl', fn () => ShadowRocketCrawl::run()); + CronStage::run('shadowrocket_publish', fn () => ShadowRocketPublish::run()); } if ($h % 6 === 0) { if ($i === 3) { - CronFreenodeCrawl::run(); // 1.爬免费节点 + CronStage::run('freenode_crawl', fn () => CronFreenodeCrawl::run()); } if ($i === 6) { - CronFreenodePool::save(); // 2.格式化入库 + CronStage::run('freenode_prepare', fn () => CronFreenodePool::save()); } if ($i === 9) { - CronFreenodeMerge::save(); // 3.加工成文件 + CronStage::run('freenode_merge', fn () => CronFreenodeMerge::save()); } if ($i === 12) { - CronFreenodeMerge::baseToSite("fn_a"); // 4.文件处理成各站版本 - CronFreenodeMerge::baseToSite("fn_b"); // 4.文件处理成各站版本 + CronStage::run('freenode_publish', function () { + CronFreenodeMerge::baseToSite('fn_a'); + CronFreenodeMerge::baseToSite('fn_b'); + }); } if ($i === 15) { - CronFreenodeSync::nodehub(); // 6.fn发布到nodehub + CronStage::run('freenode_sync', fn () => CronFreenodeSync::nodehub()); } // 发布在11点，只一次，即可 diff --git a/app/Services/Cron/Stage.php b/app/Services/Cron/Stage.php new file mode 100644 index 00000000..a6f2a942 --- /dev/null +++ b/app/Services/Cron/Stage.php @@ -0,0 +1,45 @@ + get_class($e), + 'file' => basename($e->getFile()), + 'line' => $e->getLine(), + 'reason' => str_contains($e->getMessage(), 'No space left on device') + ? 'disk_full' : 'operation_failed', + ]); + throw $e; + } + self::record($name, 'completed', ['elapsed_ms' => (int) ((microtime(true) - $started) * 1000)]); + return $result; + } + + private static function record(string $name, string $state, array $details = []): void + { + $context = ['stage' => $name, 'state' => $state, 'free_bytes' => @disk_free_space(__DIR__)] + $details; + try { + Log::build([ + 'driver' => 'daily', 'path' => storage_path('logs/cron-stage.log'), + 'days' => 7, 'permission' => 0600, + ])->log($state === 'failed' ? 'error' : 'info', 'cron_stage', $context); + } catch (\Throwable) { + // Disk exhaustion can also break the file logger; do not hide the original error. + syslog(LOG_ERR, 'master01 cron_stage '.json_encode($context)); + } + } +} diff --git a/docs/maintenance/FREENODE.md b/docs/maintenance/FREENODE.md index b9ae1674..05910884 100644 --- a/docs/maintenance/FREENODE.md +++ b/docs/maintenance/FREENODE.md @@ -211,4 +211,6 @@ Nodehub仍运行`507cbf7`，已在今天00:15正常生成20261022四站10份文 已排除的持续故障：当前两来源四格式正常请求均200；源目录在实际www-data身份下可写，dir_base与部署根一致，启用来源/文件映射正确；没有残留准备或原生校验进程。www-data每分钟执行`/usr/bin/php /www/wwwroot/master.diaodu.us/scripts/cron.php`，脚本与本地SHA一致；root的crontab主要为证书相关，不能仅检查root就认定缺少调度。没有找到本任务的持久化cron阶段日志，Laravel日志为空，Nginx现有错误记录未能还原06:03/12:03采集失败原因。 -尚未确认：采集在对应分钟未执行、执行中断还是当时上游临时失败；当前下载成功不能证明故障时上游正常。已向用户请求12:03/12:06更早一条异常作为定位证据。本次没有执行采集、准备、发布、重放同步或生产变更，未为消除告警放宽来源时效或发布门禁。调查尚未闭环，不能登记为已修复。 +后续恢复时现场复现`errno=28 No space left on device`：根盘19GB使用100%、普通运行身份可用空间0，inode仍充足。Gitea下载归档缓存`/var/lib/gitea/data/repo-archive`占3.2GB；只清理其中超过一小时未更新、没有进程打开的三个大于500MB的可再生缓存，释放2,360,516,093字节，恢复可用空间2,352,144,384字节。没有删除Git仓库、数据库或订阅批次。满盘是本次恢复中已证实的直接写入阻断；由于此前无阶段日志，不能还原06:03/12:03每个进程的确切退出点。 + +当前来源及发布清单备份在服务器`/var/backups/master01-freenode-20261022-recovery`（私有目录），www-data补跑采集成功，四个来源文件mtime更新至12:53:51。新增`Cron\Stage`包装免费节点采集、准备、加工、发布、分发以及可能在采集之前执行的Book/ShadowRocket/Umami阶段；`storage/logs/cron-stage-YYYY-MM-DD.log`保留7天、权限0600，记录阶段开始/完成/失败、耗时、剩余字节和异常类/文件/行，异常正文与节点数据不落日志；文件日志失败时退回syslog，保持原异常。没有取消来源时效或发布门禁。回归测试19项49断言通过，含实际原生内核、日志失败不阻止任务和敏感异常正文不入日志。恢复发布和上线验收结果待下文确认。 diff --git a/tests/Unit/Freenode/CronStageTest.php b/tests/Unit/Freenode/CronStageTest.php new file mode 100644 index 00000000..3d830c9d --- /dev/null +++ b/tests/Unit/Freenode/CronStageTest.php @@ -0,0 +1,55 @@ +logger = new class { + public array $events = []; + public bool $fail = false; + public function build(array $config): self { return $this; } + public function log(string $level, string $message, array $context): void { + if ($this->fail) throw new \RuntimeException('logger disk full'); + $this->events[] = $context; + } + }; + Log::swap($this->logger); + } + + protected function tearDown(): void { Log::clearResolvedInstances(); } + + public function testSuccessfulWorkKeepsItsResultAndRecordsCompletion(): void + { + self::assertSame(42, Stage::run('freenode_prepare', fn () => 42)); + self::assertSame(['started', 'completed'], array_column($this->logger->events, 'state')); + self::assertArrayHasKey('free_bytes', $this->logger->events[1]); + } + + public function testFailureKeepsOriginalExceptionWithoutLoggingSensitiveMessage(): void + { + $error = new \RuntimeException('private-node-password: No space left on device'); + try { Stage::run('freenode_crawl', fn () => throw $error); self::fail(); } + catch (\RuntimeException $caught) { self::assertSame($error, $caught); } + self::assertSame('failed', $this->logger->events[1]['state']); + self::assertSame('disk_full', $this->logger->events[1]['reason']); + self::assertStringNotContainsString('private-node-password', json_encode($this->logger->events)); + } + + public function testBrokenLoggerDoesNotPreventWorkOrReplaceItsError(): void + { + $this->logger->fail = true; + self::assertSame(42, Stage::run('freenode_prepare', fn () => 42)); + $error = new \RuntimeException('original_failure'); + try { Stage::run('freenode_prepare', fn () => throw $error); self::fail(); } + catch (\RuntimeException $caught) { self::assertSame($error, $caught); } + } +}