Record cron stage outcomes and available disk space
This commit is contained in:
hellcat
parent
dd78da819f
commit
a2bcd4267c
@ -21,6 +21,7 @@ use App\Services\Cron\NasaTestFnSubfile;
|
||||
use App\Services\Cron\ShadowRocketCrawl;
|
||||
use App\Services\Cron\ShadowRocketPublish;
|
||||
use App\Services\Cron\ReportUmami;
|
||||
use App\Services\Cron\Stage as CronStage;
|
||||
|
||||
final class Cron
|
||||
{
|
||||
@ -46,34 +47,36 @@ final class Cron
|
||||
echo "cron开始执行 \n";
|
||||
|
||||
if ($h === 8 && $i === 3) {
|
||||
(new ReportUmami())->handle();
|
||||
CronStage::run('umami', fn () => (new ReportUmami())->handle());
|
||||
}
|
||||
|
||||
if ($h % 8 === 0 && $i === 3) {
|
||||
CronBook::randbox_notify(); // note提醒
|
||||
CronStage::run('book', fn () => CronBook::randbox_notify()); // note提醒
|
||||
}
|
||||
|
||||
if ($h % 4 === 0 && $i === 3) {
|
||||
ShadowRocketCrawl::run(); // 爬取小火箭id
|
||||
ShadowRocketPublish::run(); // 小火箭id发布到site
|
||||
CronStage::run('shadowrocket_crawl', fn () => ShadowRocketCrawl::run());
|
||||
CronStage::run('shadowrocket_publish', fn () => ShadowRocketPublish::run());
|
||||
}
|
||||
|
||||
if ($h % 6 === 0) {
|
||||
if ($i === 3) {
|
||||
CronFreenodeCrawl::run(); // 1.爬免费节点
|
||||
CronStage::run('freenode_crawl', fn () => CronFreenodeCrawl::run());
|
||||
}
|
||||
if ($i === 6) {
|
||||
CronFreenodePool::save(); // 2.格式化入库
|
||||
CronStage::run('freenode_prepare', fn () => CronFreenodePool::save());
|
||||
}
|
||||
if ($i === 9) {
|
||||
CronFreenodeMerge::save(); // 3.加工成文件
|
||||
CronStage::run('freenode_merge', fn () => CronFreenodeMerge::save());
|
||||
}
|
||||
if ($i === 12) {
|
||||
CronFreenodeMerge::baseToSite("fn_a"); // 4.文件处理成各站版本
|
||||
CronFreenodeMerge::baseToSite("fn_b"); // 4.文件处理成各站版本
|
||||
CronStage::run('freenode_publish', function () {
|
||||
CronFreenodeMerge::baseToSite('fn_a');
|
||||
CronFreenodeMerge::baseToSite('fn_b');
|
||||
});
|
||||
}
|
||||
if ($i === 15) {
|
||||
CronFreenodeSync::nodehub(); // 6.fn发布到nodehub
|
||||
CronStage::run('freenode_sync', fn () => CronFreenodeSync::nodehub());
|
||||
}
|
||||
|
||||
// 发布在11点，只一次，即可
|
||||
|
||||
45
app/Services/Cron/Stage.php
Normal file
45
app/Services/Cron/Stage.php
Normal file
@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Cron;
|
||||
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/** Metadata only: never persist downloaded nodes, credentials or exception traces. */
|
||||
final class Stage
|
||||
{
|
||||
public static function run(string $name, callable $work): mixed
|
||||
{
|
||||
$started = microtime(true);
|
||||
self::record($name, 'started');
|
||||
try {
|
||||
$result = $work();
|
||||
} catch (\Throwable $e) {
|
||||
self::record($name, 'failed', [
|
||||
'exception' => get_class($e),
|
||||
'file' => basename($e->getFile()),
|
||||
'line' => $e->getLine(),
|
||||
'reason' => str_contains($e->getMessage(), 'No space left on device')
|
||||
? 'disk_full' : 'operation_failed',
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
self::record($name, 'completed', ['elapsed_ms' => (int) ((microtime(true) - $started) * 1000)]);
|
||||
return $result;
|
||||
}
|
||||
|
||||
private static function record(string $name, string $state, array $details = []): void
|
||||
{
|
||||
$context = ['stage' => $name, 'state' => $state, 'free_bytes' => @disk_free_space(__DIR__)] + $details;
|
||||
try {
|
||||
Log::build([
|
||||
'driver' => 'daily', 'path' => storage_path('logs/cron-stage.log'),
|
||||
'days' => 7, 'permission' => 0600,
|
||||
])->log($state === 'failed' ? 'error' : 'info', 'cron_stage', $context);
|
||||
} catch (\Throwable) {
|
||||
// Disk exhaustion can also break the file logger; do not hide the original error.
|
||||
syslog(LOG_ERR, 'master01 cron_stage '.json_encode($context));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -211,4 +211,6 @@ Nodehub仍运行`507cbf7`，已在今天00:15正常生成20261022四站10份文
|
||||
|
||||
已排除的持续故障：当前两来源四格式正常请求均200；源目录在实际www-data身份下可写，dir_base与部署根一致，启用来源/文件映射正确；没有残留准备或原生校验进程。www-data每分钟执行`/usr/bin/php /www/wwwroot/master.diaodu.us/scripts/cron.php`，脚本与本地SHA一致；root的crontab主要为证书相关，不能仅检查root就认定缺少调度。没有找到本任务的持久化cron阶段日志，Laravel日志为空，Nginx现有错误记录未能还原06:03/12:03采集失败原因。
|
||||
|
||||
尚未确认：采集在对应分钟未执行、执行中断还是当时上游临时失败；当前下载成功不能证明故障时上游正常。已向用户请求12:03/12:06更早一条异常作为定位证据。本次没有执行采集、准备、发布、重放同步或生产变更，未为消除告警放宽来源时效或发布门禁。调查尚未闭环，不能登记为已修复。
|
||||
后续恢复时现场复现`errno=28 No space left on device`：根盘19GB使用100%、普通运行身份可用空间0，inode仍充足。Gitea下载归档缓存`/var/lib/gitea/data/repo-archive`占3.2GB；只清理其中超过一小时未更新、没有进程打开的三个大于500MB的可再生缓存，释放2,360,516,093字节，恢复可用空间2,352,144,384字节。没有删除Git仓库、数据库或订阅批次。满盘是本次恢复中已证实的直接写入阻断；由于此前无阶段日志，不能还原06:03/12:03每个进程的确切退出点。
|
||||
|
||||
当前来源及发布清单备份在服务器`/var/backups/master01-freenode-20261022-recovery`（私有目录），www-data补跑采集成功，四个来源文件mtime更新至12:53:51。新增`Cron\Stage`包装免费节点采集、准备、加工、发布、分发以及可能在采集之前执行的Book/ShadowRocket/Umami阶段；`storage/logs/cron-stage-YYYY-MM-DD.log`保留7天、权限0600，记录阶段开始/完成/失败、耗时、剩余字节和异常类/文件/行，异常正文与节点数据不落日志；文件日志失败时退回syslog，保持原异常。没有取消来源时效或发布门禁。回归测试19项49断言通过，含实际原生内核、日志失败不阻止任务和敏感异常正文不入日志。恢复发布和上线验收结果待下文确认。
|
||||
|
||||
55
tests/Unit/Freenode/CronStageTest.php
Normal file
55
tests/Unit/Freenode/CronStageTest.php
Normal file
@ -0,0 +1,55 @@
|
||||
<?php
|
||||
namespace Tests\Unit\Freenode;
|
||||
|
||||
use App\Services\Cron\Stage;
|
||||
use Illuminate\Foundation\Application;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
final class CronStageTest extends TestCase
|
||||
{
|
||||
private object $logger;
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
new Application(dirname(__DIR__, 3));
|
||||
$this->logger = new class {
|
||||
public array $events = [];
|
||||
public bool $fail = false;
|
||||
public function build(array $config): self { return $this; }
|
||||
public function log(string $level, string $message, array $context): void {
|
||||
if ($this->fail) throw new \RuntimeException('logger disk full');
|
||||
$this->events[] = $context;
|
||||
}
|
||||
};
|
||||
Log::swap($this->logger);
|
||||
}
|
||||
|
||||
protected function tearDown(): void { Log::clearResolvedInstances(); }
|
||||
|
||||
public function testSuccessfulWorkKeepsItsResultAndRecordsCompletion(): void
|
||||
{
|
||||
self::assertSame(42, Stage::run('freenode_prepare', fn () => 42));
|
||||
self::assertSame(['started', 'completed'], array_column($this->logger->events, 'state'));
|
||||
self::assertArrayHasKey('free_bytes', $this->logger->events[1]);
|
||||
}
|
||||
|
||||
public function testFailureKeepsOriginalExceptionWithoutLoggingSensitiveMessage(): void
|
||||
{
|
||||
$error = new \RuntimeException('private-node-password: No space left on device');
|
||||
try { Stage::run('freenode_crawl', fn () => throw $error); self::fail(); }
|
||||
catch (\RuntimeException $caught) { self::assertSame($error, $caught); }
|
||||
self::assertSame('failed', $this->logger->events[1]['state']);
|
||||
self::assertSame('disk_full', $this->logger->events[1]['reason']);
|
||||
self::assertStringNotContainsString('private-node-password', json_encode($this->logger->events));
|
||||
}
|
||||
|
||||
public function testBrokenLoggerDoesNotPreventWorkOrReplaceItsError(): void
|
||||
{
|
||||
$this->logger->fail = true;
|
||||
self::assertSame(42, Stage::run('freenode_prepare', fn () => 42));
|
||||
$error = new \RuntimeException('original_failure');
|
||||
try { Stage::run('freenode_prepare', fn () => throw $error); self::fail(); }
|
||||
catch (\RuntimeException $caught) { self::assertSame($error, $caught); }
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user