#feature: add agent consumer delay monitoring

2026-09-01 15:53:29 +08:00
parent c45d68e201
commit adf728adfc
7 changed files with 352 additions and 3 deletions
@@ -0,0 +1,98 @@
+<?php
+
+namespace App\Services;
+
+use Carbon\CarbonImmutable;
+use Illuminate\Database\DatabaseManager;
+
+class AgentConsumerDelayMonitorService
+{
+    public const STATE_CONFIG_KEY = 'agent_consumer_delay_monitor.alerts';
+
+    private const DELAY_THRESHOLD_MINUTES = 5;
+
+    public function __construct(
+        private readonly DatabaseManager $database,
+        private readonly DingTalkService $dingTalkService,
+        private readonly ConfigService $configService
+    ) {}
+
+    public function check(): array
+    {
+        $now = CarbonImmutable::now();
+        $latestPending = $this->database->connection('agentslave')
+            ->table('crm_event_consumer')
+            ->select(['event_name', 'created'])
+            ->where('status', '<>', 1)
+            ->orderByDesc('id')
+            ->first();
+
+        $eventName = $latestPending?->event_name;
+        $created = $latestPending ? CarbonImmutable::parse($latestPending->created) : null;
+        $delayMinutes = $created?->diffInMinutes($now, false);
+        $isDelayed = $delayMinutes !== null && $delayMinutes > self::DELAY_THRESHOLD_MINUTES;
+
+        $previousState = $this->configService->get(self::STATE_CONFIG_KEY, []);
+        $previousState = is_array($previousState) ? $previousState : [];
+        $wasDelayed = $previousState !== [];
+        $alertedCount = 0;
+        $recoveredCount = 0;
+
+        if ($isDelayed && ! $wasDelayed
+            && $this->dingTalkService->sendText($this->buildDelayAlert($eventName, $created, $delayMinutes, $now))) {
+            $this->configService->set(self::STATE_CONFIG_KEY, [
+                'event_name' => $eventName,
+                'delayed_since' => $created->toDateTimeString(),
+            ], 'Agent 消费延迟告警状态');
+            $alertedCount = 1;
+        } elseif (! $isDelayed && $wasDelayed
+            && $this->dingTalkService->sendText($this->buildRecoveryNotice($eventName, $delayMinutes, $now))) {
+            $this->configService->set(self::STATE_CONFIG_KEY, [], 'Agent 消费延迟告警状态');
+            $recoveredCount = 1;
+        }
+
+        return [
+            'checked_at' => $now->toDateTimeString(),
+            'event_name' => $eventName,
+            'consumer_time' => $created?->toDateTimeString(),
+            'delay_minutes' => $delayMinutes === null ? null : (int) floor($delayMinutes),
+            'alerted_count' => $alertedCount,
+            'recovered_count' => $recoveredCount,
+        ];
+    }
+
+    private function buildDelayAlert(
+        string $eventName,
+        CarbonImmutable $created,
+        float $delayMinutes,
+        CarbonImmutable $now
+    ): string {
+        return implode("\n", [
+            '⚠️ 【Agent 消费延迟告警】',
+            'eventname: '.$eventName,
+            '消费到: '.$created->toDateTimeString(),
+            sprintf('延迟: %d 分钟', (int) floor($delayMinutes)),
+            '检查时间: '.$now->toDateTimeString(),
+        ]);
+    }
+
+    private function buildRecoveryNotice(
+        ?string $eventName,
+        ?float $delayMinutes,
+        CarbonImmutable $now
+    ): string {
+        $lines = [
+            '✅ 【Agent 消费延迟恢复】',
+            '检查时间: '.$now->toDateTimeString(),
+        ];
+
+        if ($eventName !== null && $delayMinutes !== null) {
+            $lines[] = '当前 eventname: '.$eventName;
+            $lines[] = sprintf('当前延迟: %d 分钟', (int) floor($delayMinutes));
+        } else {
+            $lines[] = '当前无待消费消息';
+        }
+
+        return implode("\n", $lines);
+    }
+}