#feature: add agent consumer delay monitoring

2026-09-01 15:53:29 +08:00
parent c45d68e201
commit adf728adfc
7 changed files with 352 additions and 3 deletions
@@ -0,0 +1,195 @@
+<?php
+
+namespace Tests\Unit;
+
+use App\Services\AgentConsumerDelayMonitorService;
+use App\Services\ConfigService;
+use App\Services\DingTalkService;
+use Carbon\CarbonImmutable;
+use Illuminate\Database\Connection;
+use Illuminate\Database\DatabaseManager;
+use Illuminate\Database\Query\Builder;
+use Mockery;
+use Tests\TestCase;
+
+class AgentConsumerDelayMonitorServiceTest extends TestCase
+{
+    public function test_it_alerts_when_the_latest_pending_message_is_more_than_five_minutes_old(): void
+    {
+        CarbonImmutable::setTestNow('2026-09-01 15:00:00');
+
+        try {
+            [$database, $query] = $this->mockLatestPendingQuery();
+            $dingTalk = Mockery::mock(DingTalkService::class);
+            $config = Mockery::mock(ConfigService::class);
+
+            $query->shouldReceive('first')->once()->andReturn((object) [
+                'event_name' => 'CASE_CREATE',
+                'created' => '2026-09-01 14:52:00',
+            ]);
+            $config->shouldReceive('get')
+                ->once()
+                ->with(AgentConsumerDelayMonitorService::STATE_CONFIG_KEY, [])
+                ->andReturn([]);
+            $dingTalk->shouldReceive('sendText')
+                ->once()
+                ->with(Mockery::on(fn (string $message) => str_contains($message, 'Agent 消费延迟告警')
+                    && str_contains($message, 'CASE_CREATE')
+                    && str_contains($message, '14:52:00')
+                    && str_contains($message, '8 分钟')))
+                ->andReturnTrue();
+            $config->shouldReceive('set')
+                ->once()
+                ->with(
+                    AgentConsumerDelayMonitorService::STATE_CONFIG_KEY,
+                    ['event_name' => 'CASE_CREATE', 'delayed_since' => '2026-09-01 14:52:00'],
+                    'Agent 消费延迟告警状态'
+                );
+
+            $result = (new AgentConsumerDelayMonitorService($database, $dingTalk, $config))->check();
+
+            $this->assertSame('CASE_CREATE', $result['event_name']);
+            $this->assertSame(8, $result['delay_minutes']);
+            $this->assertSame(1, $result['alerted_count']);
+            $this->assertSame(0, $result['recovered_count']);
+        } finally {
+            CarbonImmutable::setTestNow();
+        }
+    }
+
+    public function test_it_does_not_repeat_an_active_delay_alert(): void
+    {
+        CarbonImmutable::setTestNow('2026-09-01 15:00:00');
+
+        try {
+            [$database, $query] = $this->mockLatestPendingQuery();
+            $dingTalk = Mockery::mock(DingTalkService::class);
+            $config = Mockery::mock(ConfigService::class);
+            $state = ['event_name' => 'CASE_CREATE', 'delayed_since' => '2026-09-01 14:50:00'];
+
+            $query->shouldReceive('first')->once()->andReturn((object) [
+                'event_name' => 'DOCTOR_CREATE',
+                'created' => '2026-09-01 14:52:00',
+            ]);
+            $config->shouldReceive('get')
+                ->once()
+                ->with(AgentConsumerDelayMonitorService::STATE_CONFIG_KEY, [])
+                ->andReturn($state);
+            $dingTalk->shouldNotReceive('sendText');
+            $config->shouldNotReceive('set');
+
+            $result = (new AgentConsumerDelayMonitorService($database, $dingTalk, $config))->check();
+
+            $this->assertSame('DOCTOR_CREATE', $result['event_name']);
+            $this->assertSame(0, $result['alerted_count']);
+            $this->assertSame(0, $result['recovered_count']);
+        } finally {
+            CarbonImmutable::setTestNow();
+        }
+    }
+
+    public function test_it_notifies_recovery_when_the_latest_pending_message_is_within_five_minutes(): void
+    {
+        CarbonImmutable::setTestNow('2026-09-01 15:00:00');
+
+        try {
+            [$database, $query] = $this->mockLatestPendingQuery();
+            $dingTalk = Mockery::mock(DingTalkService::class);
+            $config = Mockery::mock(ConfigService::class);
+
+            $query->shouldReceive('first')->once()->andReturn((object) [
+                'event_name' => 'CASE_CREATE',
+                'created' => '2026-09-01 14:57:00',
+            ]);
+            $config->shouldReceive('get')
+                ->once()
+                ->with(AgentConsumerDelayMonitorService::STATE_CONFIG_KEY, [])
+                ->andReturn(['event_name' => 'CASE_CREATE', 'delayed_since' => '2026-09-01 14:50:00']);
+            $dingTalk->shouldReceive('sendText')
+                ->once()
+                ->with(Mockery::on(fn (string $message) => str_contains($message, 'Agent 消费延迟恢复')))
+                ->andReturnTrue();
+            $config->shouldReceive('set')
+                ->once()
+                ->with(AgentConsumerDelayMonitorService::STATE_CONFIG_KEY, [], 'Agent 消费延迟告警状态');
+
+            $result = (new AgentConsumerDelayMonitorService($database, $dingTalk, $config))->check();
+
+            $this->assertSame(0, $result['alerted_count']);
+            $this->assertSame(1, $result['recovered_count']);
+        } finally {
+            CarbonImmutable::setTestNow();
+        }
+    }
+
+    public function test_it_notifies_recovery_when_there_are_no_pending_messages(): void
+    {
+        CarbonImmutable::setTestNow('2026-09-01 15:00:00');
+
+        try {
+            [$database, $query] = $this->mockLatestPendingQuery();
+            $dingTalk = Mockery::mock(DingTalkService::class);
+            $config = Mockery::mock(ConfigService::class);
+
+            $query->shouldReceive('first')->once()->andReturnNull();
+            $config->shouldReceive('get')
+                ->once()
+                ->with(AgentConsumerDelayMonitorService::STATE_CONFIG_KEY, [])
+                ->andReturn(['event_name' => 'CASE_CREATE', 'delayed_since' => '2026-09-01 14:50:00']);
+            $dingTalk->shouldReceive('sendText')->once()->andReturnTrue();
+            $config->shouldReceive('set')
+                ->once()
+                ->with(AgentConsumerDelayMonitorService::STATE_CONFIG_KEY, [], 'Agent 消费延迟告警状态');
+
+            $result = (new AgentConsumerDelayMonitorService($database, $dingTalk, $config))->check();
+
+            $this->assertNull($result['event_name']);
+            $this->assertSame(1, $result['recovered_count']);
+        } finally {
+            CarbonImmutable::setTestNow();
+        }
+    }
+
+    public function test_it_retries_notifications_when_dingtalk_sending_fails(): void
+    {
+        CarbonImmutable::setTestNow('2026-09-01 15:00:00');
+
+        try {
+            [$database, $query] = $this->mockLatestPendingQuery();
+            $dingTalk = Mockery::mock(DingTalkService::class);
+            $config = Mockery::mock(ConfigService::class);
+
+            $query->shouldReceive('first')->once()->andReturn((object) [
+                'event_name' => 'CASE_CREATE',
+                'created' => '2026-09-01 14:50:00',
+            ]);
+            $config->shouldReceive('get')
+                ->once()
+                ->with(AgentConsumerDelayMonitorService::STATE_CONFIG_KEY, [])
+                ->andReturn([]);
+            $dingTalk->shouldReceive('sendText')->once()->andReturnFalse();
+            $config->shouldNotReceive('set');
+
+            $result = (new AgentConsumerDelayMonitorService($database, $dingTalk, $config))->check();
+
+            $this->assertSame(0, $result['alerted_count']);
+        } finally {
+            CarbonImmutable::setTestNow();
+        }
+    }
+
+    private function mockLatestPendingQuery(): array
+    {
+        $database = Mockery::mock(DatabaseManager::class);
+        $connection = Mockery::mock(Connection::class);
+        $query = Mockery::mock(Builder::class);
+
+        $database->shouldReceive('connection')->once()->with('agentslave')->andReturn($connection);
+        $connection->shouldReceive('table')->once()->with('crm_event_consumer')->andReturn($query);
+        $query->shouldReceive('select')->once()->with(['event_name', 'created'])->andReturnSelf();
+        $query->shouldReceive('where')->once()->with('status', '<>', 1)->andReturnSelf();
+        $query->shouldReceive('orderByDesc')->once()->with('id')->andReturnSelf();
+
+        return [$database, $query];
+    }
+}