AiDOPWarehouse/server/Plugins/Admin.NET.Plugin.AiDOP/Job/MdpOutboxDeadLetterAlertJob.cs

using Admin.NET.Core;
using Admin.NET.Plugin.AiDOP.Entity.DataPlatform;
using Furion.Schedule;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;

namespace Admin.NET.Plugin.AiDOP.Job;

/// <summary>
/// WP10 S4b / D12：Outbox 死信与积压告警（LogWarning + 站内 SysNotice）。
/// 每 5 分钟扫描；同条件告警间隔不少于 30 分钟，避免刷屏。无钉钉。
/// </summary>
[JobDetail("job_mdp_outbox_dl_alert", Description = "MDP Outbox 死信/积压告警", GroupName = "default", Concurrent = false)]
[PeriodSeconds(300, TriggerId = "trigger_mdp_outbox_dl_alert", Description = "每 5 分钟扫描 Outbox 告警", RunOnStart = false)]
public class MdpOutboxDeadLetterAlertJob : IJob
{
    private const long NoticeReceiverUserId = 1300000000101L;
    private const string NoticeReceiverUserName = "超级管理员";
    private static readonly TimeSpan AlertCooldown = TimeSpan.FromMinutes(30);
    private static readonly TimeSpan StalePendingThreshold = TimeSpan.FromMinutes(15);

    private static readonly object Gate = new();
    private static DateTime? _lastAlertUtc;
    private static DateTime? _lastScanUtc;
    private static long _lastSeenDeadMaxId;

    private readonly IServiceScopeFactory _scopeFactory;
    private readonly ILogger _logger;

    public MdpOutboxDeadLetterAlertJob(IServiceScopeFactory scopeFactory, ILoggerFactory loggerFactory)
    {
        _scopeFactory = scopeFactory;
        _logger = loggerFactory.CreateLogger(nameof(MdpOutboxDeadLetterAlertJob));
    }

    public async Task ExecuteAsync(JobExecutingContext context, CancellationToken stoppingToken)
    {
        if (!AidopJobGate.ShouldRun(nameof(MdpOutboxDeadLetterAlertJob), _logger)) return;

        using var scope = _scopeFactory.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>();

        var now = DateTime.Now;
        var scanSince = _lastScanUtc?.ToLocalTime() ?? now.AddMinutes(-5);

        var newDeadCount = await db.Queryable<MdpOutbox>()
            .CountAsync(x => x.Status == 2 && x.UpdateTime >= scanSince, stoppingToken);

        var maxDeadId = await db.Queryable<MdpOutbox>()
            .Where(x => x.Status == 2)
            .MaxAsync(x => (long?)x.Id, stoppingToken) ?? 0L;
        // SqlSugar 表达式树不能取私有静态字段的值，先落到局部变量再进 lambda
        var lastSeenDeadMaxId = _lastSeenDeadMaxId;
        var newDeadById = maxDeadId > lastSeenDeadMaxId
            ? await db.Queryable<MdpOutbox>()
                .CountAsync(x => x.Status == 2 && x.Id > lastSeenDeadMaxId, stoppingToken)
            : 0;

        var hasNewDead = newDeadCount > 0 || newDeadById > 0;

        double oldestPendingMinutes = 0;
        var pending = await db.Queryable<MdpOutbox>().CountAsync(x => x.Status == 0, stoppingToken);
        if (pending > 0)
        {
            var oldest = await db.Queryable<MdpOutbox>()
                .Where(x => x.Status == 0)
                .OrderBy(x => x.CreateTime)
                .Select(x => x.CreateTime)
                .FirstAsync(stoppingToken);
            oldestPendingMinutes = (now - oldest).TotalMinutes;
        }

        var stalePending = oldestPendingMinutes > StalePendingThreshold.TotalMinutes;
        _lastScanUtc = DateTime.UtcNow;
        if (maxDeadId > lastSeenDeadMaxId)
            _lastSeenDeadMaxId = maxDeadId;

        if (!hasNewDead && !stalePending)
            return;

        lock (Gate)
        {
            if (_lastAlertUtc.HasValue && DateTime.UtcNow - _lastAlertUtc.Value < AlertCooldown)
                return;
            _lastAlertUtc = DateTime.UtcNow;
        }

        var deadTotal = await db.Queryable<MdpOutbox>().CountAsync(x => x.Status == 2, stoppingToken);
        var msg =
            $"Outbox 告警：新增死信≈{Math.Max(newDeadCount, newDeadById)}（累计死信={deadTotal}），" +
            $"待推={pending}，最老待推≈{oldestPendingMinutes:F1} 分钟。" +
            $"请到「出站回写队列」查看 /aidop/data-platform/outbox。";

        _logger.LogWarning("[MdpOutboxDeadLetterAlertJob] {Message}", msg);

        try
        {
            // 后台作业无登录态，直接落 SysNotice + 一条 SysNoticeUser（超管），不走 SysNoticeService.InitNoticeInfo
            var notice = new SysNotice
            {
                Title = "Outbox死信/积压告警",
                Content = msg,
                Type = NoticeTypeEnum.NOTICE,
                PublicUserId = NoticeReceiverUserId,
                PublicUserName = NoticeReceiverUserName,
                PublicTime = now,
                Status = NoticeStatusEnum.PUBLIC,
                CreateTime = now,
                CreateUserId = NoticeReceiverUserId,
                CreateUserName = NoticeReceiverUserName,
            };
            var noticeId = await db.Insertable(notice).ExecuteReturnSnowflakeIdAsync(stoppingToken);
            await db.Insertable(new SysNoticeUser
            {
                NoticeId = noticeId,
                UserId = NoticeReceiverUserId,
                ReadStatus = NoticeUserStatusEnum.UNREAD,
            }).ExecuteCommandAsync(stoppingToken);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "[MdpOutboxDeadLetterAlertJob] SysNotice 写入失败");
        }
    }
}