فهرست منبع

feat(s8): scan orphan approval flow instances (N-2)

S8ActiveFlowWatchService now flags ApprovalFlowInstance rows of
EXCEPTION_ESCALATION/EXCEPTION_CLOSURE BizType that are Running,
exceed ThresholdHours, and have no AdoS8Exception referencing them
via ActiveFlowInstanceId — catches OnFlowStarted-failed orphans
that the existing stuck-exception scan misses.

Alerts use a dedicated channel s8-orphan-flow-instance; cooldown
dedup parses flowInstanceId back from payload (exception_id is
null for orphan rows so existing per-exception dedup cannot apply).

Reuses existing ThresholdHours / AlertCooldownMinutes / BatchSize
config; no schema changes.

Tests cover TryExtractFlowInstanceId pure logic; SqlSugar query
path follows project convention of run-once e2e validation.
YY968XX 2 ماه پیش
والد
کامیت
94cab1d87a

+ 47 - 0
server/Admin.NET.Test/S8/S8ActiveFlowWatchServiceTests.cs

@@ -0,0 +1,47 @@
+using Admin.NET.Plugin.AiDOP.Service.S8;
+using Xunit;
+
+namespace Admin.NET.Test.S8;
+
+/// <summary>
+/// N-2 孤立审批实例告警的纯逻辑回归。
+/// 仅覆盖 payload→flowInstanceId 解析路径;SqlSugar 查询路径走 run-once 端到端验证。
+/// </summary>
+public class S8ActiveFlowWatchServiceTests
+{
+    [Fact]
+    public void TryExtractFlowInstanceId_Returns_Id_For_Valid_Payload()
+    {
+        var payload = """{"type":"S8_ORPHAN_FLOW_INSTANCE","flowInstanceId":796332066504773,"bizType":"EXCEPTION_ESCALATION"}""";
+        var id = S8ActiveFlowWatchService.TryExtractFlowInstanceId(payload);
+        Assert.Equal(796332066504773L, id);
+    }
+
+    [Fact]
+    public void TryExtractFlowInstanceId_Returns_Null_When_Field_Missing()
+    {
+        var payload = """{"type":"S8_ACTIVE_FLOW_STUCK","exceptionId":42}""";
+        Assert.Null(S8ActiveFlowWatchService.TryExtractFlowInstanceId(payload));
+    }
+
+    [Fact]
+    public void TryExtractFlowInstanceId_Returns_Null_When_Field_Wrong_Type()
+    {
+        var payload = """{"flowInstanceId":"not-a-number"}""";
+        Assert.Null(S8ActiveFlowWatchService.TryExtractFlowInstanceId(payload));
+    }
+
+    [Fact]
+    public void TryExtractFlowInstanceId_Returns_Null_For_Invalid_Json()
+    {
+        Assert.Null(S8ActiveFlowWatchService.TryExtractFlowInstanceId("{not json"));
+    }
+
+    [Fact]
+    public void TryExtractFlowInstanceId_Returns_Null_For_Null_Or_Empty()
+    {
+        Assert.Null(S8ActiveFlowWatchService.TryExtractFlowInstanceId(null));
+        Assert.Null(S8ActiveFlowWatchService.TryExtractFlowInstanceId(""));
+        Assert.Null(S8ActiveFlowWatchService.TryExtractFlowInstanceId("   "));
+    }
+}

+ 126 - 0
server/Plugins/Admin.NET.Plugin.AiDOP/Service/S8/S8ActiveFlowWatchService.cs

@@ -1,4 +1,5 @@
 using Admin.NET.Plugin.AiDOP.Entity.S8;
+using Admin.NET.Plugin.ApprovalFlow;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using SqlSugar;
@@ -12,9 +13,13 @@ namespace Admin.NET.Plugin.AiDOP.Service.S8;
 public class S8ActiveFlowWatchService : ITransient
 {
     public const string AlertChannel = "s8-active-flow-stuck";
+    public const string AlertChannelOrphan = "s8-orphan-flow-instance";
+
+    private static readonly string[] S8FlowBizTypes = new[] { "EXCEPTION_ESCALATION", "EXCEPTION_CLOSURE" };
 
     private readonly SqlSugarRepository<AdoS8Exception> _exceptionRep;
     private readonly SqlSugarRepository<AdoS8NotificationLog> _notificationLogRep;
+    private readonly SqlSugarRepository<ApprovalFlowInstance> _flowInstanceRep;
     private readonly S8NotificationService _notificationService;
     private readonly S8ActiveFlowWatchOptions _options;
     private readonly ILogger<S8ActiveFlowWatchService> _logger;
@@ -22,12 +27,14 @@ public class S8ActiveFlowWatchService : ITransient
     public S8ActiveFlowWatchService(
         SqlSugarRepository<AdoS8Exception> exceptionRep,
         SqlSugarRepository<AdoS8NotificationLog> notificationLogRep,
+        SqlSugarRepository<ApprovalFlowInstance> flowInstanceRep,
         S8NotificationService notificationService,
         IOptions<S8ActiveFlowWatchOptions> options,
         ILogger<S8ActiveFlowWatchService> logger)
     {
         _exceptionRep = exceptionRep;
         _notificationLogRep = notificationLogRep;
+        _flowInstanceRep = flowInstanceRep;
         _notificationService = notificationService;
         _options = options.Value;
         _logger = logger;
@@ -121,6 +128,125 @@ public class S8ActiveFlowWatchService : ITransient
                 staleExceptions.Count);
         }
 
+        var orphanAlertCount = await ScanOrphanFlowInstancesAsync(staleBefore, alertedAfter, batchSize, now, thresholdHours, cancellationToken);
+
+        return alertCount + orphanAlertCount;
+    }
+
+    /// <summary>
+    /// N-2:扫描孤立的 S8 审批实例(FlowInstance 存在但无任何 AdoS8Exception 引用),疑似 OnFlowStarted 失败或回调链路丢失。
+    /// </summary>
+    private async Task<int> ScanOrphanFlowInstancesAsync(
+        DateTime staleBefore,
+        DateTime alertedAfter,
+        int batchSize,
+        DateTime now,
+        int thresholdHours,
+        CancellationToken cancellationToken)
+    {
+        var candidates = await _flowInstanceRep.AsQueryable()
+            .Where(fi => S8FlowBizTypes.Contains(fi.BizType)
+                         && fi.Status == FlowInstanceStatusEnum.Running
+                         && fi.StartTime < staleBefore)
+            .OrderBy(fi => fi.StartTime, OrderByType.Asc)
+            .Take(batchSize)
+            .ToListAsync();
+
+        if (candidates.Count == 0)
+            return 0;
+
+        var bizIds = candidates.Select(fi => fi.BizId).ToHashSet();
+        var instanceIds = candidates.Select(fi => fi.Id).ToHashSet();
+        var linked = await _exceptionRep.AsQueryable()
+            .Where(e => !e.IsDeleted
+                        && bizIds.Contains(e.Id)
+                        && e.ActiveFlowInstanceId.HasValue
+                        && instanceIds.Contains(e.ActiveFlowInstanceId!.Value))
+            .Select(e => new { e.Id, FlowId = e.ActiveFlowInstanceId!.Value })
+            .ToListAsync();
+        var linkedPairs = linked.Select(x => (x.Id, x.FlowId)).ToHashSet();
+
+        var orphans = candidates.Where(fi => !linkedPairs.Contains((fi.BizId, fi.Id))).ToList();
+        if (orphans.Count == 0)
+            return 0;
+
+        var recentPayloads = await _notificationLogRep.AsQueryable()
+            .Where(x => x.Channel == AlertChannelOrphan && x.CreatedAt >= alertedAfter)
+            .Select(x => x.Payload)
+            .ToListAsync();
+        var alertedInstanceIds = new HashSet<long>();
+        foreach (var p in recentPayloads)
+        {
+            var id = TryExtractFlowInstanceId(p);
+            if (id.HasValue) alertedInstanceIds.Add(id.Value);
+        }
+
+        var alertCount = 0;
+        foreach (var fi in orphans.Where(x => !alertedInstanceIds.Contains(x.Id)))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var payload = new
+            {
+                type = "S8_ORPHAN_FLOW_INSTANCE",
+                message = "S8 审批实例存在但未被任何业务异常单据引用,疑似 OnFlowStarted 失败或回调链路丢失",
+                flowInstanceId = fi.Id,
+                bizType = fi.BizType,
+                bizId = fi.BizId,
+                bizNo = fi.BizNo,
+                flowStatus = fi.Status.ToString(),
+                startTime = fi.StartTime,
+                thresholdHours,
+                detectedAt = now
+            };
+
+            try
+            {
+                await _notificationService.SendAsync(0, 0, null, AlertChannelOrphan, payload);
+                _logger.LogWarning(
+                    "S8 孤立审批实例: FlowInstanceId={InstanceId}, BizType={BizType}, BizId={BizId}, StartTime={StartTime}",
+                    fi.Id, fi.BizType, fi.BizId, fi.StartTime);
+                alertCount++;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex,
+                    "S8 孤立审批实例告警写入失败: FlowInstanceId={InstanceId}, BizType={BizType}, BizId={BizId}",
+                    fi.Id, fi.BizType, fi.BizId);
+            }
+        }
+
+        if (alertCount > 0)
+        {
+            _logger.LogInformation(
+                "S8 孤立审批实例扫描完成,本轮新增告警 {AlertCount} 条,候选 {CandidateCount} 条",
+                alertCount, orphans.Count);
+        }
+
         return alertCount;
     }
+
+    /// <summary>
+    /// 从历史告警 payload JSON 中提取 flowInstanceId 用于冷却去重。失败返回 null。
+    /// </summary>
+    internal static long? TryExtractFlowInstanceId(string? payload)
+    {
+        if (string.IsNullOrWhiteSpace(payload))
+            return null;
+        try
+        {
+            using var doc = System.Text.Json.JsonDocument.Parse(payload);
+            if (doc.RootElement.TryGetProperty("flowInstanceId", out var prop)
+                && prop.ValueKind == System.Text.Json.JsonValueKind.Number
+                && prop.TryGetInt64(out var id))
+            {
+                return id;
+            }
+        }
+        catch
+        {
+            // payload 非法 JSON 时忽略,不影响后续告警。
+        }
+        return null;
+    }
 }