|
|
@@ -1,4 +1,5 @@
|
|
|
using Admin.NET.Plugin.AiDOP.Entity.S8;
|
|
|
+using Admin.NET.Plugin.ApprovalFlow;
|
|
|
using Microsoft.Extensions.Logging;
|
|
|
using Microsoft.Extensions.Options;
|
|
|
using SqlSugar;
|
|
|
@@ -12,9 +13,13 @@ namespace Admin.NET.Plugin.AiDOP.Service.S8;
|
|
|
public class S8ActiveFlowWatchService : ITransient
|
|
|
{
|
|
|
public const string AlertChannel = "s8-active-flow-stuck";
|
|
|
+ public const string AlertChannelOrphan = "s8-orphan-flow-instance";
|
|
|
+
|
|
|
+ private static readonly string[] S8FlowBizTypes = new[] { "EXCEPTION_ESCALATION", "EXCEPTION_CLOSURE" };
|
|
|
|
|
|
private readonly SqlSugarRepository<AdoS8Exception> _exceptionRep;
|
|
|
private readonly SqlSugarRepository<AdoS8NotificationLog> _notificationLogRep;
|
|
|
+ private readonly SqlSugarRepository<ApprovalFlowInstance> _flowInstanceRep;
|
|
|
private readonly S8NotificationService _notificationService;
|
|
|
private readonly S8ActiveFlowWatchOptions _options;
|
|
|
private readonly ILogger<S8ActiveFlowWatchService> _logger;
|
|
|
@@ -22,12 +27,14 @@ public class S8ActiveFlowWatchService : ITransient
|
|
|
public S8ActiveFlowWatchService(
|
|
|
SqlSugarRepository<AdoS8Exception> exceptionRep,
|
|
|
SqlSugarRepository<AdoS8NotificationLog> notificationLogRep,
|
|
|
+ SqlSugarRepository<ApprovalFlowInstance> flowInstanceRep,
|
|
|
S8NotificationService notificationService,
|
|
|
IOptions<S8ActiveFlowWatchOptions> options,
|
|
|
ILogger<S8ActiveFlowWatchService> logger)
|
|
|
{
|
|
|
_exceptionRep = exceptionRep;
|
|
|
_notificationLogRep = notificationLogRep;
|
|
|
+ _flowInstanceRep = flowInstanceRep;
|
|
|
_notificationService = notificationService;
|
|
|
_options = options.Value;
|
|
|
_logger = logger;
|
|
|
@@ -121,6 +128,125 @@ public class S8ActiveFlowWatchService : ITransient
|
|
|
staleExceptions.Count);
|
|
|
}
|
|
|
|
|
|
+ var orphanAlertCount = await ScanOrphanFlowInstancesAsync(staleBefore, alertedAfter, batchSize, now, thresholdHours, cancellationToken);
|
|
|
+
|
|
|
+ return alertCount + orphanAlertCount;
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// N-2:扫描孤立的 S8 审批实例(FlowInstance 存在但无任何 AdoS8Exception 引用),疑似 OnFlowStarted 失败或回调链路丢失。
|
|
|
+ /// </summary>
|
|
|
+ private async Task<int> ScanOrphanFlowInstancesAsync(
|
|
|
+ DateTime staleBefore,
|
|
|
+ DateTime alertedAfter,
|
|
|
+ int batchSize,
|
|
|
+ DateTime now,
|
|
|
+ int thresholdHours,
|
|
|
+ CancellationToken cancellationToken)
|
|
|
+ {
|
|
|
+ var candidates = await _flowInstanceRep.AsQueryable()
|
|
|
+ .Where(fi => S8FlowBizTypes.Contains(fi.BizType)
|
|
|
+ && fi.Status == FlowInstanceStatusEnum.Running
|
|
|
+ && fi.StartTime < staleBefore)
|
|
|
+ .OrderBy(fi => fi.StartTime, OrderByType.Asc)
|
|
|
+ .Take(batchSize)
|
|
|
+ .ToListAsync();
|
|
|
+
|
|
|
+ if (candidates.Count == 0)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ var bizIds = candidates.Select(fi => fi.BizId).ToHashSet();
|
|
|
+ var instanceIds = candidates.Select(fi => fi.Id).ToHashSet();
|
|
|
+ var linked = await _exceptionRep.AsQueryable()
|
|
|
+ .Where(e => !e.IsDeleted
|
|
|
+ && bizIds.Contains(e.Id)
|
|
|
+ && e.ActiveFlowInstanceId.HasValue
|
|
|
+ && instanceIds.Contains(e.ActiveFlowInstanceId!.Value))
|
|
|
+ .Select(e => new { e.Id, FlowId = e.ActiveFlowInstanceId!.Value })
|
|
|
+ .ToListAsync();
|
|
|
+ var linkedPairs = linked.Select(x => (x.Id, x.FlowId)).ToHashSet();
|
|
|
+
|
|
|
+ var orphans = candidates.Where(fi => !linkedPairs.Contains((fi.BizId, fi.Id))).ToList();
|
|
|
+ if (orphans.Count == 0)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ var recentPayloads = await _notificationLogRep.AsQueryable()
|
|
|
+ .Where(x => x.Channel == AlertChannelOrphan && x.CreatedAt >= alertedAfter)
|
|
|
+ .Select(x => x.Payload)
|
|
|
+ .ToListAsync();
|
|
|
+ var alertedInstanceIds = new HashSet<long>();
|
|
|
+ foreach (var p in recentPayloads)
|
|
|
+ {
|
|
|
+ var id = TryExtractFlowInstanceId(p);
|
|
|
+ if (id.HasValue) alertedInstanceIds.Add(id.Value);
|
|
|
+ }
|
|
|
+
|
|
|
+ var alertCount = 0;
|
|
|
+ foreach (var fi in orphans.Where(x => !alertedInstanceIds.Contains(x.Id)))
|
|
|
+ {
|
|
|
+ cancellationToken.ThrowIfCancellationRequested();
|
|
|
+
|
|
|
+ var payload = new
|
|
|
+ {
|
|
|
+ type = "S8_ORPHAN_FLOW_INSTANCE",
|
|
|
+ message = "S8 审批实例存在但未被任何业务异常单据引用,疑似 OnFlowStarted 失败或回调链路丢失",
|
|
|
+ flowInstanceId = fi.Id,
|
|
|
+ bizType = fi.BizType,
|
|
|
+ bizId = fi.BizId,
|
|
|
+ bizNo = fi.BizNo,
|
|
|
+ flowStatus = fi.Status.ToString(),
|
|
|
+ startTime = fi.StartTime,
|
|
|
+ thresholdHours,
|
|
|
+ detectedAt = now
|
|
|
+ };
|
|
|
+
|
|
|
+ try
|
|
|
+ {
|
|
|
+ await _notificationService.SendAsync(0, 0, null, AlertChannelOrphan, payload);
|
|
|
+ _logger.LogWarning(
|
|
|
+ "S8 孤立审批实例: FlowInstanceId={InstanceId}, BizType={BizType}, BizId={BizId}, StartTime={StartTime}",
|
|
|
+ fi.Id, fi.BizType, fi.BizId, fi.StartTime);
|
|
|
+ alertCount++;
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ _logger.LogError(ex,
|
|
|
+ "S8 孤立审批实例告警写入失败: FlowInstanceId={InstanceId}, BizType={BizType}, BizId={BizId}",
|
|
|
+ fi.Id, fi.BizType, fi.BizId);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (alertCount > 0)
|
|
|
+ {
|
|
|
+ _logger.LogInformation(
|
|
|
+ "S8 孤立审批实例扫描完成,本轮新增告警 {AlertCount} 条,候选 {CandidateCount} 条",
|
|
|
+ alertCount, orphans.Count);
|
|
|
+ }
|
|
|
+
|
|
|
return alertCount;
|
|
|
}
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// 从历史告警 payload JSON 中提取 flowInstanceId 用于冷却去重。失败返回 null。
|
|
|
+ /// </summary>
|
|
|
+ internal static long? TryExtractFlowInstanceId(string? payload)
|
|
|
+ {
|
|
|
+ if (string.IsNullOrWhiteSpace(payload))
|
|
|
+ return null;
|
|
|
+ try
|
|
|
+ {
|
|
|
+ using var doc = System.Text.Json.JsonDocument.Parse(payload);
|
|
|
+ if (doc.RootElement.TryGetProperty("flowInstanceId", out var prop)
|
|
|
+ && prop.ValueKind == System.Text.Json.JsonValueKind.Number
|
|
|
+ && prop.TryGetInt64(out var id))
|
|
|
+ {
|
|
|
+ return id;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ catch
|
|
|
+ {
|
|
|
+ // payload 非法 JSON 时忽略,不影响后续告警。
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
}
|