S8TimeoutAutoEscalationService.cs 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. using Admin.NET.Plugin.AiDOP.Entity.S8;
  2. using Microsoft.Extensions.Logging;
  3. namespace Admin.NET.Plugin.AiDOP.Service.S8;
  4. /// <summary>
  5. /// S8-TIMEOUT-AUTO-ESCALATION-JOB-1(P4-1):扫描 sla_deadline 已超时且未关闭/未已升级的异常,
  6. /// 通过 <see cref="S8TaskFlowService.UpgradeAsync"/> 启动 EXCEPTION_ESCALATION ApprovalFlow,与人工升级链路 100% 等价。
  7. ///
  8. /// 设计要点:
  9. /// - 不依赖 timeout_flag;扫描公式与读端 IsCurrentlyTimeout 一致:sla_deadline IS NOT NULL AND sla_deadline &lt; now
  10. /// AND status NOT IN ('CLOSED','RECOVERED','ESCALATED')。
  11. /// - status 进一步限制在 ASSIGNED / IN_PROGRESS(与 <see cref="Infrastructure.S8.S8StatusRules"/> 允许 ESCALATED 的转移一致);
  12. /// NEW / PENDING_VERIFICATION / REJECTED 等状态不通过本 Job 自动升级。
  13. /// - 已有 active_flow_instance_id 的异常跳过(防重);UpgradeAsync 内部还会再校验一次,双层保险。
  14. /// - exception_type.escalate_role_code 为空 / 非法时跳过并 LogInformation;不写脏数据,不补默认。
  15. /// - 状态 / Timeline 由 <see cref="ExceptionEscalationBizHandler"/>.OnFlowStarted 写入;本服务不重复维护。
  16. /// - 通知层走 <see cref="S8NotificationLayerResolver"/>;当前 baseline notify_channel="log",无外部副作用。
  17. /// </summary>
  18. public class S8TimeoutAutoEscalationService : ITransient
  19. {
  20. private readonly SqlSugarRepository<AdoS8Exception> _rep;
  21. private readonly SqlSugarRepository<AdoS8ExceptionType> _typeRep;
  22. private readonly S8TaskFlowService _taskFlow;
  23. private readonly S8NotificationLayerResolver _layerResolver;
  24. private readonly ILogger<S8TimeoutAutoEscalationService> _logger;
  25. public S8TimeoutAutoEscalationService(
  26. SqlSugarRepository<AdoS8Exception> rep,
  27. SqlSugarRepository<AdoS8ExceptionType> typeRep,
  28. S8TaskFlowService taskFlow,
  29. S8NotificationLayerResolver layerResolver,
  30. ILogger<S8TimeoutAutoEscalationService> logger)
  31. {
  32. _rep = rep;
  33. _typeRep = typeRep;
  34. _taskFlow = taskFlow;
  35. _layerResolver = layerResolver;
  36. _logger = logger;
  37. }
  38. /// <summary>
  39. /// 扫描一次。返回成功触发升级的异常数量;调用方负责调度 / 限流。
  40. /// </summary>
  41. public async Task<int> RunOnceAsync(int batchSize = 50, CancellationToken ct = default)
  42. {
  43. var now = DateTime.Now;
  44. var candidates = await _rep.AsQueryable()
  45. .Where(x => !x.IsDeleted
  46. && x.SlaDeadline != null
  47. && x.SlaDeadline < now
  48. && (x.Status == "ASSIGNED" || x.Status == "IN_PROGRESS")
  49. && (x.ActiveFlowInstanceId == null || x.ActiveFlowInstanceId == 0))
  50. .OrderBy(x => x.SlaDeadline)
  51. .Take(batchSize)
  52. .ToListAsync();
  53. if (candidates.Count == 0) return 0;
  54. // exception_type.escalate_role_code 批量映射(factory 优先以与现有 ResolveModuleCodeAsync 一致)
  55. var typeCodes = candidates
  56. .Select(c => c.ExceptionTypeCode)
  57. .Where(c => !string.IsNullOrWhiteSpace(c))
  58. .Distinct()
  59. .Select(c => c!)
  60. .ToList();
  61. var typeMap = typeCodes.Count == 0
  62. ? new Dictionary<string, AdoS8ExceptionType>()
  63. : (await _typeRep.AsQueryable().ClearFilter()
  64. .Where(t => typeCodes.Contains(t.TypeCode))
  65. .OrderByDescending(t => t.FactoryId)
  66. .ToListAsync())
  67. .GroupBy(t => t.TypeCode)
  68. .ToDictionary(g => g.Key, g => g.First());
  69. var processed = 0;
  70. foreach (var e in candidates)
  71. {
  72. ct.ThrowIfCancellationRequested();
  73. if (string.IsNullOrWhiteSpace(e.ExceptionTypeCode)
  74. || !typeMap.TryGetValue(e.ExceptionTypeCode, out var type)
  75. || string.IsNullOrWhiteSpace(type.EscalateRoleCode))
  76. {
  77. _logger.LogInformation(
  78. "s8_timeout_auto_escalate_skip exceptionId={Id} exceptionCode={Code} reason=escalate_role_empty typeCode={TypeCode}",
  79. e.Id, e.ExceptionCode, e.ExceptionTypeCode);
  80. continue;
  81. }
  82. try
  83. {
  84. // UpgradeAsync 已内置 ActiveFlowInstanceId / IsAllowedTransition 二次校验;
  85. // 与 manual upgrade 100% 等价,状态/timeline 由 OnFlowStarted 写入。
  86. var remark = $"[AUTO] SLA deadline exceeded; auto escalation triggered. sla_deadline={e.SlaDeadline:yyyy-MM-dd HH:mm:ss}; escalate_role_code={type.EscalateRoleCode}";
  87. await _taskFlow.UpgradeAsync(e.Id, e.TenantId, e.FactoryId, remark);
  88. processed++;
  89. _logger.LogInformation(
  90. "s8_timeout_auto_escalate_started exceptionId={Id} exceptionCode={Code} typeCode={TypeCode} escalateRoleCode={Role}",
  91. e.Id, e.ExceptionCode, e.ExceptionTypeCode, type.EscalateRoleCode);
  92. await TryDispatchAsync(e);
  93. }
  94. catch (Exception ex)
  95. {
  96. _logger.LogWarning(ex,
  97. "s8_timeout_auto_escalate_failed exceptionId={Id} status={Status}", e.Id, e.Status);
  98. }
  99. }
  100. if (processed > 0 || candidates.Count > 0)
  101. _logger.LogInformation(
  102. "s8_timeout_auto_escalate_summary processed={Processed} candidates={Total}", processed, candidates.Count);
  103. return processed;
  104. }
  105. private async Task TryDispatchAsync(AdoS8Exception e)
  106. {
  107. try
  108. {
  109. await _layerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  110. {
  111. TenantId = e.TenantId,
  112. FactoryId = e.FactoryId,
  113. ExceptionId = e.Id,
  114. ExceptionNo = e.ExceptionCode,
  115. // 优先 module_code(S1-S7 严格基线),保持与 NotificationLayer baseline 同口径。
  116. SceneCode = string.IsNullOrWhiteSpace(e.ModuleCode) ? e.SceneCode : e.ModuleCode!,
  117. Severity = e.Severity,
  118. Status = "ESCALATED",
  119. Title = $"[AUTO] 异常升级 - {e.ExceptionCode}",
  120. Content = "SLA 已超时,系统自动触发升级。",
  121. SourceRuleCode = e.SourceRuleCode,
  122. });
  123. }
  124. catch (Exception ex)
  125. {
  126. _logger.LogWarning(ex, "s8_timeout_auto_escalate_dispatch_failed exceptionId={Id}", e.Id);
  127. }
  128. }
  129. }