S8TimeoutAutoEscalationService.cs 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143
  1. using Admin.NET.Plugin.AiDOP.Entity.S8;
  2. using Microsoft.Extensions.Logging;
  3. namespace Admin.NET.Plugin.AiDOP.Service.S8;
  4. /// <summary>
  5. /// S8-TIMEOUT-AUTO-ESCALATION-JOB-1(P4-1):扫描 sla_deadline 已超时且未关闭/未已升级的异常,
  6. /// 通过 <see cref="S8TaskFlowService.UpgradeAsync"/> 启动 EXCEPTION_ESCALATION ApprovalFlow,与人工升级链路 100% 等价。
  7. ///
  8. /// 设计要点:
  9. /// - 不依赖 timeout_flag;扫描公式与读端 IsCurrentlyTimeout 一致:sla_deadline IS NOT NULL AND sla_deadline &lt; now
  10. /// AND status NOT IN ('CLOSED','RECOVERED','ESCALATED')。
  11. /// - status 进一步限制在 ASSIGNED / IN_PROGRESS(与 <see cref="Infrastructure.S8.S8StatusRules"/> 允许 ESCALATED 的转移一致);
  12. /// NEW / PENDING_VERIFICATION / REJECTED 等状态不通过本 Job 自动升级。
  13. /// - 已有 active_flow_instance_id 的异常跳过(防重);UpgradeAsync 内部还会再校验一次,双层保险。
  14. /// - exception_type.escalate_role_code 为空 / 非法时跳过并 LogInformation;不写脏数据,不补默认。
  15. /// - 状态 / Timeline 由 <see cref="ExceptionEscalationBizHandler"/>.OnFlowStarted 写入;本服务不重复维护。
  16. /// - 通知层走 <see cref="S8NotificationLayerResolver"/>;当前 baseline notify_channel="log",无外部副作用。
  17. /// </summary>
  18. public class S8TimeoutAutoEscalationService : ITransient
  19. {
  20. private readonly SqlSugarRepository<AdoS8Exception> _rep;
  21. private readonly SqlSugarRepository<AdoS8ExceptionType> _typeRep;
  22. private readonly S8TaskFlowService _taskFlow;
  23. private readonly S8NotificationLayerResolver _layerResolver;
  24. private readonly ILogger<S8TimeoutAutoEscalationService> _logger;
  25. // S8StatusRules 中 ESCALATED 仅允许从 ASSIGNED / IN_PROGRESS 转入。
  26. private static readonly string[] EscalatableStatuses = { "ASSIGNED", "IN_PROGRESS" };
  27. public S8TimeoutAutoEscalationService(
  28. SqlSugarRepository<AdoS8Exception> rep,
  29. SqlSugarRepository<AdoS8ExceptionType> typeRep,
  30. S8TaskFlowService taskFlow,
  31. S8NotificationLayerResolver layerResolver,
  32. ILogger<S8TimeoutAutoEscalationService> logger)
  33. {
  34. _rep = rep;
  35. _typeRep = typeRep;
  36. _taskFlow = taskFlow;
  37. _layerResolver = layerResolver;
  38. _logger = logger;
  39. }
  40. /// <summary>
  41. /// 扫描一次。返回成功触发升级的异常数量;调用方负责调度 / 限流。
  42. /// </summary>
  43. public async Task<int> RunOnceAsync(int batchSize = 50, CancellationToken ct = default)
  44. {
  45. var now = DateTime.Now;
  46. var candidates = await _rep.AsQueryable()
  47. .Where(x => !x.IsDeleted
  48. && x.SlaDeadline != null
  49. && x.SlaDeadline < now
  50. && EscalatableStatuses.Contains(x.Status)
  51. && (x.ActiveFlowInstanceId == null || x.ActiveFlowInstanceId == 0))
  52. .OrderBy(x => x.SlaDeadline)
  53. .Take(batchSize)
  54. .ToListAsync();
  55. if (candidates.Count == 0) return 0;
  56. // exception_type.escalate_role_code 批量映射(factory 优先以与现有 ResolveModuleCodeAsync 一致)
  57. var typeCodes = candidates
  58. .Select(c => c.ExceptionTypeCode)
  59. .Where(c => !string.IsNullOrWhiteSpace(c))
  60. .Distinct()
  61. .Select(c => c!)
  62. .ToList();
  63. var typeMap = typeCodes.Count == 0
  64. ? new Dictionary<string, AdoS8ExceptionType>()
  65. : (await _typeRep.AsQueryable().ClearFilter()
  66. .Where(t => typeCodes.Contains(t.TypeCode))
  67. .OrderByDescending(t => t.FactoryId)
  68. .ToListAsync())
  69. .GroupBy(t => t.TypeCode)
  70. .ToDictionary(g => g.Key, g => g.First());
  71. var processed = 0;
  72. foreach (var e in candidates)
  73. {
  74. ct.ThrowIfCancellationRequested();
  75. if (string.IsNullOrWhiteSpace(e.ExceptionTypeCode)
  76. || !typeMap.TryGetValue(e.ExceptionTypeCode, out var type)
  77. || string.IsNullOrWhiteSpace(type.EscalateRoleCode))
  78. {
  79. _logger.LogInformation(
  80. "s8_timeout_auto_escalate_skip exceptionId={Id} exceptionCode={Code} reason=escalate_role_empty typeCode={TypeCode}",
  81. e.Id, e.ExceptionCode, e.ExceptionTypeCode);
  82. continue;
  83. }
  84. try
  85. {
  86. // UpgradeAsync 已内置 ActiveFlowInstanceId / IsAllowedTransition 二次校验;
  87. // 与 manual upgrade 100% 等价,状态/timeline 由 OnFlowStarted 写入。
  88. var remark = $"[AUTO] SLA deadline exceeded; auto escalation triggered. sla_deadline={e.SlaDeadline:yyyy-MM-dd HH:mm:ss}; escalate_role_code={type.EscalateRoleCode}";
  89. await _taskFlow.UpgradeAsync(e.Id, e.TenantId, e.FactoryId, remark);
  90. processed++;
  91. _logger.LogInformation(
  92. "s8_timeout_auto_escalate_started exceptionId={Id} exceptionCode={Code} typeCode={TypeCode} escalateRoleCode={Role}",
  93. e.Id, e.ExceptionCode, e.ExceptionTypeCode, type.EscalateRoleCode);
  94. await TryDispatchAsync(e);
  95. }
  96. catch (Exception ex)
  97. {
  98. _logger.LogWarning(ex,
  99. "s8_timeout_auto_escalate_failed exceptionId={Id} status={Status}", e.Id, e.Status);
  100. }
  101. }
  102. if (processed > 0 || candidates.Count > 0)
  103. _logger.LogInformation(
  104. "s8_timeout_auto_escalate_summary processed={Processed} candidates={Total}", processed, candidates.Count);
  105. return processed;
  106. }
  107. private async Task TryDispatchAsync(AdoS8Exception e)
  108. {
  109. try
  110. {
  111. await _layerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  112. {
  113. TenantId = e.TenantId,
  114. FactoryId = e.FactoryId,
  115. ExceptionId = e.Id,
  116. ExceptionNo = e.ExceptionCode,
  117. // 优先 module_code(S1-S7 严格基线),保持与 NotificationLayer baseline 同口径。
  118. SceneCode = string.IsNullOrWhiteSpace(e.ModuleCode) ? e.SceneCode : e.ModuleCode!,
  119. Severity = e.Severity,
  120. Status = "ESCALATED",
  121. Title = $"[AUTO] 异常升级 - {e.ExceptionCode}",
  122. Content = "SLA 已超时,系统自动触发升级。",
  123. SourceRuleCode = e.SourceRuleCode,
  124. });
  125. }
  126. catch (Exception ex)
  127. {
  128. _logger.LogWarning(ex, "s8_timeout_auto_escalate_dispatch_failed exceptionId={Id}", e.Id);
  129. }
  130. }
  131. }