S8TimeoutAutoEscalationService.cs 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154
  1. using Admin.NET.Plugin.AiDOP.Const.S8;
  2. using Admin.NET.Plugin.AiDOP.Entity.S8;
  3. using Admin.NET.Plugin.AiDOP.Infrastructure;
  4. using Microsoft.Extensions.Logging;
  5. namespace Admin.NET.Plugin.AiDOP.Service.S8;
  6. /// <summary>
  7. /// S8-TIMEOUT-AUTO-ESCALATION-JOB-1(P4-1):扫描 sla_deadline 已超时且未关闭/未已升级的异常,
  8. /// 通过 <see cref="S8TaskFlowService.UpgradeAsync"/> 启动 EXCEPTION_ESCALATION ApprovalFlow,与人工升级链路 100% 等价。
  9. ///
  10. /// 设计要点:
  11. /// - 不依赖 timeout_flag;扫描公式与读端 IsCurrentlyTimeout 一致:sla_deadline IS NOT NULL AND sla_deadline &lt; now
  12. /// AND status NOT IN ('CLOSED','RECOVERED','ESCALATED')。
  13. /// - status 进一步限制在 ASSIGNED / IN_PROGRESS(与 <see cref="Infrastructure.S8.S8StatusRules"/> 允许 ESCALATED 的转移一致);
  14. /// NEW / PENDING_VERIFICATION / REJECTED 等状态不通过本 Job 自动升级。
  15. /// - 已有 active_flow_instance_id 的异常跳过(防重);UpgradeAsync 内部还会再校验一次,双层保险。
  16. /// - exception_type.escalate_role_code 为空 / 非法时跳过并 LogInformation;不写脏数据,不补默认。
  17. /// - 状态 / Timeline 由 <see cref="ExceptionEscalationBizHandler"/>.OnFlowStarted 写入;本服务不重复维护。
  18. /// - 通知层走 <see cref="S8NotificationLayerResolver"/>;当前 baseline notify_channel="log",无外部副作用。
  19. /// </summary>
  20. public class S8TimeoutAutoEscalationService : ITransient
  21. {
  22. private readonly SqlSugarRepository<AdoS8Exception> _rep;
  23. private readonly SqlSugarRepository<AdoS8ExceptionType> _typeRep;
  24. private readonly S8TaskFlowService _taskFlow;
  25. private readonly S8NotificationLayerResolver _layerResolver;
  26. private readonly ILogger<S8TimeoutAutoEscalationService> _logger;
  27. public S8TimeoutAutoEscalationService(
  28. SqlSugarRepository<AdoS8Exception> rep,
  29. SqlSugarRepository<AdoS8ExceptionType> typeRep,
  30. S8TaskFlowService taskFlow,
  31. S8NotificationLayerResolver layerResolver,
  32. ILogger<S8TimeoutAutoEscalationService> logger)
  33. {
  34. _rep = rep;
  35. _typeRep = typeRep;
  36. _taskFlow = taskFlow;
  37. _layerResolver = layerResolver;
  38. _logger = logger;
  39. }
  40. /// <summary>
  41. /// 扫描一次。返回成功触发升级的异常数量;调用方负责调度 / 限流。
  42. /// </summary>
  43. public async Task<int> RunOnceAsync(int batchSize = 50, CancellationToken ct = default)
  44. {
  45. var now = DateTime.Now;
  46. var candidates = await _rep.AsQueryable()
  47. .Where(x => !x.IsDeleted
  48. && x.SlaDeadline != null
  49. && x.SlaDeadline < now
  50. && (x.Status == "ASSIGNED" || x.Status == "IN_PROGRESS")
  51. && (x.ActiveFlowInstanceId == null || x.ActiveFlowInstanceId == 0))
  52. .OrderBy(x => x.SlaDeadline)
  53. .Take(batchSize)
  54. .ToListAsync();
  55. if (candidates.Count == 0) return 0;
  56. // exception_type.escalate_role_code 批量映射(本租户覆盖优先于平台默认)
  57. var typeCodes = candidates
  58. .Select(c => c.ExceptionTypeCode)
  59. .Where(c => !string.IsNullOrWhiteSpace(c))
  60. .Distinct()
  61. .Select(c => c!)
  62. .ToList();
  63. // S8-P0-1-SCHEDULER-TRUSTED-SCOPE-1:原实现按 TypeCode 全库取「factory_id 最大」的一行,
  64. // 无租户/工厂谓词 —— 本 Job 是跨租户扫描,会把 B 租户的 escalate_role_code 用到 A 租户的异常上。
  65. // 现改为逐异常按其自身 TenantId 解析:候选 = 本租户覆盖 ∪ 平台默认 (tenant_id = 0)。
  66. //
  67. // S8-TENANT-ONLY-BATCH6:precedence 由 OrderByDescending(FactoryId) 改为「租户行优先」。
  68. // 本批之后租户覆盖行的 factory_id 恒为 0,与平台默认相等,按 factory 排序会退化成任意序,
  69. // 可能拿平台默认的 escalate_role_code 去升级一条本租户已自定义的异常。
  70. var typeRows = typeCodes.Count == 0
  71. ? new List<AdoS8ExceptionType>()
  72. : (await _typeRep.AsQueryable().ClearFilter()
  73. .Where(t => typeCodes.Contains(t.TypeCode))
  74. .ToListAsync())
  75. .OrderByDescending(t => t.TenantId != S8ConfigScope.GlobalTenantId)
  76. .ToList();
  77. AdoS8ExceptionType? ResolveTypeForScope(string typeCode, long tenantId) =>
  78. typeRows.FirstOrDefault(t => t.TypeCode == typeCode
  79. && (t.TenantId == tenantId || t.TenantId == S8ConfigScope.GlobalTenantId));
  80. var processed = 0;
  81. foreach (var e in candidates)
  82. {
  83. ct.ThrowIfCancellationRequested();
  84. var type = string.IsNullOrWhiteSpace(e.ExceptionTypeCode)
  85. ? null
  86. : ResolveTypeForScope(e.ExceptionTypeCode!, e.TenantId);
  87. if (type == null || string.IsNullOrWhiteSpace(type.EscalateRoleCode))
  88. {
  89. _logger.LogInformation(
  90. "s8_timeout_auto_escalate_skip exceptionId={Id} exceptionCode={Code} reason=escalate_role_empty typeCode={TypeCode}",
  91. e.Id, e.ExceptionCode, e.ExceptionTypeCode);
  92. continue;
  93. }
  94. try
  95. {
  96. // UpgradeAsync 已内置 ActiveFlowInstanceId / IsAllowedTransition 二次校验;
  97. // 与 manual upgrade 100% 等价,状态/timeline 由 OnFlowStarted 写入。
  98. var remark = $"[AUTO] SLA deadline exceeded; auto escalation triggered. sla_deadline={e.SlaDeadline:yyyy-MM-dd HH:mm:ss}; escalate_role_code={type.EscalateRoleCode}";
  99. await _taskFlow.UpgradeAsync(e.Id, e.TenantId, remark);
  100. processed++;
  101. _logger.LogInformation(
  102. "s8_timeout_auto_escalate_started exceptionId={Id} exceptionCode={Code} typeCode={TypeCode} escalateRoleCode={Role}",
  103. e.Id, e.ExceptionCode, e.ExceptionTypeCode, type.EscalateRoleCode);
  104. await TryDispatchAsync(e);
  105. }
  106. catch (Exception ex)
  107. {
  108. _logger.LogWarning(ex,
  109. "s8_timeout_auto_escalate_failed exceptionId={Id} status={Status}", e.Id, e.Status);
  110. }
  111. }
  112. if (processed > 0 || candidates.Count > 0)
  113. _logger.LogInformation(
  114. "s8_timeout_auto_escalate_summary processed={Processed} candidates={Total}", processed, candidates.Count);
  115. return processed;
  116. }
  117. private async Task TryDispatchAsync(AdoS8Exception e)
  118. {
  119. try
  120. {
  121. await _layerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  122. {
  123. TenantId = e.TenantId,
  124. ExceptionId = e.Id,
  125. ExceptionNo = e.ExceptionCode,
  126. // 优先 module_code(S1-S7 严格基线),保持与 NotificationLayer baseline 同口径。
  127. SceneCode = string.IsNullOrWhiteSpace(e.ModuleCode) ? e.SceneCode : e.ModuleCode!,
  128. Severity = e.Severity,
  129. Status = "ESCALATED",
  130. EventCode = S8NotifyEventCode.EscalationTriggered,
  131. ExceptionRef = e,
  132. Title = $"[AUTO] 异常升级 - {e.ExceptionCode}",
  133. Content = "SLA 已超时,系统自动触发升级。",
  134. SourceRuleCode = e.SourceRuleCode,
  135. });
  136. }
  137. catch (Exception ex)
  138. {
  139. _logger.LogWarning(ex, "s8_timeout_auto_escalate_dispatch_failed exceptionId={Id}", e.Id);
  140. }
  141. }
  142. }