S8TimeoutAutoEscalationService.cs 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151
  1. using Admin.NET.Plugin.AiDOP.Entity.S8;
  2. using Admin.NET.Plugin.AiDOP.Infrastructure;
  3. using Microsoft.Extensions.Logging;
  4. namespace Admin.NET.Plugin.AiDOP.Service.S8;
  5. /// <summary>
  6. /// S8-TIMEOUT-AUTO-ESCALATION-JOB-1(P4-1):扫描 sla_deadline 已超时且未关闭/未已升级的异常,
  7. /// 通过 <see cref="S8TaskFlowService.UpgradeAsync"/> 启动 EXCEPTION_ESCALATION ApprovalFlow,与人工升级链路 100% 等价。
  8. ///
  9. /// 设计要点:
  10. /// - 不依赖 timeout_flag;扫描公式与读端 IsCurrentlyTimeout 一致:sla_deadline IS NOT NULL AND sla_deadline &lt; now
  11. /// AND status NOT IN ('CLOSED','RECOVERED','ESCALATED')。
  12. /// - status 进一步限制在 ASSIGNED / IN_PROGRESS(与 <see cref="Infrastructure.S8.S8StatusRules"/> 允许 ESCALATED 的转移一致);
  13. /// NEW / PENDING_VERIFICATION / REJECTED 等状态不通过本 Job 自动升级。
  14. /// - 已有 active_flow_instance_id 的异常跳过(防重);UpgradeAsync 内部还会再校验一次,双层保险。
  15. /// - exception_type.escalate_role_code 为空 / 非法时跳过并 LogInformation;不写脏数据,不补默认。
  16. /// - 状态 / Timeline 由 <see cref="ExceptionEscalationBizHandler"/>.OnFlowStarted 写入;本服务不重复维护。
  17. /// - 通知层走 <see cref="S8NotificationLayerResolver"/>;当前 baseline notify_channel="log",无外部副作用。
  18. /// </summary>
  19. public class S8TimeoutAutoEscalationService : ITransient
  20. {
  21. private readonly SqlSugarRepository<AdoS8Exception> _rep;
  22. private readonly SqlSugarRepository<AdoS8ExceptionType> _typeRep;
  23. private readonly S8TaskFlowService _taskFlow;
  24. private readonly S8NotificationLayerResolver _layerResolver;
  25. private readonly ILogger<S8TimeoutAutoEscalationService> _logger;
  26. public S8TimeoutAutoEscalationService(
  27. SqlSugarRepository<AdoS8Exception> rep,
  28. SqlSugarRepository<AdoS8ExceptionType> typeRep,
  29. S8TaskFlowService taskFlow,
  30. S8NotificationLayerResolver layerResolver,
  31. ILogger<S8TimeoutAutoEscalationService> logger)
  32. {
  33. _rep = rep;
  34. _typeRep = typeRep;
  35. _taskFlow = taskFlow;
  36. _layerResolver = layerResolver;
  37. _logger = logger;
  38. }
  39. /// <summary>
  40. /// 扫描一次。返回成功触发升级的异常数量;调用方负责调度 / 限流。
  41. /// </summary>
  42. public async Task<int> RunOnceAsync(int batchSize = 50, CancellationToken ct = default)
  43. {
  44. var now = DateTime.Now;
  45. var candidates = await _rep.AsQueryable()
  46. .Where(x => !x.IsDeleted
  47. && x.SlaDeadline != null
  48. && x.SlaDeadline < now
  49. && (x.Status == "ASSIGNED" || x.Status == "IN_PROGRESS")
  50. && (x.ActiveFlowInstanceId == null || x.ActiveFlowInstanceId == 0))
  51. .OrderBy(x => x.SlaDeadline)
  52. .Take(batchSize)
  53. .ToListAsync();
  54. if (candidates.Count == 0) return 0;
  55. // exception_type.escalate_role_code 批量映射(本租户覆盖优先于平台默认)
  56. var typeCodes = candidates
  57. .Select(c => c.ExceptionTypeCode)
  58. .Where(c => !string.IsNullOrWhiteSpace(c))
  59. .Distinct()
  60. .Select(c => c!)
  61. .ToList();
  62. // S8-P0-1-SCHEDULER-TRUSTED-SCOPE-1:原实现按 TypeCode 全库取「factory_id 最大」的一行,
  63. // 无租户/工厂谓词 —— 本 Job 是跨租户扫描,会把 B 租户的 escalate_role_code 用到 A 租户的异常上。
  64. // 现改为逐异常按其自身 TenantId 解析:候选 = 本租户覆盖 ∪ 平台默认 (tenant_id = 0)。
  65. //
  66. // S8-TENANT-ONLY-BATCH6:precedence 由 OrderByDescending(FactoryId) 改为「租户行优先」。
  67. // 本批之后租户覆盖行的 factory_id 恒为 0,与平台默认相等,按 factory 排序会退化成任意序,
  68. // 可能拿平台默认的 escalate_role_code 去升级一条本租户已自定义的异常。
  69. var typeRows = typeCodes.Count == 0
  70. ? new List<AdoS8ExceptionType>()
  71. : (await _typeRep.AsQueryable().ClearFilter()
  72. .Where(t => typeCodes.Contains(t.TypeCode))
  73. .ToListAsync())
  74. .OrderByDescending(t => t.TenantId != S8ConfigScope.GlobalTenantId)
  75. .ToList();
  76. AdoS8ExceptionType? ResolveTypeForScope(string typeCode, long tenantId) =>
  77. typeRows.FirstOrDefault(t => t.TypeCode == typeCode
  78. && (t.TenantId == tenantId || t.TenantId == S8ConfigScope.GlobalTenantId));
  79. var processed = 0;
  80. foreach (var e in candidates)
  81. {
  82. ct.ThrowIfCancellationRequested();
  83. var type = string.IsNullOrWhiteSpace(e.ExceptionTypeCode)
  84. ? null
  85. : ResolveTypeForScope(e.ExceptionTypeCode!, e.TenantId);
  86. if (type == null || string.IsNullOrWhiteSpace(type.EscalateRoleCode))
  87. {
  88. _logger.LogInformation(
  89. "s8_timeout_auto_escalate_skip exceptionId={Id} exceptionCode={Code} reason=escalate_role_empty typeCode={TypeCode}",
  90. e.Id, e.ExceptionCode, e.ExceptionTypeCode);
  91. continue;
  92. }
  93. try
  94. {
  95. // UpgradeAsync 已内置 ActiveFlowInstanceId / IsAllowedTransition 二次校验;
  96. // 与 manual upgrade 100% 等价,状态/timeline 由 OnFlowStarted 写入。
  97. var remark = $"[AUTO] SLA deadline exceeded; auto escalation triggered. sla_deadline={e.SlaDeadline:yyyy-MM-dd HH:mm:ss}; escalate_role_code={type.EscalateRoleCode}";
  98. await _taskFlow.UpgradeAsync(e.Id, e.TenantId, remark);
  99. processed++;
  100. _logger.LogInformation(
  101. "s8_timeout_auto_escalate_started exceptionId={Id} exceptionCode={Code} typeCode={TypeCode} escalateRoleCode={Role}",
  102. e.Id, e.ExceptionCode, e.ExceptionTypeCode, type.EscalateRoleCode);
  103. await TryDispatchAsync(e);
  104. }
  105. catch (Exception ex)
  106. {
  107. _logger.LogWarning(ex,
  108. "s8_timeout_auto_escalate_failed exceptionId={Id} status={Status}", e.Id, e.Status);
  109. }
  110. }
  111. if (processed > 0 || candidates.Count > 0)
  112. _logger.LogInformation(
  113. "s8_timeout_auto_escalate_summary processed={Processed} candidates={Total}", processed, candidates.Count);
  114. return processed;
  115. }
  116. private async Task TryDispatchAsync(AdoS8Exception e)
  117. {
  118. try
  119. {
  120. await _layerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  121. {
  122. TenantId = e.TenantId,
  123. ExceptionId = e.Id,
  124. ExceptionNo = e.ExceptionCode,
  125. // 优先 module_code(S1-S7 严格基线),保持与 NotificationLayer baseline 同口径。
  126. SceneCode = string.IsNullOrWhiteSpace(e.ModuleCode) ? e.SceneCode : e.ModuleCode!,
  127. Severity = e.Severity,
  128. Status = "ESCALATED",
  129. Title = $"[AUTO] 异常升级 - {e.ExceptionCode}",
  130. Content = "SLA 已超时,系统自动触发升级。",
  131. SourceRuleCode = e.SourceRuleCode,
  132. });
  133. }
  134. catch (Exception ex)
  135. {
  136. _logger.LogWarning(ex, "s8_timeout_auto_escalate_dispatch_failed exceptionId={Id}", e.Id);
  137. }
  138. }
  139. }