S8WatchSchedulerService.cs 79 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669
  1. using Admin.NET.Plugin.AiDOP.Const.S8;
  2. using Admin.NET.Plugin.AiDOP.Entity.S8;
  3. using Admin.NET.Plugin.AiDOP.Infrastructure.S8;
  4. using Admin.NET.Plugin.AiDOP.Service.S8.Rules;
  5. using Admin.NET.Plugin.AiDOP.Service.S8.Rules.DataAccess;
  6. using Admin.NET.Plugin.AiDOP.Service.S8.Rules.Definitions;
  7. using Microsoft.Extensions.Logging;
  8. using SqlSugar;
  9. using System.Globalization;
  10. using System.Text.Json;
  11. namespace Admin.NET.Plugin.AiDOP.Service.S8;
  12. /// <summary>
  13. /// 监视规则轮询调度服务(首轮存根)。
  14. /// 后续接入 Admin.NET 定时任务机制后,由调度器周期调用 <see cref="RunOnceAsync"/>,
  15. /// 按各规则的 PollIntervalSeconds 逐条评估并生成异常记录。
  16. /// </summary>
  17. public class S8WatchSchedulerService : ITransient
  18. {
  19. private readonly SqlSugarRepository<AdoS8WatchRule> _ruleRep;
  20. private readonly SqlSugarRepository<AdoS8Exception> _exceptionRep;
  21. private readonly SqlSugarRepository<AdoS8ExceptionType> _exceptionTypeRep;
  22. private readonly S8NotificationService _notificationService;
  23. private readonly S8NotificationLayerResolver _notificationLayerResolver;
  24. private readonly S8ImpactMetricsService _impactMetricsService;
  25. private readonly S8ManualReportService _manualReportService;
  26. // S8-STEP6E:Watch 主链的业务建单边界(唯一最窄 seam)。仅 :1482 的自动建单走它;
  27. // legacy debug 主链的 CreateFromWatchAsync 保持直连 _manualReportService,行为不变。
  28. private readonly IS8ExceptionReportDispatcher _reportDispatcher;
  29. private readonly S8TimeoutRuleEvaluator _timeoutEvaluator;
  30. private readonly S8ShortageRuleEvaluator _shortageEvaluator;
  31. private readonly S8OutOfRangeRuleEvaluator _outOfRangeEvaluator;
  32. private readonly ILogger<S8WatchSchedulerService> _logger;
  33. private readonly SqlSugarRepository<AdoS8DetectionLog> _detectionLogRep;
  34. private readonly SqlSugarRepository<AdoS8RuleDetectionState> _detectionStateRep;
  35. private readonly IS8RuleCatalog _ruleCatalog;
  36. // S8-RULE-READINESS-1:与 Enable / RunNow 共用的就绪门禁。
  37. private readonly S8RuleReadinessGate _readinessGate;
  38. private readonly Rules.Health.IS8AuthorityHealthResolver _authorityHealthResolver;
  39. private const string DetectionTriggerSource = "WATCH_SCHEDULER";
  40. private const string DetectResultCreated = "CREATED";
  41. private const string DetectResultRefreshed = "REFRESHED";
  42. private const string DetectResultRecovered = "RECOVERED";
  43. /// <summary>系统恢复时间线的动作码。与人工动作码同一命名空间,但永远不带操作人。</summary>
  44. internal const string SystemRecoveredActionCode = "SYSTEM_RECOVERED";
  45. private const string DetectResultNoHit = "NO_HIT";
  46. private const string DetectResultEvaluateFailed = "EVALUATE_FAILED";
  47. /// <summary>
  48. /// 本轮因 Authority 不可信而跳过了恢复判定。
  49. /// <para>必须落库而不是只写 text log —— 这道门本身是静默的:跳过恢复不产生任何用户可见变化,
  50. /// 事后必须能从库里回答「为什么这一轮没做恢复」。</para>
  51. /// </summary>
  52. private const string DetectResultRecoveryBlocked = "RECOVERY_BLOCKED";
  53. // S8-DETECTION-LOG-WRITE-REDUCE-P1-1:REFRESHED 明细写入限频窗口。
  54. // 同 (tenant, factory, rule_code, dedup_key) 在 10 分钟内最多写一条 REFRESHED detection_log;
  55. // 业务侧 RefreshDetectionAsync / BackfillLegacyExceptionAsync 不受限频影响。
  56. private const int RefreshedRateLimitMinutes = 10;
  57. private const string DefaultTriggerType = "VALUE_DEVIATION";
  58. private const string SqlDataSourceType = "SQL";
  59. // G01-05 未闭环状态集合:复用自 S8ExceptionService 当前 pendingStatuses 事实口径
  60. // (见 S8ExceptionService.GetPagedAsync 中 pendingStatuses 的定义,两处必须保持一致)。
  61. // 这不是“自定义未闭环集合”;若现有口径调整,两处需同步修改。
  62. private static readonly string[] UnclosedExceptionStatuses =
  63. { "NEW", "ASSIGNED", "IN_PROGRESS", "PENDING_VERIFICATION" };
  64. public S8WatchSchedulerService(
  65. SqlSugarRepository<AdoS8WatchRule> ruleRep,
  66. SqlSugarRepository<AdoS8Exception> exceptionRep,
  67. SqlSugarRepository<AdoS8ExceptionType> exceptionTypeRep,
  68. S8NotificationService notificationService,
  69. S8NotificationLayerResolver notificationLayerResolver,
  70. S8ImpactMetricsService impactMetricsService,
  71. S8ManualReportService manualReportService,
  72. IS8ExceptionReportDispatcher reportDispatcher,
  73. S8TimeoutRuleEvaluator timeoutEvaluator,
  74. S8ShortageRuleEvaluator shortageEvaluator,
  75. S8OutOfRangeRuleEvaluator outOfRangeEvaluator,
  76. ILogger<S8WatchSchedulerService> logger,
  77. SqlSugarRepository<AdoS8DetectionLog> detectionLogRep,
  78. SqlSugarRepository<AdoS8RuleDetectionState> detectionStateRep,
  79. IS8RuleCatalog ruleCatalog,
  80. S8RuleReadinessGate readinessGate,
  81. Rules.Health.IS8AuthorityHealthResolver authorityHealthResolver)
  82. {
  83. _ruleRep = ruleRep;
  84. _exceptionRep = exceptionRep;
  85. _exceptionTypeRep = exceptionTypeRep;
  86. _notificationService = notificationService;
  87. _notificationLayerResolver = notificationLayerResolver;
  88. _impactMetricsService = impactMetricsService;
  89. _manualReportService = manualReportService;
  90. _reportDispatcher = reportDispatcher;
  91. _timeoutEvaluator = timeoutEvaluator;
  92. _shortageEvaluator = shortageEvaluator;
  93. _outOfRangeEvaluator = outOfRangeEvaluator;
  94. _logger = logger;
  95. _detectionLogRep = detectionLogRep;
  96. _detectionStateRep = detectionStateRep;
  97. _ruleCatalog = ruleCatalog;
  98. _readinessGate = readinessGate;
  99. _authorityHealthResolver = authorityHealthResolver;
  100. }
  101. /// <summary>
  102. /// S8-TENANT-ONLY-BATCH5:调度遍历改为 <b>Tenant → Rule</b>。
  103. ///
  104. /// <para>原实现返回 DISTINCT (tenant_id, factory_id),于是同一租户在 N 个工厂下会被 tick N 次;
  105. /// 而规则的运行策略按 Batch 2 的口径每租户只有一条,重复 tick 只会互相抢同一把租约。</para>
  106. ///
  107. /// <para>同时删掉 <c>AND r.factory_id &gt; 0</c>:Batch 2 供给出来的运行策略行
  108. /// <c>factory_id</c> 恒为 0,那条谓词会让它们**永远不被调度、且不报任何错**
  109. /// —— 页面显示已启用,实际从不执行。这是本批必须与供给同时闭合的缺口。</para>
  110. /// </summary>
  111. public async Task<List<long>> ListEnabledTenantsAsync()
  112. {
  113. return await _ruleRep.Context.Ado.SqlQueryAsync<long>(
  114. """
  115. SELECT DISTINCT r.tenant_id
  116. FROM ado_s8_watch_rule r
  117. INNER JOIN SysTenant t ON t.Id = r.tenant_id AND t.Status = 1
  118. WHERE r.enabled = 1
  119. AND r.tenant_id > 0
  120. ORDER BY r.tenant_id
  121. """);
  122. }
  123. // 取任意一条匹配的未闭环异常 Id 作为“是否存在重复单”的拦截依据。
  124. // 首版只需要“存在性”,不关心“最早 / 最新”;不在 G01-05 处理排序语义。
  125. /// <summary>
  126. /// 自动建单入口(debug run-once 使用;生产调度走 <see cref="RunDispatchTickAsync"/>)。
  127. ///
  128. /// S8-LEGACY-SQL-RESIDUAL-CLEANUP-3:原先此处还有一段旧 AlertRule 兼容主链
  129. /// (EvaluateDedupAsync → EvaluateHitsAsync → QueryDeviceRowsAsync → 直连 SQL),
  130. /// 该链自行开 SqlSugarScope 执行 rule.expression,
  131. /// 且只装载 rule_type 为空的未分类历史规则 + 需同场景恰好一条可运行 AlertRule。
  132. /// 经证实其唯一入口是 404 门禁的 debug controller,且 LoadExecutionRulesAsync 的过滤条件
  133. /// 在当前数据下恒返回空,已整体删除。本方法现在只保留三类正式 evaluator 路径。
  134. /// </summary>
  135. public async Task<List<S8WatchCreationResult>> CreateExceptionsAsync(long tenantId)
  136. {
  137. var results = new List<S8WatchCreationResult>();
  138. // R6 RunId:本次 CreateExceptionsAsync 调用对应的统一关联 id,落入 detection_log。
  139. var runId = Guid.NewGuid().ToString("N").Substring(0, 16);
  140. results.AddRange(await ProcessRulesByTypeAsync(tenantId, _timeoutEvaluator, S8TimeoutRuleEvaluator.RuleTypeCode, runId));
  141. results.AddRange(await ProcessRulesByTypeAsync(tenantId, _shortageEvaluator, S8ShortageRuleEvaluator.RuleTypeCode, runId));
  142. results.AddRange(await ProcessRulesByTypeAsync(tenantId, _outOfRangeEvaluator, S8OutOfRangeRuleEvaluator.RuleTypeCode, runId));
  143. return results;
  144. }
  145. /// <summary>
  146. /// R2 TIMEOUT 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。RunId 由内部生成。
  147. /// </summary>
  148. public Task<List<S8WatchCreationResult>> ProcessTimeoutRulesAsync(long tenantId) =>
  149. ProcessRulesByTypeAsync(tenantId, _timeoutEvaluator, S8TimeoutRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
  150. /// <summary>
  151. /// R3 SHORTAGE 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。RunId 由内部生成。
  152. /// </summary>
  153. public Task<List<S8WatchCreationResult>> ProcessShortageRulesAsync(long tenantId) =>
  154. ProcessRulesByTypeAsync(tenantId, _shortageEvaluator, S8ShortageRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
  155. /// <summary>
  156. /// R3-OUT_OF_RANGE-REWRITE-1:OUT_OF_RANGE 类规则主链。
  157. /// 复用 <see cref="ProcessRulesByTypeAsync"/>,并对历史 dedup_key=NULL 的旧记录做 compat fallback:
  158. /// (source_rule_id=rule.Id AND related_object_code=hit AND status!=CLOSED AND dedup_key IS NULL AND is_deleted=0)
  159. /// 命中则 backfill 6 列,避免重复建单。RunId 由内部生成。
  160. /// </summary>
  161. public Task<List<S8WatchCreationResult>> ProcessOutOfRangeRulesAsync(long tenantId) =>
  162. ProcessRulesByTypeAsync(tenantId, _outOfRangeEvaluator, S8OutOfRangeRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
  163. /// <summary>
  164. /// R2/R3 通用规则主链。S8-SCHED-CLEANUP-LEGACY-PATH-1:本方法已收敛为 thin wrapper,
  165. /// 单规则处理(evaluator → reconcile → hit 循环 → CREATED/REFRESHED/NO_HIT/EVALUATE_FAILED 日志)
  166. /// 全部下沉至 <see cref="ProcessSingleRuleAsync"/>,与 Job tick 路径
  167. /// (<see cref="RunDispatchTickAsync"/> → <see cref="RunSingleRuleAsync"/> → ProcessSingleRuleAsync)
  168. /// 共享同一份逻辑,避免双维护。
  169. ///
  170. /// 调用方仅有 <see cref="CreateExceptionsAsync"/>(debug run-once / Process*RulesAsync 公共薄包装)。
  171. /// debug run-once 不持 lease(保留"手动立即跑"语义),但本方法在每条规则结束时通过
  172. /// <see cref="ApplyRunOnceCompletionAsync"/> 同步更新 watch_rule.last_run_at / last_status /
  173. /// last_error / last_duration_ms / last_run_id / consecutive_failure_count,缩小 run-once 与
  174. /// Job tick 之间的 last_* 状态分裂;不动 lock_token / running_started_at / paused_until,避免
  175. /// 与正在持锁运行的 Job 撕扯。
  176. /// </summary>
  177. private async Task<List<S8WatchCreationResult>> ProcessRulesByTypeAsync(
  178. long tenantId, IS8RuleEvaluator evaluator, string ruleType, string runId)
  179. {
  180. var aggregate = new List<S8WatchCreationResult>();
  181. var rules = await _ruleRep.AsQueryable()
  182. .Where(x => x.TenantId == tenantId
  183. && x.Enabled
  184. && x.RuleType == ruleType)
  185. .ToListAsync();
  186. if (rules.Count == 0) return aggregate;
  187. foreach (var rule in rules.OrderBy(x => x.Id))
  188. {
  189. // S8-RULE-GOVERNANCE-BATCH1:无代码定义的规则不进入执行链(与 Job tick 路径同一口径)。
  190. if (!_ruleCatalog.IsDefined(rule.RuleCode))
  191. {
  192. _logger.LogWarning(
  193. "run_once_skip_no_definition ruleId={RuleId} ruleCode={RuleCode}", rule.Id, rule.RuleCode);
  194. aggregate.Add(BuildSkipResult(rule, S8RuleCatalog.ReasonNotFound, null));
  195. continue;
  196. }
  197. // S8-RUN-ONCE-LEASE-AWARENESS-1:debug run-once 不持 lease,但若该 rule 已被 Scheduler Job
  198. // 通过 PickReadyRulesAsync 抢锁(lock_token 非空且 lock_until > now),run-once 跳过该 rule,
  199. // 不写 last_*、不写 detection_log、不动锁,以避免与 Job tick 并发评估同一 rule 而互相覆盖运行态。
  200. // 锁状态用 ToListAsync 的初始快照判定;run-once 与 Job 的微秒级竞速无法在不抢 lease 的前提下
  201. // 完全消除(已登记为 follow-up 风险)。
  202. if (!string.IsNullOrEmpty(rule.LockToken)
  203. && rule.LockUntil.HasValue
  204. && rule.LockUntil.Value > DateTime.Now)
  205. {
  206. _logger.LogInformation(
  207. "run_once_skip_locked ruleId={RuleId} ruleCode={RuleCode} ruleType={RuleType} lockToken={Token} lockedBy={By} lockUntil={Until}",
  208. rule.Id, rule.RuleCode, ruleType, rule.LockToken, rule.LockedBy, rule.LockUntil);
  209. aggregate.Add(BuildSkipResult(rule, "rule_locked_by_scheduler", null));
  210. continue;
  211. }
  212. var sw = System.Diagnostics.Stopwatch.StartNew();
  213. S8RuleRunResult completion;
  214. try
  215. {
  216. var ruleResults = await ProcessSingleRuleAsync(tenantId, rule, ruleType, evaluator, runId);
  217. aggregate.AddRange(ruleResults);
  218. completion = new S8RuleRunResult
  219. {
  220. Success = true,
  221. Stats = new S8RuleRunStats
  222. {
  223. Hits = ruleResults.Count,
  224. Created = ruleResults.Count(r => r.Created),
  225. Refreshed = ruleResults.Count(r => r.Reason == "duplicate_pending"),
  226. Pending = ruleResults.Count(r => r.Reason == "antiflap_pending_hit"),
  227. Failed = ruleResults.Count(r => r.Reason == "create_failed" || r.Reason == "refresh_failed" || r.Reason == "antiflap_failed" || r.Reason == "evaluate_failed")
  228. }
  229. };
  230. }
  231. catch (Exception ex)
  232. {
  233. // ProcessSingleRuleAsync 在 evaluator 抛 S8RuleEvaluatorException 时已写 EVALUATE_FAILED
  234. // detection_log 后再 throw(供 RunSingleRuleAsync 标 Success=false)。本路径无 lease,
  235. // 吞异常以保留"逐规则失败不影响其他规则"的旧 ProcessRulesByTypeAsync 语义。
  236. _logger.LogWarning(ex, "process_rule_failed ruleCode={RuleCode} ruleType={RuleType}", rule.RuleCode, ruleType);
  237. aggregate.Add(BuildSkipResult(rule, "evaluate_failed", ex.Message));
  238. completion = new S8RuleRunResult { Success = false, ErrorMessage = ex.Message, Stats = new() };
  239. }
  240. sw.Stop();
  241. try
  242. {
  243. await ApplyRunOnceCompletionAsync(rule, completion, (int)sw.ElapsedMilliseconds, runId);
  244. }
  245. catch (Exception ex)
  246. {
  247. _logger.LogWarning(ex, "run_once_completion_write_failed ruleCode={RuleCode} ruleType={RuleType}", rule.RuleCode, ruleType);
  248. }
  249. }
  250. return aggregate;
  251. }
  252. /// <summary>
  253. /// S8-SCHED-CLEANUP-LEGACY-PATH-1:debug run-once 的 last_* 状态回写(无 lease 版)。
  254. /// 与 <see cref="OnRuleCompletedAsync"/> 的语义平行,差异在于:
  255. /// - 不要求 lock_token 匹配(run-once 不持 lease)
  256. /// - 不写 lock_token / locked_by / lock_until / running_started_at(不与 Job lease 撕扯)
  257. /// - 不做 consecutive_failure_count 阈值的 paused_until 自动暂停(debug 路径不应自动暂停 demo rule)
  258. /// 写入:last_run_at / next_run_at / last_status / last_error / last_duration_ms / last_run_id /
  259. /// consecutive_failure_count / updated_at。
  260. /// </summary>
  261. private async Task ApplyRunOnceCompletionAsync(AdoS8WatchRule rule, S8RuleRunResult result, int durationMs, string runId)
  262. {
  263. var now = DateTime.Now;
  264. var nextRunAt = now.AddSeconds(NormalizePollInterval(rule.PollIntervalSeconds));
  265. if (result.Success)
  266. {
  267. await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  268. .SetColumns(x => new AdoS8WatchRule
  269. {
  270. LastRunAt = now,
  271. NextRunAt = nextRunAt,
  272. LastStatus = "SUCCESS",
  273. LastError = null,
  274. LastDurationMs = durationMs,
  275. LastRunId = runId,
  276. ConsecutiveFailureCount = 0,
  277. UpdatedAt = now
  278. })
  279. .Where(x => x.Id == rule.Id)
  280. .ExecuteCommandAsync();
  281. }
  282. else
  283. {
  284. var errorTrunc = Truncate(result.ErrorMessage, 500);
  285. await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  286. .SetColumns(x => new AdoS8WatchRule
  287. {
  288. LastRunAt = now,
  289. NextRunAt = nextRunAt,
  290. LastStatus = "FAILED",
  291. LastError = errorTrunc,
  292. LastDurationMs = durationMs,
  293. LastRunId = runId,
  294. ConsecutiveFailureCount = x.ConsecutiveFailureCount + 1,
  295. UpdatedAt = now
  296. })
  297. .Where(x => x.Id == rule.Id)
  298. .ExecuteCommandAsync();
  299. }
  300. }
  301. // S8-SCHED-EXEC-1:trigger / recover 抗抖计数兜底,null / <1 / >10 一律按 1,避免非法配置导致永远不建单 / 永远不恢复。
  302. private static int NormalizeAntiflapCount(int raw)
  303. {
  304. if (raw < 1 || raw > 10) return 1;
  305. return raw;
  306. }
  307. /// <summary>
  308. /// 命中时累加 detection_state.consecutive_hit_count;未存在则插入 hitCount=1。
  309. /// 返回当前 state 行(含 Id)以及命中后的 hitCount。
  310. /// 注意:本函数不消费 trigger_count_required;上游决定是否进入 CreateFromHitAsync。
  311. /// </summary>
  312. private async Task<(AdoS8RuleDetectionState? state, int hitCount)> UpsertDetectionStateOnHitAsync(
  313. long tenantId, AdoS8WatchRule rule, S8RuleHit hit)
  314. {
  315. var now = DateTime.Now;
  316. var existing = await _detectionStateRep.AsQueryable()
  317. .Where(x => x.TenantId == tenantId
  318. && x.RuleCode == rule.RuleCode
  319. && x.DedupKey == hit.DedupKey)
  320. .FirstAsync();
  321. if (existing == null)
  322. {
  323. var fresh = new AdoS8RuleDetectionState
  324. {
  325. TenantId = tenantId,
  326. FactoryId = rule.FactoryId,
  327. RuleCode = rule.RuleCode,
  328. DedupKey = hit.DedupKey,
  329. SourceObjectType = string.IsNullOrEmpty(hit.SourceObjectType) ? null : hit.SourceObjectType,
  330. SourceObjectId = string.IsNullOrEmpty(hit.SourceObjectId) ? null : hit.SourceObjectId,
  331. ConsecutiveHitCount = 1,
  332. ConsecutiveMissCount = 0,
  333. LastSeenAt = now,
  334. LastHitAt = now,
  335. CreatedAt = now,
  336. UpdatedAt = now
  337. };
  338. // BUG-S8-DETECTION-STATE-ACTIVE-EXC-ID-TRIGGER1-001:必须回填 fresh.Id,否则
  339. // trigger=1 首 tick 建单后 UPDATE state SET active_exception_id WHERE id=state.Id
  340. // 命中 0 行(state.Id 为默认 0)。沿用 S8ManualReportService 既定模式。
  341. fresh = await _detectionStateRep.AsInsertable(fresh).ExecuteReturnEntityAsync();
  342. return (fresh, 1);
  343. }
  344. var newHitCount = existing.ConsecutiveHitCount + 1;
  345. await _detectionStateRep.Context.Updateable<AdoS8RuleDetectionState>()
  346. .SetColumns(x => new AdoS8RuleDetectionState
  347. {
  348. ConsecutiveHitCount = x.ConsecutiveHitCount + 1,
  349. ConsecutiveMissCount = 0,
  350. LastSeenAt = now,
  351. LastHitAt = now,
  352. SourceObjectType = string.IsNullOrEmpty(hit.SourceObjectType) ? existing.SourceObjectType : hit.SourceObjectType,
  353. SourceObjectId = string.IsNullOrEmpty(hit.SourceObjectId) ? existing.SourceObjectId : hit.SourceObjectId,
  354. UpdatedAt = now
  355. })
  356. .Where(x => x.Id == existing.Id)
  357. .ExecuteCommandAsync();
  358. existing.ConsecutiveHitCount = newHitCount;
  359. return (existing, newHitCount);
  360. }
  361. private async Task<long> FindOpenExceptionByDedupKeyAsync(long tenantId, string dedupKey)
  362. {
  363. var ids = await _exceptionRep.AsQueryable()
  364. .Where(x => x.TenantId == tenantId
  365. && !x.IsDeleted
  366. && x.Status != "CLOSED"
  367. && x.DedupKey == dedupKey)
  368. .Select(x => x.Id)
  369. .Take(1)
  370. .ToListAsync();
  371. return ids.Count > 0 ? ids[0] : 0L;
  372. }
  373. /// <summary>
  374. /// R3 OUT_OF_RANGE compat fallback 查找:用 (source_rule_id, related_object_code, status!=CLOSED,
  375. /// dedup_key IS NULL, is_deleted=0) 严格条件定位旧 AlertRule 主链留下的历史记录。
  376. /// </summary>
  377. private async Task<long> FindLegacyOutOfRangeExceptionAsync(long tenantId, long sourceRuleId, string relatedObjectCode)
  378. {
  379. if (string.IsNullOrWhiteSpace(relatedObjectCode)) return 0L;
  380. var ids = await _exceptionRep.AsQueryable()
  381. .Where(x => x.TenantId == tenantId
  382. && !x.IsDeleted
  383. && x.Status != "CLOSED"
  384. && x.DedupKey == null
  385. && x.SourceRuleId == sourceRuleId
  386. && x.RelatedObjectCode == relatedObjectCode)
  387. .Select(x => x.Id)
  388. .Take(1)
  389. .ToListAsync();
  390. return ids.Count > 0 ? ids[0] : 0L;
  391. }
  392. /// <summary>
  393. /// R3 OUT_OF_RANGE compat fallback backfill:把历史记录的 R1 新 6 列(dedup_key/source_rule_code/
  394. /// source_object_type/source_object_id/source_payload/last_detected_at)写入,并刷新 updated_at。
  395. /// </summary>
  396. private async Task BackfillLegacyExceptionAsync(long exceptionId, S8RuleHit hit)
  397. {
  398. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  399. .SetColumns(x => new AdoS8Exception
  400. {
  401. DedupKey = hit.DedupKey,
  402. SourceRuleCode = hit.SourceRuleCode,
  403. SourceObjectType = hit.SourceObjectType,
  404. SourceObjectId = hit.SourceObjectId,
  405. SourcePayload = hit.SourcePayload,
  406. LastDetectedAt = hit.DetectedAt,
  407. UpdatedAt = DateTime.Now
  408. })
  409. .Where(x => x.Id == exceptionId)
  410. .ExecuteCommandAsync();
  411. }
  412. /// <summary>
  413. /// R5 恢复时间最小闭环:对当前 rule 下未关闭、有 dedup_key、recovered_at 仍为 NULL 的异常,
  414. /// 凡不在本轮 hits.dedup_key 集合内的,写入 recovered_at = now、updated_at = now。
  415. /// 仅写这 2 列;不动 status / assignee / verifier / source_payload / last_detected_at;
  416. /// recovered_at 一旦写入,本轮不做复发清空。
  417. /// R6 返回 recoveredIds 供上游决定是否写 NO_HIT 日志,并对每个 recovered exception 写一条 RECOVERED 日志。
  418. /// </summary>
  419. private async Task<List<long>> ReconcileRecoveriesForRuleAsync(
  420. long tenantId, AdoS8WatchRule rule, string ruleType, List<S8RuleHit> hits, string runId)
  421. {
  422. var hitDedupKeys = hits
  423. .Where(h => !string.IsNullOrWhiteSpace(h.DedupKey))
  424. .Select(h => h.DedupKey)
  425. .ToHashSet(StringComparer.Ordinal);
  426. var candidates = await _exceptionRep.AsQueryable()
  427. .Where(x => x.TenantId == tenantId
  428. && !x.IsDeleted
  429. && x.Status != "CLOSED"
  430. && x.SourceRuleCode == rule.RuleCode
  431. && x.DedupKey != null
  432. && x.RecoveredAt == null)
  433. .Select(x => new { x.Id, x.DedupKey, x.SourceObjectType, x.SourceObjectId, x.RelatedObjectCode, x.ConsecutiveMissCount })
  434. .ToListAsync();
  435. if (candidates.Count == 0) return new List<long>();
  436. var now = DateTime.Now;
  437. var recoverRequired = NormalizeAntiflapCount(rule.RecoverCountRequired);
  438. var recoveredIds = new List<long>();
  439. foreach (var c in candidates)
  440. {
  441. if (hitDedupKeys.Contains(c.DedupKey!)) continue;
  442. // S8-SCHED-EXEC-1:恢复抗抖累计。
  443. // 1) 每次未命中:异常 ConsecutiveMissCount += 1,ConsecutiveHitCount 清零;
  444. // 2) miss < recover_count_required:仅累计,不写 recovered_at、不写 RECOVERED;
  445. // 3) miss >= recover_count_required:写 recovered_at、写 RECOVERED 日志。
  446. var newMissCount = c.ConsecutiveMissCount + 1;
  447. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  448. .SetColumns(x => new AdoS8Exception
  449. {
  450. ConsecutiveMissCount = x.ConsecutiveMissCount + 1,
  451. ConsecutiveHitCount = 0,
  452. UpdatedAt = now
  453. })
  454. .Where(x => x.Id == c.Id)
  455. .ExecuteCommandAsync();
  456. // detection_state 同步累计 miss(建单后 state.active_exception_id 仍指向 c.Id)。
  457. await _detectionStateRep.Context.Updateable<AdoS8RuleDetectionState>()
  458. .SetColumns(x => new AdoS8RuleDetectionState
  459. {
  460. ConsecutiveMissCount = x.ConsecutiveMissCount + 1,
  461. ConsecutiveHitCount = 0,
  462. LastSeenAt = now,
  463. LastMissAt = now,
  464. UpdatedAt = now
  465. })
  466. .Where(x => x.TenantId == tenantId
  467. && x.RuleCode == rule.RuleCode
  468. && x.DedupKey == c.DedupKey)
  469. .ExecuteCommandAsync();
  470. if (newMissCount < recoverRequired)
  471. {
  472. _logger.LogInformation(
  473. "antiflap_pending_recovery ruleCode={RuleCode} dedupKey={DedupKey} missCount={Miss} recoverRequired={Required}",
  474. rule.RuleCode, c.DedupKey, newMissCount, recoverRequired);
  475. continue;
  476. }
  477. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  478. .SetColumns(x => new AdoS8Exception
  479. {
  480. RecoveredAt = now,
  481. UpdatedAt = now
  482. })
  483. .Where(x => x.Id == c.Id)
  484. .ExecuteCommandAsync();
  485. recoveredIds.Add(c.Id);
  486. await WriteDetectionLogAsync(new AdoS8DetectionLog
  487. {
  488. TenantId = tenantId, FactoryId = rule.FactoryId,
  489. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  490. SourceObjectType = c.SourceObjectType, SourceObjectId = c.SourceObjectId,
  491. RelatedObjectCode = c.RelatedObjectCode, DedupKey = c.DedupKey,
  492. DetectResult = DetectResultRecovered,
  493. ExceptionId = c.Id,
  494. DetectedAt = now,
  495. PayloadSnapshot = JsonSerializer.Serialize(new { ruleId = rule.Id, ruleCode = rule.RuleCode, reason = "no_longer_hit", missCount = newMissCount, recoverRequired }),
  496. RunId = runId, TriggerSource = DetectionTriggerSource,
  497. Remark = "Rule no longer hit; recovered_at marked"
  498. });
  499. // S8-SYSTEM-RECOVERED-TIMELINE-1:恢复也要在时间线上留痕。
  500. // 此前 recovered_at 与 RECOVERED detection_log 都有了,唯独用户能看见的那条时间线没有 ——
  501. // 于是页面上异常"自己不见了",没人知道是系统判定恢复还是有人做了什么。
  502. await TryWriteSystemRecoveredTimelineAsync(c.Id, now, newMissCount, recoverRequired);
  503. // S8-NOTIFY-WIRE-RECOVERED-1:detection_log 已写入、recovered_at 已落库后挂入恢复通知。
  504. // 通知失败仅 LogWarning,绝不影响恢复状态/检测日志。
  505. await TryDispatchRecoveredLayerNotificationAsync(c.Id);
  506. }
  507. if (recoveredIds.Count > 0)
  508. {
  509. _logger.LogInformation(
  510. "rule_recovered ruleCode={RuleCode} ruleType={RuleType} recoveredCount={Count} recoveredIds={Ids}",
  511. rule.RuleCode, ruleType, recoveredIds.Count, string.Join(",", recoveredIds));
  512. }
  513. return recoveredIds;
  514. }
  515. /// <summary>R6 通用 hit 日志构造(CREATED / REFRESHED 共用)。</summary>
  516. private static AdoS8DetectionLog BuildHitLog(
  517. long tenantId, AdoS8WatchRule rule, string ruleType, S8RuleHit hit,
  518. string detectResult, long exceptionId, string runId) => new()
  519. {
  520. TenantId = tenantId, FactoryId = rule.FactoryId,
  521. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  522. SourceObjectType = hit.SourceObjectType, SourceObjectId = hit.SourceObjectId,
  523. RelatedObjectCode = hit.RelatedObjectCode, DedupKey = hit.DedupKey,
  524. DetectResult = detectResult,
  525. ExceptionId = exceptionId,
  526. DetectedAt = hit.DetectedAt,
  527. PayloadSnapshot = hit.SourcePayload,
  528. RunId = runId,
  529. TriggerSource = DetectionTriggerSource
  530. };
  531. /// <summary>R6 日志写入:失败仅 LogWarning,不阻断主链;不抛异常。</summary>
  532. private async Task WriteDetectionLogAsync(AdoS8DetectionLog log)
  533. {
  534. try
  535. {
  536. await _detectionLogRep.InsertAsync(log);
  537. }
  538. catch (Exception ex)
  539. {
  540. _logger.LogWarning(ex,
  541. "detection_log_write_failed runId={RunId} detectResult={Result} ruleCode={RuleCode} exceptionId={ExceptionId}",
  542. log.RunId, log.DetectResult, log.RuleCode, log.ExceptionId);
  543. }
  544. }
  545. /// <summary>
  546. /// S8-DETECTION-LOG-WRITE-REDUCE-P1-1:REFRESHED 明细限频判定。
  547. /// 查询 cutoff = detectedAt - RefreshedRateLimitMinutes 之后,
  548. /// 是否已有同 (tenant_id, factory_id, rule_code, dedup_key, DetectResult=REFRESHED) 的 detection_log;
  549. /// 命中既有索引 idx_s8_detection_log_dedup_time。查询失败仅 LogWarning 后返回 false,
  550. /// 宁可多写一条 REFRESHED 明细,也不阻断调度主链路;ruleCode / dedupKey 空白时同样返回 false(保留写入)。
  551. /// 业务侧 RefreshDetectionAsync / BackfillLegacyExceptionAsync 不在限频范围。
  552. /// </summary>
  553. private async Task<bool> HasRecentRefreshedDetectionLogAsync(
  554. long tenantId, string ruleCode, string dedupKey, DateTime detectedAt)
  555. {
  556. if (string.IsNullOrWhiteSpace(ruleCode) || string.IsNullOrWhiteSpace(dedupKey))
  557. return false;
  558. var cutoff = detectedAt.AddMinutes(-RefreshedRateLimitMinutes);
  559. try
  560. {
  561. return await _detectionLogRep.AsQueryable()
  562. .Where(x => x.TenantId == tenantId
  563. && x.DedupKey == dedupKey
  564. && x.RuleCode == ruleCode
  565. && x.DetectResult == DetectResultRefreshed
  566. && x.DetectedAt >= cutoff)
  567. .AnyAsync();
  568. }
  569. catch (Exception ex)
  570. {
  571. _logger.LogWarning(ex,
  572. "detection_log_refreshed_ratelimit_check_failed ruleCode={RuleCode} dedupKey={DedupKey}",
  573. ruleCode, dedupKey);
  574. return false;
  575. }
  576. }
  577. /// <summary>
  578. /// S8-SYSTEM-RECOVERED-TIMELINE-1:写一条系统恢复时间线。
  579. ///
  580. /// <para><b>不自动关闭异常</b>:恢复只表示"规则不再命中",是否闭环仍由人判断。
  581. /// 既有语义(recovered_at + RECOVERED detection_log,状态不变)原样保留。</para>
  582. ///
  583. /// <para><b>operator_user_id 写 NULL</b>:这一步没有真人操作者。
  584. /// 绝不伪造 1 / EmployeeId / 某个 SysUserId —— 时间线的操作人列刚在
  585. /// S8-SYSUSER-ONLY-1 里从"混装两套 ID 空间"收拾干净,不能又塞进一个假值。</para>
  586. ///
  587. /// <para><b>幂等判据</b>:不存在 <c>created_at</c> 晚于本异常最近一次命中时间
  588. /// (<c>last_detected_at</c>)的 SYSTEM_RECOVERED 行。
  589. /// <list type="bullet">
  590. /// <item>首次恢复:最近命中之后还没有恢复行 → 写入;</item>
  591. /// <item>调度器重跑:刚写的那行已经晚于 last_detected_at → 跳过,不重复追加;</item>
  592. /// <item>复发后再恢复:<c>RefreshDetectionAsync</c> 会把 last_detected_at 推到
  593. /// 旧恢复行之后 → 写入新的一条,这是<b>真实的新恢复周期</b>,本就该留痕。</item>
  594. /// </list>
  595. /// 只用既有列表达,不新增状态字段。</para>
  596. ///
  597. /// <para>失败仅 LogWarning:时间线写不进去不该把已经判定的恢复状态回滚。</para>
  598. /// </summary>
  599. private async Task TryWriteSystemRecoveredTimelineAsync(long exceptionId, DateTime now, int missCount, int recoverRequired)
  600. {
  601. try
  602. {
  603. var lastDetectedAt = await _exceptionRep.AsQueryable().ClearFilter()
  604. .Where(x => x.Id == exceptionId)
  605. .Select(x => x.LastDetectedAt)
  606. .FirstAsync();
  607. var already = await _exceptionRep.Context.Queryable<AdoS8ExceptionTimeline>()
  608. .Where(t => t.ExceptionId == exceptionId && t.ActionCode == SystemRecoveredActionCode)
  609. .WhereIF(lastDetectedAt != null, t => t.CreatedAt > lastDetectedAt!.Value)
  610. .AnyAsync();
  611. if (already)
  612. {
  613. _logger.LogInformation(
  614. "system_recovered_timeline_skip exceptionId={Id} reason=already_recorded_for_current_cycle", exceptionId);
  615. return;
  616. }
  617. await _exceptionRep.Context.Insertable(new AdoS8ExceptionTimeline
  618. {
  619. ExceptionId = exceptionId,
  620. ActionCode = SystemRecoveredActionCode,
  621. ActionLabel = "系统恢复",
  622. FromStatus = null,
  623. ToStatus = null, // 恢复不改状态机,故不写状态迁移
  624. OperatorUserId = null, // 系统动作无真人操作者,绝不伪造
  625. OperatorName = null,
  626. ActionRemark = $"系统检测到异常已恢复(连续未命中 {missCount} 次,达到恢复阈值 {recoverRequired})。异常未自动关闭,仍需人工确认闭环。",
  627. CreatedAt = now
  628. }).ExecuteCommandAsync();
  629. _logger.LogInformation("system_recovered_timeline_written exceptionId={Id} missCount={Miss}", exceptionId, missCount);
  630. }
  631. catch (Exception ex)
  632. {
  633. _logger.LogWarning(ex, "system_recovered_timeline_failed exceptionId={Id}", exceptionId);
  634. }
  635. }
  636. private static string Truncate(string? s, int max) =>
  637. string.IsNullOrEmpty(s) ? string.Empty : (s.Length <= max ? s : s.Substring(0, max));
  638. private async Task RefreshDetectionAsync(long exceptionId, S8RuleHit hit)
  639. {
  640. // S8-SCHED-EXEC-1:刷新阶段抗抖累计 + 复发清空 recovered_at。
  641. // ConsecutiveHitCount += 1(用 SetColumns 内表达式完成原子自增);ConsecutiveMissCount 归零。
  642. // RecoveredAt 不为 NULL 时(复发)一并清空,保持业务对"再次命中即视为活跃"的预期。
  643. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  644. .SetColumns(x => new AdoS8Exception
  645. {
  646. LastDetectedAt = hit.DetectedAt,
  647. SourcePayload = hit.SourcePayload,
  648. ConsecutiveHitCount = x.ConsecutiveHitCount + 1,
  649. ConsecutiveMissCount = 0,
  650. RecoveredAt = null,
  651. UpdatedAt = DateTime.Now
  652. })
  653. .Where(x => x.Id == exceptionId)
  654. .ExecuteCommandAsync();
  655. }
  656. private static S8WatchCreationResult BuildCreatedResult(AdoS8WatchRule rule, S8RuleHit hit, long exceptionId) =>
  657. new()
  658. {
  659. DedupResult = new S8WatchDedupResult
  660. {
  661. Hit = ToWatchHit(rule, hit),
  662. CanCreate = true,
  663. MatchedExceptionId = null,
  664. Reason = "no_pending"
  665. },
  666. Created = true,
  667. Skipped = false,
  668. CreatedExceptionId = exceptionId,
  669. Reason = "auto_created",
  670. ErrorMessage = null
  671. };
  672. private static S8WatchCreationResult BuildSkippedDuplicate(AdoS8WatchRule rule, S8RuleHit hit, long matchedId) =>
  673. new()
  674. {
  675. DedupResult = new S8WatchDedupResult
  676. {
  677. Hit = ToWatchHit(rule, hit),
  678. CanCreate = false,
  679. MatchedExceptionId = matchedId,
  680. Reason = "duplicate_pending"
  681. },
  682. Created = false,
  683. Skipped = true,
  684. CreatedExceptionId = null,
  685. Reason = "duplicate_pending",
  686. ErrorMessage = null
  687. };
  688. private static S8WatchCreationResult BuildSkipResult(AdoS8WatchRule rule, string reason, string? error, S8RuleHit? hit = null) =>
  689. new()
  690. {
  691. DedupResult = new S8WatchDedupResult
  692. {
  693. Hit = hit != null ? ToWatchHit(rule, hit) : new S8WatchHitResult { SourceRuleId = rule.Id, SourceRuleCode = rule.RuleCode },
  694. CanCreate = false,
  695. MatchedExceptionId = null,
  696. Reason = reason
  697. },
  698. Created = false,
  699. Skipped = true,
  700. CreatedExceptionId = null,
  701. Reason = reason,
  702. ErrorMessage = error
  703. };
  704. private static S8WatchHitResult ToWatchHit(AdoS8WatchRule rule, S8RuleHit hit) => new()
  705. {
  706. SourceRuleId = hit.SourceRuleId == 0 ? rule.Id : hit.SourceRuleId,
  707. SourceRuleCode = string.IsNullOrEmpty(hit.SourceRuleCode) ? rule.RuleCode : hit.SourceRuleCode,
  708. RelatedObjectCode = hit.RelatedObjectCode,
  709. Severity = hit.Severity,
  710. OccurrenceDeptId = hit.OccurrenceDeptId,
  711. ResponsibleDeptId = hit.ResponsibleDeptId,
  712. SourcePayload = hit.SourcePayload
  713. };
  714. // G01-04 首版最小比较符集合:>, >=, <, <=。
  715. // 允许首尾空格;非此集合的一律视为“比较符非法”,由调用方跳过。
  716. private static string NormalizeColumnName(string columnName) =>
  717. columnName.Replace("_", string.Empty, StringComparison.Ordinal).Trim().ToUpperInvariant();
  718. // ============================================================
  719. // S8-SCHED-EXEC-1:DB 驱动调度执行层
  720. // ============================================================
  721. private const int LeaseDurationMinutes = 5;
  722. private const int AutoPauseFailureThreshold = 3;
  723. private const int AutoPauseDurationHours = 1;
  724. private const int DefaultPollIntervalSeconds = 300;
  725. private const int MinPollIntervalSeconds = 60;
  726. private const int MaxPollIntervalSeconds = 86400;
  727. /// <summary>
  728. /// S8-SCHED-EXEC-1:释放过期 lease(lock_until &lt; NOW),不修改 last_status / last_error,仅清空 lock 三件套 + running_started_at。
  729. /// 返回释放的行数。
  730. /// </summary>
  731. public async Task<int> ResetExpiredLeasesAsync(long tenantId)
  732. {
  733. var now = DateTime.Now;
  734. var affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  735. .SetColumns(x => new AdoS8WatchRule
  736. {
  737. LockToken = null,
  738. LockedBy = null,
  739. LockUntil = null,
  740. RunningStartedAt = null,
  741. UpdatedAt = now
  742. })
  743. .Where(x => x.TenantId == tenantId
  744. && x.LockUntil != null
  745. && x.LockUntil < now)
  746. .ExecuteCommandAsync();
  747. if (affected > 0)
  748. {
  749. _logger.LogWarning(
  750. "lease_reset tenantId={Tenant} releasedCount={Count}",
  751. tenantId, affected);
  752. }
  753. return affected;
  754. }
  755. /// <summary>
  756. /// S8-SCHED-EXEC-1:到期规则候选 + 乐观 UPDATE 抢锁,返回成功抢到的 lease 列表。
  757. /// 抢锁条件:enabled=1 AND (paused_until IS NULL OR paused_until &lt;= NOW)
  758. /// AND (next_run_at IS NULL OR next_run_at &lt;= NOW)
  759. /// AND (lock_until IS NULL OR lock_until &lt;= NOW)。
  760. /// 抢锁回写:lock_token / locked_by / lock_until = NOW + 5min / running_started_at = NOW / last_run_id = runId。
  761. /// affectedRows == 1 才算抢到;后续 OnRuleCompletedAsync 必须按 lockToken 回写,避免旧进程覆盖新 lease。
  762. /// </summary>
  763. public async Task<List<S8RuleLease>> PickReadyRulesAsync(long tenantId, int batchSize, string lockedBy, string runId)
  764. {
  765. if (batchSize <= 0) batchSize = 16;
  766. var now = DateTime.Now;
  767. var candidates = await _ruleRep.AsQueryable()
  768. .Where(x => x.TenantId == tenantId
  769. && x.Enabled
  770. // S8-STANDARD-DATASET-HARD-CUTOVER-1:取数唯一依赖 dataset_code。
  771. // 保留这条谓词而非依赖 Enable Gate:Gate 只在"切换启用"那一刻生效,
  772. // 而 dataset_code 可能在此之后被清空(历史遗留行、直连改库)。
  773. // 一条没有数据集的规则跑起来只会每 tick 抛一次 dataset_code_missing。
  774. //
  775. // 刻意做成 **SQL 谓词而非内存过滤**:Take(batchSize) 在过滤之前生效,
  776. // 内存过滤会让无效候选占满名额,导致同租户的有效规则被饿死。
  777. && x.DatasetCode != null && x.DatasetCode != ""
  778. && (x.PausedUntil == null || x.PausedUntil <= now)
  779. && (x.NextRunAt == null || x.NextRunAt <= now)
  780. && (x.LockUntil == null || x.LockUntil <= now))
  781. .OrderBy(x => x.NextRunAt, OrderByType.Asc)
  782. .OrderBy(x => x.Id, OrderByType.Asc)
  783. .Take(batchSize)
  784. .Select(x => new { x.Id, x.RuleCode, x.RuleType })
  785. .ToListAsync();
  786. if (candidates.Count == 0) return new();
  787. var leases = new List<S8RuleLease>();
  788. foreach (var c in candidates)
  789. {
  790. var token = Guid.NewGuid().ToString("N");
  791. var lockUntil = DateTime.Now.AddMinutes(LeaseDurationMinutes);
  792. var runningAt = DateTime.Now;
  793. // 乐观 UPDATE:再校验一次条件,affectedRows=1 才算抢到。
  794. var affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  795. .SetColumns(x => new AdoS8WatchRule
  796. {
  797. LockToken = token,
  798. LockedBy = lockedBy,
  799. LockUntil = lockUntil,
  800. RunningStartedAt = runningAt,
  801. LastRunId = runId,
  802. UpdatedAt = runningAt
  803. })
  804. .Where(x => x.Id == c.Id
  805. && x.Enabled
  806. // 抢锁的乐观 UPDATE 复查同一条件:候选查询与本次 UPDATE 之间存在时间窗,
  807. // 期间该行的 dataset_code 可能被清空。
  808. && x.DatasetCode != null && x.DatasetCode != ""
  809. && (x.PausedUntil == null || x.PausedUntil <= runningAt)
  810. && (x.NextRunAt == null || x.NextRunAt <= runningAt)
  811. && (x.LockUntil == null || x.LockUntil <= runningAt))
  812. .ExecuteCommandAsync();
  813. if (affected == 1)
  814. {
  815. leases.Add(new S8RuleLease
  816. {
  817. RuleId = c.Id,
  818. RuleCode = c.RuleCode,
  819. RuleType = c.RuleType,
  820. LockToken = token,
  821. LockedBy = lockedBy,
  822. LockUntil = lockUntil,
  823. RunId = runId,
  824. AcquiredAt = runningAt
  825. });
  826. }
  827. }
  828. return leases;
  829. }
  830. /// <summary>
  831. /// S8-SCHED-EXEC-1:执行单条已抢锁规则的 evaluator → 抗抖去重 → 建单/刷新 → 恢复 reconcile。
  832. /// 不释放 lease(OnRuleCompletedAsync 负责);evaluator 抛异常时 Result.Success=false 并保留 ErrorMessage。
  833. /// </summary>
  834. public async Task<S8RuleRunResult> RunSingleRuleAsync(long tenantId, S8RuleLease lease)
  835. {
  836. // S8-P0-1-SCHEDULER-TRUSTED-SCOPE-1:执行入口必须按本次 tick 的可信作用域绑行。
  837. // 原实现只按 lease.RuleId 装载,不校验 rule 归属;一旦 lease 来源被改写 / 未来新增调用方,
  838. // 就会在 A 租户的 scope 上下文里执行 B 租户的规则,而下游全部以传入的 tenantId/factoryId 落库。
  839. // 这里同时做「谓词绑行」与「归属复核」两层,任一不符即 fail-fast,绝不静默降级。
  840. var rule = await _ruleRep.AsQueryable()
  841. .Where(x => x.Id == lease.RuleId && x.TenantId == tenantId)
  842. .FirstAsync();
  843. if (rule == null)
  844. {
  845. return new S8RuleRunResult { Success = false, ErrorMessage = "rule_not_found", Stats = new() };
  846. }
  847. if (rule.TenantId != tenantId)
  848. {
  849. _logger.LogError(
  850. "rule_scope_mismatch ruleId={RuleId} ruleTenant={RuleTenant} scopeTenant={ScopeTenant}",
  851. rule.Id, rule.TenantId, tenantId);
  852. return new S8RuleRunResult { Success = false, ErrorMessage = "rule_scope_mismatch", Stats = new() };
  853. }
  854. // S8-RULE-GOVERNANCE-BATCH1:分派的真源是**代码定义**,不是 DB 上的 rule_type 列。
  855. // 那一列现在只是 provisioning 维护的投影;若有人手工改它,规则的执行方式不应随之改变。
  856. // 同时这里是「没有代码定义的规则不得进入执行链」的把关点 —— 在 Create API 仍存在的
  857. // 过渡期(Batch 3 才退役),它保证业务即使造出任意 rule_code 也调不动调度器。
  858. var definition = _ruleCatalog.TryGet(rule.RuleCode);
  859. if (definition == null)
  860. {
  861. _logger.LogWarning(
  862. "rule_definition_not_found ruleId={RuleId} ruleCode={RuleCode} tenant={Tenant}",
  863. rule.Id, rule.RuleCode, tenantId);
  864. return new S8RuleRunResult { Success = false, ErrorMessage = S8RuleCatalog.ReasonNotFound, Stats = new() };
  865. }
  866. // S8-RULE-READINESS-1:第三条真实执行入口。**必须在 evaluator 之前** fail-fast。
  867. //
  868. // 为什么调度器也要查:启用那一刻合法,不代表现在仍合法 —— 默认部门可能被停用、
  869. // 删除,或主数据被改到别的租户。放行到下游的后果不是"报个错",而是
  870. // tick_rule_done status=SUCCESS hits=28 created=0 failed=28
  871. // 每一层都成功、异常一条没有,页面上完全看不出原因(Batch 7 实测)。
  872. //
  873. // 失败语义:本次 run 直接判失败并带上 reason,由 OnRuleCompletedAsync 落到
  874. // last_status=FAILED / last_error;**不自动停用规则**,配好部门后自然恢复。
  875. var readiness = await _readinessGate.CheckAsync(
  876. S8EffectiveRule.Resolve(rule, definition), tenantId);
  877. if (!readiness.Ok)
  878. {
  879. _logger.LogWarning(
  880. "rule_not_ready ruleId={RuleId} ruleCode={RuleCode} tenant={Tenant} reason={Reason} message={Message}",
  881. rule.Id, rule.RuleCode, tenantId, readiness.ReasonCode, readiness.Message);
  882. return new S8RuleRunResult
  883. {
  884. Success = false,
  885. ErrorMessage = $"{readiness.ReasonCode}: {readiness.Message}",
  886. Stats = new()
  887. };
  888. }
  889. var ruleType = definition.RuleType;
  890. IS8RuleEvaluator? evaluator = ruleType switch
  891. {
  892. S8TimeoutRuleEvaluator.RuleTypeCode => _timeoutEvaluator,
  893. S8ShortageRuleEvaluator.RuleTypeCode => _shortageEvaluator,
  894. S8OutOfRangeRuleEvaluator.RuleTypeCode => _outOfRangeEvaluator,
  895. _ => null
  896. };
  897. if (evaluator == null)
  898. {
  899. return new S8RuleRunResult { Success = false, ErrorMessage = $"unsupported_rule_type:{ruleType}", Stats = new() };
  900. }
  901. try
  902. {
  903. var results = await ProcessSingleRuleAsync(tenantId, rule, ruleType, evaluator, lease.RunId);
  904. var stats = new S8RuleRunStats
  905. {
  906. Hits = results.Count,
  907. Created = results.Count(r => r.Created),
  908. Refreshed = results.Count(r => r.Reason == "duplicate_pending"),
  909. Pending = results.Count(r => r.Reason == "antiflap_pending_hit"),
  910. Failed = results.Count(r => r.Reason == "create_failed" || r.Reason == "refresh_failed" || r.Reason == "antiflap_failed" || r.Reason == "evaluate_failed")
  911. };
  912. return new S8RuleRunResult { Success = true, Stats = stats };
  913. }
  914. catch (Exception ex)
  915. {
  916. return new S8RuleRunResult
  917. {
  918. Success = false,
  919. ErrorMessage = ex.Message,
  920. Stats = new()
  921. };
  922. }
  923. }
  924. /// <summary>
  925. /// S8-SCHED-EXEC-1:单规则处理(evaluator → reconcile → hit 循环)。
  926. /// 与 ProcessRulesByTypeAsync 内单规则循环体语义一致;此处抽出便于新调度路径直接调用单条 rule。
  927. /// </summary>
  928. private async Task<List<S8WatchCreationResult>> ProcessSingleRuleAsync(
  929. long tenantId, AdoS8WatchRule rule, string ruleType,
  930. IS8RuleEvaluator evaluator, string runId)
  931. {
  932. var results = new List<S8WatchCreationResult>();
  933. List<S8RuleHit> hits;
  934. try
  935. {
  936. hits = await evaluator.EvaluateAsync(tenantId, rule);
  937. }
  938. catch (Exception ex)
  939. {
  940. var failureReason = ex is S8RuleEvaluatorException sre ? sre.Reason : ex.GetType().Name;
  941. await WriteDetectionLogAsync(new AdoS8DetectionLog
  942. {
  943. TenantId = tenantId, FactoryId = rule.FactoryId,
  944. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  945. SourceObjectType = rule.SourceObjectType,
  946. DetectResult = DetectResultEvaluateFailed,
  947. DetectedAt = DateTime.Now,
  948. FailureReason = failureReason,
  949. FailureMessage = Truncate(ex.Message, 1000),
  950. RunId = runId, TriggerSource = DetectionTriggerSource
  951. });
  952. results.Add(BuildSkipResult(rule, "evaluate_failed", ex.Message));
  953. throw;
  954. }
  955. // ── Authority Health Gate ───────────────────────────────────────────────────────
  956. // 恢复判定是唯一一处「从缺席做负面推断」的地方:某个 dedup_key 不在本轮 hits 里,
  957. // 就认定它已经恢复。这条推断只有在 Authority 可信时才成立 ——
  958. // 中台断供、快照没发布成功、跑批只写了一半,都会让本该命中的行凭空消失,
  959. // 于是真实异常被静默标成「已恢复」,且没有任何人会收到通知。
  960. //
  961. // 门禁**只挡恢复这一件事**:上面的建单与刷新照常执行。
  962. // 因为 hit 是「这一行确实存在且越界」的正面证据 —— 数据不可信不会凭空造出一行,
  963. // 却完全可能让一行消失。风险不对称,处置也不对称。
  964. //
  965. // 注意这里没有、也不允许有 hits.Count 的分支:Authority 可信而本轮 0 行,
  966. // 是「业务对象全部合法退场」,必须放行恢复。挡空集会把这种情况一起冻死。
  967. var health = await _authorityHealthResolver.ResolveAsync(tenantId, rule.DatasetCode ?? string.Empty);
  968. List<long> recoveredIds;
  969. if (!health.AllowsRecovery)
  970. {
  971. recoveredIds = new();
  972. await WriteDetectionLogAsync(new AdoS8DetectionLog
  973. {
  974. TenantId = tenantId, FactoryId = rule.FactoryId,
  975. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  976. SourceObjectType = rule.SourceObjectType,
  977. DetectResult = DetectResultRecoveryBlocked,
  978. DetectedAt = DateTime.Now,
  979. FailureReason = health.ReasonCode,
  980. FailureMessage = Truncate(health.Reason, 1000),
  981. PayloadSnapshot = JsonSerializer.Serialize(new
  982. {
  983. datasetCode = health.DatasetCode,
  984. state = health.State,
  985. reasonCode = health.ReasonCode,
  986. observedAt = health.ObservedAt,
  987. lastSuccessAt = health.LastSuccessAt,
  988. staleAfter = health.StaleAfter,
  989. hitCount = hits.Count
  990. }),
  991. RunId = runId, TriggerSource = DetectionTriggerSource
  992. });
  993. _logger.LogInformation(
  994. "recovery_blocked ruleCode={RuleCode} tenant={Tenant} dataset={Dataset} " +
  995. "state={State} reason={Reason} lastSuccessAt={LastSuccessAt}",
  996. rule.RuleCode, tenantId, health.DatasetCode, health.State, health.ReasonCode, health.LastSuccessAt);
  997. // 刻意不做的三件事:
  998. // · 不动 ConsecutiveMissCount —— 已累计的 miss 都来自可信轮次,仍然有效;
  999. // 本轮既不加也不减,Authority 恢复后接着原节奏推进即可。
  1000. // · 不把本次 rule run 标成失败 —— 那会累加 ConsecutiveFailureCount,
  1001. // 达到阈值即自动暂停规则一小时;中台跑一次批就能把规则停掉。
  1002. // · 不抛异常打断本轮 —— 正面观测已经处理完,没有理由回滚它们。
  1003. }
  1004. else
  1005. {
  1006. try
  1007. {
  1008. recoveredIds = await ReconcileRecoveriesForRuleAsync(tenantId, rule, ruleType, hits, runId);
  1009. }
  1010. catch (Exception ex)
  1011. {
  1012. _logger.LogWarning(ex, "recovery_reconcile_failed ruleCode={RuleCode} ruleType={RuleType}", rule.RuleCode, ruleType);
  1013. recoveredIds = new();
  1014. }
  1015. }
  1016. // S8-DETECTION-LOG-WRITE-REDUCE-P1-1:NO_HIT 不再写 detection_log 明细。
  1017. // 规则级"被定期评估"信号由 ado_s8_watch_rule.LastRunAt / LastStatus / LastRunId / LastDurationMs
  1018. // 在 OnRuleCompletedAsync / ApplyRunOnceCompletionAsync 中承接,无需 detection_log 冗余记录。
  1019. // DetectResultNoHit 常量保留(历史数据反查 / 未来如需恢复明细写入)。
  1020. foreach (var hit in hits)
  1021. {
  1022. if (string.IsNullOrWhiteSpace(hit.DedupKey))
  1023. {
  1024. results.Add(BuildSkipResult(rule, "missing_dedup_key", null, hit));
  1025. continue;
  1026. }
  1027. long matchedId;
  1028. try
  1029. {
  1030. matchedId = await FindOpenExceptionByDedupKeyAsync(tenantId, hit.DedupKey);
  1031. }
  1032. catch (Exception ex)
  1033. {
  1034. results.Add(BuildSkipResult(rule, "query_failed", ex.Message, hit));
  1035. continue;
  1036. }
  1037. if (matchedId > 0)
  1038. {
  1039. try
  1040. {
  1041. await RefreshDetectionAsync(matchedId, hit);
  1042. if (!await HasRecentRefreshedDetectionLogAsync(tenantId, rule.RuleCode, hit.DedupKey, hit.DetectedAt))
  1043. {
  1044. await WriteDetectionLogAsync(BuildHitLog(tenantId, rule, ruleType, hit, DetectResultRefreshed, matchedId, runId));
  1045. }
  1046. results.Add(BuildSkippedDuplicate(rule, hit, matchedId));
  1047. }
  1048. catch (Exception ex)
  1049. {
  1050. results.Add(BuildSkipResult(rule, "refresh_failed", ex.Message, hit));
  1051. }
  1052. continue;
  1053. }
  1054. if (string.Equals(ruleType, S8OutOfRangeRuleEvaluator.RuleTypeCode, StringComparison.OrdinalIgnoreCase))
  1055. {
  1056. long compatId;
  1057. try
  1058. {
  1059. compatId = await FindLegacyOutOfRangeExceptionAsync(tenantId, rule.Id, hit.RelatedObjectCode);
  1060. }
  1061. catch (Exception ex)
  1062. {
  1063. results.Add(BuildSkipResult(rule, "query_failed", ex.Message, hit));
  1064. continue;
  1065. }
  1066. if (compatId > 0)
  1067. {
  1068. try
  1069. {
  1070. await BackfillLegacyExceptionAsync(compatId, hit);
  1071. if (!await HasRecentRefreshedDetectionLogAsync(tenantId, rule.RuleCode, hit.DedupKey, hit.DetectedAt))
  1072. {
  1073. await WriteDetectionLogAsync(BuildHitLog(tenantId, rule, ruleType, hit, DetectResultRefreshed, compatId, runId));
  1074. }
  1075. results.Add(BuildSkippedDuplicate(rule, hit, compatId));
  1076. }
  1077. catch (Exception ex)
  1078. {
  1079. results.Add(BuildSkipResult(rule, "refresh_failed", ex.Message, hit));
  1080. }
  1081. continue;
  1082. }
  1083. }
  1084. // S8-RULE-LIFECYCLE-CREATE-GATE-1:首次立案闸门。
  1085. //
  1086. // 位置是本机制的全部要害,两边都不能挪:
  1087. //
  1088. // · 必须在**已有异常刷新之后**(上面两个 continue 分支)——
  1089. // 一条预警在跨过到期日之后 CreateEligible 会变成 false,但它对应的异常
  1090. // 仍需正常 refresh 才能保持 active。闸门挪到刷新之前,跨期的既有预警
  1091. // 会连 last_detected_at 都停止推进,等于换一种方式把它放掉。
  1092. //
  1093. // · 必须在**首次建单之前**(下面的 exception_type / 抗抖 / CreateFromHitAsync)——
  1094. // 否则历史风险行会在规则首次启用时被一次性倒成一批"预警",
  1095. // 而它们的风险早已兑现,不是预警。
  1096. //
  1097. // 注意这条 hit 已经被 ReconcileRecoveriesForRuleAsync 消费过了
  1098. // (它在 foreach 之前执行、拿的是完整 hits 列表),因此 CreateEligible=false
  1099. // 的命中天然保护了对应异常不被误判恢复 —— 恢复侧一行都不用改。
  1100. if (!hit.CreateEligible)
  1101. {
  1102. _logger.LogInformation(
  1103. "create_not_eligible ruleCode={RuleCode} dedupKey={DedupKey} reason=hit_outside_create_window",
  1104. rule.RuleCode, hit.DedupKey);
  1105. results.Add(BuildSkipResult(rule, "create_not_eligible", null, hit));
  1106. continue;
  1107. }
  1108. bool typeExists;
  1109. try
  1110. {
  1111. typeExists = await _exceptionTypeRep.AsQueryable()
  1112. // S8-TENANT-ONLY-BATCH5:去掉工厂条件。这是一次**存在性**检查,
  1113. // 少一个条件只会更宽松:factory_id=0 的供给规则也能看到本租户的工厂级异常类型行,
  1114. // 否则它只能匹配到平台默认行、找不到就整条规则 exception_type_missing。
  1115. // 注意 t.TenantId == 0 是平台默认哨兵(S8ConfigScope),不是工厂语义,保留。
  1116. .Where(t => t.TypeCode == hit.ExceptionTypeCode
  1117. && (t.TenantId == 0 || t.TenantId == tenantId)
  1118. && t.Enabled)
  1119. .AnyAsync();
  1120. }
  1121. catch (Exception ex)
  1122. {
  1123. results.Add(BuildSkipResult(rule, "query_failed", ex.Message, hit));
  1124. continue;
  1125. }
  1126. if (!typeExists)
  1127. {
  1128. results.Add(BuildSkipResult(rule, "exception_type_missing", null, hit));
  1129. continue;
  1130. }
  1131. int hitCount;
  1132. AdoS8RuleDetectionState? state;
  1133. try
  1134. {
  1135. (state, hitCount) = await UpsertDetectionStateOnHitAsync(tenantId, rule, hit);
  1136. }
  1137. catch (Exception ex)
  1138. {
  1139. results.Add(BuildSkipResult(rule, "antiflap_failed", ex.Message, hit));
  1140. continue;
  1141. }
  1142. var triggerRequired = NormalizeAntiflapCount(rule.TriggerCountRequired);
  1143. if (hitCount < triggerRequired)
  1144. {
  1145. _logger.LogInformation(
  1146. "antiflap_pending_hit ruleCode={RuleCode} dedupKey={DedupKey} hitCount={HitCount} trigger={Trigger}",
  1147. rule.RuleCode, hit.DedupKey, hitCount, triggerRequired);
  1148. results.Add(BuildSkipResult(rule, "antiflap_pending_hit", null, hit));
  1149. continue;
  1150. }
  1151. try
  1152. {
  1153. var entity = await _reportDispatcher.CreateFromHitAsync(tenantId, hit);
  1154. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  1155. .SetColumns(x => new AdoS8Exception
  1156. {
  1157. ConsecutiveHitCount = hitCount,
  1158. ConsecutiveMissCount = 0,
  1159. UpdatedAt = DateTime.Now
  1160. })
  1161. .Where(x => x.Id == entity.Id)
  1162. .ExecuteCommandAsync();
  1163. if (state != null)
  1164. {
  1165. await _detectionStateRep.Context.Updateable<AdoS8RuleDetectionState>()
  1166. .SetColumns(x => new AdoS8RuleDetectionState
  1167. {
  1168. ActiveExceptionId = entity.Id,
  1169. UpdatedAt = DateTime.Now
  1170. })
  1171. .Where(x => x.Id == state.Id)
  1172. .ExecuteCommandAsync();
  1173. }
  1174. await WriteDetectionLogAsync(BuildHitLog(tenantId, rule, ruleType, hit, DetectResultCreated, entity.Id, runId));
  1175. await TryDispatchLayerNotificationAsync(entity);
  1176. results.Add(BuildCreatedResult(rule, hit, entity.Id));
  1177. }
  1178. catch (Exception ex)
  1179. {
  1180. results.Add(BuildSkipResult(rule, "create_failed", ex.Message, hit));
  1181. }
  1182. }
  1183. return results;
  1184. }
  1185. /// <summary>
  1186. /// S8-SCHED-EXEC-1:lease 执行完成回写。
  1187. /// 必须 WHERE id = lease.RuleId AND lock_token = lease.LockToken;affectedRows = 0 视为 lease 丢失,记录 Warning,不覆盖状态。
  1188. /// 失败 ≥ 阈值(默认 3)写 paused_until = NOW + 1h。
  1189. /// </summary>
  1190. public async Task OnRuleCompletedAsync(long tenantId, S8RuleLease lease, S8RuleRunResult result, int durationMs)
  1191. {
  1192. var rule = await _ruleRep.AsQueryable().Where(x => x.Id == lease.RuleId).FirstAsync();
  1193. if (rule == null)
  1194. {
  1195. _logger.LogWarning("lease_complete_rule_missing ruleId={RuleId}", lease.RuleId);
  1196. return;
  1197. }
  1198. var now = DateTime.Now;
  1199. var effectiveInterval = NormalizePollInterval(rule.PollIntervalSeconds);
  1200. var nextRunAt = now.AddSeconds(effectiveInterval);
  1201. int affected;
  1202. if (result.Success)
  1203. {
  1204. affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  1205. .SetColumns(x => new AdoS8WatchRule
  1206. {
  1207. LastRunAt = now,
  1208. NextRunAt = nextRunAt,
  1209. LastStatus = "SUCCESS",
  1210. LastError = null,
  1211. LastDurationMs = durationMs,
  1212. ConsecutiveFailureCount = 0,
  1213. LockToken = null,
  1214. LockedBy = null,
  1215. LockUntil = null,
  1216. RunningStartedAt = null,
  1217. UpdatedAt = now
  1218. })
  1219. .Where(x => x.Id == lease.RuleId && x.LockToken == lease.LockToken)
  1220. .ExecuteCommandAsync();
  1221. }
  1222. else
  1223. {
  1224. var errorTrunc = Truncate(result.ErrorMessage, 500);
  1225. var newFailures = rule.ConsecutiveFailureCount + 1;
  1226. DateTime? pausedUntil = rule.PausedUntil;
  1227. string? pauseReason = rule.PauseReason;
  1228. if (newFailures >= AutoPauseFailureThreshold)
  1229. {
  1230. pausedUntil = now.AddHours(AutoPauseDurationHours);
  1231. pauseReason = Truncate($"AUTO_PAUSED_AFTER_{AutoPauseFailureThreshold}_FAILURES: {errorTrunc}", 64);
  1232. }
  1233. affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  1234. .SetColumns(x => new AdoS8WatchRule
  1235. {
  1236. LastRunAt = now,
  1237. NextRunAt = nextRunAt,
  1238. LastStatus = "FAILED",
  1239. LastError = errorTrunc,
  1240. LastDurationMs = durationMs,
  1241. ConsecutiveFailureCount = x.ConsecutiveFailureCount + 1,
  1242. PausedUntil = pausedUntil,
  1243. PauseReason = pauseReason,
  1244. LockToken = null,
  1245. LockedBy = null,
  1246. LockUntil = null,
  1247. RunningStartedAt = null,
  1248. UpdatedAt = now
  1249. })
  1250. .Where(x => x.Id == lease.RuleId && x.LockToken == lease.LockToken)
  1251. .ExecuteCommandAsync();
  1252. }
  1253. if (affected == 0)
  1254. {
  1255. _logger.LogWarning(
  1256. "lease_lost_on_complete ruleId={RuleId} ruleCode={RuleCode} lockToken={LockToken}",
  1257. lease.RuleId, lease.RuleCode, lease.LockToken);
  1258. }
  1259. }
  1260. /// <summary>
  1261. /// S8-SCHED-EXEC-1:单 tick 完整流程。Job / debug 调度入口。
  1262. /// 1) ResetExpiredLeasesAsync
  1263. /// 2) PickReadyRulesAsync(batchSize)
  1264. /// 3) 每条 rule 独立 try/catch 调用 RunSingleRuleAsync + OnRuleCompletedAsync
  1265. /// 单条规则失败不影响其他规则;整 tick 不抛异常。
  1266. /// </summary>
  1267. public async Task<S8DispatchTickResult> RunDispatchTickAsync(long tenantId, int batchSize, string lockedBy)
  1268. {
  1269. var tickId = Guid.NewGuid().ToString("N").Substring(0, 8);
  1270. var runId = Guid.NewGuid().ToString("N").Substring(0, 16);
  1271. var summary = new S8DispatchTickResult { TickId = tickId, RunId = runId };
  1272. try
  1273. {
  1274. summary.LeaseReleased = await ResetExpiredLeasesAsync(tenantId);
  1275. }
  1276. catch (Exception ex)
  1277. {
  1278. _logger.LogError(ex, "tick_reset_lease_failed tickId={TickId}", tickId);
  1279. }
  1280. List<S8RuleLease> leases;
  1281. try
  1282. {
  1283. leases = await PickReadyRulesAsync(tenantId, batchSize, lockedBy, runId);
  1284. }
  1285. catch (Exception ex)
  1286. {
  1287. _logger.LogError(ex, "tick_pick_failed tickId={TickId}", tickId);
  1288. return summary;
  1289. }
  1290. summary.Picked = leases.Count;
  1291. foreach (var lease in leases)
  1292. {
  1293. var sw = System.Diagnostics.Stopwatch.StartNew();
  1294. S8RuleRunResult runResult;
  1295. try
  1296. {
  1297. runResult = await RunSingleRuleAsync(tenantId, lease);
  1298. }
  1299. catch (Exception ex)
  1300. {
  1301. runResult = new S8RuleRunResult { Success = false, ErrorMessage = ex.Message, Stats = new() };
  1302. }
  1303. sw.Stop();
  1304. try
  1305. {
  1306. await OnRuleCompletedAsync(tenantId, lease, runResult, (int)sw.ElapsedMilliseconds);
  1307. }
  1308. catch (Exception ex)
  1309. {
  1310. _logger.LogError(ex, "tick_complete_failed tickId={TickId} ruleId={RuleId} ruleCode={RuleCode}", tickId, lease.RuleId, lease.RuleCode);
  1311. }
  1312. _logger.LogInformation(
  1313. "tick_rule_done tickId={TickId} runId={RunId} ruleId={RuleId} ruleCode={RuleCode} status={Status} durationMs={Dur} hits={Hits} created={Created} refreshed={Refreshed} pending={Pending} failed={Failed} error={Error}",
  1314. tickId, runId, lease.RuleId, lease.RuleCode,
  1315. runResult.Success ? "SUCCESS" : "FAILED",
  1316. sw.ElapsedMilliseconds,
  1317. runResult.Stats.Hits, runResult.Stats.Created, runResult.Stats.Refreshed, runResult.Stats.Pending, runResult.Stats.Failed,
  1318. runResult.ErrorMessage);
  1319. if (runResult.Success)
  1320. {
  1321. summary.Success++;
  1322. summary.Created += runResult.Stats.Created;
  1323. summary.Refreshed += runResult.Stats.Refreshed;
  1324. summary.Pending += runResult.Stats.Pending;
  1325. summary.PerRuleFailed += runResult.Stats.Failed;
  1326. }
  1327. else
  1328. {
  1329. summary.Failed++;
  1330. }
  1331. }
  1332. return summary;
  1333. }
  1334. private static int NormalizePollInterval(int raw)
  1335. {
  1336. if (raw < MinPollIntervalSeconds || raw > MaxPollIntervalSeconds) return DefaultPollIntervalSeconds;
  1337. return raw;
  1338. }
  1339. /// <summary>
  1340. /// S8-NOTIFY-WIRE-WATCH-1:异常自动建单成功后,非破坏性挂入通知分层路由。
  1341. /// 全程异常隔离:任何异常仅 LogWarning,不抛回主流程,不影响 detection_log / 事务 /
  1342. /// 异常状态机;该方法独立于 CreateFromWatchAsync / CreateFromHitAsync 的事务边界
  1343. /// (两者已 commit 后才返回 entity,故在此调用安全)。
  1344. /// sceneCode:先从 entity.SceneCode;空则 fallback 到 "S8_DEMO_DEFAULT"(demo 路径)。
  1345. /// severity:直接取 entity.Severity(CreateFromWatchAsync/CreateFromHitAsync 已保证非空)。
  1346. /// </summary>
  1347. private async Task TryDispatchLayerNotificationAsync(AdoS8Exception entity)
  1348. {
  1349. if (entity == null || entity.Id <= 0) return;
  1350. try
  1351. {
  1352. var sceneCode = string.IsNullOrWhiteSpace(entity.SceneCode) ? "S8_DEMO_DEFAULT" : entity.SceneCode;
  1353. // S8-SEVERITY-FOLLOW-SERIOUS-STANDARDIZE-EXEC-1:写入前 Normalize,落 FOLLOW/SERIOUS。
  1354. var severity = S8SeverityCode.Normalize(entity.Severity);
  1355. // S8-DEMO-IMPACT-SORT-NOTICE-1:附带 30 天影响统计(重复次数 / 累计损失 / 建议关注级别)。
  1356. // 计算失败不阻断通知主链路;snap==null 时 BuildNotification 仅写基础 Context。
  1357. ExceptionImpactSnapshot? snap = null;
  1358. try
  1359. {
  1360. snap = await _impactMetricsService.ComputeOneAsync(entity);
  1361. }
  1362. catch (Exception ex)
  1363. {
  1364. _logger.LogWarning(ex, "notify_impact_compute_throw exceptionId={ExceptionId}", entity.Id);
  1365. }
  1366. var baseContent =
  1367. $"异常 {entity.ExceptionCode}:{entity.Title}(场景 {sceneCode},严重度 {severity}" +
  1368. (string.IsNullOrWhiteSpace(entity.SourceRuleCode) ? "" : $",规则 {entity.SourceRuleCode}") + ")";
  1369. var content = snap == null
  1370. ? baseContent
  1371. : baseContent
  1372. + Environment.NewLine
  1373. + $"过去30天同类异常发生 {snap.RepeatCount30d} 次,已关闭累计损失 {snap.CumulativeLossHours30d.ToString("0.#")} 小时,建议关注级别:{snap.SuggestedAttentionLabel}";
  1374. await _notificationLayerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  1375. {
  1376. TenantId = entity.TenantId,
  1377. ExceptionId = entity.Id,
  1378. ExceptionNo = entity.ExceptionCode,
  1379. SceneCode = sceneCode,
  1380. Severity = severity,
  1381. EventCode = S8NotifyEventCode.ExceptionCreated,
  1382. ExceptionRef = entity,
  1383. Title = entity.Title ?? string.Empty,
  1384. Content = content,
  1385. Status = entity.Status,
  1386. SourceRuleCode = entity.SourceRuleCode,
  1387. JumpUrl = $"/aidop/s8/exceptions/{entity.Id}",
  1388. RepeatCount30d = snap?.RepeatCount30d,
  1389. CumulativeLossHours30d = snap?.CumulativeLossHours30d,
  1390. SuggestedAttentionLevel = snap?.SuggestedAttentionLevel,
  1391. SuggestedAttentionLabel = snap?.SuggestedAttentionLabel,
  1392. ImpactReason = snap?.ImpactReason,
  1393. });
  1394. }
  1395. catch (Exception ex)
  1396. {
  1397. _logger.LogWarning(ex, "notify_dispatch_throw exceptionId={ExceptionId}", entity.Id);
  1398. }
  1399. }
  1400. /// <summary>
  1401. /// S8-NOTIFY-WIRE-RECOVERED-1:异常恢复(recovered_at 已写入、RECOVERED detection_log 已落库)后,
  1402. /// 非破坏性挂入分层通知。全程异常隔离:任何异常仅 LogWarning,不抛回主流程,不影响 detection_log /
  1403. /// 状态机;call site 已在事务边界外(recovered 路径无事务)。
  1404. /// 重新读取 entity 拿场景/严重度/编号/状态/规则代码(恢复事件相对低频,1 次额外读可接受)。
  1405. /// S8-DEMO-IMPACT-SORT-NOTICE-1:恢复事件**不**追加影响统计字段,保持原有恢复语义不变。
  1406. /// </summary>
  1407. private async Task TryDispatchRecoveredLayerNotificationAsync(long exceptionId)
  1408. {
  1409. if (exceptionId <= 0) return;
  1410. try
  1411. {
  1412. var entity = await _exceptionRep.GetByIdAsync(exceptionId);
  1413. if (entity == null)
  1414. {
  1415. _logger.LogWarning("notify_recovered_dispatch_entity_missing exceptionId={ExceptionId}", exceptionId);
  1416. return;
  1417. }
  1418. var sceneCode = string.IsNullOrWhiteSpace(entity.SceneCode) ? "S8_DEMO_DEFAULT" : entity.SceneCode;
  1419. // S8-SEVERITY-FOLLOW-SERIOUS-STANDARDIZE-EXEC-1:写入前 Normalize,落 FOLLOW/SERIOUS。
  1420. var severity = S8SeverityCode.Normalize(entity.Severity);
  1421. var title = $"【已恢复】{entity.ExceptionCode}";
  1422. var content =
  1423. $"异常 {entity.ExceptionCode} 已恢复,场景 {sceneCode},严重度 {severity}" +
  1424. (string.IsNullOrWhiteSpace(entity.SourceRuleCode) ? "" : $",规则 {entity.SourceRuleCode}");
  1425. await _notificationLayerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  1426. {
  1427. TenantId = entity.TenantId,
  1428. ExceptionId = entity.Id,
  1429. ExceptionNo = entity.ExceptionCode,
  1430. SceneCode = sceneCode,
  1431. Severity = severity,
  1432. EventCode = S8NotifyEventCode.ExceptionRecovered,
  1433. ExceptionRef = entity,
  1434. Title = title,
  1435. Content = content,
  1436. Status = entity.Status,
  1437. SourceRuleCode = entity.SourceRuleCode,
  1438. JumpUrl = $"/aidop/s8/exceptions/{entity.Id}",
  1439. Recovered = true,
  1440. });
  1441. }
  1442. catch (Exception ex)
  1443. {
  1444. _logger.LogWarning(ex, "notify_recovered_dispatch_throw exceptionId={ExceptionId}", exceptionId);
  1445. }
  1446. }
  1447. }
  1448. /// <summary>
  1449. /// G01-05 去重结果对象。仅服务 G01-06 建单前拦截,由 CanCreate 单决策位决定是否建单。
  1450. /// 只服务首版唯一场景 S2(迁移后由 S2S6_PRODUCTION 切到单模块 S2)+ 唯一 trigger_type VALUE_DEVIATION + 设备对象。
  1451. /// 不预留多 trigger_type / 平台化去重扩展结构。
  1452. /// Reason 值域:no_pending / duplicate_pending / missing_dedup_key / query_failed。
  1453. /// </summary>
  1454. public sealed class S8WatchDedupResult
  1455. {
  1456. public S8WatchHitResult Hit { get; set; } = new();
  1457. public bool CanCreate { get; set; }
  1458. public long? MatchedExceptionId { get; set; }
  1459. public string Reason { get; set; } = string.Empty;
  1460. }
  1461. /// <summary>
  1462. /// G01-06 建单结果对象。仅服务 G-01 首版主线验收,由 Created / Skipped 两位决定结局。
  1463. /// 只服务首版唯一场景 S2(迁移后由 S2S6_PRODUCTION 切到单模块 S2)+ 唯一 trigger_type VALUE_DEVIATION + 设备对象。
  1464. /// 不预留多 trigger_type / 平台化工单扩展结构。
  1465. /// Reason 值域:auto_created / create_failed / 透传自 DedupResult.Reason。
  1466. /// </summary>
  1467. public sealed class S8WatchCreationResult
  1468. {
  1469. public S8WatchDedupResult DedupResult { get; set; } = new();
  1470. public bool Created { get; set; }
  1471. public bool Skipped { get; set; }
  1472. public long? CreatedExceptionId { get; set; }
  1473. public string Reason { get; set; } = string.Empty;
  1474. public string? ErrorMessage { get; set; }
  1475. }
  1476. /// <summary>
  1477. /// G01-04 命中结果对象。承载 G01-05 去重与 G01-06 建单所需最小追溯字段,
  1478. /// 仅服务首版唯一场景 S2(迁移后由 S2S6_PRODUCTION 切到单模块 S2)+ 唯一 trigger_type VALUE_DEVIATION + 设备对象。
  1479. /// 不预留多 trigger_type / 多场景 / 平台化扩展结构。
  1480. /// </summary>
  1481. public sealed class S8WatchHitResult
  1482. {
  1483. public long SourceRuleId { get; set; }
  1484. public string SourceRuleCode { get; set; } = string.Empty;
  1485. public string RelatedObjectCode { get; set; } = string.Empty;
  1486. public decimal CurrentValue { get; set; }
  1487. public decimal ThresholdValue { get; set; }
  1488. public string TriggerCondition { get; set; } = string.Empty;
  1489. public string Severity { get; set; } = string.Empty;
  1490. public long? OccurrenceDeptId { get; set; }
  1491. public long? ResponsibleDeptId { get; set; }
  1492. public string SourcePayload { get; set; } = string.Empty;
  1493. }
  1494. /// <summary>S8-SCHED-EXEC-1:lease 抢占成功后传递的最小标识对象。</summary>
  1495. public sealed class S8RuleLease
  1496. {
  1497. public long RuleId { get; set; }
  1498. public string RuleCode { get; set; } = string.Empty;
  1499. public string? RuleType { get; set; }
  1500. public string LockToken { get; set; } = string.Empty;
  1501. public string LockedBy { get; set; } = string.Empty;
  1502. public DateTime LockUntil { get; set; }
  1503. public string RunId { get; set; } = string.Empty;
  1504. public DateTime AcquiredAt { get; set; }
  1505. }
  1506. /// <summary>S8-SCHED-EXEC-1:单条规则执行结果,OnRuleCompletedAsync 据此更新状态。</summary>
  1507. public sealed class S8RuleRunResult
  1508. {
  1509. public bool Success { get; set; }
  1510. public string? ErrorMessage { get; set; }
  1511. public S8RuleRunStats Stats { get; set; } = new();
  1512. }
  1513. public sealed class S8RuleRunStats
  1514. {
  1515. public int Hits { get; set; }
  1516. public int Created { get; set; }
  1517. public int Refreshed { get; set; }
  1518. public int Pending { get; set; }
  1519. public int Failed { get; set; }
  1520. }
  1521. /// <summary>S8-SCHED-EXEC-1:单 tick 调度结果聚合。</summary>
  1522. public sealed class S8DispatchTickResult
  1523. {
  1524. public string TickId { get; set; } = string.Empty;
  1525. public string RunId { get; set; } = string.Empty;
  1526. public int LeaseReleased { get; set; }
  1527. public int Picked { get; set; }
  1528. public int Success { get; set; }
  1529. public int Failed { get; set; }
  1530. public int Created { get; set; }
  1531. public int Refreshed { get; set; }
  1532. public int Pending { get; set; }
  1533. public int PerRuleFailed { get; set; }
  1534. }