S8WatchSchedulerService.cs 82 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873
  1. using Admin.NET.Plugin.AiDOP.Entity.S8;
  2. using Admin.NET.Plugin.AiDOP.Infrastructure.S8;
  3. using Admin.NET.Plugin.AiDOP.Service.S8.Rules;
  4. using Microsoft.Extensions.Logging;
  5. using SqlSugar;
  6. using System.Data;
  7. using System.Globalization;
  8. using System.Text.Json;
  9. namespace Admin.NET.Plugin.AiDOP.Service.S8;
  10. /// <summary>
  11. /// 监视规则轮询调度服务(首轮存根)。
  12. /// 后续接入 Admin.NET 定时任务机制后,由调度器周期调用 <see cref="RunOnceAsync"/>,
  13. /// 按各规则的 PollIntervalSeconds 逐条评估并生成异常记录。
  14. /// </summary>
  15. public class S8WatchSchedulerService : ITransient
  16. {
  17. private readonly SqlSugarRepository<AdoS8WatchRule> _ruleRep;
  18. private readonly SqlSugarRepository<AdoS8AlertRule> _alertRuleRep;
  19. private readonly SqlSugarRepository<AdoS8DataSource> _dataSourceRep;
  20. private readonly SqlSugarRepository<AdoS8Exception> _exceptionRep;
  21. private readonly SqlSugarRepository<AdoS8ExceptionType> _exceptionTypeRep;
  22. private readonly S8NotificationService _notificationService;
  23. private readonly S8NotificationLayerResolver _notificationLayerResolver;
  24. private readonly S8ImpactMetricsService _impactMetricsService;
  25. private readonly S8ManualReportService _manualReportService;
  26. private readonly S8TimeoutRuleEvaluator _timeoutEvaluator;
  27. private readonly S8ShortageRuleEvaluator _shortageEvaluator;
  28. private readonly S8OutOfRangeRuleEvaluator _outOfRangeEvaluator;
  29. private readonly ILogger<S8WatchSchedulerService> _logger;
  30. private readonly SqlSugarRepository<AdoS8DetectionLog> _detectionLogRep;
  31. private readonly SqlSugarRepository<AdoS8RuleDetectionState> _detectionStateRep;
  32. private const string DetectionTriggerSource = "WATCH_SCHEDULER";
  33. private const string DetectResultCreated = "CREATED";
  34. private const string DetectResultRefreshed = "REFRESHED";
  35. private const string DetectResultRecovered = "RECOVERED";
  36. private const string DetectResultNoHit = "NO_HIT";
  37. private const string DetectResultEvaluateFailed = "EVALUATE_FAILED";
  38. private const string DefaultTriggerType = "VALUE_DEVIATION";
  39. private const string SqlDataSourceType = "SQL";
  40. // G01-05 未闭环状态集合:复用自 S8ExceptionService 当前 pendingStatuses 事实口径
  41. // (见 S8ExceptionService.GetPagedAsync 中 pendingStatuses 的定义,两处必须保持一致)。
  42. // 这不是“自定义未闭环集合”;若现有口径调整,两处需同步修改。
  43. private static readonly string[] UnclosedExceptionStatuses =
  44. { "NEW", "ASSIGNED", "IN_PROGRESS", "PENDING_VERIFICATION" };
  45. public S8WatchSchedulerService(
  46. SqlSugarRepository<AdoS8WatchRule> ruleRep,
  47. SqlSugarRepository<AdoS8AlertRule> alertRuleRep,
  48. SqlSugarRepository<AdoS8DataSource> dataSourceRep,
  49. SqlSugarRepository<AdoS8Exception> exceptionRep,
  50. SqlSugarRepository<AdoS8ExceptionType> exceptionTypeRep,
  51. S8NotificationService notificationService,
  52. S8NotificationLayerResolver notificationLayerResolver,
  53. S8ImpactMetricsService impactMetricsService,
  54. S8ManualReportService manualReportService,
  55. S8TimeoutRuleEvaluator timeoutEvaluator,
  56. S8ShortageRuleEvaluator shortageEvaluator,
  57. S8OutOfRangeRuleEvaluator outOfRangeEvaluator,
  58. ILogger<S8WatchSchedulerService> logger,
  59. SqlSugarRepository<AdoS8DetectionLog> detectionLogRep,
  60. SqlSugarRepository<AdoS8RuleDetectionState> detectionStateRep)
  61. {
  62. _ruleRep = ruleRep;
  63. _alertRuleRep = alertRuleRep;
  64. _dataSourceRep = dataSourceRep;
  65. _exceptionRep = exceptionRep;
  66. _exceptionTypeRep = exceptionTypeRep;
  67. _notificationService = notificationService;
  68. _notificationLayerResolver = notificationLayerResolver;
  69. _impactMetricsService = impactMetricsService;
  70. _manualReportService = manualReportService;
  71. _timeoutEvaluator = timeoutEvaluator;
  72. _shortageEvaluator = shortageEvaluator;
  73. _outOfRangeEvaluator = outOfRangeEvaluator;
  74. _logger = logger;
  75. _detectionLogRep = detectionLogRep;
  76. _detectionStateRep = detectionStateRep;
  77. }
  78. public async Task<List<S8WatchExecutionRule>> LoadExecutionRulesAsync(long tenantId, long factoryId)
  79. {
  80. // R3 OUT_OF_RANGE 重写后,新三类(OUT_OF_RANGE/TIMEOUT/SHORTAGE)改走 ProcessRulesByTypeAsync。
  81. // 旧 AlertRule 兼容主链此处只装载 RuleType 为空/未分类的历史规则,避免新旧双跑导致重复建单。
  82. var watchRules = await _ruleRep.AsQueryable()
  83. .Where(x => x.TenantId == tenantId
  84. && x.FactoryId == factoryId
  85. && x.Enabled
  86. && x.SceneCode == S8SceneCode.S2
  87. && (x.RuleType == null || x.RuleType == ""))
  88. .ToListAsync();
  89. var deviceRules = watchRules
  90. .Where(x => IsDeviceWatchObjectType(x.WatchObjectType))
  91. .ToList();
  92. if (deviceRules.Count == 0) return new();
  93. var dataSourceIds = deviceRules
  94. .Select(x => x.DataSourceId)
  95. .Distinct()
  96. .ToList();
  97. var dataSources = await _dataSourceRep.AsQueryable()
  98. .Where(x => x.TenantId == tenantId
  99. && x.FactoryId == factoryId
  100. && x.Enabled
  101. && dataSourceIds.Contains(x.Id))
  102. .ToListAsync();
  103. var dataSourceMap = dataSources.ToDictionary(x => x.Id);
  104. if (dataSourceMap.Count == 0) return new();
  105. var alertRules = (await _alertRuleRep.AsQueryable()
  106. .Where(x => x.TenantId == tenantId
  107. && x.FactoryId == factoryId
  108. && x.SceneCode == S8SceneCode.S2)
  109. .ToListAsync())
  110. .Where(IsSupportedAlertRule)
  111. .ToList();
  112. // G-01 首版 AlertRule 冲突口径(C 收口):
  113. // 当前场景存在多条可运行 AlertRule 时,视为“当前规则配置冲突”并跳过该规则,
  114. // 不按“首条”继续运行,也不扩大为“整场景停摆”。
  115. // 当前模型下所有 device watchRule 共享同场景 AlertRule,故冲突态下所有 device 规则均跳过,
  116. // 但此处按“逐规则跳过”的语义实现,避免被误读为“整场景 return empty 停摆”。
  117. var alertRule = alertRules.Count == 1 ? alertRules[0] : null;
  118. var executionRules = new List<S8WatchExecutionRule>();
  119. foreach (var watchRule in deviceRules.OrderBy(x => x.Id))
  120. {
  121. // 配置冲突:当前规则跳过(不停摆其他规则)。
  122. if (alertRule == null)
  123. continue;
  124. if (!dataSourceMap.TryGetValue(watchRule.DataSourceId, out var dataSource))
  125. continue;
  126. if (!IsSupportedSqlDataSource(dataSource))
  127. continue;
  128. executionRules.Add(new S8WatchExecutionRule
  129. {
  130. WatchRuleId = watchRule.Id,
  131. WatchRuleCode = watchRule.RuleCode,
  132. SceneCode = watchRule.SceneCode,
  133. TriggerType = DefaultTriggerType,
  134. WatchObjectType = watchRule.WatchObjectType.Trim(),
  135. DataSourceId = dataSource.Id,
  136. DataSourceCode = dataSource.DataSourceCode,
  137. DataSourceType = dataSource.Type,
  138. DataSourceConnection = dataSource.Endpoint?.Trim() ?? string.Empty,
  139. QueryExpression = watchRule.Expression?.Trim() ?? string.Empty,
  140. PollIntervalSeconds = watchRule.PollIntervalSeconds,
  141. AlertRuleId = alertRule.Id,
  142. AlertRuleCode = alertRule.RuleCode,
  143. TriggerCondition = alertRule.TriggerCondition!.Trim(),
  144. ThresholdValue = alertRule.ThresholdVal!.Trim(),
  145. Severity = alertRule.Severity
  146. });
  147. }
  148. return executionRules;
  149. }
  150. public async Task<List<S8WatchDeviceQueryResult>> QueryDeviceRowsAsync(long tenantId, long factoryId)
  151. {
  152. var executionRules = await LoadExecutionRulesAsync(tenantId, factoryId);
  153. var results = new List<S8WatchDeviceQueryResult>();
  154. foreach (var rule in executionRules)
  155. results.Add(await QueryDeviceRowsAsync(rule));
  156. return results;
  157. }
  158. public async Task<S8WatchDeviceQueryResult> QueryDeviceRowsAsync(S8WatchExecutionRule rule)
  159. {
  160. if (!string.Equals(rule.DataSourceType, SqlDataSourceType, StringComparison.OrdinalIgnoreCase))
  161. return S8WatchDeviceQueryResult.Fail(rule, "数据源类型不是 SQL,已跳过");
  162. if (string.IsNullOrWhiteSpace(rule.QueryExpression))
  163. return S8WatchDeviceQueryResult.Fail(rule, "查询表达式为空,已跳过");
  164. try
  165. {
  166. using var db = CreateSqlQueryScope(rule.DataSourceConnection);
  167. var table = await db.Ado.GetDataTableAsync(rule.QueryExpression);
  168. if (!HasRequiredColumns(table))
  169. return S8WatchDeviceQueryResult.Fail(rule, "查询结果缺少 required columns: related_object_code/current_value");
  170. var rows = table.Rows.Cast<DataRow>()
  171. .Select(MapDeviceRow)
  172. .Where(x => !string.IsNullOrWhiteSpace(x.RelatedObjectCode))
  173. .ToList();
  174. return S8WatchDeviceQueryResult.Ok(rule, rows);
  175. }
  176. catch (Exception ex)
  177. {
  178. return S8WatchDeviceQueryResult.Fail(rule, $"查询执行失败: {ex.Message}");
  179. }
  180. }
  181. /// <summary>
  182. /// G01-04:基于设备级结果行集做首版 VALUE_DEVIATION 单阈值判定,
  183. /// 产出命中结果对象列表,供 G01-05 去重与 G01-06 建单消费。
  184. /// 本方法不做去重、不做建单、不做严重度重算、不做时间线。
  185. /// </summary>
  186. public async Task<List<S8WatchHitResult>> EvaluateHitsAsync(long tenantId, long factoryId)
  187. {
  188. var executionRules = await LoadExecutionRulesAsync(tenantId, factoryId);
  189. var ruleMap = executionRules.ToDictionary(x => x.WatchRuleId);
  190. var queryResults = new List<S8WatchDeviceQueryResult>();
  191. foreach (var rule in executionRules)
  192. queryResults.Add(await QueryDeviceRowsAsync(rule));
  193. var hits = new List<S8WatchHitResult>();
  194. foreach (var queryResult in queryResults)
  195. {
  196. // G01-03 查询失败:跳过,不进入判定。
  197. if (!queryResult.Success) continue;
  198. if (!ruleMap.TryGetValue(queryResult.WatchRuleId, out var rule)) continue;
  199. // 判定参数缺失:跳过当前规则。
  200. if (string.IsNullOrWhiteSpace(rule.TriggerCondition)
  201. || string.IsNullOrWhiteSpace(rule.ThresholdValue))
  202. continue;
  203. // 比较符非法:跳过当前规则。
  204. var op = TryParseTriggerCondition(rule.TriggerCondition);
  205. if (op == null) continue;
  206. // ThresholdValue 非法:跳过当前规则。
  207. if (!TryParseDecimal(rule.ThresholdValue, out var threshold)) continue;
  208. foreach (var row in queryResult.Rows)
  209. {
  210. // CurrentValue 非法(null / 无法解析数值):跳过当前行,不进入判定。
  211. if (row.CurrentValue == null) continue;
  212. // 未命中:不进入后续链路。
  213. if (!EvaluateHit(row.CurrentValue.Value, op, threshold)) continue;
  214. hits.Add(new S8WatchHitResult
  215. {
  216. SourceRuleId = rule.WatchRuleId,
  217. SourceRuleCode = rule.WatchRuleCode,
  218. AlertRuleId = rule.AlertRuleId,
  219. DataSourceId = rule.DataSourceId,
  220. RelatedObjectCode = row.RelatedObjectCode,
  221. CurrentValue = row.CurrentValue.Value,
  222. ThresholdValue = threshold,
  223. TriggerCondition = op,
  224. Severity = rule.Severity,
  225. OccurrenceDeptId = row.OccurrenceDeptId,
  226. ResponsibleDeptId = row.ResponsibleDeptId,
  227. SourcePayload = row.SourcePayload
  228. });
  229. }
  230. }
  231. return hits;
  232. }
  233. /// <summary>
  234. /// G01-05:未闭环异常去重最小实现。
  235. /// 消费 G01-04 产出的 <see cref="S8WatchHitResult"/>,按 (SourceRuleId + RelatedObjectCode)
  236. /// 在未闭环状态集合内判重,只回答“是否允许建单”。
  237. /// 首版明确不做:原单刷新 / 时间线追加 / payload 更新 / 次数累计 / 严重度重算 / 状态修复。
  238. /// </summary>
  239. public async Task<List<S8WatchDedupResult>> EvaluateDedupAsync(long tenantId, long factoryId)
  240. {
  241. var hits = await EvaluateHitsAsync(tenantId, factoryId);
  242. var results = new List<S8WatchDedupResult>(hits.Count);
  243. foreach (var hit in hits)
  244. {
  245. // 防御性分支:正常情况下 SourceRuleId 与 RelatedObjectCode 已由上游
  246. // (G01-02 规则装配 + G01-03 查询结果列校验)保证;此处仅作兜底,
  247. // 不是首版正常路径。
  248. if (hit.SourceRuleId <= 0 || string.IsNullOrWhiteSpace(hit.RelatedObjectCode))
  249. {
  250. results.Add(new S8WatchDedupResult
  251. {
  252. Hit = hit,
  253. CanCreate = false,
  254. MatchedExceptionId = null,
  255. Reason = "missing_dedup_key"
  256. });
  257. continue;
  258. }
  259. long matchedId;
  260. try
  261. {
  262. matchedId = await FindPendingExceptionIdAsync(
  263. tenantId, factoryId, hit.SourceRuleId, hit.RelatedObjectCode);
  264. }
  265. catch
  266. {
  267. // 去重查询失败:首版偏保守,宁可阻止也不重复建单,不扩展为补偿。
  268. results.Add(new S8WatchDedupResult
  269. {
  270. Hit = hit,
  271. CanCreate = false,
  272. MatchedExceptionId = null,
  273. Reason = "query_failed"
  274. });
  275. continue;
  276. }
  277. if (matchedId > 0)
  278. {
  279. // 命中已有未闭环异常 → 阻止建单。
  280. results.Add(new S8WatchDedupResult
  281. {
  282. Hit = hit,
  283. CanCreate = false,
  284. MatchedExceptionId = matchedId,
  285. Reason = "duplicate_pending"
  286. });
  287. }
  288. else
  289. {
  290. // 未命中 → 允许建单,交 G01-06 消费。
  291. results.Add(new S8WatchDedupResult
  292. {
  293. Hit = hit,
  294. CanCreate = true,
  295. MatchedExceptionId = null,
  296. Reason = "no_pending"
  297. });
  298. }
  299. }
  300. return results;
  301. }
  302. // 取任意一条匹配的未闭环异常 Id 作为“是否存在重复单”的拦截依据。
  303. // 首版只需要“存在性”,不关心“最早 / 最新”;不在 G01-05 处理排序语义。
  304. private async Task<long> FindPendingExceptionIdAsync(
  305. long tenantId, long factoryId, long sourceRuleId, string relatedObjectCode)
  306. {
  307. // SqlSugar 表达式翻译要求 Contains 数组变量必须可访问;此处将类级常量
  308. // 承接到方法内局部变量,仅为表达式翻译服务,值与 UnclosedExceptionStatuses 一致。
  309. var statuses = UnclosedExceptionStatuses;
  310. var ids = await _exceptionRep.AsQueryable()
  311. .Where(x => x.TenantId == tenantId
  312. && x.FactoryId == factoryId
  313. && !x.IsDeleted
  314. && x.SourceRuleId == sourceRuleId
  315. && x.RelatedObjectCode == relatedObjectCode
  316. && statuses.Contains(x.Status))
  317. .Select(x => x.Id)
  318. .Take(1)
  319. .ToListAsync();
  320. return ids.Count > 0 ? ids[0] : 0L;
  321. }
  322. /// <summary>
  323. /// G01-06:自动建单入口。消费 G01-05 去重结果,对 CanCreate==true 的命中
  324. /// 复用 S8ManualReportService.CreateFromWatchAsync(同一主链的自动建单分支)落成标准 AdoS8Exception。
  325. /// CanCreate==false 直接跳过;创建失败返回最小失败结果,不补偿、不重试、不对账。
  326. /// </summary>
  327. public async Task<List<S8WatchCreationResult>> CreateExceptionsAsync(long tenantId, long factoryId)
  328. {
  329. var dedupResults = await EvaluateDedupAsync(tenantId, factoryId);
  330. var results = new List<S8WatchCreationResult>(dedupResults.Count);
  331. foreach (var dedup in dedupResults)
  332. {
  333. if (!dedup.CanCreate)
  334. {
  335. results.Add(new S8WatchCreationResult
  336. {
  337. DedupResult = dedup,
  338. Created = false,
  339. Skipped = true,
  340. CreatedExceptionId = null,
  341. Reason = dedup.Reason,
  342. ErrorMessage = null
  343. });
  344. continue;
  345. }
  346. try
  347. {
  348. var entity = await _manualReportService.CreateFromWatchAsync(dedup.Hit);
  349. await TryDispatchLayerNotificationAsync(entity);
  350. results.Add(new S8WatchCreationResult
  351. {
  352. DedupResult = dedup,
  353. Created = true,
  354. Skipped = false,
  355. CreatedExceptionId = entity.Id,
  356. Reason = "auto_created",
  357. ErrorMessage = null
  358. });
  359. }
  360. catch (Exception ex)
  361. {
  362. results.Add(new S8WatchCreationResult
  363. {
  364. DedupResult = dedup,
  365. Created = false,
  366. Skipped = false,
  367. CreatedExceptionId = null,
  368. Reason = "create_failed",
  369. ErrorMessage = ex.Message
  370. });
  371. }
  372. }
  373. // R3-OUT_OF_RANGE-REWRITE-1:三类正式 evaluator 路径。
  374. // 旧 AlertRule 兼容主链(上方 dedupResults 循环)已在 LoadExecutionRulesAsync 处过滤掉
  375. // OUT_OF_RANGE/TIMEOUT/SHORTAGE,只保留未分类历史规则;故新旧不双跑。
  376. // R6 RunId:本次 CreateExceptionsAsync 调用对应的统一关联 id,落入 detection_log。
  377. var runId = Guid.NewGuid().ToString("N").Substring(0, 16);
  378. results.AddRange(await ProcessRulesByTypeAsync(tenantId, factoryId, _timeoutEvaluator, S8TimeoutRuleEvaluator.RuleTypeCode, runId));
  379. results.AddRange(await ProcessRulesByTypeAsync(tenantId, factoryId, _shortageEvaluator, S8ShortageRuleEvaluator.RuleTypeCode, runId));
  380. results.AddRange(await ProcessRulesByTypeAsync(tenantId, factoryId, _outOfRangeEvaluator, S8OutOfRangeRuleEvaluator.RuleTypeCode, runId));
  381. return results;
  382. }
  383. /// <summary>
  384. /// R2 TIMEOUT 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。RunId 由内部生成。
  385. /// </summary>
  386. public Task<List<S8WatchCreationResult>> ProcessTimeoutRulesAsync(long tenantId, long factoryId) =>
  387. ProcessRulesByTypeAsync(tenantId, factoryId, _timeoutEvaluator, S8TimeoutRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
  388. /// <summary>
  389. /// R3 SHORTAGE 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。RunId 由内部生成。
  390. /// </summary>
  391. public Task<List<S8WatchCreationResult>> ProcessShortageRulesAsync(long tenantId, long factoryId) =>
  392. ProcessRulesByTypeAsync(tenantId, factoryId, _shortageEvaluator, S8ShortageRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
  393. /// <summary>
  394. /// R3-OUT_OF_RANGE-REWRITE-1:OUT_OF_RANGE 类规则主链。
  395. /// 复用 <see cref="ProcessRulesByTypeAsync"/>,并对历史 dedup_key=NULL 的旧记录做 compat fallback:
  396. /// (source_rule_id=rule.Id AND related_object_code=hit AND status!=CLOSED AND dedup_key IS NULL AND is_deleted=0)
  397. /// 命中则 backfill 6 列,避免重复建单。RunId 由内部生成。
  398. /// </summary>
  399. public Task<List<S8WatchCreationResult>> ProcessOutOfRangeRulesAsync(long tenantId, long factoryId) =>
  400. ProcessRulesByTypeAsync(tenantId, factoryId, _outOfRangeEvaluator, S8OutOfRangeRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
  401. /// <summary>
  402. /// R2/R3 通用规则主链。S8-SCHED-CLEANUP-LEGACY-PATH-1:本方法已收敛为 thin wrapper,
  403. /// 单规则处理(evaluator → reconcile → hit 循环 → CREATED/REFRESHED/NO_HIT/EVALUATE_FAILED 日志)
  404. /// 全部下沉至 <see cref="ProcessSingleRuleAsync"/>,与 Job tick 路径
  405. /// (<see cref="RunDispatchTickAsync"/> → <see cref="RunSingleRuleAsync"/> → ProcessSingleRuleAsync)
  406. /// 共享同一份逻辑,避免双维护。
  407. ///
  408. /// 调用方仅有 <see cref="CreateExceptionsAsync"/>(debug run-once / Process*RulesAsync 公共薄包装)。
  409. /// debug run-once 不持 lease(保留"手动立即跑"语义),但本方法在每条规则结束时通过
  410. /// <see cref="ApplyRunOnceCompletionAsync"/> 同步更新 watch_rule.last_run_at / last_status /
  411. /// last_error / last_duration_ms / last_run_id / consecutive_failure_count,缩小 run-once 与
  412. /// Job tick 之间的 last_* 状态分裂;不动 lock_token / running_started_at / paused_until,避免
  413. /// 与正在持锁运行的 Job 撕扯。
  414. /// </summary>
  415. private async Task<List<S8WatchCreationResult>> ProcessRulesByTypeAsync(
  416. long tenantId, long factoryId, IS8RuleEvaluator evaluator, string ruleType, string runId)
  417. {
  418. var aggregate = new List<S8WatchCreationResult>();
  419. var rules = await _ruleRep.AsQueryable()
  420. .Where(x => x.TenantId == tenantId
  421. && x.FactoryId == factoryId
  422. && x.Enabled
  423. && x.RuleType == ruleType)
  424. .ToListAsync();
  425. if (rules.Count == 0) return aggregate;
  426. var alertRules = (await _alertRuleRep.AsQueryable()
  427. .Where(x => x.TenantId == tenantId && x.FactoryId == factoryId)
  428. .ToListAsync()).AsReadOnly();
  429. foreach (var rule in rules.OrderBy(x => x.Id))
  430. {
  431. // S8-RUN-ONCE-LEASE-AWARENESS-1:debug run-once 不持 lease,但若该 rule 已被 Scheduler Job
  432. // 通过 PickReadyRulesAsync 抢锁(lock_token 非空且 lock_until > now),run-once 跳过该 rule,
  433. // 不写 last_*、不写 detection_log、不动锁,以避免与 Job tick 并发评估同一 rule 而互相覆盖运行态。
  434. // 锁状态用 ToListAsync 的初始快照判定;run-once 与 Job 的微秒级竞速无法在不抢 lease 的前提下
  435. // 完全消除(已登记为 follow-up 风险)。
  436. if (!string.IsNullOrEmpty(rule.LockToken)
  437. && rule.LockUntil.HasValue
  438. && rule.LockUntil.Value > DateTime.Now)
  439. {
  440. _logger.LogInformation(
  441. "run_once_skip_locked ruleId={RuleId} ruleCode={RuleCode} ruleType={RuleType} lockToken={Token} lockedBy={By} lockUntil={Until}",
  442. rule.Id, rule.RuleCode, ruleType, rule.LockToken, rule.LockedBy, rule.LockUntil);
  443. aggregate.Add(BuildSkipResult(rule, "rule_locked_by_scheduler", null));
  444. continue;
  445. }
  446. var sw = System.Diagnostics.Stopwatch.StartNew();
  447. S8RuleRunResult completion;
  448. try
  449. {
  450. var ruleResults = await ProcessSingleRuleAsync(tenantId, factoryId, rule, ruleType, evaluator, alertRules, runId);
  451. aggregate.AddRange(ruleResults);
  452. completion = new S8RuleRunResult
  453. {
  454. Success = true,
  455. Stats = new S8RuleRunStats
  456. {
  457. Hits = ruleResults.Count,
  458. Created = ruleResults.Count(r => r.Created),
  459. Refreshed = ruleResults.Count(r => r.Reason == "duplicate_pending"),
  460. Pending = ruleResults.Count(r => r.Reason == "antiflap_pending_hit"),
  461. Failed = ruleResults.Count(r => r.Reason == "create_failed" || r.Reason == "refresh_failed" || r.Reason == "antiflap_failed" || r.Reason == "evaluate_failed")
  462. }
  463. };
  464. }
  465. catch (Exception ex)
  466. {
  467. // ProcessSingleRuleAsync 在 evaluator 抛 S8RuleEvaluatorException 时已写 EVALUATE_FAILED
  468. // detection_log 后再 throw(供 RunSingleRuleAsync 标 Success=false)。本路径无 lease,
  469. // 吞异常以保留"逐规则失败不影响其他规则"的旧 ProcessRulesByTypeAsync 语义。
  470. _logger.LogWarning(ex, "process_rule_failed ruleCode={RuleCode} ruleType={RuleType}", rule.RuleCode, ruleType);
  471. aggregate.Add(BuildSkipResult(rule, "evaluate_failed", ex.Message));
  472. completion = new S8RuleRunResult { Success = false, ErrorMessage = ex.Message, Stats = new() };
  473. }
  474. sw.Stop();
  475. try
  476. {
  477. await ApplyRunOnceCompletionAsync(rule, completion, (int)sw.ElapsedMilliseconds, runId);
  478. }
  479. catch (Exception ex)
  480. {
  481. _logger.LogWarning(ex, "run_once_completion_write_failed ruleCode={RuleCode} ruleType={RuleType}", rule.RuleCode, ruleType);
  482. }
  483. }
  484. return aggregate;
  485. }
  486. /// <summary>
  487. /// S8-SCHED-CLEANUP-LEGACY-PATH-1:debug run-once 的 last_* 状态回写(无 lease 版)。
  488. /// 与 <see cref="OnRuleCompletedAsync"/> 的语义平行,差异在于:
  489. /// - 不要求 lock_token 匹配(run-once 不持 lease)
  490. /// - 不写 lock_token / locked_by / lock_until / running_started_at(不与 Job lease 撕扯)
  491. /// - 不做 consecutive_failure_count 阈值的 paused_until 自动暂停(debug 路径不应自动暂停 demo rule)
  492. /// 写入:last_run_at / next_run_at / last_status / last_error / last_duration_ms / last_run_id /
  493. /// consecutive_failure_count / updated_at。
  494. /// </summary>
  495. private async Task ApplyRunOnceCompletionAsync(AdoS8WatchRule rule, S8RuleRunResult result, int durationMs, string runId)
  496. {
  497. var now = DateTime.Now;
  498. var nextRunAt = now.AddSeconds(NormalizePollInterval(rule.PollIntervalSeconds));
  499. if (result.Success)
  500. {
  501. await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  502. .SetColumns(x => new AdoS8WatchRule
  503. {
  504. LastRunAt = now,
  505. NextRunAt = nextRunAt,
  506. LastStatus = "SUCCESS",
  507. LastError = null,
  508. LastDurationMs = durationMs,
  509. LastRunId = runId,
  510. ConsecutiveFailureCount = 0,
  511. UpdatedAt = now
  512. })
  513. .Where(x => x.Id == rule.Id)
  514. .ExecuteCommandAsync();
  515. }
  516. else
  517. {
  518. var errorTrunc = Truncate(result.ErrorMessage, 500);
  519. await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  520. .SetColumns(x => new AdoS8WatchRule
  521. {
  522. LastRunAt = now,
  523. NextRunAt = nextRunAt,
  524. LastStatus = "FAILED",
  525. LastError = errorTrunc,
  526. LastDurationMs = durationMs,
  527. LastRunId = runId,
  528. ConsecutiveFailureCount = x.ConsecutiveFailureCount + 1,
  529. UpdatedAt = now
  530. })
  531. .Where(x => x.Id == rule.Id)
  532. .ExecuteCommandAsync();
  533. }
  534. }
  535. // S8-SCHED-EXEC-1:trigger / recover 抗抖计数兜底,null / <1 / >10 一律按 1,避免非法配置导致永远不建单 / 永远不恢复。
  536. private static int NormalizeAntiflapCount(int raw)
  537. {
  538. if (raw < 1 || raw > 10) return 1;
  539. return raw;
  540. }
  541. /// <summary>
  542. /// 命中时累加 detection_state.consecutive_hit_count;未存在则插入 hitCount=1。
  543. /// 返回当前 state 行(含 Id)以及命中后的 hitCount。
  544. /// 注意:本函数不消费 trigger_count_required;上游决定是否进入 CreateFromHitAsync。
  545. /// </summary>
  546. private async Task<(AdoS8RuleDetectionState? state, int hitCount)> UpsertDetectionStateOnHitAsync(
  547. long tenantId, long factoryId, AdoS8WatchRule rule, S8RuleHit hit)
  548. {
  549. var now = DateTime.Now;
  550. var existing = await _detectionStateRep.AsQueryable()
  551. .Where(x => x.TenantId == tenantId
  552. && x.FactoryId == factoryId
  553. && x.RuleCode == rule.RuleCode
  554. && x.DedupKey == hit.DedupKey)
  555. .FirstAsync();
  556. if (existing == null)
  557. {
  558. var fresh = new AdoS8RuleDetectionState
  559. {
  560. TenantId = tenantId,
  561. FactoryId = factoryId,
  562. RuleCode = rule.RuleCode,
  563. DedupKey = hit.DedupKey,
  564. SourceObjectType = string.IsNullOrEmpty(hit.SourceObjectType) ? null : hit.SourceObjectType,
  565. SourceObjectId = string.IsNullOrEmpty(hit.SourceObjectId) ? null : hit.SourceObjectId,
  566. ConsecutiveHitCount = 1,
  567. ConsecutiveMissCount = 0,
  568. LastSeenAt = now,
  569. LastHitAt = now,
  570. CreatedAt = now,
  571. UpdatedAt = now
  572. };
  573. // BUG-S8-DETECTION-STATE-ACTIVE-EXC-ID-TRIGGER1-001:必须回填 fresh.Id,否则
  574. // trigger=1 首 tick 建单后 UPDATE state SET active_exception_id WHERE id=state.Id
  575. // 命中 0 行(state.Id 为默认 0)。沿用 S8ManualReportService 既定模式。
  576. fresh = await _detectionStateRep.AsInsertable(fresh).ExecuteReturnEntityAsync();
  577. return (fresh, 1);
  578. }
  579. var newHitCount = existing.ConsecutiveHitCount + 1;
  580. await _detectionStateRep.Context.Updateable<AdoS8RuleDetectionState>()
  581. .SetColumns(x => new AdoS8RuleDetectionState
  582. {
  583. ConsecutiveHitCount = x.ConsecutiveHitCount + 1,
  584. ConsecutiveMissCount = 0,
  585. LastSeenAt = now,
  586. LastHitAt = now,
  587. SourceObjectType = string.IsNullOrEmpty(hit.SourceObjectType) ? existing.SourceObjectType : hit.SourceObjectType,
  588. SourceObjectId = string.IsNullOrEmpty(hit.SourceObjectId) ? existing.SourceObjectId : hit.SourceObjectId,
  589. UpdatedAt = now
  590. })
  591. .Where(x => x.Id == existing.Id)
  592. .ExecuteCommandAsync();
  593. existing.ConsecutiveHitCount = newHitCount;
  594. return (existing, newHitCount);
  595. }
  596. private async Task<long> FindOpenExceptionByDedupKeyAsync(long tenantId, long factoryId, string dedupKey)
  597. {
  598. var ids = await _exceptionRep.AsQueryable()
  599. .Where(x => x.TenantId == tenantId
  600. && x.FactoryId == factoryId
  601. && !x.IsDeleted
  602. && x.Status != "CLOSED"
  603. && x.DedupKey == dedupKey)
  604. .Select(x => x.Id)
  605. .Take(1)
  606. .ToListAsync();
  607. return ids.Count > 0 ? ids[0] : 0L;
  608. }
  609. /// <summary>
  610. /// R3 OUT_OF_RANGE compat fallback 查找:用 (source_rule_id, related_object_code, status!=CLOSED,
  611. /// dedup_key IS NULL, is_deleted=0) 严格条件定位旧 AlertRule 主链留下的历史记录。
  612. /// </summary>
  613. private async Task<long> FindLegacyOutOfRangeExceptionAsync(long tenantId, long factoryId, long sourceRuleId, string relatedObjectCode)
  614. {
  615. if (string.IsNullOrWhiteSpace(relatedObjectCode)) return 0L;
  616. var ids = await _exceptionRep.AsQueryable()
  617. .Where(x => x.TenantId == tenantId
  618. && x.FactoryId == factoryId
  619. && !x.IsDeleted
  620. && x.Status != "CLOSED"
  621. && x.DedupKey == null
  622. && x.SourceRuleId == sourceRuleId
  623. && x.RelatedObjectCode == relatedObjectCode)
  624. .Select(x => x.Id)
  625. .Take(1)
  626. .ToListAsync();
  627. return ids.Count > 0 ? ids[0] : 0L;
  628. }
  629. /// <summary>
  630. /// R3 OUT_OF_RANGE compat fallback backfill:把历史记录的 R1 新 6 列(dedup_key/source_rule_code/
  631. /// source_object_type/source_object_id/source_payload/last_detected_at)写入,并刷新 updated_at。
  632. /// </summary>
  633. private async Task BackfillLegacyExceptionAsync(long exceptionId, S8RuleHit hit)
  634. {
  635. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  636. .SetColumns(x => new AdoS8Exception
  637. {
  638. DedupKey = hit.DedupKey,
  639. SourceRuleCode = hit.SourceRuleCode,
  640. SourceObjectType = hit.SourceObjectType,
  641. SourceObjectId = hit.SourceObjectId,
  642. SourcePayload = hit.SourcePayload,
  643. LastDetectedAt = hit.DetectedAt,
  644. UpdatedAt = DateTime.Now
  645. })
  646. .Where(x => x.Id == exceptionId)
  647. .ExecuteCommandAsync();
  648. }
  649. /// <summary>
  650. /// R5 恢复时间最小闭环:对当前 rule 下未关闭、有 dedup_key、recovered_at 仍为 NULL 的异常,
  651. /// 凡不在本轮 hits.dedup_key 集合内的,写入 recovered_at = now、updated_at = now。
  652. /// 仅写这 2 列;不动 status / assignee / verifier / source_payload / last_detected_at;
  653. /// recovered_at 一旦写入,本轮不做复发清空。
  654. /// R6 返回 recoveredIds 供上游决定是否写 NO_HIT 日志,并对每个 recovered exception 写一条 RECOVERED 日志。
  655. /// </summary>
  656. private async Task<List<long>> ReconcileRecoveriesForRuleAsync(
  657. long tenantId, long factoryId, AdoS8WatchRule rule, string ruleType, List<S8RuleHit> hits, string runId)
  658. {
  659. var hitDedupKeys = hits
  660. .Where(h => !string.IsNullOrWhiteSpace(h.DedupKey))
  661. .Select(h => h.DedupKey)
  662. .ToHashSet(StringComparer.Ordinal);
  663. var candidates = await _exceptionRep.AsQueryable()
  664. .Where(x => x.TenantId == tenantId
  665. && x.FactoryId == factoryId
  666. && !x.IsDeleted
  667. && x.Status != "CLOSED"
  668. && x.SourceRuleCode == rule.RuleCode
  669. && x.DedupKey != null
  670. && x.RecoveredAt == null)
  671. .Select(x => new { x.Id, x.DedupKey, x.SourceObjectType, x.SourceObjectId, x.RelatedObjectCode, x.ConsecutiveMissCount })
  672. .ToListAsync();
  673. if (candidates.Count == 0) return new List<long>();
  674. var now = DateTime.Now;
  675. var recoverRequired = NormalizeAntiflapCount(rule.RecoverCountRequired);
  676. var recoveredIds = new List<long>();
  677. foreach (var c in candidates)
  678. {
  679. if (hitDedupKeys.Contains(c.DedupKey!)) continue;
  680. // S8-SCHED-EXEC-1:恢复抗抖累计。
  681. // 1) 每次未命中:异常 ConsecutiveMissCount += 1,ConsecutiveHitCount 清零;
  682. // 2) miss < recover_count_required:仅累计,不写 recovered_at、不写 RECOVERED;
  683. // 3) miss >= recover_count_required:写 recovered_at、写 RECOVERED 日志。
  684. var newMissCount = c.ConsecutiveMissCount + 1;
  685. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  686. .SetColumns(x => new AdoS8Exception
  687. {
  688. ConsecutiveMissCount = x.ConsecutiveMissCount + 1,
  689. ConsecutiveHitCount = 0,
  690. UpdatedAt = now
  691. })
  692. .Where(x => x.Id == c.Id)
  693. .ExecuteCommandAsync();
  694. // detection_state 同步累计 miss(建单后 state.active_exception_id 仍指向 c.Id)。
  695. await _detectionStateRep.Context.Updateable<AdoS8RuleDetectionState>()
  696. .SetColumns(x => new AdoS8RuleDetectionState
  697. {
  698. ConsecutiveMissCount = x.ConsecutiveMissCount + 1,
  699. ConsecutiveHitCount = 0,
  700. LastSeenAt = now,
  701. LastMissAt = now,
  702. UpdatedAt = now
  703. })
  704. .Where(x => x.TenantId == tenantId
  705. && x.FactoryId == factoryId
  706. && x.RuleCode == rule.RuleCode
  707. && x.DedupKey == c.DedupKey)
  708. .ExecuteCommandAsync();
  709. if (newMissCount < recoverRequired)
  710. {
  711. _logger.LogInformation(
  712. "antiflap_pending_recovery ruleCode={RuleCode} dedupKey={DedupKey} missCount={Miss} recoverRequired={Required}",
  713. rule.RuleCode, c.DedupKey, newMissCount, recoverRequired);
  714. continue;
  715. }
  716. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  717. .SetColumns(x => new AdoS8Exception
  718. {
  719. RecoveredAt = now,
  720. UpdatedAt = now
  721. })
  722. .Where(x => x.Id == c.Id)
  723. .ExecuteCommandAsync();
  724. recoveredIds.Add(c.Id);
  725. await WriteDetectionLogAsync(new AdoS8DetectionLog
  726. {
  727. TenantId = tenantId, FactoryId = factoryId,
  728. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  729. SourceObjectType = c.SourceObjectType, SourceObjectId = c.SourceObjectId,
  730. RelatedObjectCode = c.RelatedObjectCode, DedupKey = c.DedupKey,
  731. DetectResult = DetectResultRecovered,
  732. ExceptionId = c.Id,
  733. DetectedAt = now,
  734. PayloadSnapshot = JsonSerializer.Serialize(new { ruleId = rule.Id, ruleCode = rule.RuleCode, reason = "no_longer_hit", missCount = newMissCount, recoverRequired }),
  735. RunId = runId, TriggerSource = DetectionTriggerSource,
  736. Remark = "Rule no longer hit; recovered_at marked"
  737. });
  738. // S8-NOTIFY-WIRE-RECOVERED-1:detection_log 已写入、recovered_at 已落库后挂入恢复通知。
  739. // 通知失败仅 LogWarning,绝不影响恢复状态/检测日志。
  740. await TryDispatchRecoveredLayerNotificationAsync(c.Id);
  741. }
  742. if (recoveredIds.Count > 0)
  743. {
  744. _logger.LogInformation(
  745. "rule_recovered ruleCode={RuleCode} ruleType={RuleType} recoveredCount={Count} recoveredIds={Ids}",
  746. rule.RuleCode, ruleType, recoveredIds.Count, string.Join(",", recoveredIds));
  747. }
  748. return recoveredIds;
  749. }
  750. /// <summary>R6 通用 hit 日志构造(CREATED / REFRESHED 共用)。</summary>
  751. private static AdoS8DetectionLog BuildHitLog(
  752. long tenantId, long factoryId, AdoS8WatchRule rule, string ruleType, S8RuleHit hit,
  753. string detectResult, long exceptionId, string runId) => new()
  754. {
  755. TenantId = tenantId, FactoryId = factoryId,
  756. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  757. SourceObjectType = hit.SourceObjectType, SourceObjectId = hit.SourceObjectId,
  758. RelatedObjectCode = hit.RelatedObjectCode, DedupKey = hit.DedupKey,
  759. DetectResult = detectResult,
  760. ExceptionId = exceptionId,
  761. DetectedAt = hit.DetectedAt,
  762. PayloadSnapshot = hit.SourcePayload,
  763. RunId = runId,
  764. TriggerSource = DetectionTriggerSource
  765. };
  766. /// <summary>R6 日志写入:失败仅 LogWarning,不阻断主链;不抛异常。</summary>
  767. private async Task WriteDetectionLogAsync(AdoS8DetectionLog log)
  768. {
  769. try
  770. {
  771. await _detectionLogRep.InsertAsync(log);
  772. }
  773. catch (Exception ex)
  774. {
  775. _logger.LogWarning(ex,
  776. "detection_log_write_failed runId={RunId} detectResult={Result} ruleCode={RuleCode} exceptionId={ExceptionId}",
  777. log.RunId, log.DetectResult, log.RuleCode, log.ExceptionId);
  778. }
  779. }
  780. private static string Truncate(string? s, int max) =>
  781. string.IsNullOrEmpty(s) ? string.Empty : (s.Length <= max ? s : s.Substring(0, max));
  782. private async Task RefreshDetectionAsync(long exceptionId, S8RuleHit hit)
  783. {
  784. // S8-SCHED-EXEC-1:刷新阶段抗抖累计 + 复发清空 recovered_at。
  785. // ConsecutiveHitCount += 1(用 SetColumns 内表达式完成原子自增);ConsecutiveMissCount 归零。
  786. // RecoveredAt 不为 NULL 时(复发)一并清空,保持业务对"再次命中即视为活跃"的预期。
  787. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  788. .SetColumns(x => new AdoS8Exception
  789. {
  790. LastDetectedAt = hit.DetectedAt,
  791. SourcePayload = hit.SourcePayload,
  792. ConsecutiveHitCount = x.ConsecutiveHitCount + 1,
  793. ConsecutiveMissCount = 0,
  794. RecoveredAt = null,
  795. UpdatedAt = DateTime.Now
  796. })
  797. .Where(x => x.Id == exceptionId)
  798. .ExecuteCommandAsync();
  799. }
  800. private static S8WatchCreationResult BuildCreatedResult(AdoS8WatchRule rule, S8RuleHit hit, long exceptionId) =>
  801. new()
  802. {
  803. DedupResult = new S8WatchDedupResult
  804. {
  805. Hit = ToWatchHit(rule, hit),
  806. CanCreate = true,
  807. MatchedExceptionId = null,
  808. Reason = "no_pending"
  809. },
  810. Created = true,
  811. Skipped = false,
  812. CreatedExceptionId = exceptionId,
  813. Reason = "auto_created",
  814. ErrorMessage = null
  815. };
  816. private static S8WatchCreationResult BuildSkippedDuplicate(AdoS8WatchRule rule, S8RuleHit hit, long matchedId) =>
  817. new()
  818. {
  819. DedupResult = new S8WatchDedupResult
  820. {
  821. Hit = ToWatchHit(rule, hit),
  822. CanCreate = false,
  823. MatchedExceptionId = matchedId,
  824. Reason = "duplicate_pending"
  825. },
  826. Created = false,
  827. Skipped = true,
  828. CreatedExceptionId = null,
  829. Reason = "duplicate_pending",
  830. ErrorMessage = null
  831. };
  832. private static S8WatchCreationResult BuildSkipResult(AdoS8WatchRule rule, string reason, string? error, S8RuleHit? hit = null) =>
  833. new()
  834. {
  835. DedupResult = new S8WatchDedupResult
  836. {
  837. Hit = hit != null ? ToWatchHit(rule, hit) : new S8WatchHitResult { SourceRuleId = rule.Id, SourceRuleCode = rule.RuleCode },
  838. CanCreate = false,
  839. MatchedExceptionId = null,
  840. Reason = reason
  841. },
  842. Created = false,
  843. Skipped = true,
  844. CreatedExceptionId = null,
  845. Reason = reason,
  846. ErrorMessage = error
  847. };
  848. private static S8WatchHitResult ToWatchHit(AdoS8WatchRule rule, S8RuleHit hit) => new()
  849. {
  850. SourceRuleId = hit.SourceRuleId == 0 ? rule.Id : hit.SourceRuleId,
  851. SourceRuleCode = string.IsNullOrEmpty(hit.SourceRuleCode) ? rule.RuleCode : hit.SourceRuleCode,
  852. DataSourceId = hit.DataSourceId,
  853. RelatedObjectCode = hit.RelatedObjectCode,
  854. Severity = hit.Severity,
  855. OccurrenceDeptId = hit.OccurrenceDeptId,
  856. ResponsibleDeptId = hit.ResponsibleDeptId,
  857. SourcePayload = hit.SourcePayload
  858. };
  859. /// <summary>
  860. /// 单次轮询入口。当前仅完成规则读取与组装,返回可执行规则数量,不做实际数据采集。
  861. /// </summary>
  862. public async Task<int> RunOnceAsync()
  863. {
  864. var executionRules = await LoadExecutionRulesAsync(1, 1);
  865. return executionRules.Count;
  866. }
  867. // G01-04 首版最小比较符集合:>, >=, <, <=。
  868. // 允许首尾空格;非此集合的一律视为“比较符非法”,由调用方跳过。
  869. private static string? TryParseTriggerCondition(string raw)
  870. {
  871. var normalized = raw.Trim();
  872. return normalized switch
  873. {
  874. ">" => ">",
  875. ">=" => ">=",
  876. "<" => "<",
  877. "<=" => "<=",
  878. _ => null
  879. };
  880. }
  881. private static bool TryParseDecimal(string raw, out decimal value) =>
  882. decimal.TryParse(raw.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out value);
  883. private static bool EvaluateHit(decimal current, string op, decimal threshold) => op switch
  884. {
  885. ">" => current > threshold,
  886. ">=" => current >= threshold,
  887. "<" => current < threshold,
  888. "<=" => current <= threshold,
  889. _ => false
  890. };
  891. private static bool IsSupportedAlertRule(AdoS8AlertRule alertRule) =>
  892. !string.IsNullOrWhiteSpace(alertRule.TriggerCondition)
  893. && !string.IsNullOrWhiteSpace(alertRule.ThresholdVal);
  894. private static bool IsSupportedSqlDataSource(AdoS8DataSource dataSource) =>
  895. dataSource.Enabled
  896. && string.Equals(dataSource.Type?.Trim(), SqlDataSourceType, StringComparison.OrdinalIgnoreCase)
  897. && !string.IsNullOrWhiteSpace(dataSource.Endpoint);
  898. private static bool IsDeviceWatchObjectType(string? watchObjectType)
  899. {
  900. if (string.IsNullOrWhiteSpace(watchObjectType)) return false;
  901. var normalized = watchObjectType.Trim().ToUpperInvariant();
  902. return normalized is "DEVICE" or "EQUIPMENT" || watchObjectType.Trim() == "设备";
  903. }
  904. private SqlSugarScope CreateSqlQueryScope(string connectionString)
  905. {
  906. var dbType = _ruleRep.Context.CurrentConnectionConfig.DbType;
  907. return new SqlSugarScope(new ConnectionConfig
  908. {
  909. ConfigId = $"s8-watch-sql-{Guid.NewGuid():N}",
  910. DbType = dbType,
  911. ConnectionString = connectionString,
  912. InitKeyType = InitKeyType.Attribute,
  913. IsAutoCloseConnection = true
  914. });
  915. }
  916. private static bool HasRequiredColumns(DataTable table) =>
  917. TryGetColumnName(table.Columns, "related_object_code") != null
  918. && TryGetColumnName(table.Columns, "current_value") != null;
  919. private static S8WatchDeviceRow MapDeviceRow(DataRow row)
  920. {
  921. var columns = row.Table.Columns;
  922. var relatedObjectCodeColumn = TryGetColumnName(columns, "related_object_code");
  923. var currentValueColumn = TryGetColumnName(columns, "current_value");
  924. var occurrenceDeptIdColumn = TryGetColumnName(columns, "occurrence_dept_id");
  925. var responsibleDeptIdColumn = TryGetColumnName(columns, "responsible_dept_id");
  926. return new S8WatchDeviceRow
  927. {
  928. RelatedObjectCode = ReadString(row, relatedObjectCodeColumn),
  929. CurrentValue = ReadDecimal(row, currentValueColumn),
  930. OccurrenceDeptId = ReadLong(row, occurrenceDeptIdColumn),
  931. ResponsibleDeptId = ReadLong(row, responsibleDeptIdColumn),
  932. SourcePayload = BuildSourcePayload(row)
  933. };
  934. }
  935. private static string? TryGetColumnName(DataColumnCollection columns, string expectedName)
  936. {
  937. var normalizedExpected = NormalizeColumnName(expectedName);
  938. foreach (DataColumn column in columns)
  939. {
  940. if (NormalizeColumnName(column.ColumnName) == normalizedExpected)
  941. return column.ColumnName;
  942. }
  943. return null;
  944. }
  945. private static string NormalizeColumnName(string columnName) =>
  946. columnName.Replace("_", string.Empty, StringComparison.Ordinal).Trim().ToUpperInvariant();
  947. private static string ReadString(DataRow row, string? columnName)
  948. {
  949. if (string.IsNullOrWhiteSpace(columnName)) return string.Empty;
  950. var value = row[columnName];
  951. return value == DBNull.Value ? string.Empty : Convert.ToString(value)?.Trim() ?? string.Empty;
  952. }
  953. private static long? ReadLong(DataRow row, string? columnName)
  954. {
  955. if (string.IsNullOrWhiteSpace(columnName)) return null;
  956. var value = row[columnName];
  957. if (value == DBNull.Value) return null;
  958. return long.TryParse(Convert.ToString(value, CultureInfo.InvariantCulture), out var result) ? result : null;
  959. }
  960. private static decimal? ReadDecimal(DataRow row, string? columnName)
  961. {
  962. if (string.IsNullOrWhiteSpace(columnName)) return null;
  963. var value = row[columnName];
  964. if (value == DBNull.Value) return null;
  965. return decimal.TryParse(Convert.ToString(value, CultureInfo.InvariantCulture), NumberStyles.Any, CultureInfo.InvariantCulture, out var result)
  966. ? result
  967. : null;
  968. }
  969. private static string BuildSourcePayload(DataRow row)
  970. {
  971. var payload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
  972. foreach (DataColumn column in row.Table.Columns)
  973. {
  974. var value = row[column];
  975. payload[column.ColumnName] = value == DBNull.Value ? null : value;
  976. }
  977. return JsonSerializer.Serialize(payload);
  978. }
  979. // ============================================================
  980. // S8-SCHED-EXEC-1:DB 驱动调度执行层
  981. // ============================================================
  982. private const int LeaseDurationMinutes = 5;
  983. private const int AutoPauseFailureThreshold = 3;
  984. private const int AutoPauseDurationHours = 1;
  985. private const int DefaultPollIntervalSeconds = 300;
  986. private const int MinPollIntervalSeconds = 60;
  987. private const int MaxPollIntervalSeconds = 86400;
  988. /// <summary>
  989. /// S8-SCHED-EXEC-1:释放过期 lease(lock_until &lt; NOW),不修改 last_status / last_error,仅清空 lock 三件套 + running_started_at。
  990. /// 返回释放的行数。
  991. /// </summary>
  992. public async Task<int> ResetExpiredLeasesAsync(long tenantId, long factoryId)
  993. {
  994. var now = DateTime.Now;
  995. var affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  996. .SetColumns(x => new AdoS8WatchRule
  997. {
  998. LockToken = null,
  999. LockedBy = null,
  1000. LockUntil = null,
  1001. RunningStartedAt = null,
  1002. UpdatedAt = now
  1003. })
  1004. .Where(x => x.TenantId == tenantId
  1005. && x.FactoryId == factoryId
  1006. && x.LockUntil != null
  1007. && x.LockUntil < now)
  1008. .ExecuteCommandAsync();
  1009. if (affected > 0)
  1010. {
  1011. _logger.LogWarning(
  1012. "lease_reset tenantId={Tenant} factoryId={Factory} releasedCount={Count}",
  1013. tenantId, factoryId, affected);
  1014. }
  1015. return affected;
  1016. }
  1017. /// <summary>
  1018. /// S8-SCHED-EXEC-1:到期规则候选 + 乐观 UPDATE 抢锁,返回成功抢到的 lease 列表。
  1019. /// 抢锁条件:enabled=1 AND (paused_until IS NULL OR paused_until &lt;= NOW)
  1020. /// AND (next_run_at IS NULL OR next_run_at &lt;= NOW)
  1021. /// AND (lock_until IS NULL OR lock_until &lt;= NOW)。
  1022. /// 抢锁回写:lock_token / locked_by / lock_until = NOW + 5min / running_started_at = NOW / last_run_id = runId。
  1023. /// affectedRows == 1 才算抢到;后续 OnRuleCompletedAsync 必须按 lockToken 回写,避免旧进程覆盖新 lease。
  1024. /// </summary>
  1025. public async Task<List<S8RuleLease>> PickReadyRulesAsync(long tenantId, long factoryId, int batchSize, string lockedBy, string runId)
  1026. {
  1027. if (batchSize <= 0) batchSize = 16;
  1028. var now = DateTime.Now;
  1029. var candidates = await _ruleRep.AsQueryable()
  1030. .Where(x => x.TenantId == tenantId
  1031. && x.FactoryId == factoryId
  1032. && x.Enabled
  1033. && (x.PausedUntil == null || x.PausedUntil <= now)
  1034. && (x.NextRunAt == null || x.NextRunAt <= now)
  1035. && (x.LockUntil == null || x.LockUntil <= now))
  1036. .OrderBy(x => x.NextRunAt, OrderByType.Asc)
  1037. .OrderBy(x => x.Id, OrderByType.Asc)
  1038. .Take(batchSize)
  1039. .Select(x => new { x.Id, x.RuleCode, x.RuleType })
  1040. .ToListAsync();
  1041. if (candidates.Count == 0) return new();
  1042. var leases = new List<S8RuleLease>();
  1043. foreach (var c in candidates)
  1044. {
  1045. var token = Guid.NewGuid().ToString("N");
  1046. var lockUntil = DateTime.Now.AddMinutes(LeaseDurationMinutes);
  1047. var runningAt = DateTime.Now;
  1048. // 乐观 UPDATE:再校验一次条件,affectedRows=1 才算抢到。
  1049. var affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  1050. .SetColumns(x => new AdoS8WatchRule
  1051. {
  1052. LockToken = token,
  1053. LockedBy = lockedBy,
  1054. LockUntil = lockUntil,
  1055. RunningStartedAt = runningAt,
  1056. LastRunId = runId,
  1057. UpdatedAt = runningAt
  1058. })
  1059. .Where(x => x.Id == c.Id
  1060. && x.Enabled
  1061. && (x.PausedUntil == null || x.PausedUntil <= runningAt)
  1062. && (x.NextRunAt == null || x.NextRunAt <= runningAt)
  1063. && (x.LockUntil == null || x.LockUntil <= runningAt))
  1064. .ExecuteCommandAsync();
  1065. if (affected == 1)
  1066. {
  1067. leases.Add(new S8RuleLease
  1068. {
  1069. RuleId = c.Id,
  1070. RuleCode = c.RuleCode,
  1071. RuleType = c.RuleType,
  1072. LockToken = token,
  1073. LockedBy = lockedBy,
  1074. LockUntil = lockUntil,
  1075. RunId = runId,
  1076. AcquiredAt = runningAt
  1077. });
  1078. }
  1079. }
  1080. return leases;
  1081. }
  1082. /// <summary>
  1083. /// S8-SCHED-EXEC-1:执行单条已抢锁规则的 evaluator → 抗抖去重 → 建单/刷新 → 恢复 reconcile。
  1084. /// 不释放 lease(OnRuleCompletedAsync 负责);evaluator 抛异常时 Result.Success=false 并保留 ErrorMessage。
  1085. /// </summary>
  1086. public async Task<S8RuleRunResult> RunSingleRuleAsync(long tenantId, long factoryId, S8RuleLease lease)
  1087. {
  1088. var rule = await _ruleRep.AsQueryable()
  1089. .Where(x => x.Id == lease.RuleId)
  1090. .FirstAsync();
  1091. if (rule == null)
  1092. {
  1093. return new S8RuleRunResult { Success = false, ErrorMessage = "rule_not_found", Stats = new() };
  1094. }
  1095. var ruleType = rule.RuleType;
  1096. if (string.IsNullOrWhiteSpace(ruleType))
  1097. {
  1098. // 未分类的旧规则不在新调度路径承载;标 SKIPPED 但不视为失败。
  1099. return new S8RuleRunResult { Success = true, ErrorMessage = "rule_type_empty_skipped", Stats = new() };
  1100. }
  1101. IS8RuleEvaluator? evaluator = ruleType switch
  1102. {
  1103. S8TimeoutRuleEvaluator.RuleTypeCode => _timeoutEvaluator,
  1104. S8ShortageRuleEvaluator.RuleTypeCode => _shortageEvaluator,
  1105. S8OutOfRangeRuleEvaluator.RuleTypeCode => _outOfRangeEvaluator,
  1106. _ => null
  1107. };
  1108. if (evaluator == null)
  1109. {
  1110. return new S8RuleRunResult { Success = false, ErrorMessage = $"unsupported_rule_type:{ruleType}", Stats = new() };
  1111. }
  1112. try
  1113. {
  1114. var alertRules = (await _alertRuleRep.AsQueryable()
  1115. .Where(x => x.TenantId == tenantId && x.FactoryId == factoryId)
  1116. .ToListAsync()).AsReadOnly();
  1117. var results = await ProcessSingleRuleAsync(tenantId, factoryId, rule, ruleType, evaluator, alertRules, lease.RunId);
  1118. var stats = new S8RuleRunStats
  1119. {
  1120. Hits = results.Count,
  1121. Created = results.Count(r => r.Created),
  1122. Refreshed = results.Count(r => r.Reason == "duplicate_pending"),
  1123. Pending = results.Count(r => r.Reason == "antiflap_pending_hit"),
  1124. Failed = results.Count(r => r.Reason == "create_failed" || r.Reason == "refresh_failed" || r.Reason == "antiflap_failed" || r.Reason == "evaluate_failed")
  1125. };
  1126. return new S8RuleRunResult { Success = true, Stats = stats };
  1127. }
  1128. catch (Exception ex)
  1129. {
  1130. return new S8RuleRunResult
  1131. {
  1132. Success = false,
  1133. ErrorMessage = ex.Message,
  1134. Stats = new()
  1135. };
  1136. }
  1137. }
  1138. /// <summary>
  1139. /// S8-SCHED-EXEC-1:单规则处理(evaluator → reconcile → hit 循环)。
  1140. /// 与 ProcessRulesByTypeAsync 内单规则循环体语义一致;此处抽出便于新调度路径直接调用单条 rule。
  1141. /// </summary>
  1142. private async Task<List<S8WatchCreationResult>> ProcessSingleRuleAsync(
  1143. long tenantId, long factoryId, AdoS8WatchRule rule, string ruleType,
  1144. IS8RuleEvaluator evaluator, IReadOnlyList<AdoS8AlertRule> alertRules, string runId)
  1145. {
  1146. var results = new List<S8WatchCreationResult>();
  1147. List<S8RuleHit> hits;
  1148. try
  1149. {
  1150. hits = await evaluator.EvaluateAsync(tenantId, factoryId, rule, alertRules);
  1151. }
  1152. catch (Exception ex)
  1153. {
  1154. var failureReason = ex is S8RuleEvaluatorException sre ? sre.Reason : ex.GetType().Name;
  1155. await WriteDetectionLogAsync(new AdoS8DetectionLog
  1156. {
  1157. TenantId = tenantId, FactoryId = factoryId,
  1158. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  1159. SourceObjectType = rule.SourceObjectType,
  1160. DetectResult = DetectResultEvaluateFailed,
  1161. DetectedAt = DateTime.Now,
  1162. FailureReason = failureReason,
  1163. FailureMessage = Truncate(ex.Message, 1000),
  1164. RunId = runId, TriggerSource = DetectionTriggerSource
  1165. });
  1166. results.Add(BuildSkipResult(rule, "evaluate_failed", ex.Message));
  1167. throw;
  1168. }
  1169. List<long> recoveredIds;
  1170. try
  1171. {
  1172. recoveredIds = await ReconcileRecoveriesForRuleAsync(tenantId, factoryId, rule, ruleType, hits, runId);
  1173. }
  1174. catch (Exception ex)
  1175. {
  1176. _logger.LogWarning(ex, "recovery_reconcile_failed ruleCode={RuleCode} ruleType={RuleType}", rule.RuleCode, ruleType);
  1177. recoveredIds = new();
  1178. }
  1179. if (hits.Count == 0 && recoveredIds.Count == 0)
  1180. {
  1181. await WriteDetectionLogAsync(new AdoS8DetectionLog
  1182. {
  1183. TenantId = tenantId, FactoryId = factoryId,
  1184. RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
  1185. SourceObjectType = rule.SourceObjectType,
  1186. DetectResult = DetectResultNoHit,
  1187. DetectedAt = DateTime.Now,
  1188. RunId = runId, TriggerSource = DetectionTriggerSource
  1189. });
  1190. }
  1191. foreach (var hit in hits)
  1192. {
  1193. if (string.IsNullOrWhiteSpace(hit.DedupKey))
  1194. {
  1195. results.Add(BuildSkipResult(rule, "missing_dedup_key", null, hit));
  1196. continue;
  1197. }
  1198. long matchedId;
  1199. try
  1200. {
  1201. matchedId = await FindOpenExceptionByDedupKeyAsync(tenantId, factoryId, hit.DedupKey);
  1202. }
  1203. catch (Exception ex)
  1204. {
  1205. results.Add(BuildSkipResult(rule, "query_failed", ex.Message, hit));
  1206. continue;
  1207. }
  1208. if (matchedId > 0)
  1209. {
  1210. try
  1211. {
  1212. await RefreshDetectionAsync(matchedId, hit);
  1213. await WriteDetectionLogAsync(BuildHitLog(tenantId, factoryId, rule, ruleType, hit, DetectResultRefreshed, matchedId, runId));
  1214. results.Add(BuildSkippedDuplicate(rule, hit, matchedId));
  1215. }
  1216. catch (Exception ex)
  1217. {
  1218. results.Add(BuildSkipResult(rule, "refresh_failed", ex.Message, hit));
  1219. }
  1220. continue;
  1221. }
  1222. if (string.Equals(ruleType, S8OutOfRangeRuleEvaluator.RuleTypeCode, StringComparison.OrdinalIgnoreCase))
  1223. {
  1224. long compatId;
  1225. try
  1226. {
  1227. compatId = await FindLegacyOutOfRangeExceptionAsync(tenantId, factoryId, rule.Id, hit.RelatedObjectCode);
  1228. }
  1229. catch (Exception ex)
  1230. {
  1231. results.Add(BuildSkipResult(rule, "query_failed", ex.Message, hit));
  1232. continue;
  1233. }
  1234. if (compatId > 0)
  1235. {
  1236. try
  1237. {
  1238. await BackfillLegacyExceptionAsync(compatId, hit);
  1239. await WriteDetectionLogAsync(BuildHitLog(tenantId, factoryId, rule, ruleType, hit, DetectResultRefreshed, compatId, runId));
  1240. results.Add(BuildSkippedDuplicate(rule, hit, compatId));
  1241. }
  1242. catch (Exception ex)
  1243. {
  1244. results.Add(BuildSkipResult(rule, "refresh_failed", ex.Message, hit));
  1245. }
  1246. continue;
  1247. }
  1248. }
  1249. bool typeExists;
  1250. try
  1251. {
  1252. typeExists = await _exceptionTypeRep.AsQueryable()
  1253. .Where(t => t.TypeCode == hit.ExceptionTypeCode
  1254. && (t.TenantId == 0 || t.TenantId == tenantId)
  1255. && (t.FactoryId == 0 || t.FactoryId == factoryId)
  1256. && t.Enabled)
  1257. .AnyAsync();
  1258. }
  1259. catch (Exception ex)
  1260. {
  1261. results.Add(BuildSkipResult(rule, "query_failed", ex.Message, hit));
  1262. continue;
  1263. }
  1264. if (!typeExists)
  1265. {
  1266. results.Add(BuildSkipResult(rule, "exception_type_missing", null, hit));
  1267. continue;
  1268. }
  1269. int hitCount;
  1270. AdoS8RuleDetectionState? state;
  1271. try
  1272. {
  1273. (state, hitCount) = await UpsertDetectionStateOnHitAsync(tenantId, factoryId, rule, hit);
  1274. }
  1275. catch (Exception ex)
  1276. {
  1277. results.Add(BuildSkipResult(rule, "antiflap_failed", ex.Message, hit));
  1278. continue;
  1279. }
  1280. var triggerRequired = NormalizeAntiflapCount(rule.TriggerCountRequired);
  1281. if (hitCount < triggerRequired)
  1282. {
  1283. _logger.LogInformation(
  1284. "antiflap_pending_hit ruleCode={RuleCode} dedupKey={DedupKey} hitCount={HitCount} trigger={Trigger}",
  1285. rule.RuleCode, hit.DedupKey, hitCount, triggerRequired);
  1286. results.Add(BuildSkipResult(rule, "antiflap_pending_hit", null, hit));
  1287. continue;
  1288. }
  1289. try
  1290. {
  1291. var entity = await _manualReportService.CreateFromHitAsync(hit);
  1292. await _exceptionRep.Context.Updateable<AdoS8Exception>()
  1293. .SetColumns(x => new AdoS8Exception
  1294. {
  1295. ConsecutiveHitCount = hitCount,
  1296. ConsecutiveMissCount = 0,
  1297. UpdatedAt = DateTime.Now
  1298. })
  1299. .Where(x => x.Id == entity.Id)
  1300. .ExecuteCommandAsync();
  1301. if (state != null)
  1302. {
  1303. await _detectionStateRep.Context.Updateable<AdoS8RuleDetectionState>()
  1304. .SetColumns(x => new AdoS8RuleDetectionState
  1305. {
  1306. ActiveExceptionId = entity.Id,
  1307. UpdatedAt = DateTime.Now
  1308. })
  1309. .Where(x => x.Id == state.Id)
  1310. .ExecuteCommandAsync();
  1311. }
  1312. await WriteDetectionLogAsync(BuildHitLog(tenantId, factoryId, rule, ruleType, hit, DetectResultCreated, entity.Id, runId));
  1313. await TryDispatchLayerNotificationAsync(entity);
  1314. results.Add(BuildCreatedResult(rule, hit, entity.Id));
  1315. }
  1316. catch (Exception ex)
  1317. {
  1318. results.Add(BuildSkipResult(rule, "create_failed", ex.Message, hit));
  1319. }
  1320. }
  1321. return results;
  1322. }
  1323. /// <summary>
  1324. /// S8-SCHED-EXEC-1:lease 执行完成回写。
  1325. /// 必须 WHERE id = lease.RuleId AND lock_token = lease.LockToken;affectedRows = 0 视为 lease 丢失,记录 Warning,不覆盖状态。
  1326. /// 失败 ≥ 阈值(默认 3)写 paused_until = NOW + 1h。
  1327. /// </summary>
  1328. public async Task OnRuleCompletedAsync(long tenantId, long factoryId, S8RuleLease lease, S8RuleRunResult result, int durationMs)
  1329. {
  1330. var rule = await _ruleRep.AsQueryable().Where(x => x.Id == lease.RuleId).FirstAsync();
  1331. if (rule == null)
  1332. {
  1333. _logger.LogWarning("lease_complete_rule_missing ruleId={RuleId}", lease.RuleId);
  1334. return;
  1335. }
  1336. var now = DateTime.Now;
  1337. var effectiveInterval = NormalizePollInterval(rule.PollIntervalSeconds);
  1338. var nextRunAt = now.AddSeconds(effectiveInterval);
  1339. int affected;
  1340. if (result.Success)
  1341. {
  1342. affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  1343. .SetColumns(x => new AdoS8WatchRule
  1344. {
  1345. LastRunAt = now,
  1346. NextRunAt = nextRunAt,
  1347. LastStatus = "SUCCESS",
  1348. LastError = null,
  1349. LastDurationMs = durationMs,
  1350. ConsecutiveFailureCount = 0,
  1351. LockToken = null,
  1352. LockedBy = null,
  1353. LockUntil = null,
  1354. RunningStartedAt = null,
  1355. UpdatedAt = now
  1356. })
  1357. .Where(x => x.Id == lease.RuleId && x.LockToken == lease.LockToken)
  1358. .ExecuteCommandAsync();
  1359. }
  1360. else
  1361. {
  1362. var errorTrunc = Truncate(result.ErrorMessage, 500);
  1363. var newFailures = rule.ConsecutiveFailureCount + 1;
  1364. DateTime? pausedUntil = rule.PausedUntil;
  1365. string? pauseReason = rule.PauseReason;
  1366. if (newFailures >= AutoPauseFailureThreshold)
  1367. {
  1368. pausedUntil = now.AddHours(AutoPauseDurationHours);
  1369. pauseReason = Truncate($"AUTO_PAUSED_AFTER_{AutoPauseFailureThreshold}_FAILURES: {errorTrunc}", 64);
  1370. }
  1371. affected = await _ruleRep.Context.Updateable<AdoS8WatchRule>()
  1372. .SetColumns(x => new AdoS8WatchRule
  1373. {
  1374. LastRunAt = now,
  1375. NextRunAt = nextRunAt,
  1376. LastStatus = "FAILED",
  1377. LastError = errorTrunc,
  1378. LastDurationMs = durationMs,
  1379. ConsecutiveFailureCount = x.ConsecutiveFailureCount + 1,
  1380. PausedUntil = pausedUntil,
  1381. PauseReason = pauseReason,
  1382. LockToken = null,
  1383. LockedBy = null,
  1384. LockUntil = null,
  1385. RunningStartedAt = null,
  1386. UpdatedAt = now
  1387. })
  1388. .Where(x => x.Id == lease.RuleId && x.LockToken == lease.LockToken)
  1389. .ExecuteCommandAsync();
  1390. }
  1391. if (affected == 0)
  1392. {
  1393. _logger.LogWarning(
  1394. "lease_lost_on_complete ruleId={RuleId} ruleCode={RuleCode} lockToken={LockToken}",
  1395. lease.RuleId, lease.RuleCode, lease.LockToken);
  1396. }
  1397. }
  1398. /// <summary>
  1399. /// S8-SCHED-EXEC-1:单 tick 完整流程。Job / debug 调度入口。
  1400. /// 1) ResetExpiredLeasesAsync
  1401. /// 2) PickReadyRulesAsync(batchSize)
  1402. /// 3) 每条 rule 独立 try/catch 调用 RunSingleRuleAsync + OnRuleCompletedAsync
  1403. /// 单条规则失败不影响其他规则;整 tick 不抛异常。
  1404. /// </summary>
  1405. public async Task<S8DispatchTickResult> RunDispatchTickAsync(long tenantId, long factoryId, int batchSize, string lockedBy)
  1406. {
  1407. var tickId = Guid.NewGuid().ToString("N").Substring(0, 8);
  1408. var runId = Guid.NewGuid().ToString("N").Substring(0, 16);
  1409. var summary = new S8DispatchTickResult { TickId = tickId, RunId = runId };
  1410. try
  1411. {
  1412. summary.LeaseReleased = await ResetExpiredLeasesAsync(tenantId, factoryId);
  1413. }
  1414. catch (Exception ex)
  1415. {
  1416. _logger.LogError(ex, "tick_reset_lease_failed tickId={TickId}", tickId);
  1417. }
  1418. List<S8RuleLease> leases;
  1419. try
  1420. {
  1421. leases = await PickReadyRulesAsync(tenantId, factoryId, batchSize, lockedBy, runId);
  1422. }
  1423. catch (Exception ex)
  1424. {
  1425. _logger.LogError(ex, "tick_pick_failed tickId={TickId}", tickId);
  1426. return summary;
  1427. }
  1428. summary.Picked = leases.Count;
  1429. foreach (var lease in leases)
  1430. {
  1431. var sw = System.Diagnostics.Stopwatch.StartNew();
  1432. S8RuleRunResult runResult;
  1433. try
  1434. {
  1435. runResult = await RunSingleRuleAsync(tenantId, factoryId, lease);
  1436. }
  1437. catch (Exception ex)
  1438. {
  1439. runResult = new S8RuleRunResult { Success = false, ErrorMessage = ex.Message, Stats = new() };
  1440. }
  1441. sw.Stop();
  1442. try
  1443. {
  1444. await OnRuleCompletedAsync(tenantId, factoryId, lease, runResult, (int)sw.ElapsedMilliseconds);
  1445. }
  1446. catch (Exception ex)
  1447. {
  1448. _logger.LogError(ex, "tick_complete_failed tickId={TickId} ruleId={RuleId} ruleCode={RuleCode}", tickId, lease.RuleId, lease.RuleCode);
  1449. }
  1450. _logger.LogInformation(
  1451. "tick_rule_done tickId={TickId} runId={RunId} ruleId={RuleId} ruleCode={RuleCode} status={Status} durationMs={Dur} hits={Hits} created={Created} refreshed={Refreshed} pending={Pending} failed={Failed} error={Error}",
  1452. tickId, runId, lease.RuleId, lease.RuleCode,
  1453. runResult.Success ? "SUCCESS" : "FAILED",
  1454. sw.ElapsedMilliseconds,
  1455. runResult.Stats.Hits, runResult.Stats.Created, runResult.Stats.Refreshed, runResult.Stats.Pending, runResult.Stats.Failed,
  1456. runResult.ErrorMessage);
  1457. if (runResult.Success)
  1458. {
  1459. summary.Success++;
  1460. summary.Created += runResult.Stats.Created;
  1461. summary.Refreshed += runResult.Stats.Refreshed;
  1462. summary.Pending += runResult.Stats.Pending;
  1463. summary.PerRuleFailed += runResult.Stats.Failed;
  1464. }
  1465. else
  1466. {
  1467. summary.Failed++;
  1468. }
  1469. }
  1470. return summary;
  1471. }
  1472. private static int NormalizePollInterval(int raw)
  1473. {
  1474. if (raw < MinPollIntervalSeconds || raw > MaxPollIntervalSeconds) return DefaultPollIntervalSeconds;
  1475. return raw;
  1476. }
  1477. /// <summary>
  1478. /// S8-NOTIFY-WIRE-WATCH-1:异常自动建单成功后,非破坏性挂入通知分层路由。
  1479. /// 全程异常隔离:任何异常仅 LogWarning,不抛回主流程,不影响 detection_log / 事务 /
  1480. /// 异常状态机;该方法独立于 CreateFromWatchAsync / CreateFromHitAsync 的事务边界
  1481. /// (两者已 commit 后才返回 entity,故在此调用安全)。
  1482. /// sceneCode:先从 entity.SceneCode;空则 fallback 到 "S8_DEMO_DEFAULT"(demo 路径)。
  1483. /// severity:直接取 entity.Severity(CreateFromWatchAsync/CreateFromHitAsync 已保证非空)。
  1484. /// </summary>
  1485. private async Task TryDispatchLayerNotificationAsync(AdoS8Exception entity)
  1486. {
  1487. if (entity == null || entity.Id <= 0) return;
  1488. try
  1489. {
  1490. var sceneCode = string.IsNullOrWhiteSpace(entity.SceneCode) ? "S8_DEMO_DEFAULT" : entity.SceneCode;
  1491. // S8-SEVERITY-FOLLOW-SERIOUS-STANDARDIZE-EXEC-1:写入前 Normalize,落 FOLLOW/SERIOUS。
  1492. var severity = S8SeverityCode.Normalize(entity.Severity);
  1493. // S8-DEMO-IMPACT-SORT-NOTICE-1:附带 30 天影响统计(重复次数 / 累计损失 / 建议关注级别)。
  1494. // 计算失败不阻断通知主链路;snap==null 时 BuildNotification 仅写基础 Context。
  1495. ExceptionImpactSnapshot? snap = null;
  1496. try
  1497. {
  1498. snap = await _impactMetricsService.ComputeOneAsync(entity);
  1499. }
  1500. catch (Exception ex)
  1501. {
  1502. _logger.LogWarning(ex, "notify_impact_compute_throw exceptionId={ExceptionId}", entity.Id);
  1503. }
  1504. var baseContent =
  1505. $"异常 {entity.ExceptionCode}:{entity.Title}(场景 {sceneCode},严重度 {severity}" +
  1506. (string.IsNullOrWhiteSpace(entity.SourceRuleCode) ? "" : $",规则 {entity.SourceRuleCode}") + ")";
  1507. var content = snap == null
  1508. ? baseContent
  1509. : baseContent
  1510. + Environment.NewLine
  1511. + $"过去30天同类异常发生 {snap.RepeatCount30d} 次,已关闭累计损失 {snap.CumulativeLossHours30d.ToString("0.#")} 小时,建议关注级别:{snap.SuggestedAttentionLabel}";
  1512. await _notificationLayerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  1513. {
  1514. TenantId = entity.TenantId,
  1515. FactoryId = entity.FactoryId,
  1516. ExceptionId = entity.Id,
  1517. ExceptionNo = entity.ExceptionCode,
  1518. SceneCode = sceneCode,
  1519. Severity = severity,
  1520. Title = entity.Title ?? string.Empty,
  1521. Content = content,
  1522. Status = entity.Status,
  1523. SourceRuleCode = entity.SourceRuleCode,
  1524. JumpUrl = $"/aidop/s8/exceptions/{entity.Id}",
  1525. RepeatCount30d = snap?.RepeatCount30d,
  1526. CumulativeLossHours30d = snap?.CumulativeLossHours30d,
  1527. SuggestedAttentionLevel = snap?.SuggestedAttentionLevel,
  1528. SuggestedAttentionLabel = snap?.SuggestedAttentionLabel,
  1529. ImpactReason = snap?.ImpactReason,
  1530. });
  1531. }
  1532. catch (Exception ex)
  1533. {
  1534. _logger.LogWarning(ex, "notify_dispatch_throw exceptionId={ExceptionId}", entity.Id);
  1535. }
  1536. }
  1537. /// <summary>
  1538. /// S8-NOTIFY-WIRE-RECOVERED-1:异常恢复(recovered_at 已写入、RECOVERED detection_log 已落库)后,
  1539. /// 非破坏性挂入分层通知。全程异常隔离:任何异常仅 LogWarning,不抛回主流程,不影响 detection_log /
  1540. /// 状态机;call site 已在事务边界外(recovered 路径无事务)。
  1541. /// 重新读取 entity 拿场景/严重度/编号/状态/规则代码(恢复事件相对低频,1 次额外读可接受)。
  1542. /// S8-DEMO-IMPACT-SORT-NOTICE-1:恢复事件**不**追加影响统计字段,保持原有恢复语义不变。
  1543. /// </summary>
  1544. private async Task TryDispatchRecoveredLayerNotificationAsync(long exceptionId)
  1545. {
  1546. if (exceptionId <= 0) return;
  1547. try
  1548. {
  1549. var entity = await _exceptionRep.GetByIdAsync(exceptionId);
  1550. if (entity == null)
  1551. {
  1552. _logger.LogWarning("notify_recovered_dispatch_entity_missing exceptionId={ExceptionId}", exceptionId);
  1553. return;
  1554. }
  1555. var sceneCode = string.IsNullOrWhiteSpace(entity.SceneCode) ? "S8_DEMO_DEFAULT" : entity.SceneCode;
  1556. // S8-SEVERITY-FOLLOW-SERIOUS-STANDARDIZE-EXEC-1:写入前 Normalize,落 FOLLOW/SERIOUS。
  1557. var severity = S8SeverityCode.Normalize(entity.Severity);
  1558. var title = $"【已恢复】{entity.ExceptionCode}";
  1559. var content =
  1560. $"异常 {entity.ExceptionCode} 已恢复,场景 {sceneCode},严重度 {severity}" +
  1561. (string.IsNullOrWhiteSpace(entity.SourceRuleCode) ? "" : $",规则 {entity.SourceRuleCode}");
  1562. await _notificationLayerResolver.DispatchByLayerAsync(new S8NotificationLayerResolver.DispatchByLayerInput
  1563. {
  1564. TenantId = entity.TenantId,
  1565. FactoryId = entity.FactoryId,
  1566. ExceptionId = entity.Id,
  1567. ExceptionNo = entity.ExceptionCode,
  1568. SceneCode = sceneCode,
  1569. Severity = severity,
  1570. Title = title,
  1571. Content = content,
  1572. Status = entity.Status,
  1573. SourceRuleCode = entity.SourceRuleCode,
  1574. JumpUrl = $"/aidop/s8/exceptions/{entity.Id}",
  1575. Recovered = true,
  1576. });
  1577. }
  1578. catch (Exception ex)
  1579. {
  1580. _logger.LogWarning(ex, "notify_recovered_dispatch_throw exceptionId={ExceptionId}", exceptionId);
  1581. }
  1582. }
  1583. }
  1584. public sealed class S8WatchExecutionRule
  1585. {
  1586. public long WatchRuleId { get; set; }
  1587. public string WatchRuleCode { get; set; } = string.Empty;
  1588. public string SceneCode { get; set; } = string.Empty;
  1589. public string TriggerType { get; set; } = string.Empty;
  1590. public string WatchObjectType { get; set; } = string.Empty;
  1591. public long DataSourceId { get; set; }
  1592. public string DataSourceCode { get; set; } = string.Empty;
  1593. public string DataSourceType { get; set; } = string.Empty;
  1594. public string DataSourceConnection { get; set; } = string.Empty;
  1595. public string QueryExpression { get; set; } = string.Empty;
  1596. public int PollIntervalSeconds { get; set; }
  1597. public long AlertRuleId { get; set; }
  1598. public string AlertRuleCode { get; set; } = string.Empty;
  1599. public string TriggerCondition { get; set; } = string.Empty;
  1600. public string ThresholdValue { get; set; } = string.Empty;
  1601. public string Severity { get; set; } = string.Empty;
  1602. }
  1603. public sealed class S8WatchDeviceQueryResult
  1604. {
  1605. public long WatchRuleId { get; set; }
  1606. public string WatchRuleCode { get; set; } = string.Empty;
  1607. public bool Success { get; set; }
  1608. public string? FailureReason { get; set; }
  1609. public List<S8WatchDeviceRow> Rows { get; set; } = new();
  1610. public static S8WatchDeviceQueryResult Ok(S8WatchExecutionRule rule, List<S8WatchDeviceRow> rows) =>
  1611. new()
  1612. {
  1613. WatchRuleId = rule.WatchRuleId,
  1614. WatchRuleCode = rule.WatchRuleCode,
  1615. Success = true,
  1616. Rows = rows
  1617. };
  1618. public static S8WatchDeviceQueryResult Fail(S8WatchExecutionRule rule, string reason) =>
  1619. new()
  1620. {
  1621. WatchRuleId = rule.WatchRuleId,
  1622. WatchRuleCode = rule.WatchRuleCode,
  1623. Success = false,
  1624. FailureReason = reason
  1625. };
  1626. }
  1627. public sealed class S8WatchDeviceRow
  1628. {
  1629. public string RelatedObjectCode { get; set; } = string.Empty;
  1630. public decimal? CurrentValue { get; set; }
  1631. public long? OccurrenceDeptId { get; set; }
  1632. public long? ResponsibleDeptId { get; set; }
  1633. public string SourcePayload { get; set; } = string.Empty;
  1634. }
  1635. /// <summary>
  1636. /// G01-05 去重结果对象。仅服务 G01-06 建单前拦截,由 CanCreate 单决策位决定是否建单。
  1637. /// 只服务首版唯一场景 S2(迁移后由 S2S6_PRODUCTION 切到单模块 S2)+ 唯一 trigger_type VALUE_DEVIATION + 设备对象。
  1638. /// 不预留多 trigger_type / 平台化去重扩展结构。
  1639. /// Reason 值域:no_pending / duplicate_pending / missing_dedup_key / query_failed。
  1640. /// </summary>
  1641. public sealed class S8WatchDedupResult
  1642. {
  1643. public S8WatchHitResult Hit { get; set; } = new();
  1644. public bool CanCreate { get; set; }
  1645. public long? MatchedExceptionId { get; set; }
  1646. public string Reason { get; set; } = string.Empty;
  1647. }
  1648. /// <summary>
  1649. /// G01-06 建单结果对象。仅服务 G-01 首版主线验收,由 Created / Skipped 两位决定结局。
  1650. /// 只服务首版唯一场景 S2(迁移后由 S2S6_PRODUCTION 切到单模块 S2)+ 唯一 trigger_type VALUE_DEVIATION + 设备对象。
  1651. /// 不预留多 trigger_type / 平台化工单扩展结构。
  1652. /// Reason 值域:auto_created / create_failed / 透传自 DedupResult.Reason。
  1653. /// </summary>
  1654. public sealed class S8WatchCreationResult
  1655. {
  1656. public S8WatchDedupResult DedupResult { get; set; } = new();
  1657. public bool Created { get; set; }
  1658. public bool Skipped { get; set; }
  1659. public long? CreatedExceptionId { get; set; }
  1660. public string Reason { get; set; } = string.Empty;
  1661. public string? ErrorMessage { get; set; }
  1662. }
  1663. /// <summary>
  1664. /// G01-04 命中结果对象。承载 G01-05 去重与 G01-06 建单所需最小追溯字段,
  1665. /// 仅服务首版唯一场景 S2(迁移后由 S2S6_PRODUCTION 切到单模块 S2)+ 唯一 trigger_type VALUE_DEVIATION + 设备对象。
  1666. /// 不预留多 trigger_type / 多场景 / 平台化扩展结构。
  1667. /// </summary>
  1668. public sealed class S8WatchHitResult
  1669. {
  1670. public long SourceRuleId { get; set; }
  1671. public string SourceRuleCode { get; set; } = string.Empty;
  1672. public long AlertRuleId { get; set; }
  1673. public long DataSourceId { get; set; }
  1674. public string RelatedObjectCode { get; set; } = string.Empty;
  1675. public decimal CurrentValue { get; set; }
  1676. public decimal ThresholdValue { get; set; }
  1677. public string TriggerCondition { get; set; } = string.Empty;
  1678. public string Severity { get; set; } = string.Empty;
  1679. public long? OccurrenceDeptId { get; set; }
  1680. public long? ResponsibleDeptId { get; set; }
  1681. public string SourcePayload { get; set; } = string.Empty;
  1682. }
  1683. /// <summary>S8-SCHED-EXEC-1:lease 抢占成功后传递的最小标识对象。</summary>
  1684. public sealed class S8RuleLease
  1685. {
  1686. public long RuleId { get; set; }
  1687. public string RuleCode { get; set; } = string.Empty;
  1688. public string? RuleType { get; set; }
  1689. public string LockToken { get; set; } = string.Empty;
  1690. public string LockedBy { get; set; } = string.Empty;
  1691. public DateTime LockUntil { get; set; }
  1692. public string RunId { get; set; } = string.Empty;
  1693. public DateTime AcquiredAt { get; set; }
  1694. }
  1695. /// <summary>S8-SCHED-EXEC-1:单条规则执行结果,OnRuleCompletedAsync 据此更新状态。</summary>
  1696. public sealed class S8RuleRunResult
  1697. {
  1698. public bool Success { get; set; }
  1699. public string? ErrorMessage { get; set; }
  1700. public S8RuleRunStats Stats { get; set; } = new();
  1701. }
  1702. public sealed class S8RuleRunStats
  1703. {
  1704. public int Hits { get; set; }
  1705. public int Created { get; set; }
  1706. public int Refreshed { get; set; }
  1707. public int Pending { get; set; }
  1708. public int Failed { get; set; }
  1709. }
  1710. /// <summary>S8-SCHED-EXEC-1:单 tick 调度结果聚合。</summary>
  1711. public sealed class S8DispatchTickResult
  1712. {
  1713. public string TickId { get; set; } = string.Empty;
  1714. public string RunId { get; set; } = string.Empty;
  1715. public int LeaseReleased { get; set; }
  1716. public int Picked { get; set; }
  1717. public int Success { get; set; }
  1718. public int Failed { get; set; }
  1719. public int Created { get; set; }
  1720. public int Refreshed { get; set; }
  1721. public int Pending { get; set; }
  1722. public int PerRuleFailed { get; set; }
  1723. }