|
|
@@ -27,6 +27,14 @@ public class S8WatchSchedulerService : ITransient
|
|
|
private readonly S8ShortageRuleEvaluator _shortageEvaluator;
|
|
|
private readonly S8OutOfRangeRuleEvaluator _outOfRangeEvaluator;
|
|
|
private readonly ILogger<S8WatchSchedulerService> _logger;
|
|
|
+ private readonly SqlSugarRepository<AdoS8DetectionLog> _detectionLogRep;
|
|
|
+
|
|
|
+ private const string DetectionTriggerSource = "WATCH_SCHEDULER";
|
|
|
+ private const string DetectResultCreated = "CREATED";
|
|
|
+ private const string DetectResultRefreshed = "REFRESHED";
|
|
|
+ private const string DetectResultRecovered = "RECOVERED";
|
|
|
+ private const string DetectResultNoHit = "NO_HIT";
|
|
|
+ private const string DetectResultEvaluateFailed = "EVALUATE_FAILED";
|
|
|
|
|
|
private const string DefaultTriggerType = "VALUE_DEVIATION";
|
|
|
private const string SqlDataSourceType = "SQL";
|
|
|
@@ -48,7 +56,8 @@ public class S8WatchSchedulerService : ITransient
|
|
|
S8TimeoutRuleEvaluator timeoutEvaluator,
|
|
|
S8ShortageRuleEvaluator shortageEvaluator,
|
|
|
S8OutOfRangeRuleEvaluator outOfRangeEvaluator,
|
|
|
- ILogger<S8WatchSchedulerService> logger)
|
|
|
+ ILogger<S8WatchSchedulerService> logger,
|
|
|
+ SqlSugarRepository<AdoS8DetectionLog> detectionLogRep)
|
|
|
{
|
|
|
_ruleRep = ruleRep;
|
|
|
_alertRuleRep = alertRuleRep;
|
|
|
@@ -61,6 +70,7 @@ public class S8WatchSchedulerService : ITransient
|
|
|
_shortageEvaluator = shortageEvaluator;
|
|
|
_outOfRangeEvaluator = outOfRangeEvaluator;
|
|
|
_logger = logger;
|
|
|
+ _detectionLogRep = detectionLogRep;
|
|
|
}
|
|
|
|
|
|
public async Task<List<S8WatchExecutionRule>> LoadExecutionRulesAsync(long tenantId, long factoryId)
|
|
|
@@ -393,33 +403,35 @@ public class S8WatchSchedulerService : ITransient
|
|
|
// R3-OUT_OF_RANGE-REWRITE-1:三类正式 evaluator 路径。
|
|
|
// 旧 AlertRule 兼容主链(上方 dedupResults 循环)已在 LoadExecutionRulesAsync 处过滤掉
|
|
|
// OUT_OF_RANGE/TIMEOUT/SHORTAGE,只保留未分类历史规则;故新旧不双跑。
|
|
|
- results.AddRange(await ProcessTimeoutRulesAsync(tenantId, factoryId));
|
|
|
- results.AddRange(await ProcessShortageRulesAsync(tenantId, factoryId));
|
|
|
- results.AddRange(await ProcessOutOfRangeRulesAsync(tenantId, factoryId));
|
|
|
+ // R6 RunId:本次 CreateExceptionsAsync 调用对应的统一关联 id,落入 detection_log。
|
|
|
+ var runId = Guid.NewGuid().ToString("N").Substring(0, 16);
|
|
|
+ results.AddRange(await ProcessRulesByTypeAsync(tenantId, factoryId, _timeoutEvaluator, S8TimeoutRuleEvaluator.RuleTypeCode, runId));
|
|
|
+ results.AddRange(await ProcessRulesByTypeAsync(tenantId, factoryId, _shortageEvaluator, S8ShortageRuleEvaluator.RuleTypeCode, runId));
|
|
|
+ results.AddRange(await ProcessRulesByTypeAsync(tenantId, factoryId, _outOfRangeEvaluator, S8OutOfRangeRuleEvaluator.RuleTypeCode, runId));
|
|
|
|
|
|
return results;
|
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
|
- /// R2 TIMEOUT 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。
|
|
|
+ /// R2 TIMEOUT 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。RunId 由内部生成。
|
|
|
/// </summary>
|
|
|
public Task<List<S8WatchCreationResult>> ProcessTimeoutRulesAsync(long tenantId, long factoryId) =>
|
|
|
- ProcessRulesByTypeAsync(tenantId, factoryId, _timeoutEvaluator, S8TimeoutRuleEvaluator.RuleTypeCode);
|
|
|
+ ProcessRulesByTypeAsync(tenantId, factoryId, _timeoutEvaluator, S8TimeoutRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
|
|
|
|
|
|
/// <summary>
|
|
|
- /// R3 SHORTAGE 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。
|
|
|
+ /// R3 SHORTAGE 类规则主链:薄包装,复用 <see cref="ProcessRulesByTypeAsync"/>。RunId 由内部生成。
|
|
|
/// </summary>
|
|
|
public Task<List<S8WatchCreationResult>> ProcessShortageRulesAsync(long tenantId, long factoryId) =>
|
|
|
- ProcessRulesByTypeAsync(tenantId, factoryId, _shortageEvaluator, S8ShortageRuleEvaluator.RuleTypeCode);
|
|
|
+ ProcessRulesByTypeAsync(tenantId, factoryId, _shortageEvaluator, S8ShortageRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
|
|
|
|
|
|
/// <summary>
|
|
|
/// R3-OUT_OF_RANGE-REWRITE-1:OUT_OF_RANGE 类规则主链。
|
|
|
/// 复用 <see cref="ProcessRulesByTypeAsync"/>,并对历史 dedup_key=NULL 的旧记录做 compat fallback:
|
|
|
/// (source_rule_id=rule.Id AND related_object_code=hit AND status!=CLOSED AND dedup_key IS NULL AND is_deleted=0)
|
|
|
- /// 命中则 backfill 6 列,避免重复建单。
|
|
|
+ /// 命中则 backfill 6 列,避免重复建单。RunId 由内部生成。
|
|
|
/// </summary>
|
|
|
public Task<List<S8WatchCreationResult>> ProcessOutOfRangeRulesAsync(long tenantId, long factoryId) =>
|
|
|
- ProcessRulesByTypeAsync(tenantId, factoryId, _outOfRangeEvaluator, S8OutOfRangeRuleEvaluator.RuleTypeCode);
|
|
|
+ ProcessRulesByTypeAsync(tenantId, factoryId, _outOfRangeEvaluator, S8OutOfRangeRuleEvaluator.RuleTypeCode, Guid.NewGuid().ToString("N").Substring(0, 16));
|
|
|
|
|
|
/// <summary>
|
|
|
/// R2/R3 通用规则主链:装载 enabled WatchRule.RuleType=ruleType → evaluator → dedup_key 去重 → 建单/刷新。
|
|
|
@@ -429,7 +441,7 @@ public class S8WatchSchedulerService : ITransient
|
|
|
/// 不做 SLA 升级、不做事件触发、不做 RecoveredAt。
|
|
|
/// </summary>
|
|
|
private async Task<List<S8WatchCreationResult>> ProcessRulesByTypeAsync(
|
|
|
- long tenantId, long factoryId, IS8RuleEvaluator evaluator, string ruleType)
|
|
|
+ long tenantId, long factoryId, IS8RuleEvaluator evaluator, string ruleType, string runId)
|
|
|
{
|
|
|
var results = new List<S8WatchCreationResult>();
|
|
|
|
|
|
@@ -454,6 +466,19 @@ public class S8WatchSchedulerService : ITransient
|
|
|
}
|
|
|
catch (Exception ex)
|
|
|
{
|
|
|
+ // R6 EVALUATE_FAILED 日志:evaluator 抛 S8RuleEvaluatorException 走这里,包含 reason+message。
|
|
|
+ var failureReason = ex is S8RuleEvaluatorException sre ? sre.Reason : ex.GetType().Name;
|
|
|
+ await WriteDetectionLogAsync(new AdoS8DetectionLog
|
|
|
+ {
|
|
|
+ TenantId = tenantId, FactoryId = factoryId,
|
|
|
+ RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
|
|
|
+ SourceObjectType = rule.SourceObjectType,
|
|
|
+ DetectResult = DetectResultEvaluateFailed,
|
|
|
+ DetectedAt = DateTime.Now,
|
|
|
+ FailureReason = failureReason,
|
|
|
+ FailureMessage = Truncate(ex.Message, 1000),
|
|
|
+ RunId = runId, TriggerSource = DetectionTriggerSource
|
|
|
+ });
|
|
|
results.Add(BuildSkipResult(rule, "evaluate_failed", ex.Message));
|
|
|
continue;
|
|
|
}
|
|
|
@@ -461,15 +486,31 @@ public class S8WatchSchedulerService : ITransient
|
|
|
// R5-RECOVERY-MINIMAL-1:evaluator 成功执行后调和 recovered_at。
|
|
|
// 仅在 evaluator 成功(throw 不会到这里)时才能判定"未命中"。仅写 recovered_at + updated_at,
|
|
|
// 绝不动 status / assignee / verifier / source_payload / last_detected_at。
|
|
|
+ // R6 reconcile 返回 recoveredIds 用于决定是否写 NO_HIT。
|
|
|
+ List<long> recoveredIds = new();
|
|
|
try
|
|
|
{
|
|
|
- await ReconcileRecoveriesForRuleAsync(tenantId, factoryId, rule, ruleType, hits);
|
|
|
+ recoveredIds = await ReconcileRecoveriesForRuleAsync(tenantId, factoryId, rule, ruleType, hits, runId);
|
|
|
}
|
|
|
catch (Exception ex)
|
|
|
{
|
|
|
_logger.LogWarning(ex, "recovery_reconcile_failed ruleCode={RuleCode} ruleType={RuleType}", rule.RuleCode, ruleType);
|
|
|
}
|
|
|
|
|
|
+ // R6 NO_HIT 日志:evaluator 成功且无 hit、且本轮 reconcile 未写任何 RECOVERED → 每条 rule 一条 NO_HIT。
|
|
|
+ if (hits.Count == 0 && recoveredIds.Count == 0)
|
|
|
+ {
|
|
|
+ await WriteDetectionLogAsync(new AdoS8DetectionLog
|
|
|
+ {
|
|
|
+ TenantId = tenantId, FactoryId = factoryId,
|
|
|
+ RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
|
|
|
+ SourceObjectType = rule.SourceObjectType,
|
|
|
+ DetectResult = DetectResultNoHit,
|
|
|
+ DetectedAt = DateTime.Now,
|
|
|
+ RunId = runId, TriggerSource = DetectionTriggerSource
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
foreach (var hit in hits)
|
|
|
{
|
|
|
if (string.IsNullOrWhiteSpace(hit.DedupKey))
|
|
|
@@ -494,6 +535,7 @@ public class S8WatchSchedulerService : ITransient
|
|
|
try
|
|
|
{
|
|
|
await RefreshDetectionAsync(matchedId, hit);
|
|
|
+ await WriteDetectionLogAsync(BuildHitLog(tenantId, factoryId, rule, ruleType, hit, DetectResultRefreshed, matchedId, runId));
|
|
|
results.Add(BuildSkippedDuplicate(rule, hit, matchedId));
|
|
|
}
|
|
|
catch (Exception ex)
|
|
|
@@ -522,6 +564,7 @@ public class S8WatchSchedulerService : ITransient
|
|
|
try
|
|
|
{
|
|
|
await BackfillLegacyExceptionAsync(compatId, hit);
|
|
|
+ await WriteDetectionLogAsync(BuildHitLog(tenantId, factoryId, rule, ruleType, hit, DetectResultRefreshed, compatId, runId));
|
|
|
results.Add(BuildSkippedDuplicate(rule, hit, compatId));
|
|
|
}
|
|
|
catch (Exception ex)
|
|
|
@@ -557,6 +600,7 @@ public class S8WatchSchedulerService : ITransient
|
|
|
try
|
|
|
{
|
|
|
var entity = await _manualReportService.CreateFromHitAsync(hit);
|
|
|
+ await WriteDetectionLogAsync(BuildHitLog(tenantId, factoryId, rule, ruleType, hit, DetectResultCreated, entity.Id, runId));
|
|
|
results.Add(BuildCreatedResult(rule, hit, entity.Id));
|
|
|
}
|
|
|
catch (Exception ex)
|
|
|
@@ -630,9 +674,10 @@ public class S8WatchSchedulerService : ITransient
|
|
|
/// 凡不在本轮 hits.dedup_key 集合内的,写入 recovered_at = now、updated_at = now。
|
|
|
/// 仅写这 2 列;不动 status / assignee / verifier / source_payload / last_detected_at;
|
|
|
/// recovered_at 一旦写入,本轮不做复发清空。
|
|
|
+ /// R6 返回 recoveredIds 供上游决定是否写 NO_HIT 日志,并对每个 recovered exception 写一条 RECOVERED 日志。
|
|
|
/// </summary>
|
|
|
- private async Task ReconcileRecoveriesForRuleAsync(
|
|
|
- long tenantId, long factoryId, AdoS8WatchRule rule, string ruleType, List<S8RuleHit> hits)
|
|
|
+ private async Task<List<long>> ReconcileRecoveriesForRuleAsync(
|
|
|
+ long tenantId, long factoryId, AdoS8WatchRule rule, string ruleType, List<S8RuleHit> hits, string runId)
|
|
|
{
|
|
|
var hitDedupKeys = hits
|
|
|
.Where(h => !string.IsNullOrWhiteSpace(h.DedupKey))
|
|
|
@@ -647,9 +692,9 @@ public class S8WatchSchedulerService : ITransient
|
|
|
&& x.SourceRuleCode == rule.RuleCode
|
|
|
&& x.DedupKey != null
|
|
|
&& x.RecoveredAt == null)
|
|
|
- .Select(x => new { x.Id, x.DedupKey })
|
|
|
+ .Select(x => new { x.Id, x.DedupKey, x.SourceObjectType, x.SourceObjectId, x.RelatedObjectCode })
|
|
|
.ToListAsync();
|
|
|
- if (candidates.Count == 0) return;
|
|
|
+ if (candidates.Count == 0) return new List<long>();
|
|
|
|
|
|
var now = DateTime.Now;
|
|
|
var recoveredIds = new List<long>();
|
|
|
@@ -665,6 +710,20 @@ public class S8WatchSchedulerService : ITransient
|
|
|
.Where(x => x.Id == c.Id)
|
|
|
.ExecuteCommandAsync();
|
|
|
recoveredIds.Add(c.Id);
|
|
|
+
|
|
|
+ await WriteDetectionLogAsync(new AdoS8DetectionLog
|
|
|
+ {
|
|
|
+ TenantId = tenantId, FactoryId = factoryId,
|
|
|
+ RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
|
|
|
+ SourceObjectType = c.SourceObjectType, SourceObjectId = c.SourceObjectId,
|
|
|
+ RelatedObjectCode = c.RelatedObjectCode, DedupKey = c.DedupKey,
|
|
|
+ DetectResult = DetectResultRecovered,
|
|
|
+ ExceptionId = c.Id,
|
|
|
+ DetectedAt = now,
|
|
|
+ PayloadSnapshot = JsonSerializer.Serialize(new { ruleId = rule.Id, ruleCode = rule.RuleCode, reason = "no_longer_hit" }),
|
|
|
+ RunId = runId, TriggerSource = DetectionTriggerSource,
|
|
|
+ Remark = "Rule no longer hit; recovered_at marked"
|
|
|
+ });
|
|
|
}
|
|
|
|
|
|
if (recoveredIds.Count > 0)
|
|
|
@@ -673,8 +732,44 @@ public class S8WatchSchedulerService : ITransient
|
|
|
"rule_recovered ruleCode={RuleCode} ruleType={RuleType} recoveredCount={Count} recoveredIds={Ids}",
|
|
|
rule.RuleCode, ruleType, recoveredIds.Count, string.Join(",", recoveredIds));
|
|
|
}
|
|
|
+ return recoveredIds;
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>R6 通用 hit 日志构造(CREATED / REFRESHED 共用)。</summary>
|
|
|
+ private static AdoS8DetectionLog BuildHitLog(
|
|
|
+ long tenantId, long factoryId, AdoS8WatchRule rule, string ruleType, S8RuleHit hit,
|
|
|
+ string detectResult, long exceptionId, string runId) => new()
|
|
|
+ {
|
|
|
+ TenantId = tenantId, FactoryId = factoryId,
|
|
|
+ RuleId = rule.Id, RuleCode = rule.RuleCode, RuleType = ruleType, SceneCode = rule.SceneCode,
|
|
|
+ SourceObjectType = hit.SourceObjectType, SourceObjectId = hit.SourceObjectId,
|
|
|
+ RelatedObjectCode = hit.RelatedObjectCode, DedupKey = hit.DedupKey,
|
|
|
+ DetectResult = detectResult,
|
|
|
+ ExceptionId = exceptionId,
|
|
|
+ DetectedAt = hit.DetectedAt,
|
|
|
+ PayloadSnapshot = hit.SourcePayload,
|
|
|
+ RunId = runId,
|
|
|
+ TriggerSource = DetectionTriggerSource
|
|
|
+ };
|
|
|
+
|
|
|
+ /// <summary>R6 日志写入:失败仅 LogWarning,不阻断主链;不抛异常。</summary>
|
|
|
+ private async Task WriteDetectionLogAsync(AdoS8DetectionLog log)
|
|
|
+ {
|
|
|
+ try
|
|
|
+ {
|
|
|
+ await _detectionLogRep.InsertAsync(log);
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ _logger.LogWarning(ex,
|
|
|
+ "detection_log_write_failed runId={RunId} detectResult={Result} ruleCode={RuleCode} exceptionId={ExceptionId}",
|
|
|
+ log.RunId, log.DetectResult, log.RuleCode, log.ExceptionId);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
+ private static string Truncate(string? s, int max) =>
|
|
|
+ string.IsNullOrEmpty(s) ? string.Empty : (s.Length <= max ? s : s.Substring(0, max));
|
|
|
+
|
|
|
private async Task RefreshDetectionAsync(long exceptionId, S8RuleHit hit)
|
|
|
{
|
|
|
await _exceptionRep.Context.Updateable<AdoS8Exception>()
|