test(s8): cover recovered detection fixture

scripts/s8/recovered-edge-regression.sh

@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+# RECOVERED edge regression — S8-RECOVERED-EDGE-FIXTURE-1（dev/test only, aidopdev）。
+#
+# 四阶段验证 RECOVERED 主链 + trigger/recover 抗抖：
+#   Phase A1: 第一次命中（hit_count=1 < trigger=2）→ pending，不建单，不写 CREATED
+#   Phase A2: 第二次命中（hit_count=2 ≥ trigger=2）→ 建单，state.active_exception_id=exc.id，写 CREATED
+#   Phase B:  翻转表达式不命中（miss_count=1 < recover=2）→ recovered_at 仍 NULL，不写 RECOVERED
+#   Phase C:  再次不命中（miss_count=2 ≥ recover=2）→ recovered_at 写入，RECOVERED detection_log 写入
+#
+# 为何 trigger=2 而非 1：
+#   trigger=1 时 state 行在首 tick 由 InsertAsync 创建，但 SqlSugar 不回填 Id 到 entity；
+#   后续 UPDATE state.active_exception_id WHERE id=state.Id 命中 0 行，导致 active_exception_id
+#   永远为 NULL。这是产品侧问题，本脚本通过 trigger=2 在第二 tick 走 existing 分支（state.Id
+#   来自 query 真实可用），绕开此坑。已在风险清单中登记。
+#
+# 触发：通过 /api/aidop/s8/watch-debug/run-once（ProcessRulesByTypeAsync 路径，
+#       与 ProcessSingleRuleAsync 同样调用 ReconcileRecoveriesForRuleAsync）。
+#       /run-once 不更新 watch_rule.last_status，本脚本不对此做断言。
+#
+# 严守：
+#   - 不影响 demo rule 10/11/12
+#   - TEMP rule 测后 enabled=0；TEMP exception soft delete
+#   - 不清 detection_log / rule_detection_state / 演示数据
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=./s8-regression-common.sh
+source "${SCRIPT_DIR}/s8-regression-common.sh"
+
+auth_load
+
+RULE='TEMP_SCHED_TIMEOUT_RECOVERED'
+DEDUP_OBJ='TEMP-RECOVER-001'
+TRIGGER_REQ=2
+RECOVER_REQ=2
+
+PARAMS_JSON='{"dueAtField":"due_at","statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}'
+HIT_EXPR="SELECT '${DEDUP_OBJ}' AS related_object_code, '${DEDUP_OBJ}' AS source_object_id, DATE_SUB(NOW(), INTERVAL 1 HOUR) AS due_at, 'PENDING' AS status"
+MISS_EXPR="SELECT '${DEDUP_OBJ}' AS related_object_code, '${DEDUP_OBJ}' AS source_object_id, DATE_ADD(NOW(), INTERVAL 2 HOUR) AS due_at, 'PENDING' AS status"
+
+baseline_before=$(read_baseline)
+echo "==== recovered-edge-regression  baseline_before=${baseline_before} ===="
+
+# ---------------------------------------------------------------------------
+# Setup TEMP rule（hit 表达式 + recover_required=2）
+# ---------------------------------------------------------------------------
+echo "---- Setup TEMP rule ${RULE} (hit expression, trigger=${TRIGGER_REQ}, recover=${RECOVER_REQ}) ----"
+exists=$(mysql_run "SELECT COUNT(*) FROM ado_s8_watch_rule WHERE rule_code='${RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};")
+if [[ "${exists}" == "0" ]]; then
+  mysql_run_strict "INSERT INTO ado_s8_watch_rule
+    (tenant_id, factory_id, rule_code, scene_code, data_source_id, watch_object_type, expression, severity,
+     poll_interval_seconds, enabled, created_at, rule_type, source_object_type, params_json,
+     consecutive_failure_count, trigger_count_required, recover_count_required, next_run_at)
+    VALUES (${TENANT_ID}, ${FACTORY_ID}, '${RULE}', 'S1S7_DELIVERY', 1, 'ORDER',
+     \"${HIT_EXPR}\", 'HIGH', 60, 1, NOW(), 'TIMEOUT', 'ORDER', '${PARAMS_JSON}',
+     0, ${TRIGGER_REQ}, ${RECOVER_REQ}, NULL);" >/dev/null
+  echo "  inserted ${RULE}"
+else
+  mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=1, expression=\"${HIT_EXPR}\", params_json='${PARAMS_JSON}',
+                   trigger_count_required=${TRIGGER_REQ}, recover_count_required=${RECOVER_REQ},
+                   paused_until=NULL, pause_reason=NULL,
+                   lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL,
+                   consecutive_failure_count=0, last_status=NULL, last_error=NULL,
+                   next_run_at=NULL, updated_at=NOW()
+                 WHERE rule_code='${RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
+  echo "  re-armed ${RULE}"
+fi
+RULE_ID=$(get_rule_id_by_code "${RULE}")
+DEDUP_KEY="T${TENANT_ID}:F${FACTORY_ID}:R${RULE}:ORDER:${DEDUP_OBJ}"
+echo "  RULE_ID=${RULE_ID}  DEDUP_KEY=${DEDUP_KEY}"
+
+# 清理上次残留 detection_state / TEMP exception，保证 hit_count/miss_count 从 0 起
+mysql_run_strict "DELETE FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
+mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code='${RULE}' AND is_deleted=0;" >/dev/null
+
+marker_a=$(mysql_run "SELECT NOW();")
+sleep 1
+
+# ---------------------------------------------------------------------------
+# Phase A1: 第一次 HIT（pending，hit_count=1 < trigger=2，不建单）
+# ---------------------------------------------------------------------------
+echo "---- Phase A1: HIT 1 → pending (hit_count=1 < ${TRIGGER_REQ}) ----"
+resp_a1=$(run_once_endpoint || true)
+[[ -z "${resp_a1}" ]] && record_fail "A1: run-once returned empty"
+sleep 1
+
+a1_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0 AND status<>'CLOSED';")
+a1_state_hit=$(mysql_run "SELECT IFNULL(consecutive_hit_count,-1) FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND dedup_key='${DEDUP_KEY}';")
+a1_created_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='CREATED' AND detected_at >= '${marker_a}';")
+echo "  A1: active_excs=${a1_excs} state.hit=${a1_state_hit} CREATED_logs=${a1_created_logs}"
+
+[[ "${a1_excs}" == "0" ]] && record_pass "A1: no exception created (pending)" || record_fail "A1: unexpected exception created (count=${a1_excs})"
+[[ "${a1_state_hit}" == "1" ]] && record_pass "A1: detection_state.consecutive_hit_count=1" || record_fail "A1: expected state.hit=1, got ${a1_state_hit}"
+[[ "${a1_created_logs}" == "0" ]] && record_pass "A1: no CREATED log written yet" || record_fail "A1: unexpected ${a1_created_logs} CREATED log(s)"
+
+# ---------------------------------------------------------------------------
+# Phase A2: 第二次 HIT（fire，建单，写 CREATED + state.active_exception_id）
+# ---------------------------------------------------------------------------
+echo "---- Phase A2: HIT 2 → CREATED (hit_count=2 ≥ ${TRIGGER_REQ}) ----"
+sleep 1
+marker_a2=$(mysql_run "SELECT NOW();")
+sleep 1
+resp_a2=$(run_once_endpoint || true)
+[[ -z "${resp_a2}" ]] && record_fail "A2: run-once returned empty"
+sleep 1
+
+a_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0 AND status<>'CLOSED';")
+a_exc_id=$(mysql_run "SELECT IFNULL(MAX(id),0) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0 AND status<>'CLOSED';")
+a_recovered_at=$(mysql_run "SELECT IFNULL(recovered_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
+a_state_active=$(mysql_run "SELECT IFNULL(active_exception_id,0) FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND dedup_key='${DEDUP_KEY}';")
+a_created_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='CREATED' AND detected_at >= '${marker_a2}';")
+echo "  A2: active_excs=${a_excs} exc_id=${a_exc_id} recovered_at=${a_recovered_at} state.active=${a_state_active} CREATED_logs(since marker_a2)=${a_created_logs}"
+
+[[ "${a_excs}" == "1" ]] && record_pass "A2: 1 active exception created" || record_fail "A2: expected 1 active exception, got ${a_excs}"
+[[ "${a_recovered_at}" == "NULL" ]] && record_pass "A2: recovered_at IS NULL on creation" || record_fail "A2: recovered_at unexpectedly set: ${a_recovered_at}"
+[[ "${a_state_active}" != "0" && "${a_state_active}" == "${a_exc_id}" ]] && record_pass "A2: state.active_exception_id=${a_state_active} matches exception id" || record_fail "A2: state.active_exception_id mismatch (state=${a_state_active}, exc=${a_exc_id})"
+[[ "${a_created_logs}" -ge 1 ]] && record_pass "A2: CREATED detection_log present (${a_created_logs})" || record_fail "A2: CREATED log missing"
+
+# ---------------------------------------------------------------------------
+# Phase B: 翻转表达式 → MISS 1（< recover_required，仅累计不写 RECOVERED）
+# ---------------------------------------------------------------------------
+echo "---- Phase B: FLIP miss → miss_count=1 (< ${RECOVER_REQ}) ----"
+mysql_run_strict "UPDATE ado_s8_watch_rule SET expression=\"${MISS_EXPR}\", lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, updated_at=NOW() WHERE id=${RULE_ID};" >/dev/null
+sleep 1
+marker_b=$(mysql_run "SELECT NOW();")
+sleep 1
+
+resp_b=$(run_once_endpoint || true)
+if [[ -z "${resp_b}" ]]; then
+  record_fail "B: run-once returned empty"
+fi
+sleep 1
+
+b_miss=$(mysql_run "SELECT IFNULL(consecutive_miss_count,-1) FROM ado_s8_exception WHERE id=${a_exc_id};")
+b_hit=$(mysql_run "SELECT IFNULL(consecutive_hit_count,-1) FROM ado_s8_exception WHERE id=${a_exc_id};")
+b_recovered_at=$(mysql_run "SELECT IFNULL(recovered_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
+b_recovered_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='RECOVERED' AND detected_at >= '${marker_b}';")
+b_state_miss=$(mysql_run "SELECT IFNULL(consecutive_miss_count,-1) FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND dedup_key='${DEDUP_KEY}';")
+echo "  B: exc.miss=${b_miss} exc.hit=${b_hit} state.miss=${b_state_miss} recovered_at=${b_recovered_at} RECOVERED_logs(since marker_b)=${b_recovered_logs}"
+
+[[ "${b_miss}" == "1" ]] && record_pass "B: exception.consecutive_miss_count=1 (< ${RECOVER_REQ})" || record_fail "B: expected exc.miss_count=1, got ${b_miss}"
+[[ "${b_hit}" == "0" ]] && record_pass "B: exception.consecutive_hit_count reset to 0" || record_fail "B: expected exc.hit_count=0, got ${b_hit}"
+[[ "${b_state_miss}" == "1" ]] && record_pass "B: detection_state.consecutive_miss_count=1" || record_fail "B: expected state.miss_count=1, got ${b_state_miss}"
+[[ "${b_recovered_at}" == "NULL" ]] && record_pass "B: recovered_at still NULL (antiflap pending)" || record_fail "B: recovered_at unexpectedly set: ${b_recovered_at}"
+[[ "${b_recovered_logs}" == "0" ]] && record_pass "B: no RECOVERED log written yet" || record_fail "B: unexpected ${b_recovered_logs} RECOVERED log(s)"
+
+# ---------------------------------------------------------------------------
+# Phase C: MISS 再次 → miss_count=2 ≥ recover_required → RECOVERED
+# ---------------------------------------------------------------------------
+echo "---- Phase C: MISS again → miss_count=${RECOVER_REQ} → RECOVERED ----"
+mysql_run_strict "UPDATE ado_s8_watch_rule SET lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, updated_at=NOW() WHERE id=${RULE_ID};" >/dev/null
+sleep 1
+marker_c=$(mysql_run "SELECT NOW();")
+sleep 1
+
+resp_c=$(run_once_endpoint || true)
+if [[ -z "${resp_c}" ]]; then
+  record_fail "C: run-once returned empty"
+fi
+sleep 1
+
+c_miss=$(mysql_run "SELECT IFNULL(consecutive_miss_count,-1) FROM ado_s8_exception WHERE id=${a_exc_id};")
+c_recovered_at=$(mysql_run "SELECT IFNULL(recovered_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
+c_recovered_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='RECOVERED' AND detected_at >= '${marker_c}';")
+c_status=$(mysql_run "SELECT status FROM ado_s8_exception WHERE id=${a_exc_id};")
+c_last_detected=$(mysql_run "SELECT IFNULL(last_detected_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
+c_log_remark=$(mysql_run "SELECT IFNULL(remark,'NULL') FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='RECOVERED' AND detected_at >= '${marker_c}' ORDER BY id DESC LIMIT 1;")
+echo "  C: exc.miss=${c_miss} recovered_at=${c_recovered_at} RECOVERED_logs(since marker_c)=${c_recovered_logs} status=${c_status} last_detected_at=${c_last_detected}"
+echo "  C: RECOVERED log remark='${c_log_remark}'"
+
+[[ "${c_miss}" -ge ${RECOVER_REQ} ]] && record_pass "C: consecutive_miss_count=${c_miss} (>= ${RECOVER_REQ})" || record_fail "C: expected miss_count>=${RECOVER_REQ}, got ${c_miss}"
+[[ "${c_recovered_at}" != "NULL" ]] && record_pass "C: recovered_at written (${c_recovered_at})" || record_fail "C: recovered_at NOT written"
+[[ "${c_recovered_logs}" -ge 1 ]] && record_pass "C: RECOVERED detection_log present (${c_recovered_logs})" || record_fail "C: RECOVERED log missing"
+[[ "${c_status}" != "CLOSED" ]] && record_pass "C: exception status NOT auto-closed (${c_status})" || record_fail "C: exception unexpectedly CLOSED"
+[[ "${c_last_detected}" != "NULL" ]] && record_pass "C: last_detected_at not cleared (${c_last_detected})" || record_fail "C: last_detected_at incorrectly cleared"
+
+# ---------------------------------------------------------------------------
+# Cleanup
+# ---------------------------------------------------------------------------
+echo "---- Cleanup ----"
+mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=0, paused_until=NULL, pause_reason=NULL, lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, next_run_at=NULL, updated_at=NOW() WHERE id=${RULE_ID};" >/dev/null
+TEMP_EXC_IDS=$(mysql_run "SELECT IFNULL(GROUP_CONCAT(id),'none') FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0;")
+mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code='${RULE}' AND is_deleted=0;" >/dev/null
+echo "  disabled rule_id=${RULE_ID}; soft-deleted exception ids=[${TEMP_EXC_IDS}]"
+
+final_enabled=$(get_rule_field "${RULE_ID}" enabled)
+[[ "${final_enabled}" == "0" ]] && record_pass "cleanup: rule enabled=0" || record_fail "cleanup: rule still enabled=${final_enabled}"
+temp_visible=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0;")
+[[ "${temp_visible}" == "0" ]] && record_pass "cleanup: TEMP exception not in default list" || record_fail "cleanup: ${temp_visible} TEMP exception(s) still visible"
+
+# demo rule 10/11/12 守恒 + baseline 守恒
+demo_state=$(mysql_run "SELECT GROUP_CONCAT(CONCAT(id,':',enabled,':',IFNULL(paused_until,'NULL'),':',trigger_count_required,':',recover_count_required) ORDER BY id) FROM ado_s8_watch_rule WHERE id IN (10,11,12);")
+expected="10:1:NULL:1:1,11:1:NULL:1:1,12:1:NULL:1:1"
+[[ "${demo_state}" == "${expected}" ]] && record_pass "demo rule 10/11/12 conserved" || record_fail "demo rule drift: ${demo_state}"
+
+assert_baseline_unchanged "${baseline_before}"
+
+print_summary
+echo "TEMP rule: ${RULE} (id=${RULE_ID}); TEMP exception ids=[${TEMP_EXC_IDS}]; final exc state: id=${a_exc_id}, recovered_at=${c_recovered_at}"
+exit_by_summary

scripts/s8/rule-evaluator-regression.sh

@@ -6,7 +6,8 @@
 #   2. r2-timeout-regression.sh        — TIMEOUT 主链（默认复用 demo rule 10）
 #   3. r3-shortage-regression.sh       — SHORTAGE 主链（当前 dev 无 demo SHORTAGE，SKIP）
 #   4. r3-out-of-range-regression.sh   — OUT_OF_RANGE 主链（默认复用 demo rule 11）
-#   5. r6-detection-log-edge-regression.sh — NO_HIT/EVALUATE_FAILED edge（无 fixture，SKIP）
+#   5. r6-detection-log-edge-regression.sh — NO_HIT/EVALUATE_FAILED edge
+#   6. recovered-edge-regression.sh    — RECOVERED 主链 + recover_count_required 抗抖
 #
 # 退出码：
 #   - 仅 FAIL>0 时 exit 1；
@@ -73,6 +74,7 @@ run_child "r2-timeout"   "${SCRIPT_DIR}/r2-timeout-regression.sh"
 run_child "r3-shortage"  "${SCRIPT_DIR}/r3-shortage-regression.sh"
 run_child "r3-oor"       "${SCRIPT_DIR}/r3-out-of-range-regression.sh"
 run_child "r6-edge"      "${SCRIPT_DIR}/r6-detection-log-edge-regression.sh"
+run_child "recovered"    "${SCRIPT_DIR}/recovered-edge-regression.sh"
 
 baseline_after=$(read_baseline)