AiDOPWarehouse/scripts/s8/recovered-edge-regression.sh

#!/usr/bin/env bash
# RECOVERED edge regression — S8-RECOVERED-EDGE-FIXTURE-1（dev/test only, aidopdev）。
#
# 四阶段验证 RECOVERED 主链 + trigger/recover 抗抖：
#   Phase A1: 第一次命中（hit_count=1 < trigger=2）→ pending，不建单，不写 CREATED
#   Phase A2: 第二次命中（hit_count=2 ≥ trigger=2）→ 建单，state.active_exception_id=exc.id，写 CREATED
#   Phase B:  翻转表达式不命中（miss_count=1 < recover=2）→ recovered_at 仍 NULL，不写 RECOVERED
#   Phase C:  再次不命中（miss_count=2 ≥ recover=2）→ recovered_at 写入，RECOVERED detection_log 写入
#
# 为何 trigger=2 而非 1：
#   trigger=1 时 state 行在首 tick 由 InsertAsync 创建，但 SqlSugar 不回填 Id 到 entity；
#   后续 UPDATE state.active_exception_id WHERE id=state.Id 命中 0 行，导致 active_exception_id
#   永远为 NULL。这是产品侧问题，本脚本通过 trigger=2 在第二 tick 走 existing 分支（state.Id
#   来自 query 真实可用），绕开此坑。已在风险清单中登记。
#
# 触发：通过 /api/aidop/s8/watch-debug/run-once（ProcessRulesByTypeAsync 路径，
#       与 ProcessSingleRuleAsync 同样调用 ReconcileRecoveriesForRuleAsync）。
#       /run-once 不更新 watch_rule.last_status，本脚本不对此做断言。
#
# 严守：
#   - 不影响 demo rule 10/11/12
#   - TEMP rule 测后 enabled=0；TEMP exception soft delete
#   - 不清 detection_log / rule_detection_state / 演示数据

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./s8-regression-common.sh
source "${SCRIPT_DIR}/s8-regression-common.sh"

auth_load

RULE='TEMP_SCHED_TIMEOUT_RECOVERED'
DEDUP_OBJ='TEMP-RECOVER-001'
TRIGGER_REQ=2
RECOVER_REQ=2

PARAMS_JSON='{"dueAtField":"due_at","statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}'
HIT_EXPR="SELECT '${DEDUP_OBJ}' AS related_object_code, '${DEDUP_OBJ}' AS source_object_id, DATE_SUB(NOW(), INTERVAL 1 HOUR) AS due_at, 'PENDING' AS status"
MISS_EXPR="SELECT '${DEDUP_OBJ}' AS related_object_code, '${DEDUP_OBJ}' AS source_object_id, DATE_ADD(NOW(), INTERVAL 2 HOUR) AS due_at, 'PENDING' AS status"

baseline_before=$(read_baseline)
DEMO_SNAPSHOT=$(snapshot_demo_rule_state)
echo "==== recovered-edge-regression  baseline_before=${baseline_before} demo_snapshot=${DEMO_SNAPSHOT} ===="

# ---------------------------------------------------------------------------
# Setup TEMP rule（hit 表达式 + recover_required=2）
# ---------------------------------------------------------------------------
echo "---- Setup TEMP rule ${RULE} (hit expression, trigger=${TRIGGER_REQ}, recover=${RECOVER_REQ}) ----"
exists=$(mysql_run "SELECT COUNT(*) FROM ado_s8_watch_rule WHERE rule_code='${RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};")
if [[ "${exists}" == "0" ]]; then
  mysql_run_strict "INSERT INTO ado_s8_watch_rule
    (tenant_id, factory_id, rule_code, scene_code, data_source_id, watch_object_type, expression, severity,
     poll_interval_seconds, enabled, created_at, rule_type, source_object_type, params_json,
     consecutive_failure_count, trigger_count_required, recover_count_required, next_run_at)
    VALUES (${TENANT_ID}, ${FACTORY_ID}, '${RULE}', 'S7', 1, 'ORDER',
     \"${HIT_EXPR}\", 'HIGH', 60, 1, NOW(), 'TIMEOUT', 'ORDER', '${PARAMS_JSON}',
     0, ${TRIGGER_REQ}, ${RECOVER_REQ}, NULL);" >/dev/null
  echo "  inserted ${RULE}"
else
  mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=1, expression=\"${HIT_EXPR}\", params_json='${PARAMS_JSON}',
                   trigger_count_required=${TRIGGER_REQ}, recover_count_required=${RECOVER_REQ},
                   paused_until=NULL, pause_reason=NULL,
                   lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL,
                   consecutive_failure_count=0, last_status=NULL, last_error=NULL,
                   next_run_at=NULL, updated_at=NOW()
                 WHERE rule_code='${RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
  echo "  re-armed ${RULE}"
fi
RULE_ID=$(get_rule_id_by_code "${RULE}")
DEDUP_KEY="T${TENANT_ID}:F${FACTORY_ID}:R${RULE}:ORDER:${DEDUP_OBJ}"
echo "  RULE_ID=${RULE_ID}  DEDUP_KEY=${DEDUP_KEY}"

# 清理上次残留 detection_state / TEMP exception，保证 hit_count/miss_count 从 0 起
mysql_run_strict "DELETE FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code='${RULE}' AND is_deleted=0;" >/dev/null

marker_a=$(mysql_run "SELECT NOW();")
sleep 1

# ---------------------------------------------------------------------------
# Phase A1: 第一次 HIT（pending，hit_count=1 < trigger=2，不建单）
# ---------------------------------------------------------------------------
echo "---- Phase A1: HIT 1 → pending (hit_count=1 < ${TRIGGER_REQ}) ----"
resp_a1=$(run_once_endpoint || true)
[[ -z "${resp_a1}" ]] && record_fail "A1: run-once returned empty"
sleep 1

a1_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0 AND status<>'CLOSED';")
a1_state_hit=$(mysql_run "SELECT IFNULL(consecutive_hit_count,-1) FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND dedup_key='${DEDUP_KEY}';")
a1_created_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='CREATED' AND detected_at >= '${marker_a}';")
echo "  A1: active_excs=${a1_excs} state.hit=${a1_state_hit} CREATED_logs=${a1_created_logs}"

[[ "${a1_excs}" == "0" ]] && record_pass "A1: no exception created (pending)" || record_fail "A1: unexpected exception created (count=${a1_excs})"
[[ "${a1_state_hit}" == "1" ]] && record_pass "A1: detection_state.consecutive_hit_count=1" || record_fail "A1: expected state.hit=1, got ${a1_state_hit}"
[[ "${a1_created_logs}" == "0" ]] && record_pass "A1: no CREATED log written yet" || record_fail "A1: unexpected ${a1_created_logs} CREATED log(s)"

# ---------------------------------------------------------------------------
# Phase A2: 第二次 HIT（fire，建单，写 CREATED + state.active_exception_id）
# ---------------------------------------------------------------------------
echo "---- Phase A2: HIT 2 → CREATED (hit_count=2 ≥ ${TRIGGER_REQ}) ----"
sleep 1
marker_a2=$(mysql_run "SELECT NOW();")
sleep 1
resp_a2=$(run_once_endpoint || true)
[[ -z "${resp_a2}" ]] && record_fail "A2: run-once returned empty"
sleep 1

a_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0 AND status<>'CLOSED';")
a_exc_id=$(mysql_run "SELECT IFNULL(MAX(id),0) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0 AND status<>'CLOSED';")
a_recovered_at=$(mysql_run "SELECT IFNULL(recovered_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
a_state_active=$(mysql_run "SELECT IFNULL(active_exception_id,0) FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND dedup_key='${DEDUP_KEY}';")
a_created_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='CREATED' AND detected_at >= '${marker_a2}';")
echo "  A2: active_excs=${a_excs} exc_id=${a_exc_id} recovered_at=${a_recovered_at} state.active=${a_state_active} CREATED_logs(since marker_a2)=${a_created_logs}"

[[ "${a_excs}" == "1" ]] && record_pass "A2: 1 active exception created" || record_fail "A2: expected 1 active exception, got ${a_excs}"
[[ "${a_recovered_at}" == "NULL" ]] && record_pass "A2: recovered_at IS NULL on creation" || record_fail "A2: recovered_at unexpectedly set: ${a_recovered_at}"
[[ "${a_state_active}" != "0" && "${a_state_active}" == "${a_exc_id}" ]] && record_pass "A2: state.active_exception_id=${a_state_active} matches exception id" || record_fail "A2: state.active_exception_id mismatch (state=${a_state_active}, exc=${a_exc_id})"
[[ "${a_created_logs}" -ge 1 ]] && record_pass "A2: CREATED detection_log present (${a_created_logs})" || record_fail "A2: CREATED log missing"

# ---------------------------------------------------------------------------
# Phase B: 翻转表达式 → MISS 1（< recover_required，仅累计不写 RECOVERED）
# ---------------------------------------------------------------------------
echo "---- Phase B: FLIP miss → miss_count=1 (< ${RECOVER_REQ}) ----"
mysql_run_strict "UPDATE ado_s8_watch_rule SET expression=\"${MISS_EXPR}\", lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, updated_at=NOW() WHERE id=${RULE_ID};" >/dev/null
sleep 1
marker_b=$(mysql_run "SELECT NOW();")
sleep 1

resp_b=$(run_once_endpoint || true)
if [[ -z "${resp_b}" ]]; then
  record_fail "B: run-once returned empty"
fi
sleep 1

b_miss=$(mysql_run "SELECT IFNULL(consecutive_miss_count,-1) FROM ado_s8_exception WHERE id=${a_exc_id};")
b_hit=$(mysql_run "SELECT IFNULL(consecutive_hit_count,-1) FROM ado_s8_exception WHERE id=${a_exc_id};")
b_recovered_at=$(mysql_run "SELECT IFNULL(recovered_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
b_recovered_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='RECOVERED' AND detected_at >= '${marker_b}';")
b_state_miss=$(mysql_run "SELECT IFNULL(consecutive_miss_count,-1) FROM ado_s8_rule_detection_state WHERE rule_code='${RULE}' AND dedup_key='${DEDUP_KEY}';")
echo "  B: exc.miss=${b_miss} exc.hit=${b_hit} state.miss=${b_state_miss} recovered_at=${b_recovered_at} RECOVERED_logs(since marker_b)=${b_recovered_logs}"

[[ "${b_miss}" == "1" ]] && record_pass "B: exception.consecutive_miss_count=1 (< ${RECOVER_REQ})" || record_fail "B: expected exc.miss_count=1, got ${b_miss}"
[[ "${b_hit}" == "0" ]] && record_pass "B: exception.consecutive_hit_count reset to 0" || record_fail "B: expected exc.hit_count=0, got ${b_hit}"
[[ "${b_state_miss}" == "1" ]] && record_pass "B: detection_state.consecutive_miss_count=1" || record_fail "B: expected state.miss_count=1, got ${b_state_miss}"
[[ "${b_recovered_at}" == "NULL" ]] && record_pass "B: recovered_at still NULL (antiflap pending)" || record_fail "B: recovered_at unexpectedly set: ${b_recovered_at}"
[[ "${b_recovered_logs}" == "0" ]] && record_pass "B: no RECOVERED log written yet" || record_fail "B: unexpected ${b_recovered_logs} RECOVERED log(s)"

# ---------------------------------------------------------------------------
# Phase C: MISS 再次 → miss_count=2 ≥ recover_required → RECOVERED
# ---------------------------------------------------------------------------
echo "---- Phase C: MISS again → miss_count=${RECOVER_REQ} → RECOVERED ----"
mysql_run_strict "UPDATE ado_s8_watch_rule SET lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, updated_at=NOW() WHERE id=${RULE_ID};" >/dev/null
sleep 1
marker_c=$(mysql_run "SELECT NOW();")
sleep 1

resp_c=$(run_once_endpoint || true)
if [[ -z "${resp_c}" ]]; then
  record_fail "C: run-once returned empty"
fi
sleep 1

c_miss=$(mysql_run "SELECT IFNULL(consecutive_miss_count,-1) FROM ado_s8_exception WHERE id=${a_exc_id};")
c_recovered_at=$(mysql_run "SELECT IFNULL(recovered_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
c_recovered_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='RECOVERED' AND detected_at >= '${marker_c}';")
c_status=$(mysql_run "SELECT status FROM ado_s8_exception WHERE id=${a_exc_id};")
c_last_detected=$(mysql_run "SELECT IFNULL(last_detected_at,'NULL') FROM ado_s8_exception WHERE id=${a_exc_id};")
c_log_remark=$(mysql_run "SELECT IFNULL(remark,'NULL') FROM ado_s8_detection_log WHERE rule_code='${RULE}' AND detect_result='RECOVERED' AND detected_at >= '${marker_c}' ORDER BY id DESC LIMIT 1;")
echo "  C: exc.miss=${c_miss} recovered_at=${c_recovered_at} RECOVERED_logs(since marker_c)=${c_recovered_logs} status=${c_status} last_detected_at=${c_last_detected}"
echo "  C: RECOVERED log remark='${c_log_remark}'"

[[ "${c_miss}" -ge ${RECOVER_REQ} ]] && record_pass "C: consecutive_miss_count=${c_miss} (>= ${RECOVER_REQ})" || record_fail "C: expected miss_count>=${RECOVER_REQ}, got ${c_miss}"
[[ "${c_recovered_at}" != "NULL" ]] && record_pass "C: recovered_at written (${c_recovered_at})" || record_fail "C: recovered_at NOT written"
[[ "${c_recovered_logs}" -ge 1 ]] && record_pass "C: RECOVERED detection_log present (${c_recovered_logs})" || record_fail "C: RECOVERED log missing"
[[ "${c_status}" != "CLOSED" ]] && record_pass "C: exception status NOT auto-closed (${c_status})" || record_fail "C: exception unexpectedly CLOSED"
[[ "${c_last_detected}" != "NULL" ]] && record_pass "C: last_detected_at not cleared (${c_last_detected})" || record_fail "C: last_detected_at incorrectly cleared"

# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
echo "---- Cleanup ----"
mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=0, paused_until=NULL, pause_reason=NULL, lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, next_run_at=NULL, updated_at=NOW() WHERE id=${RULE_ID};" >/dev/null
TEMP_EXC_IDS=$(mysql_run "SELECT IFNULL(GROUP_CONCAT(id),'none') FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0;")
mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code='${RULE}' AND is_deleted=0;" >/dev/null
cleanup_temp_sched_approval_ghost_tasks
echo "  disabled rule_id=${RULE_ID}; soft-deleted exception ids=[${TEMP_EXC_IDS}]; ghost approval cancelled"

final_enabled=$(get_rule_field "${RULE_ID}" enabled)
[[ "${final_enabled}" == "0" ]] && record_pass "cleanup: rule enabled=0" || record_fail "cleanup: rule still enabled=${final_enabled}"
temp_visible=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${RULE}' AND is_deleted=0;")
[[ "${temp_visible}" == "0" ]] && record_pass "cleanup: TEMP exception not in default list" || record_fail "cleanup: ${temp_visible} TEMP exception(s) still visible"

# demo rule 10/11/12 漂移检测 + baseline 守恒
assert_demo_rule_state_unchanged "${DEMO_SNAPSHOT}"

assert_baseline_unchanged "${baseline_before}"

print_summary
echo "TEMP rule: ${RULE} (id=${RULE_ID}); TEMP exception ids=[${TEMP_EXC_IDS}]; final exc state: id=${a_exc_id}, recovered_at=${c_recovered_at}"
exit_by_summary