AiDOPWarehouse/scripts/s8/r6-detection-log-edge-regression.sh

#!/usr/bin/env bash
# R6 detection_log edge regression — S8-DETECTION-LOG-EDGE-FIXTURE-1（dev/test only, aidopdev）。
#
# 演变史：
#   - 旧版基于 G01_TEST_TIMEOUT_NO_HIT / G01_TEST_TIMEOUT_FAILED；这两条 fixture 已从 dev 移除。
#   - 上一轮 SCAN-1 改造为 SKIP（CREATED/REFRESHED 由 sched-exec 覆盖）。
#   - 本轮 EDGE-FIXTURE-1 引入 TEMP_SCHED_TIMEOUT_NO_HIT / TEMP_SCHED_TIMEOUT_EVALUATE_FAILED
#     真实覆盖 NO_HIT 与 EVALUATE_FAILED 两条 detection_log 边缘路径。
#
# 验证机制：
#   - 通过 watch_rule.next_run_at=NOW 触发 Job tick 调度，最多 90s 轮询 last_run_at 推进。
#     Job 路径（RunDispatchTickAsync→ProcessSingleRuleAsync）写 detection_log + 更新
#     watch_rule.last_status / last_error / consecutive_failure_count，CTO 任务约束 五.B 要求。
#   - NO_HIT：valid params + 表达式返回未到期行 → evaluator 0 hits → NO_HIT 日志。
#   - EVALUATE_FAILED：params_json 缺 dueAtField → evaluator 抛 S8RuleEvaluatorException
#     (reason=params_schema_invalid) → EVALUATE_FAILED 日志 + last_status=FAILED。
#
# 严守：
#   - 不影响 demo rule 10/11/12。
#   - TEMP rule 测后 enabled=0；TEMP exception 若有 soft delete（理论上两条都不应建单）。
#   - 不清 detection_log / rule_detection_state / 演示 exception。

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./s8-regression-common.sh
source "${SCRIPT_DIR}/s8-regression-common.sh"

auth_load

NO_HIT_RULE='TEMP_SCHED_TIMEOUT_NO_HIT'
FAILED_RULE='TEMP_SCHED_TIMEOUT_EVALUATE_FAILED'

# NO_HIT：合法 params + 表达式返回 1 行 due_at=NOW+2h（未到期），evaluator 应判定 0 hits
NO_HIT_PARAMS='{"dueAtField":"due_at","statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}'
NO_HIT_EXPR="SELECT 'TEMP-NOHIT-001' AS related_object_code, 'TEMP-NOHIT-001' AS source_object_id, DATE_ADD(NOW(), INTERVAL 2 HOUR) AS due_at, 'PENDING' AS status"

# EVALUATE_FAILED：params 缺 dueAtField → evaluator 抛 params_schema_invalid
FAILED_PARAMS='{"statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}'
FAILED_EXPR="SELECT 'TEMP-FAILED-001' AS related_object_code, 'TEMP-FAILED-001' AS source_object_id, DATE_SUB(NOW(), INTERVAL 1 HOUR) AS due_at, 'PENDING' AS status"

baseline_before=$(read_baseline)
DEMO_SNAPSHOT=$(snapshot_demo_rule_state)
echo "==== r6-detection-log-edge-regression  baseline_before=${baseline_before} demo_snapshot=${DEMO_SNAPSHOT} ===="

# ---------------------------------------------------------------------------
# 幂等 upsert TEMP rules（直接 DB 写入绕过 service-level 校验，进入 evaluator 才暴露 schema 错）
# ---------------------------------------------------------------------------
upsert_temp_rule() {
  local code="$1" expr="$2" params="$3"
  local exists
  exists=$(mysql_run "SELECT COUNT(*) FROM ado_s8_watch_rule WHERE rule_code='${code}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};")
  if [[ "${exists}" == "0" ]]; then
    mysql_run_strict "INSERT INTO ado_s8_watch_rule
      (tenant_id, factory_id, rule_code, scene_code, data_source_id, watch_object_type, expression, severity,
       poll_interval_seconds, enabled, created_at, rule_type, source_object_type, params_json,
       consecutive_failure_count, trigger_count_required, recover_count_required, next_run_at)
      VALUES (${TENANT_ID}, ${FACTORY_ID}, '${code}', 'S7', 1, 'ORDER',
       \"${expr}\", 'HIGH', 60, 1, NOW(), 'TIMEOUT', 'ORDER', '${params}',
       0, 1, 1, NOW());" >/dev/null
    echo "  inserted ${code}"
  else
    mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=1, expression=\"${expr}\", params_json='${params}',
                       trigger_count_required=1, recover_count_required=1,
                       paused_until=NULL, pause_reason=NULL,
                       lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL,
                       consecutive_failure_count=0, last_status=NULL, last_error=NULL,
                       next_run_at=NOW(), updated_at=NOW()
                     WHERE rule_code='${code}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
    echo "  re-armed ${code}"
  fi
}

echo "---- Setup TEMP rules ----"
upsert_temp_rule "${NO_HIT_RULE}"   "${NO_HIT_EXPR}"  "${NO_HIT_PARAMS}"
upsert_temp_rule "${FAILED_RULE}"  "${FAILED_EXPR}"  "${FAILED_PARAMS}"

NO_HIT_ID=$(get_rule_id_by_code "${NO_HIT_RULE}")
FAILED_ID=$(get_rule_id_by_code "${FAILED_RULE}")
echo "  NO_HIT_ID=${NO_HIT_ID}  FAILED_ID=${FAILED_ID}"

# 清理可能残留的 detection_state / TEMP exception（理论上两条都不应建单）
mysql_run_strict "DELETE FROM ado_s8_rule_detection_state WHERE rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;" >/dev/null

# ---------------------------------------------------------------------------
# 触发 Job tick：等待两条 TEMP rule 的 last_run_at 推进（Job 间隔 60s + 余量）
# ---------------------------------------------------------------------------
echo "---- Wait Job tick (poll up to 90s for both TEMP rules to be processed) ----"
marker=$(mysql_run "SELECT NOW();")
sleep 1
echo "  marker=${marker}"

# 触发：next_run_at=NOW（已在 upsert 时设过，但再确保一次以防 Job 已抢锁释放）
mysql_run_strict "UPDATE ado_s8_watch_rule SET next_run_at=NOW(), lock_token=NULL, locked_by=NULL, lock_until=NULL WHERE id IN (${NO_HIT_ID}, ${FAILED_ID});" >/dev/null

picked_up=0
for ((i=1; i<=18; i++)); do
  sleep 5
  nh_run_at=$(get_rule_field "${NO_HIT_ID}" last_run_at)
  fl_run_at=$(get_rule_field "${FAILED_ID}" last_run_at)
  if [[ "${nh_run_at}" != "NULL" && "${fl_run_at}" != "NULL" \
        && "${nh_run_at}" > "${marker}" && "${fl_run_at}" > "${marker}" ]]; then
    picked_up=1
    echo "  iter ${i}: both TEMP rules picked up (nh_run_at=${nh_run_at}, fl_run_at=${fl_run_at})"
    break
  fi
  echo "  iter ${i}: waiting… nh_run_at=${nh_run_at} fl_run_at=${fl_run_at}"
done

if (( picked_up == 0 )); then
  record_skip "Job did not pick up both TEMP rules within 90s — backend cron likely paused or saturated"
  # 仍走收尾流程
fi

# ---------------------------------------------------------------------------
# A. NO_HIT 验证
# ---------------------------------------------------------------------------
echo "---- A. NO_HIT verification ----"
if (( picked_up == 1 )); then
  nh_no_hit_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${NO_HIT_RULE}' AND detect_result='NO_HIT' AND detected_at >= '${marker}';")
  nh_other_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${NO_HIT_RULE}' AND detect_result IN ('CREATED','REFRESHED','EVALUATE_FAILED') AND detected_at >= '${marker}';")
  nh_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${NO_HIT_RULE}' AND is_deleted=0;")
  nh_status=$(get_rule_field "${NO_HIT_ID}" last_status)
  echo "  NO_HIT_logs=${nh_no_hit_logs} other_logs=${nh_other_logs} active_excs=${nh_excs} last_status=${nh_status}"

  [[ "${nh_no_hit_logs}" -ge 1 ]] && record_pass "A: NO_HIT detection_log present (${nh_no_hit_logs})" || record_fail "A: expected NO_HIT log >=1, got ${nh_no_hit_logs}"
  [[ "${nh_other_logs}" == "0" ]] && record_pass "A: NO_HIT rule produced no CREATED/REFRESHED/FAILED" || record_fail "A: NO_HIT rule unexpected ${nh_other_logs} other logs"
  [[ "${nh_excs}" == "0" ]] && record_pass "A: NO_HIT rule did not create exception" || record_fail "A: NO_HIT rule wrongly created ${nh_excs} exception(s)"
  [[ "${nh_status}" == "SUCCESS" ]] && record_pass "A: NO_HIT rule last_status=SUCCESS (evaluator OK with 0 hits)" || record_fail "A: NO_HIT rule last_status expected SUCCESS, got ${nh_status}"
else
  record_skip "A: NO_HIT — Job tick timeout, cannot verify"
fi

# ---------------------------------------------------------------------------
# B. EVALUATE_FAILED 验证
# ---------------------------------------------------------------------------
echo "---- B. EVALUATE_FAILED verification ----"
if (( picked_up == 1 )); then
  fl_failed_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}';")
  fl_failed_reason=$(mysql_run "SELECT failure_reason FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}' ORDER BY id DESC LIMIT 1;")
  fl_failed_msg_len=$(mysql_run "SELECT CHAR_LENGTH(IFNULL(failure_message,'')) FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}' ORDER BY id DESC LIMIT 1;")
  fl_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${FAILED_RULE}' AND is_deleted=0;")
  fl_status=$(get_rule_field "${FAILED_ID}" last_status)
  fl_error=$(get_rule_field "${FAILED_ID}" last_error)
  fl_failcount=$(get_rule_field "${FAILED_ID}" consecutive_failure_count)
  echo "  EVALUATE_FAILED_logs=${fl_failed_logs} reason='${fl_failed_reason}' msg_len=${fl_failed_msg_len} active_excs=${fl_excs}"
  echo "  last_status=${fl_status} last_error=${fl_error} consecutive_failure_count=${fl_failcount}"

  [[ "${fl_failed_logs}" -ge 1 ]] && record_pass "B: EVALUATE_FAILED detection_log present (${fl_failed_logs})" || record_fail "B: expected EVALUATE_FAILED log >=1, got ${fl_failed_logs}"
  [[ -n "${fl_failed_reason}" && "${fl_failed_reason}" != "NULL" ]] && record_pass "B: failure_reason populated ('${fl_failed_reason}')" || record_fail "B: failure_reason missing"
  [[ "${fl_failed_msg_len}" -gt 0 ]] && record_pass "B: failure_message non-empty (${fl_failed_msg_len} chars)" || record_fail "B: failure_message empty"
  [[ "${fl_excs}" == "0" ]] && record_pass "B: FAILED rule did not create exception" || record_fail "B: FAILED rule wrongly created ${fl_excs} exception(s)"
  [[ "${fl_status}" == "FAILED" ]] && record_pass "B: FAILED rule last_status=FAILED" || record_fail "B: FAILED rule last_status expected FAILED, got ${fl_status}"
  [[ "${fl_error}" != "NULL" && -n "${fl_error}" ]] && record_pass "B: last_error populated" || record_fail "B: last_error missing"
  [[ "${fl_failcount}" -ge 1 ]] && record_pass "B: consecutive_failure_count=${fl_failcount} (>=1)" || record_fail "B: consecutive_failure_count not bumped (${fl_failcount})"
else
  record_skip "B: EVALUATE_FAILED — Job tick timeout, cannot verify"
fi

# ---------------------------------------------------------------------------
# Cleanup TEMP rules
# ---------------------------------------------------------------------------
echo "---- Cleanup TEMP rules ----"
mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=0, paused_until=NULL, pause_reason=NULL, lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, updated_at=NOW() WHERE rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
TEMP_EXC_IDS=$(mysql_run "SELECT IFNULL(GROUP_CONCAT(id),'none') FROM ado_s8_exception WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;")
mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;" >/dev/null
cleanup_temp_sched_approval_ghost_tasks
echo "  disabled ${NO_HIT_RULE} (id=${NO_HIT_ID}) / ${FAILED_RULE} (id=${FAILED_ID}); soft-deleted exception ids=[${TEMP_EXC_IDS}]; ghost approval cancelled"

nh_enabled=$(get_rule_field "${NO_HIT_ID}" enabled)
fl_enabled=$(get_rule_field "${FAILED_ID}" enabled)
[[ "${nh_enabled}" == "0" ]] && record_pass "cleanup: ${NO_HIT_RULE} enabled=0" || record_fail "cleanup: ${NO_HIT_RULE} still enabled=${nh_enabled}"
[[ "${fl_enabled}" == "0" ]] && record_pass "cleanup: ${FAILED_RULE} enabled=0" || record_fail "cleanup: ${FAILED_RULE} still enabled=${fl_enabled}"

temp_visible=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;")
[[ "${temp_visible}" == "0" ]] && record_pass "cleanup: TEMP exceptions not in default list" || record_fail "cleanup: ${temp_visible} TEMP exception(s) still in default list"

# ---------------------------------------------------------------------------
# demo rule 10/11/12 漂移检测 + baseline 守恒
# ---------------------------------------------------------------------------
assert_demo_rule_state_unchanged "${DEMO_SNAPSHOT}"

assert_baseline_unchanged "${baseline_before}"

print_summary
echo "TEMP rules: ${NO_HIT_RULE} (id=${NO_HIT_ID}), ${FAILED_RULE} (id=${FAILED_ID}); TEMP exception ids=[${TEMP_EXC_IDS}]"
exit_by_summary