#!/usr/bin/env bash # R6 detection_log edge regression — S8-DETECTION-LOG-EDGE-FIXTURE-1(dev/test only, aidopdev)。 # # 演变史: # - 旧版基于 G01_TEST_TIMEOUT_NO_HIT / G01_TEST_TIMEOUT_FAILED;这两条 fixture 已从 dev 移除。 # - 上一轮 SCAN-1 改造为 SKIP(CREATED/REFRESHED 由 sched-exec 覆盖)。 # - 本轮 EDGE-FIXTURE-1 引入 TEMP_SCHED_TIMEOUT_NO_HIT / TEMP_SCHED_TIMEOUT_EVALUATE_FAILED # 真实覆盖 NO_HIT 与 EVALUATE_FAILED 两条 detection_log 边缘路径。 # # 验证机制: # - 通过 watch_rule.next_run_at=NOW 触发 Job tick 调度,最多 90s 轮询 last_run_at 推进。 # Job 路径(RunDispatchTickAsync→ProcessSingleRuleAsync)写 detection_log + 更新 # watch_rule.last_status / last_error / consecutive_failure_count,CTO 任务约束 五.B 要求。 # - NO_HIT:valid params + 表达式返回未到期行 → evaluator 0 hits → NO_HIT 日志。 # - EVALUATE_FAILED:params_json 缺 dueAtField → evaluator 抛 S8RuleEvaluatorException # (reason=params_schema_invalid) → EVALUATE_FAILED 日志 + last_status=FAILED。 # # 严守: # - 不影响 demo rule 10/11/12。 # - TEMP rule 测后 enabled=0;TEMP exception 若有 soft delete(理论上两条都不应建单)。 # - 不清 detection_log / rule_detection_state / 演示 exception。 set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=./s8-regression-common.sh source "${SCRIPT_DIR}/s8-regression-common.sh" auth_load NO_HIT_RULE='TEMP_SCHED_TIMEOUT_NO_HIT' FAILED_RULE='TEMP_SCHED_TIMEOUT_EVALUATE_FAILED' # NO_HIT:合法 params + 表达式返回 1 行 due_at=NOW+2h(未到期),evaluator 应判定 0 hits NO_HIT_PARAMS='{"dueAtField":"due_at","statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}' NO_HIT_EXPR="SELECT 'TEMP-NOHIT-001' AS related_object_code, 'TEMP-NOHIT-001' AS source_object_id, DATE_ADD(NOW(), INTERVAL 2 HOUR) AS due_at, 'PENDING' AS status" # EVALUATE_FAILED:params 缺 dueAtField → evaluator 抛 params_schema_invalid FAILED_PARAMS='{"statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}' FAILED_EXPR="SELECT 'TEMP-FAILED-001' AS related_object_code, 'TEMP-FAILED-001' AS source_object_id, DATE_SUB(NOW(), INTERVAL 1 HOUR) AS due_at, 'PENDING' AS status" baseline_before=$(read_baseline) DEMO_SNAPSHOT=$(snapshot_demo_rule_state) echo "==== r6-detection-log-edge-regression baseline_before=${baseline_before} demo_snapshot=${DEMO_SNAPSHOT} ====" # --------------------------------------------------------------------------- # 幂等 upsert TEMP rules(直接 DB 写入绕过 service-level 校验,进入 evaluator 才暴露 schema 错) # --------------------------------------------------------------------------- upsert_temp_rule() { local code="$1" expr="$2" params="$3" local exists exists=$(mysql_run "SELECT COUNT(*) FROM ado_s8_watch_rule WHERE rule_code='${code}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};") if [[ "${exists}" == "0" ]]; then mysql_run_strict "INSERT INTO ado_s8_watch_rule (tenant_id, factory_id, rule_code, scene_code, data_source_id, watch_object_type, expression, severity, poll_interval_seconds, enabled, created_at, rule_type, source_object_type, params_json, consecutive_failure_count, trigger_count_required, recover_count_required, next_run_at) VALUES (${TENANT_ID}, ${FACTORY_ID}, '${code}', 'S7', 1, 'ORDER', \"${expr}\", 'HIGH', 60, 1, NOW(), 'TIMEOUT', 'ORDER', '${params}', 0, 1, 1, NOW());" >/dev/null echo " inserted ${code}" else mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=1, expression=\"${expr}\", params_json='${params}', trigger_count_required=1, recover_count_required=1, paused_until=NULL, pause_reason=NULL, lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, consecutive_failure_count=0, last_status=NULL, last_error=NULL, next_run_at=NOW(), updated_at=NOW() WHERE rule_code='${code}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null echo " re-armed ${code}" fi } echo "---- Setup TEMP rules ----" upsert_temp_rule "${NO_HIT_RULE}" "${NO_HIT_EXPR}" "${NO_HIT_PARAMS}" upsert_temp_rule "${FAILED_RULE}" "${FAILED_EXPR}" "${FAILED_PARAMS}" NO_HIT_ID=$(get_rule_id_by_code "${NO_HIT_RULE}") FAILED_ID=$(get_rule_id_by_code "${FAILED_RULE}") echo " NO_HIT_ID=${NO_HIT_ID} FAILED_ID=${FAILED_ID}" # 清理可能残留的 detection_state / TEMP exception(理论上两条都不应建单) mysql_run_strict "DELETE FROM ado_s8_rule_detection_state WHERE rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;" >/dev/null # --------------------------------------------------------------------------- # 触发 Job tick:等待两条 TEMP rule 的 last_run_at 推进(Job 间隔 60s + 余量) # --------------------------------------------------------------------------- echo "---- Wait Job tick (poll up to 90s for both TEMP rules to be processed) ----" marker=$(mysql_run "SELECT NOW();") sleep 1 echo " marker=${marker}" # 触发:next_run_at=NOW(已在 upsert 时设过,但再确保一次以防 Job 已抢锁释放) mysql_run_strict "UPDATE ado_s8_watch_rule SET next_run_at=NOW(), lock_token=NULL, locked_by=NULL, lock_until=NULL WHERE id IN (${NO_HIT_ID}, ${FAILED_ID});" >/dev/null picked_up=0 for ((i=1; i<=18; i++)); do sleep 5 nh_run_at=$(get_rule_field "${NO_HIT_ID}" last_run_at) fl_run_at=$(get_rule_field "${FAILED_ID}" last_run_at) if [[ "${nh_run_at}" != "NULL" && "${fl_run_at}" != "NULL" \ && "${nh_run_at}" > "${marker}" && "${fl_run_at}" > "${marker}" ]]; then picked_up=1 echo " iter ${i}: both TEMP rules picked up (nh_run_at=${nh_run_at}, fl_run_at=${fl_run_at})" break fi echo " iter ${i}: waiting… nh_run_at=${nh_run_at} fl_run_at=${fl_run_at}" done if (( picked_up == 0 )); then record_skip "Job did not pick up both TEMP rules within 90s — backend cron likely paused or saturated" # 仍走收尾流程 fi # --------------------------------------------------------------------------- # A. NO_HIT 验证 # --------------------------------------------------------------------------- echo "---- A. NO_HIT verification ----" if (( picked_up == 1 )); then nh_no_hit_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${NO_HIT_RULE}' AND detect_result='NO_HIT' AND detected_at >= '${marker}';") nh_other_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${NO_HIT_RULE}' AND detect_result IN ('CREATED','REFRESHED','EVALUATE_FAILED') AND detected_at >= '${marker}';") nh_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${NO_HIT_RULE}' AND is_deleted=0;") nh_status=$(get_rule_field "${NO_HIT_ID}" last_status) echo " NO_HIT_logs=${nh_no_hit_logs} other_logs=${nh_other_logs} active_excs=${nh_excs} last_status=${nh_status}" [[ "${nh_no_hit_logs}" -ge 1 ]] && record_pass "A: NO_HIT detection_log present (${nh_no_hit_logs})" || record_fail "A: expected NO_HIT log >=1, got ${nh_no_hit_logs}" [[ "${nh_other_logs}" == "0" ]] && record_pass "A: NO_HIT rule produced no CREATED/REFRESHED/FAILED" || record_fail "A: NO_HIT rule unexpected ${nh_other_logs} other logs" [[ "${nh_excs}" == "0" ]] && record_pass "A: NO_HIT rule did not create exception" || record_fail "A: NO_HIT rule wrongly created ${nh_excs} exception(s)" [[ "${nh_status}" == "SUCCESS" ]] && record_pass "A: NO_HIT rule last_status=SUCCESS (evaluator OK with 0 hits)" || record_fail "A: NO_HIT rule last_status expected SUCCESS, got ${nh_status}" else record_skip "A: NO_HIT — Job tick timeout, cannot verify" fi # --------------------------------------------------------------------------- # B. EVALUATE_FAILED 验证 # --------------------------------------------------------------------------- echo "---- B. EVALUATE_FAILED verification ----" if (( picked_up == 1 )); then fl_failed_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}';") fl_failed_reason=$(mysql_run "SELECT failure_reason FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}' ORDER BY id DESC LIMIT 1;") fl_failed_msg_len=$(mysql_run "SELECT CHAR_LENGTH(IFNULL(failure_message,'')) FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}' ORDER BY id DESC LIMIT 1;") fl_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${FAILED_RULE}' AND is_deleted=0;") fl_status=$(get_rule_field "${FAILED_ID}" last_status) fl_error=$(get_rule_field "${FAILED_ID}" last_error) fl_failcount=$(get_rule_field "${FAILED_ID}" consecutive_failure_count) echo " EVALUATE_FAILED_logs=${fl_failed_logs} reason='${fl_failed_reason}' msg_len=${fl_failed_msg_len} active_excs=${fl_excs}" echo " last_status=${fl_status} last_error=${fl_error} consecutive_failure_count=${fl_failcount}" [[ "${fl_failed_logs}" -ge 1 ]] && record_pass "B: EVALUATE_FAILED detection_log present (${fl_failed_logs})" || record_fail "B: expected EVALUATE_FAILED log >=1, got ${fl_failed_logs}" [[ -n "${fl_failed_reason}" && "${fl_failed_reason}" != "NULL" ]] && record_pass "B: failure_reason populated ('${fl_failed_reason}')" || record_fail "B: failure_reason missing" [[ "${fl_failed_msg_len}" -gt 0 ]] && record_pass "B: failure_message non-empty (${fl_failed_msg_len} chars)" || record_fail "B: failure_message empty" [[ "${fl_excs}" == "0" ]] && record_pass "B: FAILED rule did not create exception" || record_fail "B: FAILED rule wrongly created ${fl_excs} exception(s)" [[ "${fl_status}" == "FAILED" ]] && record_pass "B: FAILED rule last_status=FAILED" || record_fail "B: FAILED rule last_status expected FAILED, got ${fl_status}" [[ "${fl_error}" != "NULL" && -n "${fl_error}" ]] && record_pass "B: last_error populated" || record_fail "B: last_error missing" [[ "${fl_failcount}" -ge 1 ]] && record_pass "B: consecutive_failure_count=${fl_failcount} (>=1)" || record_fail "B: consecutive_failure_count not bumped (${fl_failcount})" else record_skip "B: EVALUATE_FAILED — Job tick timeout, cannot verify" fi # --------------------------------------------------------------------------- # Cleanup TEMP rules # --------------------------------------------------------------------------- echo "---- Cleanup TEMP rules ----" mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=0, paused_until=NULL, pause_reason=NULL, lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, updated_at=NOW() WHERE rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null TEMP_EXC_IDS=$(mysql_run "SELECT IFNULL(GROUP_CONCAT(id),'none') FROM ado_s8_exception WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;") mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;" >/dev/null cleanup_temp_sched_approval_ghost_tasks echo " disabled ${NO_HIT_RULE} (id=${NO_HIT_ID}) / ${FAILED_RULE} (id=${FAILED_ID}); soft-deleted exception ids=[${TEMP_EXC_IDS}]; ghost approval cancelled" nh_enabled=$(get_rule_field "${NO_HIT_ID}" enabled) fl_enabled=$(get_rule_field "${FAILED_ID}" enabled) [[ "${nh_enabled}" == "0" ]] && record_pass "cleanup: ${NO_HIT_RULE} enabled=0" || record_fail "cleanup: ${NO_HIT_RULE} still enabled=${nh_enabled}" [[ "${fl_enabled}" == "0" ]] && record_pass "cleanup: ${FAILED_RULE} enabled=0" || record_fail "cleanup: ${FAILED_RULE} still enabled=${fl_enabled}" temp_visible=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;") [[ "${temp_visible}" == "0" ]] && record_pass "cleanup: TEMP exceptions not in default list" || record_fail "cleanup: ${temp_visible} TEMP exception(s) still in default list" # --------------------------------------------------------------------------- # demo rule 10/11/12 漂移检测 + baseline 守恒 # --------------------------------------------------------------------------- assert_demo_rule_state_unchanged "${DEMO_SNAPSHOT}" assert_baseline_unchanged "${baseline_before}" print_summary echo "TEMP rules: ${NO_HIT_RULE} (id=${NO_HIT_ID}), ${FAILED_RULE} (id=${FAILED_ID}); TEMP exception ids=[${TEMP_EXC_IDS}]" exit_by_summary