|
|
@@ -1,12 +1,24 @@
|
|
|
#!/usr/bin/env bash
|
|
|
-# R6 detection_log edge regression — S8-REGRESSION-FIXTURE-1 重构版(dev/test only, aidopdev)。
|
|
|
+# R6 detection_log edge regression — S8-DETECTION-LOG-EDGE-FIXTURE-1(dev/test only, aidopdev)。
|
|
|
#
|
|
|
# 演变史:
|
|
|
-# - 旧版基于 G01_TEST_TIMEOUT_NO_HIT / G01_TEST_TIMEOUT_FAILED 两条 TEMP rule。
|
|
|
-# - 当前 dev 已无这两条 G01_TEST_* fixture,且 demo 规则均稳定命中。
|
|
|
-# - 重构后:检测到旧 fixture 不存在 → record_skip + exit 0。
|
|
|
-# - 任务 CTO 约束 六.G:CREATED / REFRESHED 已由 sched-exec-regression.sh 覆盖;
|
|
|
-# NO_HIT / EVALUATE_FAILED 暂无安全 fixture,标记 SKIP 不得伪 PASS。
|
|
|
+# - 旧版基于 G01_TEST_TIMEOUT_NO_HIT / G01_TEST_TIMEOUT_FAILED;这两条 fixture 已从 dev 移除。
|
|
|
+# - 上一轮 SCAN-1 改造为 SKIP(CREATED/REFRESHED 由 sched-exec 覆盖)。
|
|
|
+# - 本轮 EDGE-FIXTURE-1 引入 TEMP_SCHED_TIMEOUT_NO_HIT / TEMP_SCHED_TIMEOUT_EVALUATE_FAILED
|
|
|
+# 真实覆盖 NO_HIT 与 EVALUATE_FAILED 两条 detection_log 边缘路径。
|
|
|
+#
|
|
|
+# 验证机制:
|
|
|
+# - 通过 watch_rule.next_run_at=NOW 触发 Job tick 调度,最多 90s 轮询 last_run_at 推进。
|
|
|
+# Job 路径(RunDispatchTickAsync→ProcessSingleRuleAsync)写 detection_log + 更新
|
|
|
+# watch_rule.last_status / last_error / consecutive_failure_count,CTO 任务约束 五.B 要求。
|
|
|
+# - NO_HIT:valid params + 表达式返回未到期行 → evaluator 0 hits → NO_HIT 日志。
|
|
|
+# - EVALUATE_FAILED:params_json 缺 dueAtField → evaluator 抛 S8RuleEvaluatorException
|
|
|
+# (reason=params_schema_invalid) → EVALUATE_FAILED 日志 + last_status=FAILED。
|
|
|
+#
|
|
|
+# 严守:
|
|
|
+# - 不影响 demo rule 10/11/12。
|
|
|
+# - TEMP rule 测后 enabled=0;TEMP exception 若有 soft delete(理论上两条都不应建单)。
|
|
|
+# - 不清 detection_log / rule_detection_state / 演示 exception。
|
|
|
|
|
|
set -uo pipefail
|
|
|
|
|
|
@@ -14,26 +26,163 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
# shellcheck source=./s8-regression-common.sh
|
|
|
source "${SCRIPT_DIR}/s8-regression-common.sh"
|
|
|
|
|
|
-NO_HIT_RULE="${NO_HIT_RULE:-G01_TEST_TIMEOUT_NO_HIT}"
|
|
|
-FAILED_RULE="${FAILED_RULE:-G01_TEST_TIMEOUT_FAILED}"
|
|
|
+auth_load
|
|
|
+
|
|
|
+NO_HIT_RULE='TEMP_SCHED_TIMEOUT_NO_HIT'
|
|
|
+FAILED_RULE='TEMP_SCHED_TIMEOUT_EVALUATE_FAILED'
|
|
|
+
|
|
|
+# NO_HIT:合法 params + 表达式返回 1 行 due_at=NOW+2h(未到期),evaluator 应判定 0 hits
|
|
|
+NO_HIT_PARAMS='{"dueAtField":"due_at","statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}'
|
|
|
+NO_HIT_EXPR="SELECT 'TEMP-NOHIT-001' AS related_object_code, 'TEMP-NOHIT-001' AS source_object_id, DATE_ADD(NOW(), INTERVAL 2 HOUR) AS due_at, 'PENDING' AS status"
|
|
|
+
|
|
|
+# EVALUATE_FAILED:params 缺 dueAtField → evaluator 抛 params_schema_invalid
|
|
|
+FAILED_PARAMS='{"statusField":"status","completedStates":["CLOSED","DONE","COMPLETED"],"objectCodeField":"related_object_code","objectIdField":"source_object_id","graceMinutes":0,"exceptionTypeCode":"DELIVERY_DELAY"}'
|
|
|
+FAILED_EXPR="SELECT 'TEMP-FAILED-001' AS related_object_code, 'TEMP-FAILED-001' AS source_object_id, DATE_SUB(NOW(), INTERVAL 1 HOUR) AS due_at, 'PENDING' AS status"
|
|
|
|
|
|
baseline_before=$(read_baseline)
|
|
|
echo "==== r6-detection-log-edge-regression baseline_before=${baseline_before} ===="
|
|
|
|
|
|
-nohit_exists=$(mysql_run "SELECT COUNT(*) FROM ado_s8_watch_rule WHERE rule_code='${NO_HIT_RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};")
|
|
|
-failed_exists=$(mysql_run "SELECT COUNT(*) FROM ado_s8_watch_rule WHERE rule_code='${FAILED_RULE}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};")
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# 幂等 upsert TEMP rules(直接 DB 写入绕过 service-level 校验,进入 evaluator 才暴露 schema 错)
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+upsert_temp_rule() {
|
|
|
+ local code="$1" expr="$2" params="$3"
|
|
|
+ local exists
|
|
|
+ exists=$(mysql_run "SELECT COUNT(*) FROM ado_s8_watch_rule WHERE rule_code='${code}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};")
|
|
|
+ if [[ "${exists}" == "0" ]]; then
|
|
|
+ mysql_run_strict "INSERT INTO ado_s8_watch_rule
|
|
|
+ (tenant_id, factory_id, rule_code, scene_code, data_source_id, watch_object_type, expression, severity,
|
|
|
+ poll_interval_seconds, enabled, created_at, rule_type, source_object_type, params_json,
|
|
|
+ consecutive_failure_count, trigger_count_required, recover_count_required, next_run_at)
|
|
|
+ VALUES (${TENANT_ID}, ${FACTORY_ID}, '${code}', 'S1S7_DELIVERY', 1, 'ORDER',
|
|
|
+ \"${expr}\", 'HIGH', 60, 1, NOW(), 'TIMEOUT', 'ORDER', '${params}',
|
|
|
+ 0, 1, 1, NOW());" >/dev/null
|
|
|
+ echo " inserted ${code}"
|
|
|
+ else
|
|
|
+ mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=1, expression=\"${expr}\", params_json='${params}',
|
|
|
+ trigger_count_required=1, recover_count_required=1,
|
|
|
+ paused_until=NULL, pause_reason=NULL,
|
|
|
+ lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL,
|
|
|
+ consecutive_failure_count=0, last_status=NULL, last_error=NULL,
|
|
|
+ next_run_at=NOW(), updated_at=NOW()
|
|
|
+ WHERE rule_code='${code}' AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
|
|
|
+ echo " re-armed ${code}"
|
|
|
+ fi
|
|
|
+}
|
|
|
+
|
|
|
+echo "---- Setup TEMP rules ----"
|
|
|
+upsert_temp_rule "${NO_HIT_RULE}" "${NO_HIT_EXPR}" "${NO_HIT_PARAMS}"
|
|
|
+upsert_temp_rule "${FAILED_RULE}" "${FAILED_EXPR}" "${FAILED_PARAMS}"
|
|
|
+
|
|
|
+NO_HIT_ID=$(get_rule_id_by_code "${NO_HIT_RULE}")
|
|
|
+FAILED_ID=$(get_rule_id_by_code "${FAILED_RULE}")
|
|
|
+echo " NO_HIT_ID=${NO_HIT_ID} FAILED_ID=${FAILED_ID}"
|
|
|
+
|
|
|
+# 清理可能残留的 detection_state / TEMP exception(理论上两条都不应建单)
|
|
|
+mysql_run_strict "DELETE FROM ado_s8_rule_detection_state WHERE rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
|
|
|
+mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;" >/dev/null
|
|
|
+
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# 触发 Job tick:等待两条 TEMP rule 的 last_run_at 推进(Job 间隔 60s + 余量)
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+echo "---- Wait Job tick (poll up to 90s for both TEMP rules to be processed) ----"
|
|
|
+marker=$(mysql_run "SELECT NOW();")
|
|
|
+sleep 1
|
|
|
+echo " marker=${marker}"
|
|
|
+
|
|
|
+# 触发:next_run_at=NOW(已在 upsert 时设过,但再确保一次以防 Job 已抢锁释放)
|
|
|
+mysql_run_strict "UPDATE ado_s8_watch_rule SET next_run_at=NOW(), lock_token=NULL, locked_by=NULL, lock_until=NULL WHERE id IN (${NO_HIT_ID}, ${FAILED_ID});" >/dev/null
|
|
|
+
|
|
|
+picked_up=0
|
|
|
+for ((i=1; i<=18; i++)); do
|
|
|
+ sleep 5
|
|
|
+ nh_run_at=$(get_rule_field "${NO_HIT_ID}" last_run_at)
|
|
|
+ fl_run_at=$(get_rule_field "${FAILED_ID}" last_run_at)
|
|
|
+ if [[ "${nh_run_at}" != "NULL" && "${fl_run_at}" != "NULL" \
|
|
|
+ && "${nh_run_at}" > "${marker}" && "${fl_run_at}" > "${marker}" ]]; then
|
|
|
+ picked_up=1
|
|
|
+ echo " iter ${i}: both TEMP rules picked up (nh_run_at=${nh_run_at}, fl_run_at=${fl_run_at})"
|
|
|
+ break
|
|
|
+ fi
|
|
|
+ echo " iter ${i}: waiting… nh_run_at=${nh_run_at} fl_run_at=${fl_run_at}"
|
|
|
+done
|
|
|
|
|
|
-if [[ "${nohit_exists}" == "0" && "${failed_exists}" == "0" ]]; then
|
|
|
- record_skip "NO_HIT/EVALUATE_FAILED edge fixtures absent in dev (${NO_HIT_RULE}/${FAILED_RULE}) — CREATED/REFRESHED already covered by sched-exec-regression.sh"
|
|
|
- assert_baseline_unchanged "${baseline_before}"
|
|
|
- print_summary
|
|
|
- exit 0
|
|
|
+if (( picked_up == 0 )); then
|
|
|
+ record_skip "Job did not pick up both TEMP rules within 90s — backend cron likely paused or saturated"
|
|
|
+ # 仍走收尾流程
|
|
|
fi
|
|
|
|
|
|
-# ---- 预留:如果未来注入了 NO_HIT/FAILED edge fixture,本段可被启用。----
|
|
|
-auth_load
|
|
|
-record_skip "NO_HIT/EVALUATE_FAILED partial fixture present (nohit=${nohit_exists}, failed=${failed_exists}) — automated edge driver not yet adapted to demo data layout"
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# A. NO_HIT 验证
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+echo "---- A. NO_HIT verification ----"
|
|
|
+if (( picked_up == 1 )); then
|
|
|
+ nh_no_hit_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${NO_HIT_RULE}' AND detect_result='NO_HIT' AND detected_at >= '${marker}';")
|
|
|
+ nh_other_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${NO_HIT_RULE}' AND detect_result IN ('CREATED','REFRESHED','EVALUATE_FAILED') AND detected_at >= '${marker}';")
|
|
|
+ nh_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${NO_HIT_RULE}' AND is_deleted=0;")
|
|
|
+ nh_status=$(get_rule_field "${NO_HIT_ID}" last_status)
|
|
|
+ echo " NO_HIT_logs=${nh_no_hit_logs} other_logs=${nh_other_logs} active_excs=${nh_excs} last_status=${nh_status}"
|
|
|
+
|
|
|
+ [[ "${nh_no_hit_logs}" -ge 1 ]] && record_pass "A: NO_HIT detection_log present (${nh_no_hit_logs})" || record_fail "A: expected NO_HIT log >=1, got ${nh_no_hit_logs}"
|
|
|
+ [[ "${nh_other_logs}" == "0" ]] && record_pass "A: NO_HIT rule produced no CREATED/REFRESHED/FAILED" || record_fail "A: NO_HIT rule unexpected ${nh_other_logs} other logs"
|
|
|
+ [[ "${nh_excs}" == "0" ]] && record_pass "A: NO_HIT rule did not create exception" || record_fail "A: NO_HIT rule wrongly created ${nh_excs} exception(s)"
|
|
|
+ [[ "${nh_status}" == "SUCCESS" ]] && record_pass "A: NO_HIT rule last_status=SUCCESS (evaluator OK with 0 hits)" || record_fail "A: NO_HIT rule last_status expected SUCCESS, got ${nh_status}"
|
|
|
+else
|
|
|
+ record_skip "A: NO_HIT — Job tick timeout, cannot verify"
|
|
|
+fi
|
|
|
+
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# B. EVALUATE_FAILED 验证
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+echo "---- B. EVALUATE_FAILED verification ----"
|
|
|
+if (( picked_up == 1 )); then
|
|
|
+ fl_failed_logs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}';")
|
|
|
+ fl_failed_reason=$(mysql_run "SELECT failure_reason FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}' ORDER BY id DESC LIMIT 1;")
|
|
|
+ fl_failed_msg_len=$(mysql_run "SELECT CHAR_LENGTH(IFNULL(failure_message,'')) FROM ado_s8_detection_log WHERE rule_code='${FAILED_RULE}' AND detect_result='EVALUATE_FAILED' AND detected_at >= '${marker}' ORDER BY id DESC LIMIT 1;")
|
|
|
+ fl_excs=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code='${FAILED_RULE}' AND is_deleted=0;")
|
|
|
+ fl_status=$(get_rule_field "${FAILED_ID}" last_status)
|
|
|
+ fl_error=$(get_rule_field "${FAILED_ID}" last_error)
|
|
|
+ fl_failcount=$(get_rule_field "${FAILED_ID}" consecutive_failure_count)
|
|
|
+ echo " EVALUATE_FAILED_logs=${fl_failed_logs} reason='${fl_failed_reason}' msg_len=${fl_failed_msg_len} active_excs=${fl_excs}"
|
|
|
+ echo " last_status=${fl_status} last_error=${fl_error} consecutive_failure_count=${fl_failcount}"
|
|
|
+
|
|
|
+ [[ "${fl_failed_logs}" -ge 1 ]] && record_pass "B: EVALUATE_FAILED detection_log present (${fl_failed_logs})" || record_fail "B: expected EVALUATE_FAILED log >=1, got ${fl_failed_logs}"
|
|
|
+ [[ -n "${fl_failed_reason}" && "${fl_failed_reason}" != "NULL" ]] && record_pass "B: failure_reason populated ('${fl_failed_reason}')" || record_fail "B: failure_reason missing"
|
|
|
+ [[ "${fl_failed_msg_len}" -gt 0 ]] && record_pass "B: failure_message non-empty (${fl_failed_msg_len} chars)" || record_fail "B: failure_message empty"
|
|
|
+ [[ "${fl_excs}" == "0" ]] && record_pass "B: FAILED rule did not create exception" || record_fail "B: FAILED rule wrongly created ${fl_excs} exception(s)"
|
|
|
+ [[ "${fl_status}" == "FAILED" ]] && record_pass "B: FAILED rule last_status=FAILED" || record_fail "B: FAILED rule last_status expected FAILED, got ${fl_status}"
|
|
|
+ [[ "${fl_error}" != "NULL" && -n "${fl_error}" ]] && record_pass "B: last_error populated" || record_fail "B: last_error missing"
|
|
|
+ [[ "${fl_failcount}" -ge 1 ]] && record_pass "B: consecutive_failure_count=${fl_failcount} (>=1)" || record_fail "B: consecutive_failure_count not bumped (${fl_failcount})"
|
|
|
+else
|
|
|
+ record_skip "B: EVALUATE_FAILED — Job tick timeout, cannot verify"
|
|
|
+fi
|
|
|
+
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# Cleanup TEMP rules
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+echo "---- Cleanup TEMP rules ----"
|
|
|
+mysql_run_strict "UPDATE ado_s8_watch_rule SET enabled=0, paused_until=NULL, pause_reason=NULL, lock_token=NULL, locked_by=NULL, lock_until=NULL, running_started_at=NULL, updated_at=NOW() WHERE rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND tenant_id=${TENANT_ID} AND factory_id=${FACTORY_ID};" >/dev/null
|
|
|
+TEMP_EXC_IDS=$(mysql_run "SELECT IFNULL(GROUP_CONCAT(id),'none') FROM ado_s8_exception WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;")
|
|
|
+mysql_run_strict "UPDATE ado_s8_exception SET is_deleted=1, updated_at=NOW() WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;" >/dev/null
|
|
|
+echo " disabled ${NO_HIT_RULE} (id=${NO_HIT_ID}) / ${FAILED_RULE} (id=${FAILED_ID}); soft-deleted exception ids=[${TEMP_EXC_IDS}]"
|
|
|
+
|
|
|
+nh_enabled=$(get_rule_field "${NO_HIT_ID}" enabled)
|
|
|
+fl_enabled=$(get_rule_field "${FAILED_ID}" enabled)
|
|
|
+[[ "${nh_enabled}" == "0" ]] && record_pass "cleanup: ${NO_HIT_RULE} enabled=0" || record_fail "cleanup: ${NO_HIT_RULE} still enabled=${nh_enabled}"
|
|
|
+[[ "${fl_enabled}" == "0" ]] && record_pass "cleanup: ${FAILED_RULE} enabled=0" || record_fail "cleanup: ${FAILED_RULE} still enabled=${fl_enabled}"
|
|
|
+
|
|
|
+temp_visible=$(mysql_run "SELECT COUNT(*) FROM ado_s8_exception WHERE source_rule_code IN ('${NO_HIT_RULE}','${FAILED_RULE}') AND is_deleted=0;")
|
|
|
+[[ "${temp_visible}" == "0" ]] && record_pass "cleanup: TEMP exceptions not in default list" || record_fail "cleanup: ${temp_visible} TEMP exception(s) still in default list"
|
|
|
+
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# demo rule 10/11/12 守恒 + baseline 守恒
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+demo_state=$(mysql_run "SELECT GROUP_CONCAT(CONCAT(id,':',enabled,':',IFNULL(paused_until,'NULL'),':',trigger_count_required,':',recover_count_required) ORDER BY id) FROM ado_s8_watch_rule WHERE id IN (10,11,12);")
|
|
|
+expected="10:1:NULL:1:1,11:1:NULL:1:1,12:1:NULL:1:1"
|
|
|
+[[ "${demo_state}" == "${expected}" ]] && record_pass "demo rule 10/11/12 conserved" || record_fail "demo rule drift: ${demo_state}"
|
|
|
|
|
|
assert_baseline_unchanged "${baseline_before}"
|
|
|
+
|
|
|
print_summary
|
|
|
+echo "TEMP rules: ${NO_HIT_RULE} (id=${NO_HIT_ID}), ${FAILED_RULE} (id=${FAILED_ID}); TEMP exception ids=[${TEMP_EXC_IDS}]"
|
|
|
exit_by_summary
|