mirror of
https://github.com/tanweai/pua.git
synced 2026-08-29 02:02:20 +08:00
xsser
b2c4f913f2
- 所有钉味输出从 ▎ Unicode 前缀改为 markdown blockquote（> ）语法 Claude Code 渲染器仅对 > 语法触发 dim ▎ + italic 灰色块样式 - 每条 blockquote 开头标注来源：《置身钉内》或《置身钉外》 - 新增独立 skills/ding/ skill 目录（SKILL.md + references/） - 新增 commands/ding.md、evidence.md、again.md、done-check.md - 新增 evals/test-trigger-regex.sh（38 条触发正则测试） - hooks/session-restore.sh、flavor-helper.sh 同步更新格式 - 版本号 3.4.6 → 3.5.0（plugin.json + 两个 marketplace manifest）
96 lines
3.4 KiB
Bash
Executable File
96 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# PUA Skill Triggering Test
|
|
# Tests whether the skill triggers on correct prompts and doesn't trigger on incorrect ones
|
|
#
|
|
# Usage: ./run-trigger-test.sh [--plugin-dir <path>]
|
|
# Requires: claude CLI
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/test-helpers.sh"
|
|
PLUGIN_DIR="${1:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
|
RESULTS_DIR="/tmp/pua-evals/$(date +%s)"
|
|
mkdir -p "$RESULTS_DIR"
|
|
EVAL_PUA_CONFIG="$RESULTS_DIR/pua-config.json"
|
|
printf '%s\n' '{"always_on":false,"feedback_frequency":0}' > "$EVAL_PUA_CONFIG"
|
|
EVAL_WORKSPACE="$RESULTS_DIR/workspace"
|
|
mkdir -p "$EVAL_WORKSPACE"
|
|
|
|
echo "=== PUA Trigger Tests ==="
|
|
echo "Plugin dir: $PLUGIN_DIR"
|
|
echo "Results: $RESULTS_DIR"
|
|
echo ""
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
test_prompt() {
|
|
local prompt="$1"
|
|
local should_trigger="$2"
|
|
local label="$3"
|
|
local outfile="$RESULTS_DIR/$(echo "$label" | tr ' ' '_').json"
|
|
|
|
(
|
|
cd "$EVAL_WORKSPACE"
|
|
PUA_CONFIG="$EVAL_PUA_CONFIG" PUA_FORCE_ON=1 run_with_timeout 120 claude -p "$prompt" \
|
|
--plugin-dir "$PLUGIN_DIR" \
|
|
--dangerously-skip-permissions \
|
|
--max-turns 2 \
|
|
--output-format stream-json \
|
|
--verbose \
|
|
> "$outfile" 2>&1 || true
|
|
)
|
|
|
|
local triggered=false
|
|
if grep -q '"skill":"pua"' "$outfile" 2>/dev/null || \
|
|
grep -q '"skill":"pua:pua"' "$outfile" 2>/dev/null; then
|
|
triggered=true
|
|
elif [ "$should_trigger" = "yes" ] && grep -qE 'PUA Skill Context|User Frustration Signal|PUA生效|3\.25|闭环|owner|颗粒度|抓手|底层逻辑|置身钉外|证据链|没跑测试|换个方法|done-check|evidence' "$outfile" 2>/dev/null; then
|
|
# Claude may apply PUA pressure from hook/context without an explicit Skill
|
|
# tool event before max-turns. Count observable PUA behavior as triggered.
|
|
triggered=true
|
|
fi
|
|
|
|
if [ "$should_trigger" = "yes" ] && [ "$triggered" = "true" ]; then
|
|
echo " ✅ PASS: $label (correctly triggered)"
|
|
PASS=$((PASS + 1))
|
|
elif [ "$should_trigger" = "no" ] && [ "$triggered" = "false" ]; then
|
|
echo " ✅ PASS: $label (correctly NOT triggered)"
|
|
PASS=$((PASS + 1))
|
|
elif [ "$should_trigger" = "yes" ] && [ "$triggered" = "false" ]; then
|
|
echo " ❌ FAIL: $label (should trigger but didn't)"
|
|
FAIL=$((FAIL + 1))
|
|
else
|
|
echo " ❌ FAIL: $label (should NOT trigger but did)"
|
|
FAIL=$((FAIL + 1))
|
|
fi
|
|
}
|
|
|
|
echo "--- Should Trigger ---"
|
|
test_prompt "你怎么又失败了？再试试" "yes" "frustration-zh"
|
|
test_prompt "Try harder, stop giving up" "yes" "frustration-en"
|
|
test_prompt "PUA模式 帮我修这个bug" "yes" "explicit-pua"
|
|
test_prompt "你这个交付质量太差了" "yes" "quality-complaint"
|
|
test_prompt "没跑测试别说完成" "yes" "done-without-test-zh"
|
|
test_prompt "证据呢？数据在哪？" "yes" "evidence-request-zh"
|
|
test_prompt "老板体感不是 oracle，给我一条置身钉外提醒" "yes" "ding-workplace-zh"
|
|
test_prompt "Change approach, don't keep tweaking the same thing" "yes" "change-approach-en"
|
|
|
|
echo ""
|
|
echo "--- Should NOT Trigger ---"
|
|
test_prompt "Help me write a sort function" "no" "simple-coding"
|
|
test_prompt "What is async await in JavaScript?" "no" "info-query"
|
|
test_prompt "帮我翻译这段话" "no" "translation"
|
|
|
|
echo ""
|
|
echo "=== Results ==="
|
|
echo "Passed: $PASS"
|
|
echo "Failed: $FAIL"
|
|
echo "Total: $((PASS + FAIL))"
|
|
echo "Results dir: $RESULTS_DIR"
|
|
|
|
if [ "$FAIL" -gt 0 ]; then
|
|
exit 1
|
|
fi
|