From 538372d53f419df7376434040f516b43b248bd9a Mon Sep 17 00:00:00 2001 From: Renchao Wu <42173584+widechaos@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:48:45 +0400 Subject: [PATCH 1/2] test(loop): exercise production functions in unit suites --- tests/unit/test_exit_detection.bats | 181 +++--------------------- tests/unit/test_rate_limiting.bats | 75 ++-------- tests/unit/test_session_continuity.bats | 71 ++-------- 3 files changed, 38 insertions(+), 289 deletions(-) diff --git a/tests/unit/test_exit_detection.bats b/tests/unit/test_exit_detection.bats index 0b082108..1b9d72c0 100644 --- a/tests/unit/test_exit_detection.bats +++ b/tests/unit/test_exit_detection.bats @@ -7,6 +7,17 @@ setup() { # Source helper functions source "$(dirname "$BATS_TEST_FILENAME")/../helpers/test_helper.bash" + # Load the production functions without running the loop or its startup code. + source "${BATS_TEST_DIRNAME}/../../lib/int_utils.sh" + local function_name function_body + for function_name in _safe_count _count_blocking_unchecked should_exit_gracefully; do + function_body=$(sed -n "/^${function_name}() {/,/^}/p" "${BATS_TEST_DIRNAME}/../../ralph_loop.sh") + [[ -n "$function_body" ]] || return 1 + eval "$function_body" + declare -F "$function_name" > /dev/null || return 1 + done + log_status() { :; } + # Set up environment with .ralph/ subfolder structure export RALPH_DIR=".ralph" export EXIT_SIGNALS_FILE="$RALPH_DIR/.exit_signals" @@ -30,110 +41,6 @@ teardown() { rm -rf "$TEST_TEMP_DIR" } -# Helper: _count_blocking_unchecked (extracted from ralph_loop.sh, Issue #239) -# Counts unchecked "- [ ]" items that block exit, skipping items under OPTIONAL_SECTIONS. -_count_blocking_unchecked() { - local file="$1" - [[ ! -f "$file" ]] && { printf '0'; return 0; } - local raw - raw=$(awk -v sections="${OPTIONAL_SECTIONS:-}" ' - BEGIN { - n = split(sections, arr, ",") - for (i = 1; i <= n; i++) { - s = arr[i] - gsub(/^[ \t]+|[ \t]+$/, "", s) - if (s != "") opt[tolower(s)] = 1 - } - } - /^[[:space:]]*#+[[:space:]]+/ { - line = $0 - sub(/^[[:space:]]+/, "", line) - level = 0 - while (substr(line, level + 1, 1) == "#") level++ - title = substr(line, level + 1) - sub(/^[[:space:]]+/, "", title) - sub(/[[:space:]]+$/, "", title) - if (optional_active && level <= optional_level) optional_active = 0 - if (tolower(title) in opt) { optional_active = 1; optional_level = level } - next - } - !optional_active && /^[[:space:]]*- \[ \]/ { count++ } - END { print count + 0 } - ' "$file" 2>/dev/null | tr -d '\r\n[:space:]' | head -c 10) - if [[ "$raw" =~ ^[0-9]+$ ]]; then - printf '%d' "$raw" - else - printf '0' - fi -} - -# Helper function: should_exit_gracefully (extracted from ralph_loop.sh) -# Updated to respect EXIT_SIGNAL from .response_analysis for completion indicators -should_exit_gracefully() { - if [[ ! -f "$EXIT_SIGNALS_FILE" ]]; then - echo "" # Return empty string instead of using return code - return 1 # Don't exit, file doesn't exist - fi - - local signals=$(cat "$EXIT_SIGNALS_FILE") - - # Count recent signals (last 5 loops) - with error handling - local recent_test_loops - local recent_done_signals - local recent_completion_indicators - - recent_test_loops=$(echo "$signals" | jq '.test_only_loops | length' 2>/dev/null || echo "0") - recent_done_signals=$(echo "$signals" | jq '.done_signals | length' 2>/dev/null || echo "0") - recent_completion_indicators=$(echo "$signals" | jq '.completion_indicators | length' 2>/dev/null || echo "0") - - # Check for exit conditions - - # 1. Too many consecutive test-only loops - if [[ $recent_test_loops -ge $MAX_CONSECUTIVE_TEST_LOOPS ]]; then - echo "test_saturation" - return 0 - fi - - # 2. Multiple "done" signals - if [[ $recent_done_signals -ge $MAX_CONSECUTIVE_DONE_SIGNALS ]]; then - echo "completion_signals" - return 0 - fi - - # 3. Strong completion indicators (only if Claude's EXIT_SIGNAL is true) - # This prevents premature exits when heuristics detect completion patterns - # but Claude explicitly indicates work is still in progress - local claude_exit_signal="false" - if [[ -f "$RESPONSE_ANALYSIS_FILE" ]]; then - claude_exit_signal=$(jq -r '.analysis.exit_signal // false' "$RESPONSE_ANALYSIS_FILE" 2>/dev/null || echo "false") - fi - - if [[ $recent_completion_indicators -ge 2 ]] && [[ "$claude_exit_signal" == "true" ]]; then - echo "project_complete" - return 0 - fi - - # 4. Check fix_plan.md for completion - # Fix #144: Only match valid markdown checkboxes, not date entries like [2026-01-29] - # Issue #239: unchecked items under OPTIONAL_SECTIONS do not block exit - if [[ -f "$RALPH_DIR/fix_plan.md" ]]; then - local uncompleted_items - local completed_items - uncompleted_items=$(_count_blocking_unchecked "$RALPH_DIR/fix_plan.md") - completed_items=$(grep -cE "^[[:space:]]*- \[[xX]\]" "$RALPH_DIR/fix_plan.md" 2>/dev/null || echo "0") - completed_items=$(echo "$completed_items" | tr -d '[:space:]') - local total_items=$((uncompleted_items + completed_items)) - - if [[ $total_items -gt 0 ]] && [[ $completed_items -eq $total_items ]]; then - echo "plan_complete" - return 0 - fi - fi - - echo "" # Return empty string instead of using return code - return 1 # Don't exit -} - # Test 1: No exit when signals are empty @test "should_exit_gracefully returns empty with no signals" { echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" @@ -556,8 +463,9 @@ EOF # Test 26: EXIT_SIGNAL=false with explicit false value in JSON @test "should_exit_gracefully handles explicit false exit_signal" { - echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": [1,2,3,4,5]}' > "$EXIT_SIGNALS_FILE" + echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": [1,2,3,4]}' > "$EXIT_SIGNALS_FILE" + # Keep below the independent five-indicator safety circuit breaker. # Explicit false value cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF' { @@ -858,59 +766,6 @@ EOF # When Claude Code is denied permission to run commands, Ralph should detect # this from the permission_denials field and halt the loop to allow user intervention. -# Helper function with permission denial support -should_exit_gracefully_with_denials() { - if [[ ! -f "$EXIT_SIGNALS_FILE" ]]; then - echo "" - return 1 - fi - - local signals=$(cat "$EXIT_SIGNALS_FILE") - - local recent_test_loops - local recent_done_signals - local recent_completion_indicators - - recent_test_loops=$(echo "$signals" | jq '.test_only_loops | length' 2>/dev/null || echo "0") - recent_done_signals=$(echo "$signals" | jq '.done_signals | length' 2>/dev/null || echo "0") - recent_completion_indicators=$(echo "$signals" | jq '.completion_indicators | length' 2>/dev/null || echo "0") - - # Check for permission denials first (highest priority - Issue #101) - if [[ -f "$RESPONSE_ANALYSIS_FILE" ]]; then - local has_permission_denials=$(jq -r '.analysis.has_permission_denials // false' "$RESPONSE_ANALYSIS_FILE" 2>/dev/null || echo "false") - if [[ "$has_permission_denials" == "true" ]]; then - echo "permission_denied" - return 0 - fi - fi - - # 1. Too many consecutive test-only loops - if [[ $recent_test_loops -ge $MAX_CONSECUTIVE_TEST_LOOPS ]]; then - echo "test_saturation" - return 0 - fi - - # 2. Multiple "done" signals - if [[ $recent_done_signals -ge $MAX_CONSECUTIVE_DONE_SIGNALS ]]; then - echo "completion_signals" - return 0 - fi - - # 3. Strong completion indicators (only if Claude's EXIT_SIGNAL is true) - local claude_exit_signal="false" - if [[ -f "$RESPONSE_ANALYSIS_FILE" ]]; then - claude_exit_signal=$(jq -r '.analysis.exit_signal // false' "$RESPONSE_ANALYSIS_FILE" 2>/dev/null || echo "false") - fi - - if [[ $recent_completion_indicators -ge 2 ]] && [[ "$claude_exit_signal" == "true" ]]; then - echo "project_complete" - return 0 - fi - - echo "" - return 1 -} - # Test 36: Exit on permission denial detected @test "should_exit_gracefully exits on permission_denied" { echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" @@ -936,7 +791,7 @@ should_exit_gracefully_with_denials() { } EOF - result=$(should_exit_gracefully_with_denials) + result=$(should_exit_gracefully) assert_equal "$result" "permission_denied" } @@ -965,7 +820,7 @@ EOF } EOF - result=$(should_exit_gracefully_with_denials || true) + result=$(should_exit_gracefully || true) assert_equal "$result" "" } @@ -988,7 +843,7 @@ EOF EOF # Permission denied should take priority - result=$(should_exit_gracefully_with_denials) + result=$(should_exit_gracefully) assert_equal "$result" "permission_denied" } @@ -1007,7 +862,7 @@ EOF } EOF - result=$(should_exit_gracefully_with_denials) + result=$(should_exit_gracefully) assert_equal "$result" "permission_denied" } @@ -1027,7 +882,7 @@ EOF } EOF - result=$(should_exit_gracefully_with_denials || true) + result=$(should_exit_gracefully || true) assert_equal "$result" "" } diff --git a/tests/unit/test_rate_limiting.bats b/tests/unit/test_rate_limiting.bats index 769272dd..f5b58393 100755 --- a/tests/unit/test_rate_limiting.bats +++ b/tests/unit/test_rate_limiting.bats @@ -3,11 +3,21 @@ load '../helpers/test_helper' -# Source ralph functions (we need to extract these first) setup() { # Source helper functions source "$(dirname "$BATS_TEST_FILENAME")/../helpers/test_helper.bash" + # Load the production functions without running the loop or its startup code. + source "${BATS_TEST_DIRNAME}/../../lib/int_utils.sh" + local function_name function_body + for function_name in extract_token_usage update_token_count can_make_call increment_call_counter; do + function_body=$(sed -n "/^${function_name}() {/,/^}/p" "${BATS_TEST_DIRNAME}/../../ralph_loop.sh") + [[ -n "$function_body" ]] || return 1 + eval "$function_body" + declare -F "$function_name" > /dev/null || return 1 + done + log_status() { :; } + # Set up environment with .ralph/ subfolder structure export RALPH_DIR=".ralph" export MAX_CALLS_PER_HOUR=100 @@ -33,69 +43,6 @@ teardown() { rm -rf "$TEST_TEMP_DIR" } -# Helper function: extract_token_usage (extracted from ralph_loop.sh) -extract_token_usage() { - local output_file=$1 - if [[ ! -f "$output_file" ]]; then - echo "0" - return - fi - local tokens - tokens=$(jq -r ' - ((.usage.input_tokens // .metadata.usage.input_tokens // 0) | - if type == "number" then . else 0 end) + - ((.usage.output_tokens // .metadata.usage.output_tokens // 0) | - if type == "number" then . else 0 end) - ' "$output_file" 2>/dev/null) - echo "${tokens:-0}" -} - -# Helper function: update_token_count (extracted from ralph_loop.sh) -update_token_count() { - local output_file=$1 - local new_tokens - new_tokens=$(extract_token_usage "$output_file") - if [[ "$new_tokens" -gt 0 ]] 2>/dev/null; then - local current - current=$(cat "$TOKEN_COUNT_FILE" 2>/dev/null || echo "0") - echo $(( current + new_tokens )) > "$TOKEN_COUNT_FILE" - fi -} - -# Helper function: can_make_call (extracted from ralph_loop.sh) -can_make_call() { - local calls_made=0 - if [[ -f "$CALL_COUNT_FILE" ]]; then - calls_made=$(cat "$CALL_COUNT_FILE") - fi - - if [[ $calls_made -ge $MAX_CALLS_PER_HOUR ]]; then - return 1 # Cannot make call — invocation limit reached - fi - - if [[ "${MAX_TOKENS_PER_HOUR:-0}" -gt 0 ]] 2>/dev/null; then - local tokens_used=0 - tokens_used=$(cat "$TOKEN_COUNT_FILE" 2>/dev/null || echo "0") - if [[ $tokens_used -ge $MAX_TOKENS_PER_HOUR ]]; then - return 1 # Cannot make call — token limit reached - fi - fi - - return 0 # Can make call -} - -# Helper function: increment_call_counter (extracted from ralph_loop.sh) -increment_call_counter() { - local calls_made=0 - if [[ -f "$CALL_COUNT_FILE" ]]; then - calls_made=$(cat "$CALL_COUNT_FILE") - fi - - ((calls_made++)) - echo "$calls_made" > "$CALL_COUNT_FILE" - echo "$calls_made" -} - # Test 1: can_make_call returns success when under limit @test "can_make_call returns success when under limit" { echo "50" > "$CALL_COUNT_FILE" diff --git a/tests/unit/test_session_continuity.bats b/tests/unit/test_session_continuity.bats index 23b837b1..95ac8979 100644 --- a/tests/unit/test_session_continuity.bats +++ b/tests/unit/test_session_continuity.bats @@ -46,6 +46,15 @@ setup() { source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh" source "${BATS_TEST_DIRNAME}/../../lib/circuit_breaker.sh" + # Exercise the real reset implementation, including its history dependency. + local function_name function_body + for function_name in reset_session log_session_transition; do + function_body=$(sed -n "/^${function_name}() {/,/^}/p" "${BATS_TEST_DIRNAME}/../../ralph_loop.sh") + [[ -n "$function_body" ]] || return 1 + eval "$function_body" + declare -F "$function_name" > /dev/null || return 1 + done + # Define color variables for log_status RED='\033[0;31m' GREEN='\033[0;32m' @@ -572,8 +581,6 @@ EOF local completion_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE") [[ "$completion_count" == "3" ]] - # Source ralph_loop.sh to get reset_session function - # We need to mock some things to prevent full initialization export RALPH_SESSION_HISTORY_FILE="$RALPH_DIR/.ralph_session_history" export RESPONSE_ANALYSIS_FILE="$RALPH_DIR/.response_analysis" @@ -581,37 +588,6 @@ EOF echo '{"analysis": {"exit_signal": true}}' > "$RESPONSE_ANALYSIS_FILE" [[ -f "$RESPONSE_ANALYSIS_FILE" ]] - # Define reset_session inline for testing (extracted from ralph_loop.sh) - reset_session() { - local reason=${1:-"manual_reset"} - local reset_timestamp - reset_timestamp=$(get_iso_timestamp) - - jq -n \ - --arg session_id "" \ - --arg created_at "" \ - --arg last_used "" \ - --arg reset_at "$reset_timestamp" \ - --arg reset_reason "$reason" \ - '{ - session_id: $session_id, - created_at: $created_at, - last_used: $last_used, - reset_at: $reset_at, - reset_reason: $reset_reason - }' > "$RALPH_SESSION_FILE" - - rm -f "$CLAUDE_SESSION_FILE" 2>/dev/null - - # Issue #91 fix: Clear exit signals - if [[ -f "$EXIT_SIGNALS_FILE" ]]; then - echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" - fi - - # Clear response analysis - rm -f "$RESPONSE_ANALYSIS_FILE" 2>/dev/null - } - # Call reset_session reset_session "test_reset" @@ -649,35 +625,6 @@ EOF local exit_signal=$(jq -r '.analysis.exit_signal' "$RESPONSE_ANALYSIS_FILE") [[ "$exit_signal" == "true" ]] - # Define reset_session with the fix - reset_session() { - local reason=${1:-"manual_reset"} - local reset_timestamp - reset_timestamp=$(get_iso_timestamp) - - jq -n \ - --arg session_id "" \ - --arg created_at "" \ - --arg last_used "" \ - --arg reset_at "$reset_timestamp" \ - --arg reset_reason "$reason" \ - '{ - session_id: $session_id, - created_at: $created_at, - last_used: $last_used, - reset_at: $reset_at, - reset_reason: $reset_reason - }' > "$RALPH_SESSION_FILE" - - rm -f "$CLAUDE_SESSION_FILE" 2>/dev/null - - # Issue #91 fix - if [[ -f "$EXIT_SIGNALS_FILE" ]]; then - echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" - fi - rm -f "$RESPONSE_ANALYSIS_FILE" 2>/dev/null - } - # User runs --reset-session reset_session "manual_reset" From 29be2aee42a224e29c80fb0064b6520e4de00cc8 Mon Sep 17 00:00:00 2001 From: Frank Bria <136862992+frankbria@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:49:38 -0700 Subject: [PATCH 2/2] test(loop): pin the five-indicator safety breaker; document real-function tests (#374) With the real should_exit_gracefully under test, a 5->500 threshold mutant of its safety breaker still passed (opencode review). Add the positive case next to the explicit-false boundary test, and record the real-function rule in CLAUDE.md's Test Suite section. --- CLAUDE.md | 1 + tests/unit/test_exit_detection.bats | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index ebd724af..e00aa8da 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -320,6 +320,7 @@ Multi-layered defense against Claude deleting Ralph's own config: Tests use bats, organized under `tests/unit/`, `tests/integration/`, and `tests/e2e/` (helpers in `tests/helpers/`). Run via `npm test` (all), `npm run test:unit` / `test:integration` / `test:e2e`, or `bats ` for one file. `npm test` reports the current count. - File naming maps to subject: e.g. `test_circuit_breaker_recovery.bats`, `test_cli_modern.bats`, `test_exit_detection.bats`, `test_enable_core.bats` — add tests to the file matching the component you changed +- Unit tests exercise the REAL `ralph_loop.sh` functions: extract with `sed -n '/^fn() {/,/^}/p' ralph_loop.sh` and `eval` in `setup()`, guarded by a non-empty body check plus `declare -F` (or source the script after `set --`). Never use a hand-maintained inline copy; copies let production regressions pass green (Issue #374) - `tests/e2e/test_full_loop.bats` runs ralph_loop.sh as a real subprocess with an executable mock `claude` CLI (`tests/e2e/helpers/e2e_helper.bash`). The mock must take >1s per call — ralph's early-failure detection treats sub-second exits as startup failures. Raw `.ralph/.call_count` assertions go through `assert_call_count`, which skips only when the run itself crossed an hour boundary (the hourly rate-limit reset legitimately zeroes the counter — Issue #285); `mock_call_count` stays the unconditional invocation proof - **Test pass rate (100%) is the quality gate.** Coverage measurement with kcov is informational only (`COVERAGE_THRESHOLD=0`): kcov cannot instrument subprocesses spawned by bats (see [bats-core#15](https://github.com/bats-core/bats-core/issues/15)) diff --git a/tests/unit/test_exit_detection.bats b/tests/unit/test_exit_detection.bats index 1b9d72c0..4173d7a1 100644 --- a/tests/unit/test_exit_detection.bats +++ b/tests/unit/test_exit_detection.bats @@ -479,6 +479,15 @@ EOF assert_equal "$result" "" } +@test "should_exit_gracefully safety breaker fires at five indicators despite explicit false" { + # The breaker is independent of EXIT_SIGNAL: one indicator past test 26's fixture forces the exit + echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": [1,2,3,4,5]}' > "$EXIT_SIGNALS_FILE" + echo '{"analysis": {"exit_signal": false}}' > "$RESPONSE_ANALYSIS_FILE" + + result=$(should_exit_gracefully) + assert_equal "$result" "safety_circuit_breaker" +} + # Test 27: EXIT_SIGNAL missing from analysis object should default to false @test "should_exit_gracefully defaults to false when exit_signal field missing" { echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": [1,2]}' > "$EXIT_SIGNALS_FILE"