From f0fde5a360e1f909c435bc45e22c4746487a449c Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Sat, 18 Jul 2026 15:09:59 -0700 Subject: [PATCH 1/3] fix: wire Actions secret into device evals --- .github/workflows/eval.yml | 3 +++ scripts/run-eval-suite.sh | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index f8c0bb3..c88928d 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -44,12 +44,15 @@ jobs: - name: Run trajectory smoke suite env: INPUT_GOALS: ${{ inputs.goals }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | set -euxo pipefail ./scripts/run-eval-suite.sh - name: Run agent benchmark if: ${{ github.event_name == 'schedule' || inputs.run_benchmark == 'true' }} + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | set -euxo pipefail ./scripts/run-agent-benchmark.sh \ diff --git a/scripts/run-eval-suite.sh b/scripts/run-eval-suite.sh index e3ca0d9..7639a0d 100755 --- a/scripts/run-eval-suite.sh +++ b/scripts/run-eval-suite.sh @@ -28,9 +28,9 @@ if ! adb get-state >/dev/null 2>&1; then exit 1 fi -api_key_file="$root/.worktree/secrets/openai_api_key" -if [[ ! -s "$api_key_file" ]]; then - echo "run-eval-suite: FAIL — missing $api_key_file" >&2 +api_key_file="${OPENPHONE_API_KEY_FILE:-$root/.worktree/secrets/openai_api_key}" +if [[ -z "${OPENAI_API_KEY:-}" && ! -s "$api_key_file" ]]; then + echo "run-eval-suite: FAIL — set OPENAI_API_KEY or provide $api_key_file" >&2 exit 1 fi @@ -62,12 +62,13 @@ for goal in "${GOALS[@]}"; do echo echo "===== smoke $i/${#GOALS[@]} =====" echo "goal: $goal" - ./scripts/run-assistant-task.sh \ - --goal "$goal" \ - --api-key-file "$api_key_file" \ - --wait 35 || { - echo "run-eval-suite: task $i failed; continuing" - } + run_args=(--goal "$goal" --wait 35) + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + run_args+=(--api-key-file "$api_key_file") + fi + ./scripts/run-assistant-task.sh "${run_args[@]}" || { + echo "run-eval-suite: task $i failed; continuing" + } # Pull the latest trajectory from the device into the artifact dir. # `pull-latest-trajectory.sh` requires an explicit Export Trace from the From 9b27ac97aa66fdd3ad71411dc7bcbe8dfb272105 Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Sat, 18 Jul 2026 15:09:59 -0700 Subject: [PATCH 2/3] feat: add fast GCP assistant build lane --- .github/workflows/gcp-cache-refresh.yml | 2 + .github/workflows/gcp-lab.yml | 27 ++++++ docs/BUILD.md | 15 +++ docs/TESTING.md | 10 ++ scripts/check.sh | 42 +++++++++ scripts/lab/gcp/common.sh | 38 +++++++- scripts/lab/gcp/create-vm.sh | 15 ++- scripts/lab/gcp/prewarm-cache.sh | 116 +++++++++++++++--------- scripts/lab/gcp/run-smoke.sh | 98 ++++++++++++++++---- 9 files changed, 304 insertions(+), 59 deletions(-) diff --git a/.github/workflows/gcp-cache-refresh.yml b/.github/workflows/gcp-cache-refresh.yml index 1610c61..d9a3315 100644 --- a/.github/workflows/gcp-cache-refresh.yml +++ b/.github/workflows/gcp-cache-refresh.yml @@ -63,6 +63,7 @@ jobs: OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }} OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }} OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }} + OPENPHONE_GCP_FALLBACK_ZONES: ${{ vars.OPENPHONE_GCP_FALLBACK_ZONES || '' }} OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }} OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }} OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }} @@ -155,6 +156,7 @@ jobs: --ref "${{ steps.plan.outputs.ref }}" --project "${OPENPHONE_GCP_PROJECT}" --zone "${OPENPHONE_GCP_ZONE}" + --fallback-zones "${OPENPHONE_GCP_FALLBACK_ZONES}" --machine-type "${OPENPHONE_GCP_MACHINE_TYPE}" --boot-disk-size "${OPENPHONE_GCP_BOOT_DISK_SIZE}" --boot-disk-type "${OPENPHONE_GCP_BOOT_DISK_TYPE}" diff --git a/.github/workflows/gcp-lab.yml b/.github/workflows/gcp-lab.yml index 6c93601..e118383 100644 --- a/.github/workflows/gcp-lab.yml +++ b/.github/workflows/gcp-lab.yml @@ -27,6 +27,7 @@ on: options: - incremental-emulator - smoke-only + - assistant-apk - export-emulator-image - custom arch: @@ -82,6 +83,14 @@ on: options: - "false" - "true" + export_assistant_apk: + description: "Custom lane only: build and upload OpenPhoneAssistant.apk" + required: true + default: "false" + type: choice + options: + - "false" + - "true" run_smoke: description: "Custom lane only: boot emulator and run smoke after build/export" required: true @@ -129,6 +138,7 @@ jobs: OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }} OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }} OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }} + OPENPHONE_GCP_FALLBACK_ZONES: ${{ vars.OPENPHONE_GCP_FALLBACK_ZONES || '' }} OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }} OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }} OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }} @@ -201,6 +211,7 @@ jobs: cache_disk="" keep_vm="false" lane="incremental-emulator" + export_assistant_apk="false" export_emulator_image="false" run_smoke="true" skip_build="false" @@ -234,21 +245,31 @@ jobs: lane="${{ inputs.lane }}" case "$lane" in incremental-emulator) + export_assistant_apk="false" export_emulator_image="false" run_smoke="true" skip_build="false" ;; smoke-only) + export_assistant_apk="false" export_emulator_image="false" run_smoke="true" skip_build="true" ;; + assistant-apk) + export_assistant_apk="true" + export_emulator_image="false" + run_smoke="false" + skip_build="false" + ;; export-emulator-image) + export_assistant_apk="false" export_emulator_image="true" run_smoke="false" skip_build="false" ;; custom) + export_assistant_apk="${{ inputs.export_assistant_apk }}" export_emulator_image="${{ inputs.export_emulator_image }}" run_smoke="${{ inputs.run_smoke }}" skip_build="${{ inputs.skip_build }}" @@ -274,6 +295,7 @@ jobs: echo "cache_disk=$cache_disk" echo "cache_source_snapshot=$cache_source_snapshot" echo "keep_vm=$keep_vm" + echo "export_assistant_apk=$export_assistant_apk" echo "export_emulator_image=$export_emulator_image" echo "run_smoke=$run_smoke" echo "skip_build=$skip_build" @@ -289,6 +311,7 @@ jobs: --ref "${{ steps.target.outputs.ref }}" --project "${OPENPHONE_GCP_PROJECT}" --zone "${OPENPHONE_GCP_ZONE}" + --fallback-zones "${OPENPHONE_GCP_FALLBACK_ZONES}" --machine-type "${{ steps.target.outputs.machine_type }}" --boot-disk-size "${{ steps.target.outputs.boot_disk_size }}" --boot-disk-type "${{ steps.target.outputs.boot_disk_type }}" @@ -304,6 +327,9 @@ jobs: if [[ "${{ steps.target.outputs.keep_vm }}" == "true" ]]; then args+=(--keep-vm) fi + if [[ "${{ steps.target.outputs.export_assistant_apk }}" == "true" ]]; then + args+=(--export-assistant-apk) + fi if [[ "${{ steps.target.outputs.export_emulator_image }}" == "true" ]]; then args+=(--export-emulator-image) fi @@ -340,6 +366,7 @@ jobs: echo "- Cache mode: \`${{ steps.target.outputs.cache_mode }}\`" echo "- Cache disk: \`${{ steps.target.outputs.cache_disk || 'none' }}\`" echo "- Cache snapshot: \`${{ steps.target.outputs.cache_source_snapshot || 'none' }}\`" + echo "- Export assistant APK: \`${{ steps.target.outputs.export_assistant_apk }}\`" echo "- Export emulator image: \`${{ steps.target.outputs.export_emulator_image }}\`" echo "- Run smoke: \`${{ steps.target.outputs.run_smoke }}\`" echo "- Skip build: \`${{ steps.target.outputs.skip_build }}\`" diff --git a/docs/BUILD.md b/docs/BUILD.md index a520ce3..2709b03 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -224,6 +224,11 @@ setting. The pre-sync scaffold check may skip the standalone Java check when the full Android build provides the authoritative compiler/toolchain validation. +When the primary Compute zone is temporarily out of the configured machine +type, set `OPENPHONE_GCP_FALLBACK_ZONES` to a comma-separated list of alternate +zones. Disposable snapshot labs and scheduled cache refreshes retry only +capacity-stockout failures. Other failures remain fail-fast. + Current generic-target status: - `OPENPHONE_BUILD_GOAL=droid ./scripts/build.sh openphone_arm64` has been @@ -281,6 +286,16 @@ OPENPHONE_BUILD_GOAL=OpenPhoneAssistant \ ./scripts/build.sh openphone_tegu-bp4a-userdebug ``` +When a persistent build host is not available, dispatch the `GCP Lab` workflow +with lane `assistant-apk`. It restores the warm Android snapshot, builds only +`OpenPhoneAssistant`, and uploads: + +```text +artifacts/assistant-apk/OpenPhoneAssistant.apk +artifacts/assistant-apk/OpenPhoneAssistant.apk.sha256 +artifacts/assistant-apk/source-ref.txt +``` + Copy the resulting APK back to the host, then push it into `/system_ext` without recovery. Because `org.openphone.assistant` is a persistent privileged app, Android rejects normal `adb install -r`; enable Developer Options -> Rooted diff --git a/docs/TESTING.md b/docs/TESTING.md index aef434e..dbd33f2 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -161,6 +161,10 @@ the legacy self-hosted runner. boots the emulator, and runs smoke. - `smoke-only`: fastest manual lane. Restores the newest warm snapshot and boots/smokes already-built outputs without rebuilding. +- `assistant-apk`: builds only the privileged assistant module and uploads + `OpenPhoneAssistant.apk`, its SHA-256 sidecar, and the source ref. Push the + result to an already-booted userdebug device with + `scripts/push-assistant-apk.sh`. - `export-emulator-image`: rebuilds and uploads the SDK system image zip without booting the emulator. - `custom`: exposes the lower-level export, smoke, and skip-build switches. @@ -169,6 +173,12 @@ Automatic maintainer-approved PR lab runs use `incremental-emulator`. Use `smoke-only` only when you intentionally want to test an already-built warm image rather than PR build output. +Set the repository variable `OPENPHONE_GCP_FALLBACK_ZONES` to a comma-separated +list such as `us-central1-a,us-central1-b,us-central1-f`. Lab runs retry those +zones only when Compute reports a capacity stockout; build, test, permission, +and configuration failures still fail immediately. Cache refreshes maintain a +zone-suffixed warm disk when they have to move away from the primary zone. + ## Device Check No physical device is supported until its `docs/devices/.md` diff --git a/scripts/check.sh b/scripts/check.sh index 15471b5..b3d4613 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -291,6 +291,32 @@ grep -q -- '--tunnel-through-iap' "$gcp_common" || { printf 'GCP lab SSH/SCP helpers must pass --tunnel-through-iap\n' >&2 exit 1 } +OPENPHONE_GCP_TUNNEL_THROUGH_IAP=TRUE \ + bash -c 'source "$1"; gcp_use_iap_tunnel' _ "$gcp_common" || { + printf 'GCP IAP setting parsing must work with the host Bash version\n' >&2 + exit 1 +} +zone_candidates="$( + bash -c 'source "$1"; gcp_zone_candidates us-central1-c "us-central1-a,us-central1-c us-central1-b"' \ + _ "$gcp_common" +)" +[[ "$zone_candidates" == $'us-central1-c\nus-central1-a\nus-central1-b' ]] || { + printf 'GCP fallback zone planning must preserve order and remove duplicates\n' >&2 + exit 1 +} +bash -c 'source "$1"; gcp_is_capacity_error "code: ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS"' \ + _ "$gcp_common" || { + printf 'GCP capacity-error detection must recognize zone stockouts\n' >&2 + exit 1 +} +grep -q 'OPENPHONE_GCP_FALLBACK_ZONES' "$root/.github/workflows/gcp-lab.yml" || { + printf 'GCP lab workflow must pass configured fallback zones\n' >&2 + exit 1 +} +grep -q 'OPENPHONE_GCP_FALLBACK_ZONES' "$root/.github/workflows/gcp-cache-refresh.yml" || { + printf 'GCP cache refresh workflow must pass configured fallback zones\n' >&2 + exit 1 +} for gcp_script in \ "$root/scripts/lab/gcp/bootstrap-vm.sh" \ "$root/scripts/lab/gcp/run-release.sh" \ @@ -341,6 +367,22 @@ grep -q 'lane:' "$root/.github/workflows/gcp-lab.yml" || { printf 'GCP lab workflow must expose explicit lab lanes\n' >&2 exit 1 } +grep -q 'assistant-apk' "$root/.github/workflows/gcp-lab.yml" || { + printf 'GCP lab workflow must expose the focused assistant APK lane\n' >&2 + exit 1 +} +grep -q -- '--export-assistant-apk' "$root/scripts/lab/gcp/run-smoke.sh" || { + printf 'GCP lab smoke helper must support focused assistant APK export\n' >&2 + exit 1 +} +grep -Fq 'artifacts/.' "$root/scripts/lab/gcp/run-smoke.sh" || { + printf 'GCP lab smoke helper must copy artifact contents without an extra directory level\n' >&2 + exit 1 +} +grep -q 'exit 75' "$root/scripts/lab/gcp/run-smoke.sh" || { + printf 'GCP lab smoke helper must preserve retryable capacity status for its callers\n' >&2 + exit 1 +} if grep -q '^ push:' "$root/.github/workflows/emulator.yml"; then printf 'legacy self-hosted emulator workflow must be manual-only; use GCP Lab for trusted gates\n' >&2 exit 1 diff --git a/scripts/lab/gcp/common.sh b/scripts/lab/gcp/common.sh index ee98426..f099de5 100644 --- a/scripts/lab/gcp/common.sh +++ b/scripts/lab/gcp/common.sh @@ -10,6 +10,7 @@ source "$root/scripts/common.sh" OPENPHONE_GCP_PROJECT="${OPENPHONE_GCP_PROJECT:-openphone-lab}" OPENPHONE_GCP_REGION="${OPENPHONE_GCP_REGION:-us-central1}" OPENPHONE_GCP_ZONE="${OPENPHONE_GCP_ZONE:-us-central1-a}" +OPENPHONE_GCP_FALLBACK_ZONES="${OPENPHONE_GCP_FALLBACK_ZONES:-}" OPENPHONE_GCP_MACHINE_TYPE="${OPENPHONE_GCP_MACHINE_TYPE:-c3-standard-22}" OPENPHONE_GCP_BOOT_DISK_SIZE="${OPENPHONE_GCP_BOOT_DISK_SIZE:-1000GB}" OPENPHONE_GCP_BOOT_DISK_TYPE="${OPENPHONE_GCP_BOOT_DISK_TYPE:-pd-ssd}" @@ -48,6 +49,39 @@ shell_quote() { printf '%q' "$1" } +gcp_zone_candidates() { + local primary_zone="${1:-$OPENPHONE_GCP_ZONE}" + local fallback_zones="${2:-$OPENPHONE_GCP_FALLBACK_ZONES}" + local candidate + local seen + local duplicate + local -a emitted=() + + fallback_zones="${fallback_zones//,/ }" + for candidate in "$primary_zone" $fallback_zones; do + [[ -n "$candidate" ]] || continue + duplicate=false + for seen in "${emitted[@]-}"; do + if [[ "$seen" == "$candidate" ]]; then + duplicate=true + break + fi + done + if [[ "$duplicate" == true ]]; then + continue + fi + emitted+=("$candidate") + printf '%s\n' "$candidate" + done +} + +gcp_is_capacity_error() { + local message="$1" + grep -Eq \ + 'ZONE_RESOURCE_POOL_EXHAUSTED|RESOURCE_POOL_EXHAUSTED|resource pool exhausted|reason: stockout|currently unavailable in the .* zone' \ + <<<"$message" +} + gcp_instance_exists() { local name="$1" project="$2" zone="$3" gcloud compute instances describe "$name" \ @@ -56,7 +90,9 @@ gcp_instance_exists() { } gcp_use_iap_tunnel() { - case "${OPENPHONE_GCP_TUNNEL_THROUGH_IAP,,}" in + local value + value="$(printf '%s' "$OPENPHONE_GCP_TUNNEL_THROUGH_IAP" | tr '[:upper:]' '[:lower:]')" + case "$value" in 1|true|yes|on) return 0 ;; 0|false|no|off) return 1 ;; *) die "OPENPHONE_GCP_TUNNEL_THROUGH_IAP must be 1/true or 0/false" ;; diff --git a/scripts/lab/gcp/create-vm.sh b/scripts/lab/gcp/create-vm.sh index 434a01a..f7d1128 100755 --- a/scripts/lab/gcp/create-vm.sh +++ b/scripts/lab/gcp/create-vm.sh @@ -11,6 +11,7 @@ usage() { Usage: scripts/lab/gcp/create-vm.sh [options] Creates one disposable OpenPhone lab VM inside the shared GCP lab project. +Returns exit status 75 when GCP reports a retryable zone-capacity stockout. Options: --name VM name. Default: generated. @@ -212,6 +213,18 @@ if [[ "$cache_mode" == "attach-disk" || "$cache_mode" == "snapshot" ]]; then fi info "Creating GCP lab VM: $name" -gcloud "${args[@]}" +set +e +create_output="$(gcloud "${args[@]}" 2>&1)" +create_status=$? +set -e +if [[ "$create_status" -ne 0 ]]; then + printf '%s\n' "$create_output" >&2 + if gcp_is_capacity_error "$create_output"; then + printf 'GCP zone capacity is temporarily unavailable: %s\n' "$zone" >&2 + exit 75 + fi + exit "$create_status" +fi +printf '%s\n' "$create_output" created_cache_disk=false printf '%s\n' "$name" diff --git a/scripts/lab/gcp/prewarm-cache.sh b/scripts/lab/gcp/prewarm-cache.sh index ce065df..e949a18 100755 --- a/scripts/lab/gcp/prewarm-cache.sh +++ b/scripts/lab/gcp/prewarm-cache.sh @@ -21,6 +21,9 @@ Options: --ref Git ref/SHA to prewarm. Default: current HEAD. --project GCP project. Default: OPENPHONE_GCP_PROJECT. --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --fallback-zones Comma-separated zones tried after a capacity + stockout. Alternate zones use a zone-suffixed + persistent cache disk seeded from the snapshot. --machine-type Machine type. Default: c3-standard-22. --boot-disk-size Boot disk size. Default: 1000GB. --boot-disk-type Boot disk type. Default: pd-ssd. @@ -47,6 +50,7 @@ repo_url="$default_repo_url" ref="$default_ref" project="$OPENPHONE_GCP_PROJECT" zone="$OPENPHONE_GCP_ZONE" +fallback_zones="$OPENPHONE_GCP_FALLBACK_ZONES" machine_type="$OPENPHONE_GCP_MACHINE_TYPE" boot_disk_size="$OPENPHONE_GCP_BOOT_DISK_SIZE" boot_disk_type="$OPENPHONE_GCP_BOOT_DISK_TYPE" @@ -98,6 +102,11 @@ while [[ $# -gt 0 ]]; do zone="$2" shift 2 ;; + --fallback-zones) + [[ $# -ge 2 ]] || die "--fallback-zones requires a value" + fallback_zones="$2" + shift 2 + ;; --machine-type) [[ $# -ge 2 ]] || die "--machine-type requires a value" machine_type="$2" @@ -186,52 +195,76 @@ if gcloud compute snapshots describe "$snapshot" \ die "snapshot already exists: $snapshot" fi -if ! gcloud compute disks describe "$cache_disk" \ - --project "$project" \ - --zone "$zone" >/dev/null 2>&1; then - create_args=( - compute disks create "$cache_disk" +short_ref="$(printf '%s' "$ref" | cut -c1-12)" +name="$(sanitize_gcp_name "openphone-lab-prewarm-${short_ref}-$(date -u +%H%M%S)")" +primary_zone="$zone" +selected_zone="" +selected_cache_disk="" +while IFS= read -r candidate_zone; do + candidate_cache_disk="$cache_disk" + if [[ "$candidate_zone" != "$primary_zone" ]]; then + candidate_cache_disk="$(sanitize_gcp_name "${cache_disk}-${candidate_zone}")" + fi + + if ! gcloud compute disks describe "$candidate_cache_disk" \ + --project "$project" \ + --zone "$candidate_zone" >/dev/null 2>&1; then + create_args=( + compute disks create "$candidate_cache_disk" + --project "$project" + --zone "$candidate_zone" + --type "$cache_disk_type" + --size "$cache_disk_size" + --labels "app=openphone,purpose=lab-cache,managed-by=codex" + ) + if [[ -n "$source_snapshot" ]]; then + create_args+=(--source-snapshot "$source_snapshot") + fi + info "Creating cache disk in $candidate_zone: $candidate_cache_disk" + gcloud "${create_args[@]}" + else + info "Reusing cache disk in $candidate_zone: $candidate_cache_disk" + fi + + run_args=( + --name "$name" + --repo-url "$repo_url" + --ref "$ref" --project "$project" - --zone "$zone" - --type "$cache_disk_type" - --size "$cache_disk_size" - --labels "app=openphone,purpose=lab-cache,managed-by=codex" + --zone "$candidate_zone" + --machine-type "$machine_type" + --boot-disk-size "$boot_disk_size" + --boot-disk-type "$boot_disk_type" + --cache-mode attach-disk + --cache-disk "$candidate_cache_disk" + --arch "$arch" + --variant "$variant" + --runtime local + --timeout "$timeout_seconds" ) - if [[ -n "$source_snapshot" ]]; then - create_args+=(--source-snapshot "$source_snapshot") + if [[ -n "$repo_sync_jobs" ]]; then + run_args+=(--repo-sync-jobs "$repo_sync_jobs") + fi + if [[ "$keep_vm" == true ]]; then + run_args+=(--keep-vm) fi - info "Creating cache disk: $cache_disk" - gcloud "${create_args[@]}" -else - info "Reusing cache disk: $cache_disk" -fi -short_ref="$(printf '%s' "$ref" | cut -c1-12)" -name="$(sanitize_gcp_name "openphone-lab-prewarm-${short_ref}-$(date -u +%H%M%S)")" -run_args=( - --name "$name" - --repo-url "$repo_url" - --ref "$ref" - --project "$project" - --zone "$zone" - --machine-type "$machine_type" - --boot-disk-size "$boot_disk_size" - --boot-disk-type "$boot_disk_type" - --cache-mode attach-disk - --cache-disk "$cache_disk" - --arch "$arch" - --variant "$variant" - --runtime local - --timeout "$timeout_seconds" -) -if [[ -n "$repo_sync_jobs" ]]; then - run_args+=(--repo-sync-jobs "$repo_sync_jobs") -fi -if [[ "$keep_vm" == true ]]; then - run_args+=(--keep-vm) -fi + if "$script_dir/run-smoke.sh" "${run_args[@]}"; then + selected_zone="$candidate_zone" + selected_cache_disk="$candidate_cache_disk" + break + else + run_status=$? + fi + if [[ "$run_status" -ne 75 ]]; then + exit "$run_status" + fi + info "Retrying cache refresh in the next fallback zone" +done < <(gcp_zone_candidates "$zone" "$fallback_zones") -"$script_dir/run-smoke.sh" "${run_args[@]}" +[[ -n "$selected_zone" ]] || die "GCP cache refresh capacity unavailable in every configured zone" +zone="$selected_zone" +cache_disk="$selected_cache_disk" if [[ "$keep_vm" == true ]]; then info "--keep-vm left the cache disk attached; skipping snapshot creation" @@ -249,6 +282,7 @@ gcloud compute snapshots create "$snapshot" \ --labels "app=openphone,purpose=lab-cache,managed-by=codex" printf 'Cache disk: %s\n' "$cache_disk" +printf 'Cache zone: %s\n' "$zone" printf 'Cache snapshot: %s\n' "$snapshot" printf 'Set this repo variable for warm PR/release labs:\n' printf ' OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT=%s\n' "$snapshot" diff --git a/scripts/lab/gcp/run-smoke.sh b/scripts/lab/gcp/run-smoke.sh index 9112f76..d6d584b 100755 --- a/scripts/lab/gcp/run-smoke.sh +++ b/scripts/lab/gcp/run-smoke.sh @@ -21,6 +21,8 @@ Options: --slot Lab slot name on the VM. Default: VM name. --project GCP project. Default: OPENPHONE_GCP_PROJECT. --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --fallback-zones Comma-separated zones tried after a capacity + stockout. Default: OPENPHONE_GCP_FALLBACK_ZONES. --machine-type Machine type. Default: c3-standard-22. --boot-disk-size Boot disk size. Default: 1000GB. --boot-disk-type Boot disk type. Default: pd-ssd. @@ -39,6 +41,8 @@ Options: --repo-sync-jobs repo sync jobs. Default: nproc. --keep-vm Leave VM running for debug. --skip-build Reuse existing Android build outputs on the VM. + --export-assistant-apk Build OpenPhoneAssistant only and copy the APK + into lab artifacts for fast device/emulator pushes. --export-emulator-image Copy sdk-repo-linux-system-images.zip into lab artifacts for local Mac/SDK installation. --skip-smoke Build/export artifacts but do not boot the @@ -61,6 +65,7 @@ ref="$default_ref" slot="" project="$OPENPHONE_GCP_PROJECT" zone="$OPENPHONE_GCP_ZONE" +fallback_zones="$OPENPHONE_GCP_FALLBACK_ZONES" machine_type="$OPENPHONE_GCP_MACHINE_TYPE" boot_disk_size="$OPENPHONE_GCP_BOOT_DISK_SIZE" boot_disk_type="$OPENPHONE_GCP_BOOT_DISK_TYPE" @@ -74,6 +79,7 @@ timeout_seconds=900 repo_sync_jobs="" keep_vm=false skip_build=false +export_assistant_apk=false export_emulator_image=false skip_smoke=false runtimes=() @@ -110,6 +116,11 @@ while [[ $# -gt 0 ]]; do zone="$2" shift 2 ;; + --fallback-zones) + [[ $# -ge 2 ]] || die "--fallback-zones requires a value" + fallback_zones="$2" + shift 2 + ;; --machine-type) [[ $# -ge 2 ]] || die "--machine-type requires a value" machine_type="$2" @@ -178,6 +189,10 @@ while [[ $# -gt 0 ]]; do skip_build=true shift ;; + --export-assistant-apk) + export_assistant_apk=true + shift + ;; --export-emulator-image) export_emulator_image=true shift @@ -245,7 +260,7 @@ remote_script="$(mktemp "${TMPDIR:-/tmp}/openphone-gcp-remote.XXXXXX")" copy_remote_artifacts() { mkdir -p "$artifact_dir" if gcp_compute_scp "$project" "$zone" --recurse \ - "$name:~/openphone-src/.worktree/lab/$slot/artifacts" \ + "$name:~/openphone-src/.worktree/lab/$slot/artifacts/." \ "$artifact_dir/" >/dev/null; then artifacts_copied=true return 0 @@ -267,24 +282,47 @@ cleanup() { } trap cleanup EXIT -create_args=( - --name "$name" - --project "$project" - --zone "$zone" - --machine-type "$machine_type" - --boot-disk-size "$boot_disk_size" - --boot-disk-type "$boot_disk_type" - --cache-mode "$cache_mode" -) -if [[ -n "$cache_disk" ]]; then - create_args+=(--cache-disk "$cache_disk") -fi -if [[ -n "$cache_source_snapshot" ]]; then - create_args+=(--cache-source-snapshot "$cache_source_snapshot") +if [[ "$cache_mode" == "attach-disk" ]]; then + fallback_zones="" fi -"$script_dir/create-vm.sh" "${create_args[@]}" +selected_zone="" +while IFS= read -r candidate_zone; do + create_args=( + --name "$name" + --project "$project" + --zone "$candidate_zone" + --machine-type "$machine_type" + --boot-disk-size "$boot_disk_size" + --boot-disk-type "$boot_disk_type" + --cache-mode "$cache_mode" + ) + if [[ -n "$cache_disk" ]]; then + create_args+=(--cache-disk "$cache_disk") + fi + if [[ -n "$cache_source_snapshot" ]]; then + create_args+=(--cache-source-snapshot "$cache_source_snapshot") + fi + + if "$script_dir/create-vm.sh" "${create_args[@]}"; then + selected_zone="$candidate_zone" + break + else + create_status=$? + fi + if [[ "$create_status" -ne 75 ]]; then + exit "$create_status" + fi + info "Retrying GCP lab VM in the next fallback zone" +done < <(gcp_zone_candidates "$zone" "$fallback_zones") + +if [[ -z "$selected_zone" ]]; then + printf 'error: GCP VM capacity unavailable in every configured zone\n' >&2 + exit 75 +fi +zone="$selected_zone" vm_created=true +info "GCP lab selected zone: $zone" "$script_dir/bootstrap-vm.sh" --name "$name" --project "$project" --zone "$zone" @@ -300,6 +338,7 @@ variant="${OPENPHONE_EMULATOR_VARIANT:?}" timeout_seconds="${OPENPHONE_EMULATOR_TIMEOUT:?}" repo_sync_jobs="${OPENPHONE_REPO_SYNC_JOBS:-}" skip_build="${OPENPHONE_SKIP_BUILD:-0}" +export_assistant_apk="${OPENPHONE_EXPORT_ASSISTANT_APK:-0}" export_emulator_image="${OPENPHONE_EXPORT_EMULATOR_IMAGE:-0}" skip_smoke="${OPENPHONE_SKIP_SMOKE:-0}" runtime_csv="${OPENPHONE_LAB_RUNTIMES:-local}" @@ -427,6 +466,28 @@ if [[ "$export_emulator_image" == "1" && "$skip_build" != "1" ]]; then skip_build=1 fi +if [[ "$export_assistant_apk" == "1" ]]; then + if [[ "$skip_build" != "1" ]]; then + OPENPHONE_BUILD_GOAL=OpenPhoneAssistant \ + ./scripts/build-emulator.sh --arch "$arch" --variant "$variant" + fi + + product_dir="$(emulator_product_dir)" + assistant_apk="$OPENPHONE_ANDROID_DIR/out/target/product/$product_dir/system_ext/priv-app/OpenPhoneAssistant/OpenPhoneAssistant.apk" + artifact_apk_dir="$HOME/openphone-src/.worktree/lab/$slot/artifacts/assistant-apk" + mkdir -p "$artifact_apk_dir" + if [[ ! -f "$assistant_apk" ]]; then + printf 'error: focused assistant APK not found: %s\n' "$assistant_apk" >&2 + exit 1 + fi + cp "$assistant_apk" "$artifact_apk_dir/OpenPhoneAssistant.apk" + ( + cd "$artifact_apk_dir" + sha256sum OpenPhoneAssistant.apk > OpenPhoneAssistant.apk.sha256 + git -C "$HOME/openphone-src" rev-parse HEAD > source-ref.txt + ) +fi + if [[ "$export_emulator_image" == "1" ]]; then product_dir="$(emulator_product_dir)" image_zip="$OPENPHONE_ANDROID_DIR/out/target/product/$product_dir/sdk-repo-linux-system-images.zip" @@ -475,6 +536,10 @@ export_emulator_image_value=0 if [[ "$export_emulator_image" == true ]]; then export_emulator_image_value=1 fi +export_assistant_apk_value=0 +if [[ "$export_assistant_apk" == true ]]; then + export_assistant_apk_value=1 +fi skip_smoke_value=0 if [[ "$skip_smoke" == true ]]; then skip_smoke_value=1 @@ -487,6 +552,7 @@ remote_command+=" OPENPHONE_EMULATOR_VARIANT=$(shell_quote "$variant")" remote_command+=" OPENPHONE_EMULATOR_TIMEOUT=$(shell_quote "$timeout_seconds")" remote_command+=" OPENPHONE_REPO_SYNC_JOBS=$(shell_quote "$repo_sync_jobs")" remote_command+=" OPENPHONE_SKIP_BUILD=$(shell_quote "$skip_build_value")" +remote_command+=" OPENPHONE_EXPORT_ASSISTANT_APK=$(shell_quote "$export_assistant_apk_value")" remote_command+=" OPENPHONE_EXPORT_EMULATOR_IMAGE=$(shell_quote "$export_emulator_image_value")" remote_command+=" OPENPHONE_SKIP_SMOKE=$(shell_quote "$skip_smoke_value")" remote_command+=" OPENPHONE_LAB_RUNTIMES=$(shell_quote "$runtime_csv")" From e9abc35f74c8d5c2ef41a9946f122842aa5a4a87 Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Sat, 18 Jul 2026 15:53:55 -0700 Subject: [PATCH 3/3] fix: harden GCP fallback retries --- .github/workflows/gcp-cache-refresh.yml | 19 ++- .github/workflows/gcp-lab.yml | 24 +++- scripts/check.sh | 9 ++ scripts/lab/gcp/common.sh | 43 +++++- scripts/lab/gcp/create-vm.sh | 43 +++--- scripts/lab/gcp/prewarm-cache.sh | 28 +++- scripts/lab/gcp/run-smoke.sh | 15 +++ scripts/test-gcp-lab-retries.sh | 165 ++++++++++++++++++++++++ 8 files changed, 324 insertions(+), 22 deletions(-) create mode 100755 scripts/test-gcp-lab-retries.sh diff --git a/.github/workflows/gcp-cache-refresh.yml b/.github/workflows/gcp-cache-refresh.yml index d9a3315..093cba5 100644 --- a/.github/workflows/gcp-cache-refresh.yml +++ b/.github/workflows/gcp-cache-refresh.yml @@ -151,6 +151,16 @@ jobs: shell: bash run: | set -euxo pipefail + result_file="${RUNNER_TEMP}/openphone-gcp-cache-refresh-result.json" + rm -f "$result_file" + capture_selection() { + if [[ -s "$result_file" ]]; then + jq -r \ + '"selected_zone=\(.selected_zone)", "selected_cache_disk=\(.selected_cache_disk)"' \ + "$result_file" >> "$GITHUB_OUTPUT" + fi + } + trap capture_selection EXIT args=( --repo-url "https://github.com/secondly-com/OpenPhone.git" --ref "${{ steps.plan.outputs.ref }}" @@ -164,6 +174,7 @@ jobs: --snapshot "${{ steps.plan.outputs.snapshot }}" --arch "${{ steps.plan.outputs.arch }}" --variant "${{ steps.plan.outputs.variant }}" + --result-file "$result_file" ) if [[ -n "${{ steps.plan.outputs.source_snapshot }}" ]]; then args+=(--source-snapshot "${{ steps.plan.outputs.source_snapshot }}") @@ -203,13 +214,19 @@ jobs: if: always() shell: bash run: | + selected_zone="${{ steps.refresh.outputs.selected_zone }}" + selected_zone="${selected_zone:-$OPENPHONE_GCP_ZONE}" + selected_cache_disk="${{ steps.refresh.outputs.selected_cache_disk }}" + selected_cache_disk="${selected_cache_disk:-${{ steps.plan.outputs.cache_disk }}}" { echo "## GCP cache refresh" echo echo "- Ref: \`${{ steps.plan.outputs.ref }}\`" echo "- Arch: \`${{ steps.plan.outputs.arch }}\`" echo "- Variant: \`${{ steps.plan.outputs.variant }}\`" - echo "- Cache disk: \`${{ steps.plan.outputs.cache_disk }}\`" + echo "- Primary zone: \`${OPENPHONE_GCP_ZONE}\`" + echo "- Selected zone: \`${selected_zone}\`" + echo "- Cache disk: \`${selected_cache_disk}\`" echo "- Snapshot: \`${{ steps.plan.outputs.snapshot }}\`" echo "- Source snapshot: \`${{ steps.plan.outputs.source_snapshot || 'none' }}\`" echo "- Latest snapshot discovery: \`GCP labels\`" diff --git a/.github/workflows/gcp-lab.yml b/.github/workflows/gcp-lab.yml index e118383..77b109a 100644 --- a/.github/workflows/gcp-lab.yml +++ b/.github/workflows/gcp-lab.yml @@ -302,9 +302,20 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Run GCP emulator lab + id: lab shell: bash run: | set -euxo pipefail + result_file="${RUNNER_TEMP}/openphone-gcp-lab-result.json" + rm -f "$result_file" + capture_selection() { + if [[ -s "$result_file" ]]; then + jq -r \ + '"selected_zone=\(.selected_zone)", "selected_cache_disk=\(.selected_cache_disk)"' \ + "$result_file" >> "$GITHUB_OUTPUT" + fi + } + trap capture_selection EXIT args=( --name "${{ steps.target.outputs.name }}" --repo-url "${{ steps.target.outputs.repo_url }}" @@ -317,6 +328,7 @@ jobs: --boot-disk-type "${{ steps.target.outputs.boot_disk_type }}" --cache-mode "${{ steps.target.outputs.cache_mode }}" --arch "${{ steps.target.outputs.arch }}" + --result-file "$result_file" ) if [[ -n "${{ steps.target.outputs.cache_disk }}" ]]; then args+=(--cache-disk "${{ steps.target.outputs.cache_disk }}") @@ -350,12 +362,20 @@ jobs: if: always() shell: bash run: | + selected_zone="${{ steps.lab.outputs.selected_zone }}" + selected_zone="${selected_zone:-$OPENPHONE_GCP_ZONE}" + selected_cache_disk="${{ steps.lab.outputs.selected_cache_disk }}" + if [[ -z "$selected_cache_disk" ]]; then + selected_cache_disk="${{ steps.target.outputs.cache_disk }}" + fi + selected_cache_disk="${selected_cache_disk:-none}" { echo "## GCP emulator lab" echo echo "- Cloud provider: \`GCP\`" echo "- Project: \`${OPENPHONE_GCP_PROJECT}\`" - echo "- Zone: \`${OPENPHONE_GCP_ZONE}\`" + echo "- Primary zone: \`${OPENPHONE_GCP_ZONE}\`" + echo "- Selected zone: \`${selected_zone}\`" echo "- VM: \`${{ steps.target.outputs.name }}\`" echo "- Source ref: \`${{ steps.target.outputs.ref }}\`" echo "- Source repo: \`${{ steps.target.outputs.repo_url }}\`" @@ -364,7 +384,7 @@ jobs: echo "- Emulator arch: \`${{ steps.target.outputs.arch }}\`" echo "- Machine type: \`${{ steps.target.outputs.machine_type }}\`" echo "- Cache mode: \`${{ steps.target.outputs.cache_mode }}\`" - echo "- Cache disk: \`${{ steps.target.outputs.cache_disk || 'none' }}\`" + echo "- Cache disk: \`${selected_cache_disk}\`" echo "- Cache snapshot: \`${{ steps.target.outputs.cache_source_snapshot || 'none' }}\`" echo "- Export assistant APK: \`${{ steps.target.outputs.export_assistant_apk }}\`" echo "- Export emulator image: \`${{ steps.target.outputs.export_emulator_image }}\`" diff --git a/scripts/check.sh b/scripts/check.sh index b3d4613..f824b1e 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -309,6 +309,7 @@ bash -c 'source "$1"; gcp_is_capacity_error "code: ZONE_RESOURCE_POOL_EXHAUSTED_ printf 'GCP capacity-error detection must recognize zone stockouts\n' >&2 exit 1 } +"$root/scripts/test-gcp-lab-retries.sh" grep -q 'OPENPHONE_GCP_FALLBACK_ZONES' "$root/.github/workflows/gcp-lab.yml" || { printf 'GCP lab workflow must pass configured fallback zones\n' >&2 exit 1 @@ -317,6 +318,14 @@ grep -q 'OPENPHONE_GCP_FALLBACK_ZONES' "$root/.github/workflows/gcp-cache-refres printf 'GCP cache refresh workflow must pass configured fallback zones\n' >&2 exit 1 } +grep -q 'steps.lab.outputs.selected_zone' "$root/.github/workflows/gcp-lab.yml" || { + printf 'GCP lab workflow must summarize the selected fallback zone\n' >&2 + exit 1 +} +grep -q 'steps.refresh.outputs.selected_zone' "$root/.github/workflows/gcp-cache-refresh.yml" || { + printf 'GCP cache refresh workflow must summarize the selected fallback zone\n' >&2 + exit 1 +} for gcp_script in \ "$root/scripts/lab/gcp/bootstrap-vm.sh" \ "$root/scripts/lab/gcp/run-release.sh" \ diff --git a/scripts/lab/gcp/common.sh b/scripts/lab/gcp/common.sh index f099de5..c79a98e 100644 --- a/scripts/lab/gcp/common.sh +++ b/scripts/lab/gcp/common.sh @@ -51,7 +51,7 @@ shell_quote() { gcp_zone_candidates() { local primary_zone="${1:-$OPENPHONE_GCP_ZONE}" - local fallback_zones="${2:-$OPENPHONE_GCP_FALLBACK_ZONES}" + local fallback_zones="${2-$OPENPHONE_GCP_FALLBACK_ZONES}" local candidate local seen local duplicate @@ -82,6 +82,47 @@ gcp_is_capacity_error() { <<<"$message" } +gcp_run_with_capacity_status() { + local output + local status + + if output="$("$@" 2>&1)"; then + printf '%s\n' "$output" + return 0 + else + status=$? + fi + + printf '%s\n' "$output" >&2 + if gcp_is_capacity_error "$output"; then + return 75 + fi + return "$status" +} + +gcp_write_selection_result() { + local result_file="$1" + local vm_name="$2" + local selected_zone="$3" + local selected_cache_disk="${4:-}" + local result_tmp + + [[ -n "$result_file" ]] || return 0 + mkdir -p "$(dirname "$result_file")" + result_tmp="${result_file}.tmp.$$" + python3 - "$vm_name" "$selected_zone" "$selected_cache_disk" >"$result_tmp" <<'PY' +import json +import sys + +print(json.dumps({ + "vm_name": sys.argv[1], + "selected_zone": sys.argv[2], + "selected_cache_disk": sys.argv[3], +}, sort_keys=True)) +PY + mv "$result_tmp" "$result_file" +} + gcp_instance_exists() { local name="$1" project="$2" zone="$3" gcloud compute instances describe "$name" \ diff --git a/scripts/lab/gcp/create-vm.sh b/scripts/lab/gcp/create-vm.sh index f7d1128..9735f57 100755 --- a/scripts/lab/gcp/create-vm.sh +++ b/scripts/lab/gcp/create-vm.sh @@ -171,14 +171,26 @@ trap cleanup_cache_disk EXIT if [[ "$cache_mode" == "snapshot" ]]; then info "Creating per-run cache disk from snapshot: $cache_disk" - gcloud compute disks create "$cache_disk" \ - --project "$project" \ - --zone "$zone" \ - --type "$cache_disk_type" \ - --size "$cache_disk_size" \ - --source-snapshot "$cache_source_snapshot" \ - --labels "$labels" created_cache_disk=true + disk_create_args=( + gcloud compute disks create "$cache_disk" + --project "$project" + --zone "$zone" + --type "$cache_disk_type" + --size "$cache_disk_size" + --source-snapshot "$cache_source_snapshot" + --labels "$labels" + ) + if gcp_run_with_capacity_status "${disk_create_args[@]}"; then + : + else + create_status=$? + if [[ "$create_status" -eq 75 ]]; then + printf 'GCP zone capacity is temporarily unavailable while creating the cache disk: %s\n' \ + "$zone" >&2 + fi + exit "$create_status" + fi fi args=( @@ -213,18 +225,15 @@ if [[ "$cache_mode" == "attach-disk" || "$cache_mode" == "snapshot" ]]; then fi info "Creating GCP lab VM: $name" -set +e -create_output="$(gcloud "${args[@]}" 2>&1)" -create_status=$? -set -e -if [[ "$create_status" -ne 0 ]]; then - printf '%s\n' "$create_output" >&2 - if gcp_is_capacity_error "$create_output"; then - printf 'GCP zone capacity is temporarily unavailable: %s\n' "$zone" >&2 - exit 75 +if gcp_run_with_capacity_status gcloud "${args[@]}"; then + : +else + create_status=$? + if [[ "$create_status" -eq 75 ]]; then + printf 'GCP zone capacity is temporarily unavailable while creating the VM: %s\n' \ + "$zone" >&2 fi exit "$create_status" fi -printf '%s\n' "$create_output" created_cache_disk=false printf '%s\n' "$name" diff --git a/scripts/lab/gcp/prewarm-cache.sh b/scripts/lab/gcp/prewarm-cache.sh index e949a18..41d59df 100755 --- a/scripts/lab/gcp/prewarm-cache.sh +++ b/scripts/lab/gcp/prewarm-cache.sh @@ -33,6 +33,7 @@ Options: --variant eng|userdebug Emulator variant. Default: eng. --timeout Emulator boot timeout. Default: 900. --repo-sync-jobs repo sync jobs. Default: nproc on the VM. + --result-file Write selected VM zone/disk metadata as JSON. --keep-vm Leave the prewarm VM running for debug. -h, --help Show this help. EOF @@ -63,6 +64,7 @@ arch="x86_64" variant="eng" timeout_seconds=900 repo_sync_jobs="" +result_file="" keep_vm=false while [[ $# -gt 0 ]]; do @@ -152,6 +154,11 @@ while [[ $# -gt 0 ]]; do repo_sync_jobs="$2" shift 2 ;; + --result-file) + [[ $# -ge 2 ]] || die "--result-file requires a value" + result_file="$2" + shift 2 + ;; --keep-vm) keep_vm=true shift @@ -194,6 +201,9 @@ if gcloud compute snapshots describe "$snapshot" \ --project "$project" >/dev/null 2>&1; then die "snapshot already exists: $snapshot" fi +if [[ -n "$result_file" ]]; then + rm -f "$result_file" +fi short_ref="$(printf '%s' "$ref" | cut -c1-12)" name="$(sanitize_gcp_name "openphone-lab-prewarm-${short_ref}-$(date -u +%H%M%S)")" @@ -221,7 +231,20 @@ while IFS= read -r candidate_zone; do create_args+=(--source-snapshot "$source_snapshot") fi info "Creating cache disk in $candidate_zone: $candidate_cache_disk" - gcloud "${create_args[@]}" + if gcp_run_with_capacity_status gcloud "${create_args[@]}"; then + : + else + disk_create_status=$? + if [[ "$disk_create_status" -eq 75 ]]; then + info "Cache disk capacity unavailable in $candidate_zone; trying the next fallback zone" + gcloud compute disks delete "$candidate_cache_disk" \ + --project "$project" \ + --zone "$candidate_zone" \ + --quiet >/dev/null 2>&1 || true + continue + fi + exit "$disk_create_status" + fi else info "Reusing cache disk in $candidate_zone: $candidate_cache_disk" fi @@ -248,6 +271,9 @@ while IFS= read -r candidate_zone; do if [[ "$keep_vm" == true ]]; then run_args+=(--keep-vm) fi + if [[ -n "$result_file" ]]; then + run_args+=(--result-file "$result_file") + fi if "$script_dir/run-smoke.sh" "${run_args[@]}"; then selected_zone="$candidate_zone" diff --git a/scripts/lab/gcp/run-smoke.sh b/scripts/lab/gcp/run-smoke.sh index d6d584b..9a9f8e3 100755 --- a/scripts/lab/gcp/run-smoke.sh +++ b/scripts/lab/gcp/run-smoke.sh @@ -39,6 +39,7 @@ Options: May be repeated. Default: local. --timeout Emulator boot timeout. Default: 900. --repo-sync-jobs repo sync jobs. Default: nproc. + --result-file Write selected VM zone/disk metadata as JSON. --keep-vm Leave VM running for debug. --skip-build Reuse existing Android build outputs on the VM. --export-assistant-apk Build OpenPhoneAssistant only and copy the APK @@ -77,6 +78,7 @@ arch="x86_64" variant="eng" timeout_seconds=900 repo_sync_jobs="" +result_file="" keep_vm=false skip_build=false export_assistant_apk=false @@ -181,6 +183,11 @@ while [[ $# -gt 0 ]]; do repo_sync_jobs="$2" shift 2 ;; + --result-file) + [[ $# -ge 2 ]] || die "--result-file requires a value" + result_file="$2" + shift 2 + ;; --keep-vm) keep_vm=true shift @@ -247,6 +254,9 @@ slot="$(printf '%s' "$slot" | tr -c 'A-Za-z0-9_.-' '-')" artifact_root="$root/.worktree/gcp-lab/$name" artifact_dir="$artifact_root/artifacts" mkdir -p "$artifact_dir" +if [[ -n "$result_file" ]]; then + rm -f "$result_file" +fi info "GCP lab target: name=$name project=$project zone=$zone ref=$ref" info "GCP lab shape: machine=$machine_type disk=$boot_disk_size/$boot_disk_type cache_mode=$cache_mode" @@ -323,6 +333,11 @@ fi zone="$selected_zone" vm_created=true info "GCP lab selected zone: $zone" +selected_cache_disk="$cache_disk" +if [[ "$cache_mode" == "snapshot" && -z "$selected_cache_disk" ]]; then + selected_cache_disk="$(sanitize_gcp_name "${name}-cache")" +fi +gcp_write_selection_result "$result_file" "$name" "$zone" "$selected_cache_disk" "$script_dir/bootstrap-vm.sh" --name "$name" --project "$project" --zone "$zone" diff --git a/scripts/test-gcp-lab-retries.sh b/scripts/test-gcp-lab-retries.sh new file mode 100755 index 0000000..7b350f6 --- /dev/null +++ b/scripts/test-gcp-lab-retries.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +gcp_common="$root/scripts/lab/gcp/common.sh" +create_vm="$root/scripts/lab/gcp/create-vm.sh" +run_smoke="$root/scripts/lab/gcp/run-smoke.sh" + +fail() { + printf 'test-gcp-lab-retries: FAIL — %s\n' "$*" >&2 + exit 1 +} + +OPENPHONE_GCP_FALLBACK_ZONES="us-central1-a,us-central1-b" +# shellcheck source=scripts/lab/gcp/common.sh +source "$gcp_common" + +inherited_candidates="$(gcp_zone_candidates us-central1-c)" +[[ "$inherited_candidates" == $'us-central1-c\nus-central1-a\nus-central1-b' ]] || { + fail "omitting fallback zones must use OPENPHONE_GCP_FALLBACK_ZONES" +} + +explicit_empty_candidates="$(gcp_zone_candidates us-central1-c "")" +[[ "$explicit_empty_candidates" == "us-central1-c" ]] || { + fail "an explicit empty fallback list must select only the primary zone" +} + +test_tmp="$(mktemp -d "${TMPDIR:-/tmp}/openphone-gcp-retries.XXXXXX")" +attach_test_name="retry-contract-attach-disk" +trap 'rm -rf "$test_tmp" "$root/.worktree/gcp-lab/$attach_test_name"' EXIT + +selection_file="$test_tmp/selection.json" +gcp_write_selection_result \ + "$selection_file" \ + "openphone-test-vm" \ + "us-central1-b" \ + "openphone-test-cache-us-central1-b" +selection="$( + python3 - "$selection_file" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")) +print("|".join([ + value["vm_name"], + value["selected_zone"], + value["selected_cache_disk"], +])) +PY +)" +[[ "$selection" == \ + "openphone-test-vm|us-central1-b|openphone-test-cache-us-central1-b" ]] || { + fail "selection result must preserve the actual VM, zone, and cache disk" +} + +gcloud() { + printf '%s\n' "$*" >>"$OPENPHONE_GCP_TEST_LOG" + case "$*" in + "compute instances describe "*) + return 1 + ;; + "compute disks describe "*) + if [[ "$OPENPHONE_GCP_TEST_SCENARIO" == "attach-capacity" ]]; then + return 0 + fi + return 1 + ;; + "compute disks create "*) + case "$OPENPHONE_GCP_TEST_SCENARIO" in + disk-capacity) + printf 'ERROR: ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS\n' >&2 + return 1 + ;; + disk-generic) + printf 'ERROR: permission denied\n' >&2 + return 23 + ;; + *) + printf 'Created test cache disk\n' + return 0 + ;; + esac + ;; + "compute instances create "*) + if [[ "$OPENPHONE_GCP_TEST_SCENARIO" == "instance-capacity" || + "$OPENPHONE_GCP_TEST_SCENARIO" == "attach-capacity" ]]; then + printf 'ERROR: RESOURCE_POOL_EXHAUSTED\n' >&2 + return 1 + fi + printf 'Created test VM\n' + return 0 + ;; + "compute disks delete "*) + return 0 + ;; + *) + printf 'unexpected gcloud call: %s\n' "$*" >&2 + return 99 + ;; + esac +} +export -f gcloud + +run_create_case() { + local scenario="$1" + local expected_status="$2" + local output + local status + + : >"$test_tmp/gcloud.log" + set +e + output="$( + OPENPHONE_GCP_TEST_SCENARIO="$scenario" \ + OPENPHONE_GCP_TEST_LOG="$test_tmp/gcloud.log" \ + "$create_vm" \ + --name "retry-contract-${scenario}" \ + --project test-project \ + --zone test-zone \ + --cache-mode snapshot \ + --cache-source-snapshot warm-snapshot 2>&1 + )" + status=$? + set -e + + if [[ "$status" -ne "$expected_status" ]]; then + printf '%s\n' "$output" >&2 + fail "$scenario returned $status; expected $expected_status" + fi +} + +run_create_case disk-capacity 75 +grep -q '^compute disks delete ' "$test_tmp/gcloud.log" || { + fail "a failed snapshot disk create must attempt cleanup" +} + +run_create_case disk-generic 23 +run_create_case instance-capacity 75 + +: >"$test_tmp/gcloud.log" +set +e +attach_output="$( + OPENPHONE_GCP_FALLBACK_ZONES="us-central1-a,us-central1-b" \ + OPENPHONE_GCP_TEST_SCENARIO="attach-capacity" \ + OPENPHONE_GCP_TEST_LOG="$test_tmp/gcloud.log" \ + "$run_smoke" \ + --name "$attach_test_name" \ + --project test-project \ + --zone us-central1-c \ + --cache-mode attach-disk \ + --cache-disk test-cache \ + --skip-build \ + --skip-smoke 2>&1 +)" +attach_status=$? +set -e +if [[ "$attach_status" -ne 75 ]]; then + printf '%s\n' "$attach_output" >&2 + fail "attach-disk capacity failure returned $attach_status; expected 75" +fi +if grep -q -- '--zone us-central1-[ab]' "$test_tmp/gcloud.log"; then + fail "attach-disk retry escaped the disk's primary zone" +fi + +printf 'GCP retry contract tests passed.\n'