Add fast GCP assistant iteration lane

.github/workflows/eval.yml

@@ -44,12 +44,15 @@ jobs:
       - name: Run trajectory smoke suite
         env:
           INPUT_GOALS: ${{ inputs.goals }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           set -euxo pipefail
           ./scripts/run-eval-suite.sh
 
       - name: Run agent benchmark
         if: ${{ github.event_name == 'schedule' || inputs.run_benchmark == 'true' }}
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           set -euxo pipefail
           ./scripts/run-agent-benchmark.sh \

.github/workflows/gcp-cache-refresh.yml

@@ -63,6 +63,7 @@ jobs:
       OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }}
       OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }}
       OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }}
+      OPENPHONE_GCP_FALLBACK_ZONES: ${{ vars.OPENPHONE_GCP_FALLBACK_ZONES || '' }}
       OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }}
       OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }}
       OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }}
@@ -150,18 +151,30 @@ jobs:
         shell: bash
         run: |
           set -euxo pipefail
+          result_file="${RUNNER_TEMP}/openphone-gcp-cache-refresh-result.json"
+          rm -f "$result_file"
+          capture_selection() {
+            if [[ -s "$result_file" ]]; then
+              jq -r \
+                '"selected_zone=\(.selected_zone)", "selected_cache_disk=\(.selected_cache_disk)"' \
+                "$result_file" >> "$GITHUB_OUTPUT"
+            fi
+          }
+          trap capture_selection EXIT
           args=(
             --repo-url "https://github.com/secondly-com/OpenPhone.git"
             --ref "${{ steps.plan.outputs.ref }}"
             --project "${OPENPHONE_GCP_PROJECT}"
             --zone "${OPENPHONE_GCP_ZONE}"
+            --fallback-zones "${OPENPHONE_GCP_FALLBACK_ZONES}"
             --machine-type "${OPENPHONE_GCP_MACHINE_TYPE}"
             --boot-disk-size "${OPENPHONE_GCP_BOOT_DISK_SIZE}"
             --boot-disk-type "${OPENPHONE_GCP_BOOT_DISK_TYPE}"
             --cache-disk "${{ steps.plan.outputs.cache_disk }}"
             --snapshot "${{ steps.plan.outputs.snapshot }}"
             --arch "${{ steps.plan.outputs.arch }}"
             --variant "${{ steps.plan.outputs.variant }}"
+            --result-file "$result_file"
           )
           if [[ -n "${{ steps.plan.outputs.source_snapshot }}" ]]; then
             args+=(--source-snapshot "${{ steps.plan.outputs.source_snapshot }}")
@@ -201,13 +214,19 @@ jobs:
         if: always()
         shell: bash
         run: |
+          selected_zone="${{ steps.refresh.outputs.selected_zone }}"
+          selected_zone="${selected_zone:-$OPENPHONE_GCP_ZONE}"
+          selected_cache_disk="${{ steps.refresh.outputs.selected_cache_disk }}"
+          selected_cache_disk="${selected_cache_disk:-${{ steps.plan.outputs.cache_disk }}}"
           {
             echo "## GCP cache refresh"
             echo
             echo "- Ref: \`${{ steps.plan.outputs.ref }}\`"
             echo "- Arch: \`${{ steps.plan.outputs.arch }}\`"
             echo "- Variant: \`${{ steps.plan.outputs.variant }}\`"
-            echo "- Cache disk: \`${{ steps.plan.outputs.cache_disk }}\`"
+            echo "- Primary zone: \`${OPENPHONE_GCP_ZONE}\`"
+            echo "- Selected zone: \`${selected_zone}\`"
+            echo "- Cache disk: \`${selected_cache_disk}\`"
             echo "- Snapshot: \`${{ steps.plan.outputs.snapshot }}\`"
             echo "- Source snapshot: \`${{ steps.plan.outputs.source_snapshot || 'none' }}\`"
             echo "- Latest snapshot discovery: \`GCP labels\`"

.github/workflows/gcp-lab.yml

@@ -27,6 +27,7 @@ on:
         options:
           - incremental-emulator
           - smoke-only
+          - assistant-apk
           - export-emulator-image
           - custom
       arch:
@@ -82,6 +83,14 @@ on:
         options:
           - "false"
           - "true"
+      export_assistant_apk:
+        description: "Custom lane only: build and upload OpenPhoneAssistant.apk"
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "false"
+          - "true"
       run_smoke:
         description: "Custom lane only: boot emulator and run smoke after build/export"
         required: true
@@ -129,6 +138,7 @@ jobs:
       OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }}
       OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }}
       OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }}
+      OPENPHONE_GCP_FALLBACK_ZONES: ${{ vars.OPENPHONE_GCP_FALLBACK_ZONES || '' }}
       OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }}
       OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }}
       OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }}
@@ -201,6 +211,7 @@ jobs:
             cache_disk=""
             keep_vm="false"
             lane="incremental-emulator"
+            export_assistant_apk="false"
             export_emulator_image="false"
             run_smoke="true"
             skip_build="false"
@@ -234,21 +245,31 @@ jobs:
             lane="${{ inputs.lane }}"
             case "$lane" in
               incremental-emulator)
+                export_assistant_apk="false"
                 export_emulator_image="false"
                 run_smoke="true"
                 skip_build="false"
                 ;;
               smoke-only)
+                export_assistant_apk="false"
                 export_emulator_image="false"
                 run_smoke="true"
                 skip_build="true"
                 ;;
+              assistant-apk)
+                export_assistant_apk="true"
+                export_emulator_image="false"
+                run_smoke="false"
+                skip_build="false"
+                ;;
               export-emulator-image)
+                export_assistant_apk="false"
                 export_emulator_image="true"
                 run_smoke="false"
                 skip_build="false"
                 ;;
               custom)
+                export_assistant_apk="${{ inputs.export_assistant_apk }}"
                 export_emulator_image="${{ inputs.export_emulator_image }}"
                 run_smoke="${{ inputs.run_smoke }}"
                 skip_build="${{ inputs.skip_build }}"
@@ -274,26 +295,40 @@ jobs:
             echo "cache_disk=$cache_disk"
             echo "cache_source_snapshot=$cache_source_snapshot"
             echo "keep_vm=$keep_vm"
+            echo "export_assistant_apk=$export_assistant_apk"
             echo "export_emulator_image=$export_emulator_image"
             echo "run_smoke=$run_smoke"
             echo "skip_build=$skip_build"
           } >> "$GITHUB_OUTPUT"
 
       - name: Run GCP emulator lab
+        id: lab
         shell: bash
         run: |
           set -euxo pipefail
+          result_file="${RUNNER_TEMP}/openphone-gcp-lab-result.json"
+          rm -f "$result_file"
+          capture_selection() {
+            if [[ -s "$result_file" ]]; then
+              jq -r \
+                '"selected_zone=\(.selected_zone)", "selected_cache_disk=\(.selected_cache_disk)"' \
+                "$result_file" >> "$GITHUB_OUTPUT"
+            fi
+          }
+          trap capture_selection EXIT
           args=(
             --name "${{ steps.target.outputs.name }}"
             --repo-url "${{ steps.target.outputs.repo_url }}"
             --ref "${{ steps.target.outputs.ref }}"
             --project "${OPENPHONE_GCP_PROJECT}"
             --zone "${OPENPHONE_GCP_ZONE}"
+            --fallback-zones "${OPENPHONE_GCP_FALLBACK_ZONES}"
             --machine-type "${{ steps.target.outputs.machine_type }}"
             --boot-disk-size "${{ steps.target.outputs.boot_disk_size }}"
             --boot-disk-type "${{ steps.target.outputs.boot_disk_type }}"
             --cache-mode "${{ steps.target.outputs.cache_mode }}"
             --arch "${{ steps.target.outputs.arch }}"
+            --result-file "$result_file"
           )
           if [[ -n "${{ steps.target.outputs.cache_disk }}" ]]; then
             args+=(--cache-disk "${{ steps.target.outputs.cache_disk }}")
@@ -304,6 +339,9 @@ jobs:
           if [[ "${{ steps.target.outputs.keep_vm }}" == "true" ]]; then
             args+=(--keep-vm)
           fi
+          if [[ "${{ steps.target.outputs.export_assistant_apk }}" == "true" ]]; then
+            args+=(--export-assistant-apk)
+          fi
           if [[ "${{ steps.target.outputs.export_emulator_image }}" == "true" ]]; then
             args+=(--export-emulator-image)
           fi
@@ -324,12 +362,20 @@ jobs:
         if: always()
         shell: bash
         run: |
+          selected_zone="${{ steps.lab.outputs.selected_zone }}"
+          selected_zone="${selected_zone:-$OPENPHONE_GCP_ZONE}"
+          selected_cache_disk="${{ steps.lab.outputs.selected_cache_disk }}"
+          if [[ -z "$selected_cache_disk" ]]; then
+            selected_cache_disk="${{ steps.target.outputs.cache_disk }}"
+          fi
+          selected_cache_disk="${selected_cache_disk:-none}"
           {
             echo "## GCP emulator lab"
             echo
             echo "- Cloud provider: \`GCP\`"
             echo "- Project: \`${OPENPHONE_GCP_PROJECT}\`"
-            echo "- Zone: \`${OPENPHONE_GCP_ZONE}\`"
+            echo "- Primary zone: \`${OPENPHONE_GCP_ZONE}\`"
+            echo "- Selected zone: \`${selected_zone}\`"
             echo "- VM: \`${{ steps.target.outputs.name }}\`"
             echo "- Source ref: \`${{ steps.target.outputs.ref }}\`"
             echo "- Source repo: \`${{ steps.target.outputs.repo_url }}\`"
@@ -338,8 +384,9 @@ jobs:
             echo "- Emulator arch: \`${{ steps.target.outputs.arch }}\`"
             echo "- Machine type: \`${{ steps.target.outputs.machine_type }}\`"
             echo "- Cache mode: \`${{ steps.target.outputs.cache_mode }}\`"
-            echo "- Cache disk: \`${{ steps.target.outputs.cache_disk || 'none' }}\`"
+            echo "- Cache disk: \`${selected_cache_disk}\`"
             echo "- Cache snapshot: \`${{ steps.target.outputs.cache_source_snapshot || 'none' }}\`"
+            echo "- Export assistant APK: \`${{ steps.target.outputs.export_assistant_apk }}\`"
             echo "- Export emulator image: \`${{ steps.target.outputs.export_emulator_image }}\`"
             echo "- Run smoke: \`${{ steps.target.outputs.run_smoke }}\`"
             echo "- Skip build: \`${{ steps.target.outputs.skip_build }}\`"

docs/BUILD.md

@@ -224,6 +224,11 @@ setting.
 The pre-sync scaffold check may skip the standalone Java check when the full
 Android build provides the authoritative compiler/toolchain validation.
 
+When the primary Compute zone is temporarily out of the configured machine
+type, set `OPENPHONE_GCP_FALLBACK_ZONES` to a comma-separated list of alternate
+zones. Disposable snapshot labs and scheduled cache refreshes retry only
+capacity-stockout failures. Other failures remain fail-fast.
+
 Current generic-target status:
 
 - `OPENPHONE_BUILD_GOAL=droid ./scripts/build.sh openphone_arm64` has been
@@ -281,6 +286,16 @@ OPENPHONE_BUILD_GOAL=OpenPhoneAssistant \
 ./scripts/build.sh openphone_tegu-bp4a-userdebug
 ```
 
+When a persistent build host is not available, dispatch the `GCP Lab` workflow
+with lane `assistant-apk`. It restores the warm Android snapshot, builds only
+`OpenPhoneAssistant`, and uploads:
+
+```text
+artifacts/assistant-apk/OpenPhoneAssistant.apk
+artifacts/assistant-apk/OpenPhoneAssistant.apk.sha256
+artifacts/assistant-apk/source-ref.txt
+```
+
 Copy the resulting APK back to the host, then push it into `/system_ext` without
 recovery. Because `org.openphone.assistant` is a persistent privileged app,
 Android rejects normal `adb install -r`; enable Developer Options -> Rooted

docs/TESTING.md

@@ -161,6 +161,10 @@ the legacy self-hosted runner.
   boots the emulator, and runs smoke.
 - `smoke-only`: fastest manual lane. Restores the newest warm snapshot and
   boots/smokes already-built outputs without rebuilding.
+- `assistant-apk`: builds only the privileged assistant module and uploads
+  `OpenPhoneAssistant.apk`, its SHA-256 sidecar, and the source ref. Push the
+  result to an already-booted userdebug device with
+  `scripts/push-assistant-apk.sh`.
 - `export-emulator-image`: rebuilds and uploads the SDK system image zip
   without booting the emulator.
 - `custom`: exposes the lower-level export, smoke, and skip-build switches.
@@ -169,6 +173,12 @@ Automatic maintainer-approved PR lab runs use `incremental-emulator`. Use
 `smoke-only` only when you intentionally want to test an already-built warm
 image rather than PR build output.
 
+Set the repository variable `OPENPHONE_GCP_FALLBACK_ZONES` to a comma-separated
+list such as `us-central1-a,us-central1-b,us-central1-f`. Lab runs retry those
+zones only when Compute reports a capacity stockout; build, test, permission,
+and configuration failures still fail immediately. Cache refreshes maintain a
+zone-suffixed warm disk when they have to move away from the primary zone.
+
 ## Device Check
 
 No physical device is supported until its `docs/devices/<codename>.md`

scripts/check.sh

@@ -291,6 +291,41 @@ grep -q -- '--tunnel-through-iap' "$gcp_common" || {
   printf 'GCP lab SSH/SCP helpers must pass --tunnel-through-iap\n' >&2
   exit 1
 }
+OPENPHONE_GCP_TUNNEL_THROUGH_IAP=TRUE \
+  bash -c 'source "$1"; gcp_use_iap_tunnel' _ "$gcp_common" || {
+  printf 'GCP IAP setting parsing must work with the host Bash version\n' >&2
+  exit 1
+}
+zone_candidates="$(
+  bash -c 'source "$1"; gcp_zone_candidates us-central1-c "us-central1-a,us-central1-c us-central1-b"' \
+    _ "$gcp_common"
+)"
+[[ "$zone_candidates" == $'us-central1-c\nus-central1-a\nus-central1-b' ]] || {
+  printf 'GCP fallback zone planning must preserve order and remove duplicates\n' >&2
+  exit 1
+}
+bash -c 'source "$1"; gcp_is_capacity_error "code: ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS"' \
+  _ "$gcp_common" || {
+  printf 'GCP capacity-error detection must recognize zone stockouts\n' >&2
+  exit 1
+}
+"$root/scripts/test-gcp-lab-retries.sh"
+grep -q 'OPENPHONE_GCP_FALLBACK_ZONES' "$root/.github/workflows/gcp-lab.yml" || {
+  printf 'GCP lab workflow must pass configured fallback zones\n' >&2
+  exit 1
+}
+grep -q 'OPENPHONE_GCP_FALLBACK_ZONES' "$root/.github/workflows/gcp-cache-refresh.yml" || {
+  printf 'GCP cache refresh workflow must pass configured fallback zones\n' >&2
+  exit 1
+}
+grep -q 'steps.lab.outputs.selected_zone' "$root/.github/workflows/gcp-lab.yml" || {
+  printf 'GCP lab workflow must summarize the selected fallback zone\n' >&2
+  exit 1
+}
+grep -q 'steps.refresh.outputs.selected_zone' "$root/.github/workflows/gcp-cache-refresh.yml" || {
+  printf 'GCP cache refresh workflow must summarize the selected fallback zone\n' >&2
+  exit 1
+}
 for gcp_script in \
   "$root/scripts/lab/gcp/bootstrap-vm.sh" \
   "$root/scripts/lab/gcp/run-release.sh" \
@@ -341,6 +376,22 @@ grep -q 'lane:' "$root/.github/workflows/gcp-lab.yml" || {
   printf 'GCP lab workflow must expose explicit lab lanes\n' >&2
   exit 1
 }
+grep -q 'assistant-apk' "$root/.github/workflows/gcp-lab.yml" || {
+  printf 'GCP lab workflow must expose the focused assistant APK lane\n' >&2
+  exit 1
+}
+grep -q -- '--export-assistant-apk' "$root/scripts/lab/gcp/run-smoke.sh" || {
+  printf 'GCP lab smoke helper must support focused assistant APK export\n' >&2
+  exit 1
+}
+grep -Fq 'artifacts/.' "$root/scripts/lab/gcp/run-smoke.sh" || {
+  printf 'GCP lab smoke helper must copy artifact contents without an extra directory level\n' >&2
+  exit 1
+}
+grep -q 'exit 75' "$root/scripts/lab/gcp/run-smoke.sh" || {
+  printf 'GCP lab smoke helper must preserve retryable capacity status for its callers\n' >&2
+  exit 1
+}
 if grep -q '^  push:' "$root/.github/workflows/emulator.yml"; then
   printf 'legacy self-hosted emulator workflow must be manual-only; use GCP Lab for trusted gates\n' >&2
   exit 1