diff --git a/02-use-cases/02-workflow-automation-agents/README.md b/02-use-cases/02-workflow-automation-agents/README.md index 0c30a4afd..69480fc9f 100644 --- a/02-use-cases/02-workflow-automation-agents/README.md +++ b/02-use-cases/02-workflow-automation-agents/README.md @@ -31,6 +31,7 @@ Agents that run without a user in the loop. They are triggered by system events | [enterprise-web-intelligence-agent](./enterprise-web-intelligence-agent/) | Market Intelligence | Intermediate | Runtime, Browser; automated web scraping pipeline implemented twice (LangGraph and Strands) for comparison | | [intelligent-event-agent](./intelligent-event-agent/) | General | Beginner | Runtime, Memory, Gateway *(in development, no README yet)* | | [multi-isv-orchestration](./multi-isv-orchestration/) | Enterprise CRM + ERP | Intermediate | Gateway (multi-target), Identity (Cognito inbound + CustomOauth2 outbound); Salesforce + SAP MCP Server through one Gateway for cross-system queries | +| [gpu-music-production-agent](./gpu-music-production-agent/) | Media & Entertainment | Advanced | Runtime (EC2 capacity provider, GPU), Memory; a generative audio model runs on the instance GPU and three collocated agents hand files to each other over a shared EBS volume, with a computed verdict that escalates to human review | ## See also diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/.dockerignore b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/.dockerignore new file mode 100644 index 000000000..2fd20e859 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/.dockerignore @@ -0,0 +1,23 @@ +# Keep the build context to the agent and what it needs. +# +# build/agentdeps is NOT excluded: deploy.py vendors the agent wheels there for +# linux/amd64 and the Dockerfiles COPY them in, which is what lets the images +# build with no RUN steps and therefore without QEMU emulation on an arm64 host. +build/compliance/ +build/*.zip +verify/ +scripts/ +dist/ +runs/ +*.md +!requirements.txt +.git/ +.gitignore +.venv/ +venv/ +__pycache__/ +*.pyc +*.pyo +.pytest_cache/ +.DS_Store +deployment_state.json diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/.gitignore b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/.gitignore new file mode 100644 index 000000000..4d7e268f6 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/.gitignore @@ -0,0 +1,8 @@ +.venv/ +build/ +__pycache__/ +*.pyc +deployment_state.json +*.zip +.DS_Store +runs/ diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/Dockerfile.composition b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/Dockerfile.composition new file mode 100644 index 000000000..b56b770a6 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/Dockerfile.composition @@ -0,0 +1,53 @@ +# Composition agent image. +# +# Deliberately has NO RUN steps. Every dependency is vendored on the build host +# with `uv --python-platform x86_64-manylinux2014` and copied in, so this builds +# for linux/amd64 from an arm64 Mac in seconds. A single `pip install` here would +# execute under QEMU emulation instead. +# +# GPU families on AgentCore capacity providers are all x86_64, so the platform is +# pinned rather than inherited. +# +# Measured: 180 MB, against the 2 GB hard cap on an AgentCore Runtime image. The +# generative stack (torch + CUDA + ACE-Step, about 6.3 GB installed) does not fit +# in any image and lives on the capacity provider's `models` volume instead; +# model_stack/prepare.py builds it there on first use. +FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:3.12-slim + +WORKDIR /app + +# Vendored wheels, unpacked by deploy.py into build/agentdeps. +COPY build/agentdeps /app/deps +ENV PYTHONPATH=/app/deps +ENV PYTHONUNBUFFERED=1 + +# AgentCore injects the NVIDIA driver into /usr/lib64 at run time. This base image +# is Debian, whose dynamic linker searches /usr/lib/x86_64-linux-gnu and /usr/lib +# but NOT /usr/lib64 -- there is no entry for it in /etc/ld.so.conf.d, and +# ldconfig cannot help because the driver appears after the image is built. +# +# Without this, libcuda.so.1 is present on disk and still unloadable, so torch +# reports cuda_available=False and silently runs on the CPU. Measured on a live +# g6.xlarge: `ctypes.CDLL("libcuda.so.1")` raises "cannot open shared object +# file", and the same call with LD_LIBRARY_PATH=/usr/lib64 succeeds and torch then +# reports one NVIDIA L4. +# +# An Amazon Linux based image would not need this, because /usr/lib64 is a +# standard library path there. +ENV LD_LIBRARY_PATH=/usr/lib64 + +COPY audio_dsp.py composition_agent.py /app/ +COPY model_stack /app/model_stack + +# Runs as root so the agent can write to the capacity provider volume. The mount +# is 2775 root:agentcore-runtime-user and the agent process holds that +# supplementary group; a command shell in the same runtime does not, which is why +# only the agent can populate the volume. Drop privileges only if you move the +# workspace to an EFS or S3 Files access point where you set the POSIX UID/GID. + +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s \ + CMD python -c "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/ping',timeout=3)" + +CMD ["python", "composition_agent.py"] diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/Dockerfile.mastering b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/Dockerfile.mastering new file mode 100644 index 000000000..d24c6d24f --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/Dockerfile.mastering @@ -0,0 +1,24 @@ +# Mastering agent image. No RUN steps, for the same reason as +# Dockerfile.composition: dependencies are vendored for linux/amd64 on the build +# host so this builds without QEMU emulation. +# +# This agent does no GPU work. It runs on the same GPU instance as the composition +# agent because collocation is what gives it access to the rendered audio, but +# mastering is filters and gain, so keeping it on the CPU leaves the GPU free for +# generation and avoids two processes competing for VRAM. +FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:3.12-slim + +WORKDIR /app + +COPY build/agentdeps /app/deps +ENV PYTHONPATH=/app/deps +ENV PYTHONUNBUFFERED=1 + +COPY audio_dsp.py mastering_agent.py /app/ + +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s \ + CMD python -c "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/ping',timeout=3)" + +CMD ["python", "mastering_agent.py"] diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/README.md b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/README.md new file mode 100644 index 000000000..7afbf82f1 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/README.md @@ -0,0 +1,826 @@ +# Music production on Amazon Bedrock AgentCore Runtime Instances + +Three AI agents collaborate on one music track, inside one long-lived session, on +**one GPU instance**. They generate real audio with a model running locally on the +instance's GPU, master it with real signal processing, and screen it against a +back-catalogue, then fix and re-check their own work when the screen objects. + +Final result is a `.wav` you can listen to, its mastered version, and three +markdown reports explaining every decision. All files are downloaded to your machine. + +| Agent | Role | Artifact | Compute | Produces | +|---|---|---|---|---| +| `composition_agent.py` | Writes a brief, then renders audio with ACE-Step | container (ECR) | **GPU** (NVIDIA L4) | `composition.wav`, `composition.md` | +| `mastering_agent.py` | Measures the mix, picks a chain, applies it, measures again | container (ECR) | CPU | `master.wav`, `mastering.md` | +| `compliance_agent.py` | Re-measures, screens for similarity, computes a verdict | **zip on S3** | CPU | `compliance.md`, `compliance.json` | + +All three run `global.anthropic.claude-sonnet-4-6` on Amazon Bedrock for reasoning. +The music itself is generated by **ACE-Step v1 3.5B** (Apache-2.0) running on the +instance GPU, not by a hosted service. + +## Contents + +**Getting started** + +1. [What it does](#what-it-does): the agents, the pipeline +2. [Before you start](#before-you-start): prerequisites +3. [Run it](#run-it): deploy, run, listen, tear down +4. [What you get](#what-you-get): where every file lands, and how to download it + +**Using it for your own work** + +5. [Customising it](#customising-it): prompts, payloads, thresholds, models +6. [Troubleshooting](#troubleshooting): every failure this sample actually hit + +**Going deeper** + +7. [How it's built](#how-its-built): every file, how artifacts are made + +--- + +# What it does + +## The three agents + +The three agents model **three teams that +ship independently**, and they differ in every way that matters operationally. + +| | composition | mastering | compliance | +|---|---|---|---| +| Needs | **an L4 GPU** | CPU | CPU | +| Packaged as | container image (ECR) | container image (ECR) | **zip on S3** | +| Time to ship a change | image build + push | image build + push | seconds | +| Volumes mounted | `/mnt/tracks` + `/mnt/models` | `/mnt/tracks` | `/mnt/tracks` | +| Calls another agent? | no | no | **yes** — composition | + +**`composition_agent.py`**: +Sonnet turns a producer's request into a structured brief, then the agent shells out +to a second Python interpreter on `/mnt/models` where ACE-Step renders actual audio +on the L4. It also owns two things nobody else does: **building the ML stack** on the +volume (`mode=prepare`), and **creating each track directory**, the other two +deliberately refuse to create it, so a mis-ordered pipeline fails loudly. + +**`mastering_agent.py`**: decides, then verifies. +Opens `composition.wav` off the shared volume, measures it, and hands **the +measurements** (not the audio) to Sonnet, which returns an EQ / compressor / +limiter chain. `audio_dsp.py` applies it and the output is measured again. The +model never touches a sample, and its plan is checked rather than trusted. + +**`compliance_agent.py`**: the auditor, which computes rather than asks. +Re-measures the audio independently, checks delivery QC against what the mastering +agent *claimed*, and screens against the back-catalogue using chroma features and +subsequence DTW across all twelve transpositions. The verdict is arithmetic (`cleared` / `review_required` / `not_cleared`) and the model only writes the +explanation. If the screen objects, it calls back into the composition runtime for a +replacement, then re-screens. + +## The pipeline + +No agent ever receives audio in a request. Each writes files to `/mnt/tracks//` and the next one opens them. + +``` + invoke.py ─── one runtimeSessionId for all seven calls ───┐ + │ + ═══ everything below runs on ONE g6.xlarge instance ══════╧════ + + [1] composition mode=prepare ─▶ /mnt/models: venv + weights + [2] composition mode=catalogue ─▶ catalogue/*.wav (the references) + [3] composition mode=compose ─▶ composition.wav ──┐ + │ + [4] mastering reads ◀──────────────────────────────┘ + measure ─▶ Sonnet picks a chain ─▶ apply ─▶ measure + ─▶ master.wav ──┐ + │ + [5] compliance reads ◀─────────────────────────────────────────┘ + re-measure ─▶ delivery QC ─▶ chroma/DTW screen + ─▶ verdict, computed in code + │ + │ flagged, or in the review band? + ▼ + [6] composition mode=remediate ◀── invoked BY compliance, + ─▶ composition_remediated.wav same session id + │ + [7] mastering re-master ◀─┘ ─▶ master.wav + compliance re-screen ─▶ compliance.md / compliance.json +``` + +| Step | What happens | +|---|---| +| 1 · prepare | Provisions the GPU instance and builds a 14.6 GB ML stack onto `/mnt/models` a venv with CUDA torch, ACE-Step from a pinned commit, and the weights from HuggingFace. Nothing musical yet | +| 2 · catalogue | Renders two short reference tracks, the fictional back-catalogue the screen compares against, generated by the same model so no third-party recording ships with the sample | +| 3 · compose | Sonnet writes the brief; ACE-Step renders on the L4. With `--with-catalogue` this render is deliberately conditioned on a catalogue track, so the screen has something real to catch | +| 4 · master | Measure → Sonnet picks the chain → DSP applies it → measure again | +| 5 · compliance | Re-measure, delivery QC, similarity screen, computed verdict | +| 6–7 · remediate | Only if step 5 objects: a replacement is rendered, re-mastered and re-screened + +Steps 6–7 exist because a remediation leaves `master.wav` describing audio that no +longer exists. Without them a run ends holding a polished master of the **rejected** +material, the one artifact a producer would actually ship. + +## The one idea everything rests on + +`InvokeAgentRuntime` takes a `runtimeSessionId`. On the Instances compute type, +**one session id maps to one EC2 instance**, and every runtime on the same capacity +provider invoked with that id is served by that instance, with the same EBS volumes +mounted. + +That is the whole coordination mechanism. There is no queue, no database, no +orchestrator. `invoke.py` generates one session id and passes it to all three +runtimes, so the mastering agent can open a file the composition agent wrote minutes +earlier. Change the session id and it gets a different instance with an empty volume. + +| Mount | Size | Holds | Lifetime | +|---|---|---|---| +| `/mnt/tracks` | 20 GiB | the track directory the agents pass files through | destroyed with the session | +| `/mnt/models` | 60 GiB | Python venv (6.3 GB) + ACE-Step weights (8.3 GB) | destroyed with the session, unless restored from a snapshot | + +Both survive an instance idling out and resuming. Neither survives deleting the +session + +--- + +# Before you start + +## What you need + +| | | +|---|---| +| **AWS credentials** | Able to create IAM, EC2, ECR, S3 and AgentCore resources | +| **Bedrock model access** | `global.anthropic.claude-sonnet-4-6` enabled in your Region| +| **`AWS_REGION` set** | Region for deployment | +| **GPU capacity** | `g6.xlarge` in your Region | +| **Python 3.10+** | with **boto3/botocore ≥ 1.43.72** | +| **[`uv`](https://docs.astral.sh/uv/)** | vendors Linux wheels for both artifact types | +| **A container CLI** | [Finch](https://runfinch.com/), Docker, nerdctl or Podman. `deploy.py` probes it with ` info` before creating anything | +| **A default VPC** | or set `CP_SUBNET_ID` and `CP_SECURITY_GROUP_ID` together | + +Notes on the container CLI: Finch on macOS runs containers in a Linux VM, so +`finch vm init` once and `finch vm start` after a reboot. With more than one CLI +installed, the first of `finch, docker, nerdctl, podman` on `PATH` wins — override +with `CONTAINER_CLI=docker`. + +The bundled AWS CLI may be *behind* boto3 for these APIs. Use the scripts rather than +`aws bedrock-agentcore-control create-capacity-provider`. + +--- + +# Run it + +```bash +export AWS_REGION=us-east-2 # a Region where you have g6.xlarge capacity + +python scripts/deploy.py # creates everything, launches nothing +python scripts/invoke.py --with-catalogue # ~7 min - the GPU starts billing here +python scripts/cleanup.py # deletes the SESSION +``` + +Each step is expanded below. + +## 1. Deploy + +```bash +export AWS_REGION=us-east-2 +python scripts/deploy.py +``` + +Creates, in order: two IAM roles → the artifact S3 bucket → two container images built +and pushed to ECR → the compliance zip uploaded to S3 → a capacity provider → three +agent runtimes. Most of the wall time is building and pushing images, so it depends on +your machine and your upload speed. + +It writes **`deployment_state.json`** — runtime ARNs, the capacity provider id, the +bucket, and every session id used since. **Do not delete this file.** It is how +`cleanup.py` finds the session, and deleting the session is what deprovisions the +instance and its volumes. + +Check it worked: + +```bash +python -c "import json;s=json.load(open('deployment_state.json'));print(json.dumps(s['runtimes'],indent=2))" +``` + +## 2. Run the pipeline + +```bash +python scripts/invoke.py --with-catalogue +``` + +**Use `--with-catalogue` for your first run.** Without it the similarity screen has +nothing to compare against and reports "no back-catalogue on the volume", so you see +four steps instead of the full loop with remediation. + +About 7 minutes on a cold volume, and under 3 once `/mnt/models` is built. Watch for +`rendered : NVIDIA L4` in step 3 — anything else means the GPU is not being used. + +The single most important line is the last one: + +``` + All 7 steps were served by one instance, as intended. +``` + +If that instead says steps landed on different hosts, a capacity retry started a new +session mid-run and the collocation demonstration did not happen. + +Other useful invocations: + +```bash +python scripts/invoke.py # no catalogue: compose, master, comply only +python scripts/invoke.py --duration 60 # 60 s of audio instead of 30 +python scripts/invoke.py --track my-song # name the track directory +python scripts/invoke.py --resume # stop the session, resume it, re-screen +python scripts/invoke.py --session <33-100 chars> # reuse an exact session id +``` + +`--resume` demonstrates that the volume outlives the process: it stops every agent in +the session, waits, then invokes again. `prepare` drops from 245 s to 13.5 s because +the venv and weights are still on `/mnt/models`, and each agent's conversation history +is reloaded from the volume rather than from memory. + +## 3. Listen to the results + +Audio is downloaded automatically into `runs//`: + +```bash +ls -la runs/*/ +afplay runs/*/composition.wav # the raw render (macOS; use aplay on Linux) +afplay runs/*/master.wav # after mastering +open runs/*/mastering.md # why that chain was chosen +open runs/*/compliance.md # the verdict and the screen +``` + +`composition.wav` should sound noticeably hotter and flatter than `master.wav`; the +mastered file is the one at −14 LUFS with a −1 dBTP ceiling. + +## 4. Tear it down + +```bash +python scripts/cleanup.py +``` + +Deletes the **session first** — the step that stops EC2 and EBS charges — then the +runtimes, capacity provider, ECR repositories, bucket, IAM roles and CloudWatch log +groups. It also reports any volumes a failed placement leaked, which cannot be deleted +even by an administrator. + +> [!CAUTION] +> **`cleanup.py` deletes the artifact bucket, and the rendered audio with it.** The +> copies in `runs//` are yours and survive. Anything still only in S3 is gone, +> download it first. + +Confirm nothing survived: + +```bash +aws ec2 describe-instances --include-managed-resources --region "$AWS_REGION" \ + --filters "Name=instance-state-name,Values=running,pending" \ + --query 'Reservations[].Instances[].[InstanceId,InstanceType]' --output text +``` + +Managed instances are **hidden without `--include-managed-resources`**, so the plain +command will reassure you falsely. + +## Expected output + +Measured on a live `g6.xlarge` in `us-east-2` — one session, one instance, seven +steps, with `--with-catalogue`. Trimmed to the interesting lines: + +``` + 1. prepare model stack (GPU instance + torch + weights) + host : ip-172-31-27-158.us-east-2.compute.internal (process 71c95af6, x86_64) + stack : ready=True # 13.5s - the volume was already built + # by an earlier track in this session. + # Cold, this step is 245s. + 2. render back-catalogue # 2 reference tracks, 37.8s + + 3. compose (renders audio on the GPU) + rendered : NVIDIA L4 in 5.9s (peak VRAM 7.63 GiB) + audio 20.062s 48000Hz 2ch -7.56 LUFS peak 0.01 dBTP LRA 1.45 LU + + 4. master (real DSP, verified by measurement) + read : composition.wav <- written by another agent + before 20.062s 48000Hz 2ch -7.56 LUFS peak 0.01 dBTP LRA 1.45 LU + after 20.062s 48000Hz 2ch -14.0 LUFS peak -2.9 dBTP LRA 1.45 LU + targets : loudness met, true peak held + + 5. compliance screen + read : composition_remediated.wav <- written by another agent + verdict : REVIEW REQUIRED + screen : 2 reference(s), closest catalogue_00.wav distance 0.0892 (review) + standout : 0.8479x the next-closest (flag below 0.6) + stale : master.wav predates the screened render - re-run mastering + remediation was requested from the composition agent + + -- remediation happened, so the master is stale: re-mastering -- + + 6. re-master the replacement + before 29.907s 48000Hz 2ch -10.94 LUFS peak -0.12 dBTP LRA 4.78 LU + after 29.907s 48000Hz 2ch -14.0 LUFS peak -1.0 dBTP LRA 5.41 LU + targets : loudness met, true peak held + + 7. re-screen the new master + read : master.wav <- written by another agent + verdict : REVIEW REQUIRED + screen : 2 reference(s), closest catalogue_00.wav distance 0.0896 (review) + + -- collocation -- + All 7 steps were served by one instance, as intended. +``` + +Four things in that output are the whole sample: + +* **One hostname for seven invocations across three runtimes.** That only works + because they shared a `runtimeSessionId` on one capacity provider. +* **`-7.56 → -14.0 LUFS` and `+0.01 → -2.9 dBTP` are measured after processing**, not + a plan a model asserted. The mix arrived a hair over full scale; the master lands + 1.9 dB below the ceiling. +* **The screen caught a deliberate derivative, asked for a replacement, and + re-screened it.** `read` shows `composition_remediated.wav`, so the verdict is for + the replacement, not the original. +* **The final verdict is REVIEW REQUIRED, not CLEARED.** `0.0896` sits in the review + band, and a track the screen wants a human to look at does not auto-clear. A demo + that ends on an honest amber is the point — see + [the compliance screen](#the-compliance-screen). + +The mastering agent's reasoning is worth reading in `runs//mastering.md`. On +step 6 it placed a 22 Hz highpass to remove a DC offset **it had measured at +0.002227**, cut 2 dB at 250 Hz, added a 1.5 dB air shelf, applied −2.14 dB of +broadband gain, and **bypassed the compressor entirely** — recording why: + +> Dynamics/compression: LRA of 4.78 LU is healthy and appropriate for the +> build-up/drop structure — the arrangement's own dynamic arc does the work a +> compressor would, so none is applied. + +Every number it reasons from is a measurement of the actual file, and the report has a +**Deliberately left alone** section — the decisions a plausible-looking generated +mastering plan never contains. + +--- + +# What you get + +Three locations, with different lifetimes: + +| Location | What lands there | Survives `cleanup.py`? | +|---|---|---| +| `runs//` on your machine | whatever each response listed as an artifact | **yes** | +| `s3://music-production-artifacts--/tracks//` | every published `.wav` / `.md` / `.json` | **no** — the bucket is deleted | +| `/mnt/tracks//` on the instance | everything, including the catalogue | **no** — dies with the session | + +## Downloaded automatically + +`invoke.py` fetches artifacts after every step, so a normal run leaves you with: + +``` +runs// + composition.wav the raw render + composition.md the brief, plus the render stats + master.wav after mastering (or after re-mastering, if remediation fired) + mastering.md the chain, and what was deliberately left alone + compliance.md the verdict, QC table and similarity screen + compliance.json the same thing as data +``` + +It prefers **your own credentials** over the presigned URL the agent returned +([invoke.py:116](scripts/invoke.py#L116)), falling back to the URL if that fails. The +URL is useful for handing a render to someone with no AWS access, but it is only valid +while the credentials that signed it are — and AgentCore rotates the execution-role +credentials it vends to agents. + +## Everything on the volume + +`S3` marks what is uploaded as it is produced. The rest exists only on the volume. + +``` +/mnt/tracks// + catalogue/catalogue_00.wav [2] composition screened against by [5] + catalogue/catalogue_01.wav [2] composition + catalogue.json [2] composition read by [5] for style tags + composition.wav S3 [3] composition read by [4] + composition.md S3 [3] composition + composition_input_params.json [3] ACE-Step's own dump of its render args + master.wav S3 [4] mastering read by [5] + mastering.md S3 [4] mastering + mastering.json [4] mastering read by [5] for its targets + composition_remediated.wav S3 [6] composition read by [7] + composition_remediated.md S3 [6] composition + compliance.md S3 [5] compliance + compliance.json S3 [5] compliance + .sessions-composition/ Strands history, per agent (mode 0700) + .sessions-mastering/ + .sessions-compliance/ +``` + +Three things worth knowing before you go hunting for a file: + +* **The catalogue audio is never uploaded.** It is an input to the screen, not a + deliverable, so `catalogue_00.wav` cannot be fetched from S3. +* **`composition_remediated.wav` is in S3, but `invoke.py` will not download it.** + Remediation happens *inside* the compliance invocation, so the composition agent's + artifact list is returned to the compliance agent, not to the script. +* **`master.wav` is overwritten.** Step 7 re-masters the replacement to the same + filename. What you keep is the master of the replacement, which is the correct + deliverable but not the only render that existed. + +## Downloading by hand + +```bash +BUCKET=$(python -c "import json;print(json.load(open('deployment_state.json'))['s3']['bucket'])") +TRACK=$(python -c "import json;print(json.load(open('deployment_state.json'))['last_run']['track'])") + +aws s3 ls "s3://$BUCKET/tracks/$TRACK/" --region "$AWS_REGION" +aws s3 sync "s3://$BUCKET/tracks/$TRACK/" "./runs/$TRACK/" --region "$AWS_REGION" +``` + +To share one render with someone who has no AWS access, mint a link yourself: + +```bash +aws s3 presign "s3://$BUCKET/tracks/$TRACK/master.wav" --expires-in 3600 \ + --region "$AWS_REGION" +``` + +**Files that were never published** — the catalogue audio, `mastering.json` — are +harder to reach. The instance has no shell you can write from: a command shell reports +`groups=0(root)` and gets `EACCES` on the mount, and there is no scp. Either add a +`publish()` call to the agent and redeploy, or +snapshot the volume and attach it to an instance +of your own. + +--- + +# Customising it + +## Changing the prompts + +The prompts live in [scripts/invoke.py](scripts/invoke.py) and there is **no +`--prompt` flag** — it is a fixed demonstration script, so changing the request means +editing it or [invoking a runtime directly](#invoking-a-runtime-directly). + +| Step | Prompt | Line | +|---|---|---| +| compose | `Create an upbeat electronic track with heavy bass and synth melodies.` | [323](scripts/invoke.py#L323) | +| compose, with `--with-catalogue` | `Create a melodic techno track with analog bass and warm pads, close to our catalogue sound.` | [329](scripts/invoke.py#L329) | +| master | `Master this for streaming.` | [340](scripts/invoke.py#L340) | +| compliance | `Screen this master for release.` | [347](scripts/invoke.py#L347) | +| re-master | `Master the replacement for streaming.` | [360](scripts/invoke.py#L360) | +| re-screen | `Screen the re-mastered replacement.` | [369](scripts/invoke.py#L369) | +| catalogue styles | `melodic techno, analog bass, warm pads, 124 bpm` and `lo-fi hip hop, dusty piano, vinyl crackle, 82 bpm` | [composition_agent.py:460](composition_agent.py#L460) | + +**Important:** the composition prompt does not control the render directly. It goes to +Sonnet, which turns it into a `CompositionBrief` — and of that brief's nine fields, +**only `style_tags` and `lyrics` reach ACE-Step.** The renderer takes no other text +input, so `key`, `tempo_bpm`, `time_signature`, `chord_progression`, +`instrumentation`, `structure` and `title` are documentation that lands in +`composition.md`. Tempo influences the audio only because the model writes "124 bpm" +into the tag string. + +So "make it sad" works only as far as the model translates it into concrete +descriptors. For exact control of the render, set `style_tags` yourself by calling the +runtime directly. + +## Invoking a runtime directly + +This is the flexible path, and the clearest look at the AgentCore API. + +```bash +ARN=$(python -c "import json;print(json.load(open('deployment_state.json'))['runtimes']['composition']['arn'])") + +aws bedrock-agentcore invoke-agent-runtime \ + --region "$AWS_REGION" \ + --agent-runtime-arn "$ARN" \ + --qualifier DEFAULT \ + --runtime-session-id "my-own-session-id-padded-to-33-chars-min" \ + --payload '{"mode":"compose","track_id":"my-song", + "prompt":"A slow, mournful piano piece in 3/4 with tape hiss.", + "duration_s":45,"seed":7}' \ + /dev/stdout +``` + +Two rules that will otherwise cost you time: + +* **The session id must be 33–100 characters.** `InvokeAgentRuntime` accepts up to + 256, but `DeleteCapacityProviderSession` caps it at 100 — so a longer id can be + invoked and then never deleted, stranding its volumes. +* **You must call `mode=prepare` once per session** before composing, or the render + fails with "the model stack on the shared volume is not ready". + +If your AWS CLI lacks `invoke-agent-runtime`, or you want the response parsed, use +boto3 as `invoke.py` does. Note the response member is named **`response`**, not +`body`, and the default 60 s read timeout is far too short for a cold start: + +```python +import json, boto3 +from botocore.config import Config + +state = json.load(open("deployment_state.json")) +client = boto3.client("bedrock-agentcore", region_name=state["region"], config=Config(read_timeout=900)) + +r = client.invoke_agent_runtime( + agentRuntimeArn=state["runtimes"]["composition"]["arn"], + qualifier="DEFAULT", + runtimeSessionId="my-own-session-id-padded-to-33-chars-min", + payload=json.dumps( + { + "mode": "compose", + "track_id": "my-song", + "prompt": "A slow, mournful piano piece in 3/4 with tape hiss.", + "duration_s": 45, + "seed": 7, + } + ).encode(), +) + +body = json.loads(r["response"].read()) +print(body["brief"]["style_tags"], body["artifacts"][0]["s3_uri"]) +``` + +## Payload reference + +Every field below is real; nothing else is read. + +**Composition runtime** + +| Field | Default | Meaning | +|---|---|---| +| `mode` | `compose` | `prepare` \| `status` \| `catalogue` \| `compose` \| `remediate` | +| `track_id` | `demo-track` | Names the directory on the shared volume | +| `prompt` | `Compose an upbeat electronic track.` | The producer's request. **Must be a string** | +| `duration_s` | `30.0` | Seconds of audio | +| `steps` | `27` | Diffusion steps. Higher is slower, not reliably better | +| `seed` | *unset* | Fix it to make a render reproducible | +| `styles` | two built-in tag strings | `catalogue` mode only: one reference per entry | +| `imitate_catalogue` | *unset* | e.g. `catalogue_00.wav` — conditions the render on that file, producing deliberately derivative audio | +| `reference_strength` | `0.75` | How strongly to follow it, 0–1 | +| `issue` | *unset* | `remediate` mode: what the screen objected to | +| `avoid` | `{}` | `remediate` mode: `{title, reference, style_tags, other_references}` — the evidence that makes a rewrite informed rather than blind | + +**Mastering runtime** + +| Field | Default | Meaning | +|---|---|---| +| `track_id` | `demo-track` | Must already hold a render | +| `platform` | `spotify` | `spotify` −14/−1 · `apple` −16/−1 · `youtube` −14/−1 · `amazon` −14/−2 · `broadcast` −23/−1 (LUFS / dBTP) | +| `prompt` | `Master this for .` | Steers the engineer's judgement, not the targets | + +**Compliance runtime** + +| Field | Default | Meaning | +|---|---|---| +| `track_id` | `demo-track` | Must already hold audio | +| `prompt` | `Review this track for release.` | Steers the explanation only — **never the verdict** | +| `auto_remediate` | `true` | Set `false` to screen without calling back into composition | + +## Configuration + +All optional except the Region. These are runtime environment variables, so changing +a model or a threshold needs a `deploy.py` run but **no image rebuild**: + +```bash +SIMILARITY_FAIL_DISTANCE=0.06 \ +CP_INSTANCE_TYPE=g5.xlarge \ +python scripts/deploy.py +``` + +| Variable | Default | Purpose | +|---|---|---| +| `AWS_REGION` | *required* | No fallback | +| `CP_INSTANCE_TYPE` | `g6.xlarge` | One type only; AgentCore gives no fallback across a list | +| `CP_SUBNET_ID` / `CP_SECURITY_GROUP_ID` | every default-VPC subnet in an AZ offering the type | Set both together, comma-separated | +| `MODELS_SIZE_GIB` | `60` | Holds a 6.3 GB venv and 8.3 GB of weights | +| `TRACKS_SIZE_GIB` | `20` | Shared workspace | +| `ROOT_FREE_GIB` | `30` | Default is 8, not enough for a 391 MB image plus scratch | +| `MODELS_SNAPSHOT_ID` | *unset* | Restore a prepared model stack and skip `prepare` | +| `IDLE_INSTANCE_TIMEOUT` | `600` | Deliberately below the service default of 900: an idle GPU is the expensive mistake | +| `IDLE_SESSION_TIMEOUT` | `600` | Per-agent session idle timeout. The *instance* goes away only once every agent on it has idled out | +| `MAX_LIFETIME` | `86400` | Max `1209600` (14 days), must be ≥ both idle timeouts | +| `CONTAINER_CLI` | first of `finch, docker, nerdctl, podman` on `PATH` | Probed with ` info` before anything is created | +| `ACESTEP_SHA` | pinned commit | The generator revision | +| `ACESTEP_WEIGHTS_REPO` | `ACE-Step/ACE-Step-v1-3.5B` | HuggingFace repo the weights come from | +| `PREPARE_TIMEOUT_S` / `RENDER_TIMEOUT_S` | `1500` / `900` | Subprocess ceilings. `RENDER_TIMEOUT_S` must stay under the service's 900 s synchronous request limit | +| `SIMILARITY_FAIL_DISTANCE` / `SIMILARITY_REVIEW_DISTANCE` | `0.045` / `0.10` | Absolute screening thresholds. **Recalibrate against your own catalogue** — see [the compliance screen](#the-compliance-screen) | +| `SIMILARITY_STANDOUT_RATIO` | `0.6` | Relative test: flag when the closest match is this much closer than the next | +| `COMPOSITION_MODEL_ID` / `MASTERING_MODEL_ID` / `COMPLIANCE_MODEL_ID` | `global.anthropic.claude-sonnet-4-6` | Prefer a `global.` or Region-matched profile: a `us.` profile will not resolve outside US Regions | + +`deploy.py` also injects `ARTIFACT_BUCKET`, `COMPOSITION_RUNTIME_ARN`, +`COMPOSITION_QUALIFIER`, `WORKSPACE_DIR` and `MODELS_DIR`. Those are +wiring, not knobs — listed only so an unfamiliar value in a runtime's environment is +not a mystery. + +## Shipping a change to one agent + +```bash +python scripts/update.py composition --restart-session +python scripts/update.py compliance # zip only: seconds, no image build +python scripts/update.py --all --restart-session +``` + +`--restart-session` matters more than it looks. **A running session keeps serving the +code it started with, silently** — no error, no warning, just the old behaviour. + +--- + +# Troubleshooting + +Every item here is a failure this sample actually hit. + +| Symptom | Cause | +|---|---| +| `Insufficient EC2 capacity` | GPU capacity, not your configuration. Retry on a fresh session id; see [GPU capacity](#gpu-capacity) | +| A deployed fix has no effect | A running session keeps serving the version it started with, silently. `update.py --restart-session`, or use a new session id | +| `torch.cuda.is_available()` is `False` on a GPU fleet | The driver is at `/usr/lib64`, which a Debian-based image does not search. Set `LD_LIBRARY_PATH=/usr/lib64` | +| `ValidationException` on `runtimeSessionId` | Minimum is **33** characters. `InvokeAgentRuntime` allows 256 but `DeleteCapacityProviderSession` caps it at **100** — a longer id can be invoked and then never deleted, stranding its volumes | +| `Failed to provision compute resources for the agent` | Opaque by design. Check CloudTrail — for a snapshot-backed volume it is the missing `ec2:CreateVolume` on the snapshot ARN | +| `describe-instances` / `describe-volumes` show nothing after cleanup | Both are EC2 *managed resources*, hidden by default. Pass `--include-managed-resources` | +| Waiting forever for status `ACTIVE` | The terminal state is **`READY`**; the API enum has no `ACTIVE`, whatever the docs prose says | +| `NoCredentialsError: Unable to locate credentials` | A long-lived container can end up with **no** vended credentials. Measured: after ~70 minutes, `boto3.Session().get_credentials()` returned `None` inside a container that had been calling Bedrock happily. AgentCore vends credentials over a private endpoint (`AWS_EC2_METADATA_SERVICE_ENDPOINT=http://100.88.0.1:`, not `169.254.169.254`). Stopping and resuming the session restores them | +| `KeyError: 'body'` reading a response | The `InvokeAgentRuntime` response member is `response`, not `body` | +| `NetworkConfiguration is not allowed when capacityProviderConfiguration is specified` | Mutually exclusive; the VPC belongs to the capacity provider | +| `DeleteCapacityProvider` fails with "attached agent runtime versions" | Versions detach asynchronously after `DeleteAgentRuntime` returns. `cleanup.py` polls | +| `Permission denied` writing to `/mnt/models` from a shell | Expected. Only the agent process holds `agentcore-runtime-user`; a command shell has `groups=0(root)` and mode `2775` falls through to `other` | +| `Agent is already processing a request` | A module-level Strands `Agent` shared across concurrent invocations. Each agent here builds its `Agent` per request | +| `No solution found ... numpy has no usable wheels` | `--python-platform x86_64-manylinux2014` is too old a baseline; numpy 2.5 and scipy 1.18 ship `manylinux_2_28` | +| `ImportError: TorchCodec is required for save_with_torchcodec` (or `load_`) | Recent torchaudio delegates file I/O to TorchCodec, which nothing installs. `generate.py` redirects both `save` and `load` to soundfile | +| `pip install ace-step` fails at metadata generation | The PyPI sdist omits the `requirements.txt` its own `setup.py` reads. Install from a pinned GitHub tarball | +| 403 `SignatureDoesNotMatch` on a presigned S3 URL | botocore presigned against the global host `bucket.s3.amazonaws.com` while scoping the signature to the Region. Pass `endpoint_url=https://s3..amazonaws.com` | +| A 24 s clip fails a "dynamics retained" check | Loudness range needs enough 3 s blocks to mean anything, and a uniform *source* is not mastering's fault. The check is skipped under 30 s and otherwise compared against the pre-master measurement | +| Base image CVEs | `python:3.12-slim` carries the usual base-image findings. Rebuild regularly; under the shared responsibility model, container image currency is yours | + +## Finding the logs + +An agent's stdout and tracebacks go to a log group named after the **runtime id**, in +a stream named after the **session id** — so the session id you passed to +`InvokeAgentRuntime` is how you find that run's logs: + +```bash +aws logs get-log-events --region "$AWS_REGION" \ + --log-group-name "/aws/bedrock-agentcore/runtimes/-DEFAULT" \ + --log-stream-name "" --query 'events[].message' --output text +``` + +Two more streams sit alongside it, `otel-rt-logs` and `spans`. `storedBytes` on the +log group lags well behind reality. `cleanup.py` deletes these groups; nothing else +does, and they are created with no retention policy. + +`mode=prepare` streams its subprocess output line by line (prefixed `prepare | `) +rather than buffering it, so a four-minute build is visible in CloudWatch as it +happens instead of looking identical to a hang. + +## GPU capacity + +`allowedInstanceTypes` gives no fallback, so a capacity provider is effectively pinned +to one instance type, and GPU capacity fluctuates hard. + +It arrives in ~13 s, so you find out quickly. `invoke.py` retries on a **fresh session +id**, because the failure is bound to the placement AgentCore chose. If that is not +enough: + +* Try another Region (`AWS_REGION=us-east-1`). +* Pick a different type (`CP_INSTANCE_TYPE=g5.xlarge`). +* Reserve capacity with an **On-Demand Capacity Reservation**; the API supports + targeting one through `capacityReservationSpecification`. This is the only way to + *guarantee* a GPU, and it bills whether or not you use it. + +--- + +# How it's built + +## The files + +16 files, ~4,300 lines of code. + +``` +gpu-music-production-agent/ +├── composition_agent.py 577 GPU agent ─┐ +├── mastering_agent.py 470 CPU agent ├─ the three agents +├── compliance_agent.py 719 CPU agent ─┘ +├── audio_dsp.py 519 shared DSP + measurement library +├── model_stack/ +│ ├── prepare.py 208 builds the venv + weights onto /mnt/models +│ └── generate.py 179 runs ACE-Step inside that venv +├── Dockerfile.composition 53 → the composition image +├── Dockerfile.mastering 24 → the mastering image +├── requirements.txt 20 agent-side deps only, never the ML stack +├── scripts/ +│ ├── deploy.py 653 stand everything up +│ ├── invoke.py 401 run the workflow, download the audio +│ ├── update.py 224 ship one agent, not the others +│ └── cleanup.py 253 tear everything down +├── .dockerignore .gitignore +└── README.md +``` + +| File | Role | +|---|---| +| `composition_agent.py` | Five modes — `prepare` (build the ML stack), `status`, `catalogue`, `compose`, `remediate`. Owns creation of each track directory. The only agent that uses the GPU, via a subprocess in a separate interpreter | +| `mastering_agent.py` | Measures the render, asks Sonnet for an EQ/compressor/limiter chain from those numbers, applies it, measures again. Holds the per-platform delivery targets | +| `compliance_agent.py` | Re-measures independently, runs delivery QC against `mastering.json`'s claims, screens against the catalogue, **computes** the three-state verdict, and calls back into the composition runtime when the screen objects. `findings_brief()` is what withholds figures from the model | +| `audio_dsp.py` | Everything numeric, implemented from the specs to avoid GPL and unlicensed dependencies: ITU-R BS.1770-4 loudness, EBU 3342 loudness range, true peak with 4× oversampling, RBJ biquads, a compressor, a look-ahead limiter, chroma features and subsequence DTW. Validated against EBU Tech 3341 | +| `model_stack/prepare.py` | Creates the venv on `/mnt/models`, installs CUDA torch and ACE-Step from a pinned GitHub tarball, downloads the weights, copies `generate.py` in beside them. Idempotent — stamps each stage so a resumed session skips finished work | +| `model_stack/generate.py` | The renderer. Runs under `/mnt/models/venv/bin/python`, not the agent's interpreter, because ACE-Step pins `transformers==4.50.0` and drags in gradio and spacy. Patches `torchaudio.save`/`load` to soundfile, and prints its result as a `__RENDER_RESULT__` JSON line the agent parses | +| `Dockerfile.composition` | `python:3.12-slim`, zero `RUN` steps. Sets `LD_LIBRARY_PATH=/usr/lib64` — without it torch silently runs on the CPU on a GPU fleet | +| `Dockerfile.mastering` | The same, minus the GPU concerns and `model_stack/` | +| `requirements.txt` | Agent-side only: Strands, the AgentCore SDK, boto3, numpy, scipy, soundfile. **Never** torch — that lives on the volume | +| `scripts/deploy.py` | IAM roles → S3 bucket → vendor wheels → build and push both images → build and upload the zip → capacity provider → three runtimes. Writes `deployment_state.json`. Creates **no instances** | +| `scripts/invoke.py` | Generates one session id, drives all seven steps through it, retries capacity failures on a fresh id, prints the collocation proof, downloads artifacts | +| `scripts/update.py` | Rebuilds and ships one agent. `--restart-session` matters: a live session keeps serving its old code silently | +| `scripts/cleanup.py` | Deletes the **session first** (the fastest way to stop EC2 and EBS billing), then runtimes, capacity provider, ECR, bucket, roles, log groups. Reports volumes that failed placements leaked | + +Written at run time: `runs//` (yours, survives teardown), `build/` (vendored +wheels and the zip), and `deployment_state.json` — **do not lose that one**, it holds +the session ids. + +## Two Dockerfiles, three agents + +There is no `Dockerfile.compliance`, and that is deliberate. AgentCore Runtime takes +two kinds of artifact, and this sample ships both onto **one** capacity provider: + +``` + composition_agent.py ─┐ + audio_dsp.py ─┼─▶ Dockerfile.composition ─▶ ECR image ─▶ GPU runtime + model_stack/ ─┤ + LD_LIBRARY_PATH=/usr/lib64 + build/agentdeps ─┘ (the CUDA trap) + + mastering_agent.py ─┐ + audio_dsp.py ─┼─▶ Dockerfile.mastering ───▶ ECR image ─▶ CPU runtime + build/agentdeps ─┘ + + compliance_agent.py ─┐ + audio_dsp.py ─┼─▶ deploy.py: zip ────────▶ S3 zip ────▶ CPU runtime + vendored wheels ─┘ 82 MB / 235 MB uncompressed + + model_stack/generate.py ──▶ copied onto /mnt/models by prepare.py at run time, + NOT into any image - it runs in the ACE-Step venv +``` + +The container agents are registered with `containerConfiguration.containerUri`; the +compliance agent with `codeConfiguration.code.s3` plus `runtime: PYTHON_3_12` and +`entryPoint: ["compliance_agent.py"]`. Measured **82.0 MB compressed, 235.0 MB +uncompressed** against service limits of 250 MB and 750 MB. It fits because compliance +needs numpy and scipy but no torch — the same reason the GPU agent *cannot* be a zip. + +Why bother? The zip path needs no container CLI, no ECR repository and no image push, +so `update.py compliance` ships in seconds. Proving the two artifact types **coexist +in one session on one instance** is part of what this sample demonstrates — and it +surfaces a wrinkle: they run as different Linux identities. + +Both Dockerfiles have **zero `RUN` steps**. Dependencies are vendored on your machine +first, so the image build is pure `COPY`. + +`audio_dsp.py` appears three times. That duplication is deliberate: the agents +ship independently and must not import from one another, but there is no reason to +transcribe ITU-R BS.1770 three times. + +--- + +## The compliance screen + +Two checks, both on the audio. + +**Delivery QC** re-measures integrated loudness (ITU-R BS.1770-4), loudness range +(EBU Tech 3342), true peak with 4× oversampling, clipping runs, DC offset and mono +compatibility, and compares them against the targets `mastering.json` claims. + +**Similarity screening** compares chroma features with subsequence DTW over all twelve +transpositions, so a copy shifted to another key still matches — which a plain +acoustic fingerprint would miss entirely. Calibrated on synthetic references: + +| | chroma-DTW distance | +|---|---| +| **Synthetic tones** — identical melody, different render | **0.0000** | +| **Synthetic tones** — same melody transposed +5 semitones | **0.0015** (shift recovered) | +| **Synthetic tones** — unrelated melody | **0.2729** | +| **Real ACE-Step audio** — audio2audio derivative | **0.029** | +| **Real ACE-Step audio** — unrelated, same genre | **0.062** and **0.122** | + +**A single absolute threshold does not work, and finding that out is the useful part.** +The two real-audio ranges overlap: `0.08` failed an unrelated track, then `0.045` +*cleared* a track that had been conditioned directly on a catalogue reference. No cut +separates them. + +What does separate them is how far the closest match **stands out from the rest of the +catalogue**. A derivative is much closer to one track than to the others; an original +is roughly equidistant: + +| | closest | next | ratio | +|---|---|---|---| +| derivative | 0.029 / 0.050 | 0.122 / 0.130 | **0.239 / 0.384** | +| original | 0.062 | 0.063 | **0.98** | +| after remediation | 0.089 | 0.105 | **0.85** | + +So the screen uses two signals — absolute distance below `SIMILARITY_FAIL_DISTANCE`, +**or** a standout ratio below `SIMILARITY_STANDOUT_RATIO` (0.6). Either one flags. + +The verdict has three states, not two: + +| Outcome | Meaning | +|---|---| +| `cleared` | QC passed and nothing flagged or in the review band | +| `review_required` | Nothing flagged, but the closest match is under the review threshold. **Does not auto-clear** | +| `not_cleared` | QC failed, or the similarity screen flagged it | + +That middle state exists because of a real bug: a track conditioned on a catalogue +reference scored 0.0498, landed in the review band, and was reported **CLEARED**. +Auto-clearing something the screen wanted a human to look at is the one thing a +compliance tool must not do. + +> [!IMPORTANT] +> **This is a screen, not a copyright clearance.** It compares against one local +> catalogue with one feature. A flag means escalate to a human or a licensed +> identification service; a pass means nothing was found in that catalogue. diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/audio_dsp.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/audio_dsp.py new file mode 100644 index 000000000..b1e946ff4 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/audio_dsp.py @@ -0,0 +1,533 @@ +"""Audio measurement and processing, in numpy and scipy only. + +This module is what turns the sample from three agents writing prose into three +agents doing verifiable work: the mastering agent applies real filters and the +compliance agent measures the result against the spec the mastering agent +claimed. Every number here is reproducible from the WAV file. + +Deliberately has no third-party DSP dependency: + +* ``pedalboard`` is GPL (it links JUCE), which rules it out for a published + sample that customers adapt. +* ``pyloudnorm`` publishes no license metadata, and true-peak measurement is not + in it anyway, so the loudness algorithm is implemented here instead. + +That leaves numpy, scipy and soundfile, all BSD-3. It also means the loudness +maths is on the page rather than behind an import, which is the point of a +sample. + +Loudness follows ITU-R BS.1770-4 and loudness range follows EBU Tech 3342. The +standard specifies its K-weighting coefficients at 48 kHz, so audio is resampled +to 48 kHz for measurement rather than the coefficients being re-derived per rate. + +A copy of this file is placed in each agent artifact at build time. The agents +ship independently and must not import from one another, but there is no reason +for the DSP to be transcribed three times. +""" + +from __future__ import annotations + +import math +from dataclasses import asdict, dataclass, field + +import numpy as np +import soundfile as sf +from scipy import signal + +# --------------------------------------------------------------------------- +# ITU-R BS.1770-4 constants +# --------------------------------------------------------------------------- + +MEASURE_RATE = 48_000 + +# Stage 1: shelving filter approximating the acoustic effect of the head. +_K_STAGE1_B = np.array([1.53512485958697, -2.69169618940638, 1.19839281085285]) +_K_STAGE1_A = np.array([1.0, -1.69065929318241, 0.73248077421585]) +# Stage 2: RLB high-pass. +_K_STAGE2_B = np.array([1.0, -2.0, 1.0]) +_K_STAGE2_A = np.array([1.0, -1.99004745483398, 0.99007225036621]) + +_ABSOLUTE_GATE_LUFS = -70.0 +_RELATIVE_GATE_LU = -10.0 +_LUFS_OFFSET = -0.691 # so that a 1 kHz sine at -20 dBFS reads -20 LUFS + +# Channel weights G_i from the standard. Stereo uses the first two. +_CHANNEL_WEIGHTS = np.array([1.0, 1.0, 1.0, 1.41, 1.41]) + + +# --------------------------------------------------------------------------- +# I/O. Internally audio is float64 shaped (samples, channels). +# --------------------------------------------------------------------------- + + +def read_audio(path: str) -> tuple[np.ndarray, int]: + """Read a soundfile into float64 (samples, channels).""" + data, rate = sf.read(path, always_2d=True, dtype="float64") + return data, int(rate) + + +def write_audio(path: str, data: np.ndarray, rate: int, subtype: str = "PCM_24") -> None: + """Write (samples, channels) audio. 24-bit PCM by default: a master should + not be delivered as 16-bit, and float WAV confuses some players.""" + if data.ndim == 1: + data = data[:, None] + sf.write(path, data, rate, subtype=subtype) + + +def resample(data: np.ndarray, src_rate: int, dst_rate: int) -> np.ndarray: + """Polyphase resample, preserving the (samples, channels) shape.""" + if src_rate == dst_rate: + return data + g = math.gcd(int(src_rate), int(dst_rate)) + up, down = dst_rate // g, src_rate // g + return signal.resample_poly(data, up, down, axis=0) + + +# --------------------------------------------------------------------------- +# Loudness and peak measurement +# --------------------------------------------------------------------------- + + +def _k_weight(data: np.ndarray) -> np.ndarray: + """Apply the two BS.1770 pre-filters. Input must be at 48 kHz.""" + out = signal.lfilter(_K_STAGE1_B, _K_STAGE1_A, data, axis=0) + return signal.lfilter(_K_STAGE2_B, _K_STAGE2_A, out, axis=0) + + +def _block_mean_squares(weighted: np.ndarray, rate: int, block_s: float, overlap: float) -> np.ndarray: + """Mean square per block per channel -> (blocks, channels).""" + block = round(block_s * rate) + step = max(1, round(block * (1.0 - overlap))) + n = weighted.shape[0] + if n < block: + return np.empty((0, weighted.shape[1])) + starts = range(0, n - block + 1, step) + return np.stack([np.mean(weighted[s : s + block] ** 2, axis=0) for s in starts]) + + +def _blocks_to_loudness(mean_squares: np.ndarray) -> np.ndarray: + """Per-block loudness in LUFS from per-channel mean squares.""" + weights = _CHANNEL_WEIGHTS[: mean_squares.shape[1]] + summed = mean_squares @ weights + with np.errstate(divide="ignore"): + return _LUFS_OFFSET + 10.0 * np.log10(np.maximum(summed, 1e-30)) + + +def integrated_loudness(data: np.ndarray, rate: int) -> float: + """Gated integrated loudness in LUFS, per BS.1770-4. + + Two-stage gating: an absolute -70 LUFS gate, then a gate 10 LU below the + ungated mean of what survived. Without the gating a track with quiet + passages measures far lower than it sounds, which is the whole reason the + standard specifies it. + """ + d = resample(data, rate, MEASURE_RATE) + weighted = _k_weight(d) + ms = _block_mean_squares(weighted, MEASURE_RATE, 0.400, 0.75) + if ms.shape[0] == 0: + return float("-inf") + loud = _blocks_to_loudness(ms) + + above_absolute = loud > _ABSOLUTE_GATE_LUFS + if not np.any(above_absolute): + return float("-inf") + + weights = _CHANNEL_WEIGHTS[: ms.shape[1]] + relative_ref = _LUFS_OFFSET + 10.0 * np.log10(max(float(np.mean(ms[above_absolute] @ weights)), 1e-30)) + keep = above_absolute & (loud > relative_ref + _RELATIVE_GATE_LU) + if not np.any(keep): + return float("-inf") + return float(_LUFS_OFFSET + 10.0 * np.log10(max(float(np.mean(ms[keep] @ weights)), 1e-30))) + + +def loudness_range(data: np.ndarray, rate: int) -> float: + """Loudness range (LRA) in LU, per EBU Tech 3342: 3 s blocks, a -20 LU + relative gate, then the span between the 10th and 95th percentiles.""" + d = resample(data, rate, MEASURE_RATE) + weighted = _k_weight(d) + ms = _block_mean_squares(weighted, MEASURE_RATE, 3.0, 2.0 / 3.0) + if ms.shape[0] < 2: + return 0.0 + loud = _blocks_to_loudness(ms) + above_absolute = loud > _ABSOLUTE_GATE_LUFS + if not np.any(above_absolute): + return 0.0 + weights = _CHANNEL_WEIGHTS[: ms.shape[1]] + ref = _LUFS_OFFSET + 10.0 * np.log10(max(float(np.mean(ms[above_absolute] @ weights)), 1e-30)) + kept = loud[above_absolute & (loud > ref - 20.0)] + if kept.size < 2: + return 0.0 + return float(np.percentile(kept, 95) - np.percentile(kept, 10)) + + +def true_peak_dbtp(data: np.ndarray, rate: int, oversample: int = 4) -> float: + """True peak in dBTP. + + Sample peak misses inter-sample peaks, which is exactly what a streaming + platform's decoder will reconstruct and then clip. BS.1770 calls for at + least 4x oversampling before taking the maximum. + """ + up = signal.resample_poly(data, oversample, 1, axis=0) + peak = float(np.max(np.abs(up))) if up.size else 0.0 + return 20.0 * math.log10(peak) if peak > 0 else float("-inf") + + +def sample_peak_dbfs(data: np.ndarray) -> float: + peak = float(np.max(np.abs(data))) if data.size else 0.0 + return 20.0 * math.log10(peak) if peak > 0 else float("-inf") + + +def clipped_runs(data: np.ndarray, threshold: float = 0.9995, min_run: int = 3) -> int: + """Count runs of consecutive samples pinned at full scale. + + A single sample at full scale is unremarkable; three or more in a row is the + signature of something that was already clipped before it reached us. + """ + mono = np.max(np.abs(data), axis=1) + hot = mono >= threshold + if not np.any(hot): + return 0 + edges = np.diff(np.concatenate(([0], hot.view(np.int8), [0]))) + starts = np.flatnonzero(edges == 1) + ends = np.flatnonzero(edges == -1) + return int(np.sum((ends - starts) >= min_run)) + + +def dc_offset(data: np.ndarray) -> float: + return float(np.max(np.abs(np.mean(data, axis=0)))) if data.size else 0.0 + + +def stereo_correlation(data: np.ndarray) -> float | None: + """Correlation between channels: ~1.0 is near-mono, negative risks + cancellation when a listener's playback folds to mono.""" + if data.shape[1] < 2: + return None + left, right = data[:, 0], data[:, 1] + if np.std(left) < 1e-9 or np.std(right) < 1e-9: + return None + return float(np.corrcoef(left, right)[0, 1]) + + +@dataclass +class Measurements: + """Everything measurable about a rendered file, as data.""" + + duration_s: float + sample_rate: int + channels: int + integrated_lufs: float + loudness_range_lu: float + true_peak_dbtp: float + sample_peak_dbfs: float + clipped_runs: int + dc_offset: float + stereo_correlation: float | None + silent: bool + + def to_dict(self) -> dict: + d = asdict(self) + # -inf is valid for digital silence but is not JSON, so report it as None. + for k, v in d.items(): + if isinstance(v, float) and not math.isfinite(v): + d[k] = None + return d + + +def measure(path: str) -> Measurements: + data, rate = read_audio(path) + return Measurements( + duration_s=round(data.shape[0] / rate, 3), + sample_rate=rate, + channels=int(data.shape[1]), + integrated_lufs=round(integrated_loudness(data, rate), 2), + loudness_range_lu=round(loudness_range(data, rate), 2), + true_peak_dbtp=round(true_peak_dbtp(data, rate), 2), + sample_peak_dbfs=round(sample_peak_dbfs(data), 2), + clipped_runs=clipped_runs(data), + dc_offset=round(dc_offset(data), 6), + stereo_correlation=(round(c, 4) if (c := stereo_correlation(data)) is not None else None), + silent=bool(np.max(np.abs(data)) < 1e-5) if data.size else True, + ) + + +# --------------------------------------------------------------------------- +# Filters. RBJ Audio EQ Cookbook biquads, returned as second-order sections. +# --------------------------------------------------------------------------- + + +def _biquad(kind: str, freq: float, rate: float, q: float = 0.707, gain_db: float = 0.0) -> np.ndarray: + w0 = 2.0 * math.pi * max(min(freq, rate * 0.49), 1.0) / rate + cos_w0, sin_w0 = math.cos(w0), math.sin(w0) + alpha = sin_w0 / (2.0 * max(q, 1e-3)) + A = 10.0 ** (gain_db / 40.0) + + if kind == "highpass": + b = [(1 + cos_w0) / 2, -(1 + cos_w0), (1 + cos_w0) / 2] + a = [1 + alpha, -2 * cos_w0, 1 - alpha] + elif kind == "lowpass": + b = [(1 - cos_w0) / 2, 1 - cos_w0, (1 - cos_w0) / 2] + a = [1 + alpha, -2 * cos_w0, 1 - alpha] + elif kind == "peaking": + b = [1 + alpha * A, -2 * cos_w0, 1 - alpha * A] + a = [1 + alpha / A, -2 * cos_w0, 1 - alpha / A] + elif kind in ("lowshelf", "highshelf"): + sq = 2.0 * math.sqrt(A) * alpha + if kind == "lowshelf": + b = [ + A * ((A + 1) - (A - 1) * cos_w0 + sq), + 2 * A * ((A - 1) - (A + 1) * cos_w0), + A * ((A + 1) - (A - 1) * cos_w0 - sq), + ] + a = [(A + 1) + (A - 1) * cos_w0 + sq, -2 * ((A - 1) + (A + 1) * cos_w0), (A + 1) + (A - 1) * cos_w0 - sq] + else: + b = [ + A * ((A + 1) + (A - 1) * cos_w0 + sq), + -2 * A * ((A - 1) + (A + 1) * cos_w0), + A * ((A + 1) + (A - 1) * cos_w0 - sq), + ] + a = [(A + 1) - (A - 1) * cos_w0 + sq, 2 * ((A - 1) - (A + 1) * cos_w0), (A + 1) - (A - 1) * cos_w0 - sq] + else: + raise ValueError(f"unknown filter kind {kind!r}") + + b = np.array(b, dtype=np.float64) / a[0] + a = np.array(a, dtype=np.float64) / a[0] + return np.concatenate([b, a])[None, :] + + +def apply_filters(data: np.ndarray, rate: int, bands: list[dict]) -> np.ndarray: + """Apply a list of ``{type, freq_hz, gain_db, q}`` bands in series. + + Uses sosfilt rather than sosfiltfilt: a mastering chain is causal, and + zero-phase filtering would smear transients backwards in time. + """ + if not bands: + return data + sos = np.vstack( + [ + _biquad( + b.get("type", "peaking"), + float(b.get("freq_hz", 1000.0)), + rate, + float(b.get("q", 0.707)), + float(b.get("gain_db", 0.0)), + ) + for b in bands + ] + ) + return signal.sosfilt(sos, data, axis=0) + + +# --------------------------------------------------------------------------- +# Dynamics +# --------------------------------------------------------------------------- + + +def _smooth_envelope(level_db: np.ndarray, rate: int, attack_ms: float, release_ms: float) -> np.ndarray: + """One-pole attack/release follower over a dB-domain level signal.""" + att = math.exp(-1.0 / max(attack_ms * 1e-3 * rate, 1.0)) + rel = math.exp(-1.0 / max(release_ms * 1e-3 * rate, 1.0)) + out = np.empty_like(level_db) + prev = level_db[0] if level_db.size else 0.0 + for i, v in enumerate(level_db): + coeff = att if v > prev else rel + prev = coeff * prev + (1.0 - coeff) * v + out[i] = prev + return out + + +def compress( + data: np.ndarray, + rate: int, + threshold_db: float = -18.0, + ratio: float = 2.0, + attack_ms: float = 20.0, + release_ms: float = 200.0, + knee_db: float = 6.0, + makeup_db: float | None = None, +) -> tuple[np.ndarray, dict]: + """Feed-forward peak compressor with a soft knee. + + Gain reduction is computed from the linked maximum across channels so the + stereo image is not pulled around by one channel ducking alone. + """ + if data.size == 0: + return data, {"max_gain_reduction_db": 0.0} + detector = np.max(np.abs(data), axis=1) + with np.errstate(divide="ignore"): + level_db = 20.0 * np.log10(np.maximum(detector, 1e-12)) + + over = level_db - threshold_db + target_reduction = np.zeros_like(over) + if knee_db > 0: + in_knee = (over > -knee_db / 2) & (over <= knee_db / 2) + above = over > knee_db / 2 + target_reduction[in_knee] = (1.0 / ratio - 1.0) * (over[in_knee] + knee_db / 2) ** 2 / (2.0 * knee_db) + target_reduction[above] = (1.0 / ratio - 1.0) * over[above] + else: + above = over > 0 + target_reduction[above] = (1.0 / ratio - 1.0) * over[above] + + smoothed = _smooth_envelope(-target_reduction, rate, attack_ms, release_ms) + gain = 10.0 ** (-smoothed / 20.0) + out = data * gain[:, None] + if makeup_db: + out = out * (10.0 ** (makeup_db / 20.0)) + return out, {"max_gain_reduction_db": round(float(np.max(smoothed)), 2)} + + +def limit( + data: np.ndarray, + rate: int, + ceiling_dbtp: float = -1.0, + lookahead_ms: float = 5.0, + release_ms: float = 50.0, + oversample: int = 4, +) -> tuple[np.ndarray, dict]: + """Look-ahead true-peak limiter. + + The gain envelope is derived from the 4x oversampled signal, because the + ceiling that matters is the true peak a decoder reconstructs, not the sample + peak we happen to store. Look-ahead means the gain is already down by the + time the transient arrives, rather than clamping it after the fact. + """ + if data.size == 0: + return data, {"max_gain_reduction_db": 0.0} + ceiling = 10.0 ** (ceiling_dbtp / 20.0) + + up = signal.resample_poly(data, oversample, 1, axis=0) + peak_up = np.max(np.abs(up), axis=1) + # Fold the oversampled envelope back to the base rate, keeping the worst + # case in each original sample period. + usable = (peak_up.size // oversample) * oversample + peak = peak_up[:usable].reshape(-1, oversample).max(axis=1) + if peak.size < data.shape[0]: + peak = np.concatenate([peak, np.repeat(peak[-1:], data.shape[0] - peak.size)]) + peak = peak[: data.shape[0]] + + needed = np.minimum(1.0, ceiling / np.maximum(peak, 1e-12)) + look = max(1, round(lookahead_ms * 1e-3 * rate)) + # Running minimum over the look-ahead window: pull gain down early. + padded = np.concatenate([needed, np.repeat(needed[-1:], look)]) + strides = np.lib.stride_tricks.sliding_window_view(padded, look + 1) + target = strides.min(axis=1)[: needed.size] + + rel = math.exp(-1.0 / max(release_ms * 1e-3 * rate, 1.0)) + gain = np.empty_like(target) + prev = 1.0 + for i, t in enumerate(target): + prev = t if t < prev else rel * prev + (1.0 - rel) * t + gain[i] = prev + + out = data * gain[:, None] + with np.errstate(divide="ignore"): + reduction = -20.0 * np.log10(np.maximum(gain.min(), 1e-12)) + return out, {"max_gain_reduction_db": round(float(reduction), 2)} + + +def normalise_loudness( + data: np.ndarray, rate: int, target_lufs: float, max_gain_db: float = 24.0 +) -> tuple[np.ndarray, dict]: + """Apply a single broadband gain so integrated loudness hits the target.""" + current = integrated_loudness(data, rate) + if not math.isfinite(current): + return data, {"applied_gain_db": 0.0, "measured_before_lufs": None} + gain_db = float(np.clip(target_lufs - current, -max_gain_db, max_gain_db)) + return data * (10.0 ** (gain_db / 20.0)), { + "applied_gain_db": round(gain_db, 2), + "measured_before_lufs": round(current, 2), + } + + +# --------------------------------------------------------------------------- +# Similarity features, for the compliance agent +# --------------------------------------------------------------------------- + + +def _chroma_filterbank(n_fft: int, rate: int) -> np.ndarray: + """Map FFT bins onto 12 pitch classes. + + Each bin is assigned to the pitch class of its centre frequency, which is + crude next to a constant-Q transform but needs no extra dependency and is + sufficient to compare harmonic content between two renders. + """ + freqs = np.fft.rfftfreq(n_fft, 1.0 / rate) + bank = np.zeros((12, freqs.size)) + with np.errstate(divide="ignore", invalid="ignore"): + midi = 69.0 + 12.0 * np.log2(np.maximum(freqs, 1e-9) / 440.0) + valid = (freqs > 55.0) & (freqs < 5000.0) + pitch_class = np.mod(np.round(midi).astype(int), 12) + for b in np.flatnonzero(valid): + bank[pitch_class[b], b] = 1.0 + return bank + + +def chroma(path: str, hop_s: float = 0.1, n_fft: int = 4096) -> np.ndarray: + """CENS-style chroma: (frames, 12), L2-normalised per frame. + + Normalising per frame makes the feature insensitive to level, so a louder + master of the same material still matches. + """ + data, rate = read_audio(path) + mono = np.mean(data, axis=1) + hop = max(1, round(hop_s * rate)) + if mono.size < n_fft: + mono = np.pad(mono, (0, n_fft - mono.size)) + window = np.hanning(n_fft) + frames = 1 + (mono.size - n_fft) // hop + bank = _chroma_filterbank(n_fft, rate) + out = np.empty((frames, 12)) + for i in range(frames): + seg = mono[i * hop : i * hop + n_fft] * window + mag = np.abs(np.fft.rfft(seg)) + v = bank @ mag + norm = np.linalg.norm(v) + out[i] = v / norm if norm > 1e-9 else 0.0 + return out + + +def chroma_dtw_distance(a: np.ndarray, b: np.ndarray, transpositions: bool = True) -> tuple[float, int]: + """Normalised DTW distance between two chromagrams, and the best rotation. + + Trying all twelve rotations of the pitch-class axis is what catches material + that was transposed rather than copied verbatim, which is the case a plain + fingerprint match misses entirely. Returns (distance in [0, 1], semitones). + """ + if a.size == 0 or b.size == 0: + return 1.0, 0 + best = (1.0, 0) + for shift in range(12) if transpositions else (0,): + rolled = np.roll(b, shift, axis=1) + # Cosine distance matrix; both inputs are already L2-normalised. + cost = 1.0 - (a @ rolled.T) + n, m = cost.shape + acc = np.full((n + 1, m + 1), np.inf) + acc[0, 0] = 0.0 + for i in range(1, n + 1): + row, prev = acc[i], acc[i - 1] + c = cost[i - 1] + for j in range(1, m + 1): + row[j] = c[j - 1] + min(prev[j], row[j - 1], prev[j - 1]) + d = float(acc[n, m] / (n + m)) + if d < best[0]: + best = (d, shift) + return best + + +@dataclass +class SimilarityHit: + reference: str + distance: float + semitone_shift: int + similarity: float = field(init=False) + + def __post_init__(self) -> None: + # Distance 0 is identical; report a friendlier 0-1 similarity too. + self.similarity = round(max(0.0, 1.0 - self.distance), 4) + + def to_dict(self) -> dict: + return { + "reference": self.reference, + "distance": round(self.distance, 4), + "semitone_shift": self.semitone_shift, + "similarity": self.similarity, + } diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/compliance_agent.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/compliance_agent.py new file mode 100644 index 000000000..6f9ec1915 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/compliance_agent.py @@ -0,0 +1,813 @@ +"""Compliance agent. + +Screens the finished master before release. Two independent checks, both on the +actual audio: + +1. **Delivery QC.** Re-measures loudness, true peak, clipping, DC offset and mono + compatibility, and compares them against the targets the mastering agent said + it was hitting. The mastering agent grading its own homework is not a control; + this is. + +2. **Similarity screening.** Compares the master's harmonic content against + the reference catalogue using chroma features and subsequence DTW over + all twelve transpositions, so a copy that was shifted to another key still + matches. An acoustic fingerprint would only catch a byte-level duplicate. + +If the screen flags the track, it calls back into the composition agent's +runtime -- same session, same instance -- for an original replacement, then +re-screens. + +**This is a screen, not a copyright clearance.** It compares against one local +catalogue with one feature. A flag means escalate to a human or a licensed +identification service; a pass means nothing was found in that catalogue. + +The verdict is computed from the measurements, not asked of the model. The model +writes the explanation and the remediation brief. That way the prose and the +verdict cannot disagree, which they can when a model is asked to do both. + +Packaged as a zip artifact on Amazon S3 rather than a container image, which is +the point of shipping it alongside the other two: artifact type is a per-team +choice and mixed artifacts coexist on one capacity provider. It stays a zip +because its dependencies are numpy, scipy and soundfile -- about 54 MB against +the 250 MB compressed limit. A torch-based embedding model would not fit and +would force it to become a container. + +Note on trust: agents collocated on one instance are not isolated from each other +and can read each other's credentials, so only mutually trusted agents belong in +a session. All three here are first-party. A third-party vendor's agent belongs +on its own capacity provider. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import platform +import uuid +from pathlib import Path + +import audio_dsp as audio +import boto3 +from bedrock_agentcore.runtime import BedrockAgentCoreApp +from botocore.config import Config +from pydantic import BaseModel, Field, field_validator +from strands import Agent +from strands.models import BedrockModel +from strands.session import FileSessionManager + +AGENT_NAME = "compliance" +PROCESS_ID = uuid.uuid4().hex[:8] + +WORKSPACE = Path(os.environ.get("WORKSPACE_DIR", "/mnt/tracks")) +MODEL_ID = os.environ.get("MODEL_ID", "global.anthropic.claude-sonnet-4-6") +REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") +ARTIFACT_BUCKET = os.environ.get("ARTIFACT_BUCKET") + +# Set by deploy.py once the composition runtime exists. Its ARN cannot be +# hand-written: CreateAgentRuntime appends a random 10-character suffix. +COMPOSITION_RUNTIME_ARN = os.environ.get("COMPOSITION_RUNTIME_ARN") +COMPOSITION_QUALIFIER = os.environ.get("COMPOSITION_QUALIFIER", "DEFAULT") + +# Absolute distance thresholds. These are empirical and, on their own, NOT +# sufficient -- which is the honest lesson from calibrating them: +# +# synthetic melodies : identical 0.000 | transposed 0.002 | unrelated 0.273 +# real generated audio, derivative : 0.029, 0.050 +# real generated audio, unrelated : 0.062, 0.063, 0.093, 0.122 +# +# Those two real-audio ranges OVERLAP, so no single cut separates a copy from an +# original: 0.08 failed an unrelated track, and 0.045 cleared a track that had been +# conditioned directly on a catalogue reference. The relative STANDOUT_RATIO test +# below is what actually discriminates; these absolutes are the coarse first pass. +# Recalibrate all three against your own catalogue before trusting them. +FAIL_DISTANCE = float(os.environ.get("SIMILARITY_FAIL_DISTANCE", "0.045")) +REVIEW_DISTANCE = float(os.environ.get("SIMILARITY_REVIEW_DISTANCE", "0.10")) + +# The relative test. If the closest match is less than this fraction of the +# next-closest distance, one catalogue track stands out and that is suspicious +# regardless of the absolute number. Measured derivative ratios: 0.239 and 0.384. +# Measured original: 0.98. Needs at least two references to mean anything. +STANDOUT_RATIO = float(os.environ.get("SIMILARITY_STANDOUT_RATIO", "0.6")) + +SYSTEM_PROMPT = """You are a music copyright and delivery compliance analyst. + +You are given measurements of a finished master and the output of a similarity +screen against the reference catalogue. Explain what the numbers mean for +release readiness, in plain language, for a producer who is not an engineer. + +Rules: +- The screen has already decided every comparison. State its conclusions and + explain what they mean musically. Do not re-derive a verdict, and do not compare + a measurement against a threshold -- the report prints the figures itself. +- Do not speculate about infringement the screen did not find, and do not dismiss + a flag it did raise. +- Common chord progressions and standard song forms are not infringement. +- If a match was flagged or needs review, describe precisely what a replacement + must change: the harmonic movement, the melodic contour, the key. +- Be explicit that this is a screen against one catalogue, not a legal clearance. + +Under 300 words.""" + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(AGENT_NAME) + +app = BedrockAgentCoreApp() + + +class WorkflowError(Exception): + """A recoverable problem in the pipeline, reported to the caller as data.""" + + +class Review(BaseModel): + """The model's explanation. It does not decide the verdict.""" + + summary: str = Field(description="Two sentences on release readiness.") + explanation: str = Field(default="", description="What the measurements mean.") + remediation_instruction: str = Field( + default="", description="If a flag was raised, precisely what a replacement must change." + ) + + @field_validator("summary", "explanation", "remediation_instruction", mode="before") + @classmethod + def _stringify(cls, v): + if v is None: + return "" + if isinstance(v, (list, dict)): + return json.dumps(v) + return str(v) + + +# --------------------------------------------------------------------- workspace + + +def track_path(track_id: str) -> Path: + safe = "".join(c for c in track_id if c.isalnum() or c in "-_")[:64] or "track" + return WORKSPACE / safe + + +def require_track_dir(track_id: str) -> Path: + """Fail loudly if the composition agent has not created the track yet. + + Deliberately does not create it: this agent ships as a zip and runs as a real + host user, while the container agents run as a namespaced root. Whichever + identity creates a directory first can lock the other out, so creation + belongs to the composition agent alone. + """ + p = track_path(track_id) + if not p.is_dir(): + raise WorkflowError( + f"No workspace for track '{track_id}'. Invoke the composition agent first, with the same runtimeSessionId." + ) + return p + + +def read_text(track_id: str, name: str) -> str | None: + p = track_path(track_id) / name + return p.read_text(encoding="utf-8") if p.exists() else None + + +def write_text(track_id: str, name: str, text: str) -> Path: + p = track_path(track_id) / name + p.write_text(text, encoding="utf-8") + try: + p.chmod(0o664) + except PermissionError: + pass + logger.info("wrote %s (%d bytes)", p, len(text)) + return p + + +def list_artifacts(track_id: str) -> list[str]: + p = track_path(track_id) + return sorted(x.name for x in p.iterdir() if x.is_file()) if p.is_dir() else [] + + +def subject_audio(track_id: str) -> tuple[str, Path, bool]: + """What is actually under review, and whether the master is stale. + + The master is the right subject only while it is newer than the newest render. + After a remediation the composition agent has written a fresh + composition_remediated.wav and nothing has re-mastered it, so screening + master.wav again would re-judge the very material that was just replaced -- + measured: a remediated track stayed NOT CLEARED because the stale master was + screened a second time. + + Returns (name, path, master_is_stale). + """ + directory = track_path(track_id) + renders = [ + p + for p in (directory / "composition_remediated.wav", directory / "composition.wav") + if p.exists() and p.stat().st_size > 0 + ] + master = directory / "master.wav" + has_master = master.exists() and master.stat().st_size > 0 + + if has_master and renders: + newest_render = max(renders, key=lambda p: p.stat().st_mtime) + if newest_render.stat().st_mtime > master.stat().st_mtime: + # The master predates the current render: screen the render and say so. + return newest_render.name, newest_render, True + return master.name, master, False + if has_master: + return master.name, master, False + if renders: + newest = max(renders, key=lambda p: p.stat().st_mtime) + return newest.name, newest, False + raise WorkflowError( + "No audio to review in the shared workspace. Invoke the composition and " + "mastering agents first, with the same runtimeSessionId." + ) + + +def publish(track_id: str, path: Path) -> dict: + """Copy an artifact to S3 and hand back a link the caller can open. + + The volume lives inside a managed instance with no shell and is destroyed + with the session, so S3 is the only route by which a report reaches whoever + invoked us. + """ + info: dict = { + "name": path.name, + "bytes": path.stat().st_size, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest()[:16], + } + if not ARTIFACT_BUCKET: + return info + key = f"tracks/{track_id}/{path.name}" + # The regional endpoint is explicit on purpose. Left to itself botocore + # presigned against the global host (bucket.s3.amazonaws.com) while scoping + # the signature to us-east-2, and every URL came back 403 + # SignatureDoesNotMatch. Measured on a live run. + s3 = boto3.client("s3", region_name=REGION, endpoint_url=f"https://s3.{REGION}.amazonaws.com") + s3.upload_file(str(path), ARTIFACT_BUCKET, key) + info["s3_uri"] = f"s3://{ARTIFACT_BUCKET}/{key}" + info["url"] = s3.generate_presigned_url( + "get_object", Params={"Bucket": ARTIFACT_BUCKET, "Key": key}, ExpiresIn=86400 + ) + logger.info("published %s", info["s3_uri"]) + return info + + +def host_info() -> dict: + """Facts that make collocation observable from the response.""" + u = platform.uname() + return {"process_id": PROCESS_ID, "hostname": u.node, "architecture": u.machine, "cpus": os.cpu_count()} + + +# -------------------------------------------------------------------- the checks + + +def delivery_qc(path: Path, track_id: str, is_master: bool = True) -> dict: + """Re-measure the subject and check it against what mastering claimed. + + ``is_master`` matters. Delivery targets only apply to a master; asserting them + against a raw render makes the verdict guaranteed-fail after any remediation, + because an unmastered file is of course not at -14 LUFS. Measured case: a + freshly remediated render was failed for being -16.88 LUFS against a target it + had never been through mastering to meet. Those two checks become + informational when the subject is not the master. + """ + m = audio.measure(str(path)).to_dict() + claimed: dict = {} + source_lra: float | None = None + raw = read_text(track_id, "mastering.json") + if raw: + try: + result = json.loads(raw).get("result") or {} + claimed = result.get("targets") or {} + # What the mix measured BEFORE mastering, so the dynamics check can + # ask whether mastering crushed the material rather than whether the + # material had dynamics in the first place. + source_lra = (result.get("before") or {}).get("loudness_range_lu") + except json.JSONDecodeError: + claimed = {} + target_lufs = claimed.get("lufs") + target_peak = claimed.get("true_peak_dbtp") + + checks: list[dict] = [] + + def check(name: str, ok: bool, detail: str) -> None: + checks.append({"check": name, "pass": bool(ok), "detail": detail}) + + if target_lufs is not None and m["integrated_lufs"] is not None: + err = abs(m["integrated_lufs"] - target_lufs) + detail = f"{m['integrated_lufs']} LUFS against a {target_lufs} LUFS target (error {err:.2f} LU, tolerance 1.0)" + if is_master: + check("integrated_loudness", err <= 1.0, detail) + else: + checks.append( + { + "check": "integrated_loudness", + "pass": True, + "detail": f"not applicable to an unmastered render: {detail}", + } + ) + if target_peak is not None and m["true_peak_dbtp"] is not None: + detail = f"{m['true_peak_dbtp']} dBTP against a {target_peak} dBTP ceiling" + if is_master: + check("true_peak_ceiling", m["true_peak_dbtp"] <= target_peak + 0.1, detail) + else: + checks.append( + { + "check": "true_peak_ceiling", + "pass": True, + "detail": f"not applicable to an unmastered render: {detail}", + } + ) + check("no_clipping", m["clipped_runs"] == 0, f"{m['clipped_runs']} run(s) of consecutive full-scale samples") + check("dc_offset", m["dc_offset"] < 0.005, f"DC offset {m['dc_offset']}") + check("not_silent", not m["silent"], "audio is present") + if m["stereo_correlation"] is not None: + check("mono_compatible", m["stereo_correlation"] > -0.2, f"inter-channel correlation {m['stereo_correlation']}") + # Loudness range needs enough 3-second blocks to mean anything, and this + # agent's job is to catch a master that destroyed the mix, not to fail a mix + # that was uniform to begin with. Measured case that got this wrong: a 24 s + # generated loop came in at 0.47 LU and went out at 0.6 LU, and an absolute + # 1.0 LU floor blamed mastering for the source's own uniformity. + if m["loudness_range_lu"] is None or (m["duration_s"] or 0) < 30: + checks.append( + { + "check": "dynamics_retained", + "pass": True, + "detail": f"not assessed: {m['duration_s']}s is too short for a meaningful loudness range", + } + ) + elif source_lra is not None: + floor = max(0.0, source_lra * 0.5 - 0.1) + check( + "dynamics_retained", + m["loudness_range_lu"] >= floor, + f"loudness range {m['loudness_range_lu']} LU against {source_lra} LU in the mix (at least half expected)", + ) + else: + check( + "dynamics_retained", + m["loudness_range_lu"] >= 1.0, + f"loudness range {m['loudness_range_lu']} LU (no pre-master measurement available to compare against)", + ) + + return { + "measurements": m, + "claimed_targets": claimed, + "source_lra": source_lra, + "checks": checks, + "passed": all(c["pass"] for c in checks), + } + + +def catalogue_metadata(track_id: str) -> dict[str, dict]: + """What the catalogue tracks were made from, keyed by filename. + + Read so that a flag can tell the composition agent *what to diverge from* + rather than just "be different". catalogue.json is written by the composition + agent's mode=catalogue and holds the style_tags and title of each reference. + """ + raw = read_text(track_id, "catalogue.json") + if not raw: + return {} + try: + entries = json.loads(raw) + except json.JSONDecodeError: + return {} + out: dict[str, dict] = {} + for e in entries if isinstance(entries, list) else []: + name = e.get("file") + if name: + out[name] = {"title": e.get("title"), "style_tags": e.get("style_tags")} + return out + + +def similarity_screen(subject: Path, track_id: str) -> dict: + """Compare the subject against the local back-catalogue.""" + cat_dir = track_path(track_id) / "catalogue" + references = sorted(cat_dir.glob("*.wav")) if cat_dir.is_dir() else [] + if not references: + return { + "references": 0, + "hits": [], + "flagged": False, + "note": "no back-catalogue on the volume; similarity was not screened", + } + + metadata = catalogue_metadata(track_id) + subject_chroma = audio.chroma(str(subject)) + hits = [] + for ref in references: + distance, shift = audio.chroma_dtw_distance(subject_chroma, audio.chroma(str(ref))) + hit = audio.SimilarityHit(reference=ref.name, distance=distance, semitone_shift=shift).to_dict() + hit["verdict"] = ( + "near_duplicate" if distance < FAIL_DISTANCE else "review" if distance < REVIEW_DISTANCE else "clear" + ) + # Carried through so remediation can be told what to move away from. + hit.update(metadata.get(ref.name, {})) + hits.append(hit) + hits.sort(key=lambda h: h["distance"]) + closest = hits[0] + + # A second, relative signal, because absolute distance alone does not work. + # Measured across runs: a genuine derivative scored 0.029 and 0.050 while + # unrelated same-genre material scored 0.062 to 0.122 -- overlapping ranges, so + # any single cut either misses copies or fails originals. + # + # What does separate them is how far the closest match stands out from the rest + # of the catalogue. A derivative is much closer to one track than to the others + # (ratios 0.239 and 0.384); an original is roughly equidistant (0.98). + standout_ratio = None + if len(hits) >= 2 and hits[1]["distance"] > 0: + standout_ratio = round(closest["distance"] / hits[1]["distance"], 4) + standout = standout_ratio is not None and standout_ratio < STANDOUT_RATIO + + below_fail = closest["distance"] < FAIL_DISTANCE + return { + "references": len(references), + "hits": hits, + "closest": closest, + # Either signal is enough to flag: an outright close match, or one track + # that stands out sharply from the rest of the catalogue. + "flagged": bool(below_fail or standout), + "flag_reason": ( + "distance below the fail threshold" + if below_fail + else "closest match stands out from the catalogue" + if standout + else None + ), + "needs_review": bool(not below_fail and not standout and closest["distance"] < REVIEW_DISTANCE), + "standout_ratio": standout_ratio, + "thresholds": { + "fail_below": FAIL_DISTANCE, + "review_below": REVIEW_DISTANCE, + "standout_ratio_below": STANDOUT_RATIO, + }, + } + + +def remediation_available() -> bool: + return bool(COMPOSITION_RUNTIME_ARN) + + +def call_composition_runtime(session_id: str, track_id: str, issue: str, avoid: dict | None = None) -> tuple[bool, str]: + """Invoke the composition agent's runtime for remediation. + + Uses the caller's own session id, so AgentCore routes the request to the + composition agent already running on this instance instead of provisioning + another one. + + Returns (remediated, message). The boolean matters: "the screen asked for a + replacement" and "a replacement now exists" are different facts, and only the + second licenses the entrypoint to go looking for the new file. + """ + if not COMPOSITION_RUNTIME_ARN: + return False, "Remediation is unavailable: COMPOSITION_RUNTIME_ARN is not set." + client = boto3.client( + "bedrock-agentcore", + region_name=REGION, + # A nested invocation runs inside this request's 15-minute budget, so the + # read timeout must comfortably exceed the inner render. Retries stay on + # to absorb RetryableConflictException (409). + config=Config(read_timeout=600, retries={"max_attempts": 3, "mode": "standard"}), + ) + response = client.invoke_agent_runtime( + agentRuntimeArn=COMPOSITION_RUNTIME_ARN, + qualifier=COMPOSITION_QUALIFIER, + runtimeSessionId=session_id, + payload=json.dumps( + { + "mode": "remediate", + "track_id": track_id, + "issue": issue, + # The evidence, not just the complaint. Without this the + # composition agent is rewriting blind: measured, a blind + # rewrite moved chroma distance only 0.029 -> 0.093. + "avoid": avoid or {}, + } + ).encode(), + ) + # The response body member is named "response", not "body". + body = json.loads(response["response"].read()) + if body.get("status") != "ok": + return False, f"Remediation failed: {body.get('error', 'unknown error')}" + logger.info("remediation produced %s", [a.get("name") for a in body.get("artifacts", [])]) + return True, body.get("result", "") + + +def build_agent(session_id: str, track_id: str) -> Agent: + kwargs: dict = {"model_id": MODEL_ID} + if REGION: + kwargs["region_name"] = REGION + return Agent( + name=AGENT_NAME, + model=BedrockModel(**kwargs), + system_prompt=SYSTEM_PROMPT, + session_manager=FileSessionManager( + session_id=f"{session_id}-{AGENT_NAME}", + # Per-agent: FileSessionManager creates its storage directory with + # mode 0700, which locks out agents running as another identity. + storage_dir=str(track_path(track_id) / f".sessions-{AGENT_NAME}"), + ), + ) + + +def screen(track_id: str) -> dict: + """Run both checks and compute the verdict from them.""" + name, subject, master_stale = subject_audio(track_id) + qc = delivery_qc(subject, track_id, is_master=(name == "master.wav")) + sim = similarity_screen(subject, track_id) + # Three outcomes, not two. Auto-clearing anything the screen wanted a human to + # look at is the one thing a compliance tool must not do -- measured: a track + # conditioned directly on a catalogue reference scored 0.0498, landed in the + # review band, and was reported CLEARED. + if not qc["passed"] or sim["flagged"]: + outcome = "not_cleared" + elif sim.get("needs_review"): + outcome = "review_required" + else: + outcome = "cleared" + + return { + "subject": name, + "master_is_stale": master_stale, + "delivery_qc": qc, + "similarity": sim, + # The verdict is arithmetic on the measurements, never the model's call. + # A stale master is not a release blocker; it is a "re-master this" signal, + # so it does not fail the verdict on its own. + "outcome": outcome, + "passed": outcome == "cleared", + } + + +def findings_brief(result: dict) -> str: + """Render what the screen decided, in words, with no figures to compare. + + Every comparison in this pipeline is already resolved in code: each hit carries + a verdict, the screen carries flagged / needs_review / flag_reason, and the + outcome is computed arithmetic. Handing the model raw distances next to raw + thresholds invites it to re-derive what has already been derived, and a smaller + model did exactly that -- it wrote "0.0892 remains below the automatic fail + threshold of 0.045" into a delivery report, conflating the fail and review + thresholds it had been given by name. The conclusion happened to be right and + the premise was false; only the computed verdict kept the report honest. + + Choosing a stronger model makes that rarer. Not asking makes it impossible, so + the figures are withheld here -- render_review prints them from the same dict, + straight out of the measurements, where no model can restate them wrongly. + """ + qc, sim = result["delivery_qc"], result["similarity"] + lines = [ + f"Subject file: {result['subject']}", + ( + f"DECIDED OUTCOME: {result.get('outcome')} -- final, computed from the " + "measurements. Explain it; do not re-derive it." + ), + "", + f"Delivery QC: {'passed' if qc['passed'] else 'FAILED'}.", + ] + lines += [f" - {c['check']}: {'pass' if c['pass'] else 'FAIL'} -- {c['detail']}" for c in qc.get("checks", [])] + + lines += ["", "Similarity screen:"] + if not sim.get("references"): + lines.append(f" {sim.get('note', 'not screened')}") + return "\n".join(lines) + + lines.append(f" {sim['references']} catalogue reference(s), all 12 transpositions.") + if sim.get("flagged"): + lines.append(f" FLAGGED, and the reason is: {sim['flag_reason']}.") + elif sim.get("needs_review"): + # needs_review is decided on the closest hit, but more than one can land in + # the band -- count them rather than saying "one match" and being wrong. + n = sum(1 for h in sim.get("hits", []) if h.get("verdict") == "review") + lines.append( + f" Not flagged for blocking, but {n} match(es) fall in the band " + "that requires manual review before release." + ) + else: + lines.append(" Nothing flagged, and nothing in the manual-review band.") + + states = {"near_duplicate": "near-duplicate", "review": "needs manual review", "clear": "cleared"} + for h in sim.get("hits", []): + # The transposition is musical information, not a threshold comparison, so + # it stays. Distances and ratios do not. + # catalogue.json may be absent, in which case there is no title to show and + # repeating the filename in quotes reads like a bug. + named = f' ("{h["title"]}")' if h.get("title") else "" + line = ( + f" - {h['reference']}{named}: " + f"{states.get(h.get('verdict'), h.get('verdict'))}, " + f"best alignment at {int(h.get('semitone_shift', 0)):+d} semitones" + ) + if h.get("style_tags"): + line += f", generated from: {h['style_tags']}" + lines.append(line) + return "\n".join(lines) + + +def render_review(result: dict, review: Review, remediated: bool) -> str: + qc, sim = result["delivery_qc"], result["similarity"] + label = {"cleared": "CLEARED", "review_required": "REVIEW REQUIRED", "not_cleared": "NOT CLEARED"}[ + result.get("outcome", "not_cleared") + ] + lines = [ + f"# Compliance Review: `{result['subject']}`", + "", + f"**Verdict: {label}**", + "", + review.summary, + "", + "## Delivery QC", + "", + "| check | result | detail |", + "|---|---|---|", + ] + for c in qc["checks"]: + lines.append(f"| {c['check']} | {'pass' if c['pass'] else 'FAIL'} | {c['detail']} |") + lines += ["", "## Similarity screen", ""] + if not sim.get("references"): + lines.append(f"_{sim.get('note')}_") + else: + preamble = f"Screened against {sim['references']} catalogue reference(s), all 12 transpositions." + if sim.get("standout_ratio") is not None: + preamble += ( + f" Closest match is {sim['standout_ratio']}x the next-closest " + f"distance (flagged below {sim['thresholds']['standout_ratio_below']})." + ) + if sim.get("flag_reason"): + preamble += f" **Flagged: {sim['flag_reason']}.**" + lines += [preamble, "", "| reference | distance | similarity | shift | verdict |", "|---|---|---|---|---|"] + for h in sim["hits"]: + lines.append( + f"| {h['reference']} | {h['distance']} | {h['similarity']} | " + f"{h['semitone_shift']:+d} st | {h['verdict']} |" + ) + if review.explanation: + lines += ["", "## Analysis", "", review.explanation] + if not result["passed"] and review.remediation_instruction: + lines += ["", "## Required changes", "", review.remediation_instruction] + if remediated: + lines += [ + "", + ( + "_A replacement was requested from the composition agent and " + "re-screened; the verdict above is for the replacement._" + ), + ] + if result.get("master_is_stale"): + lines += [ + "", + ( + "_Note: `master.wav` predates the render screened above, so it " + "is stale. Re-run the mastering agent before release._" + ), + ] + lines += [ + "", + "---", + "", + ( + "_This is a screen against one local catalogue using one harmonic " + "feature. It is not a copyright clearance. A flag means escalate to " + "a human or a licensed identification service._" + ), + ] + return "\n".join(lines) + + +@app.entrypoint +def invoke(payload, context): + session_id = getattr(context, "session_id", None) or "local-session" + track_id = payload.get("track_id", "demo-track") + prompt = payload.get("prompt") or "Review this track for release." + if not isinstance(prompt, str): + return {"status": "error", "agent": AGENT_NAME, "error": "'prompt' must be a string", "host": host_info()} + + logger.info( + "invoke: track=%s session=%s model=%s remediation=%s", + track_id, + session_id, + MODEL_ID, + "on" if remediation_available() else "off", + ) + + try: + require_track_dir(track_id) + result = screen(track_id) + logger.info("screen: passed=%s closest=%s", result["passed"], result["similarity"].get("closest")) + + remediated = False + if ( + not result["passed"] + and result["similarity"]["flagged"] + and remediation_available() + and payload.get("auto_remediate", True) + ): + closest = result["similarity"]["closest"] + issue = ( + f"The rendered track is harmonically near-identical to catalogue " + f"reference {closest['reference']} (chroma-DTW distance " + f"{closest['distance']}, transposed {closest['semitone_shift']:+d} " + f"semitones, similarity {closest['similarity']}). The screen fails " + f"anything below {FAIL_DISTANCE}; a replacement needs to score above " + f"{REVIEW_DISTANCE} to clear without review." + ) + # Hand over what the flagged reference actually is, so the rewrite can + # move away from something specific instead of guessing. + avoid = { + "reference": closest["reference"], + "title": closest.get("title"), + "style_tags": closest.get("style_tags"), + "distance": closest["distance"], + "semitone_shift": closest["semitone_shift"], + "fail_below": FAIL_DISTANCE, + "clear_above": REVIEW_DISTANCE, + "other_references": [ + {"reference": h["reference"], "style_tags": h.get("style_tags"), "distance": h["distance"]} + for h in result["similarity"]["hits"][1:] + ], + } + remediated, _ = call_composition_runtime(session_id, track_id, issue, avoid) + if remediated: + replacement = track_path(track_id) / "composition_remediated.wav" + if not replacement.exists(): + raise WorkflowError( + "The composition agent reported a successful remediation but " + "composition_remediated.wav is not on the shared volume - the " + "two agents are not seeing the same filesystem. Check that " + "both runtimes mount the same capacity provider volume and " + "were invoked with the same runtimeSessionId." + ) + # Re-screen. The subject is now the replacement, and because the + # verdict is computed from measurements it cannot inherit the + # earlier failure. + result = screen(track_id) + logger.info("re-screen after remediation: passed=%s", result["passed"]) + + agent = build_agent(session_id, track_id) + analysis = agent( + f"Explain these compliance results for the producer.\n\n{findings_brief(result)}\n\nRequest: {prompt}", + structured_output_model=Review, + ) + review = analysis.structured_output or Review( + summary="The model did not return an explanation; the computed verdict stands." + ) + + markdown = render_review(result, review, remediated) + write_text(track_id, "compliance.md", markdown) + report = { + "verdict": { + "passed": result["passed"], + "outcome": result.get("outcome"), + "remediation_requested": remediated, + }, + "subject": result["subject"], + "master_is_stale": result.get("master_is_stale", False), + "delivery_qc": result["delivery_qc"], + "similarity": result["similarity"], + "review": review.model_dump(), + "reviewed_files": list_artifacts(track_id), + } + write_text(track_id, "compliance.json", json.dumps(report, indent=2)) + + artifacts = [ + publish(track_id, track_path(track_id) / "compliance.md"), + publish(track_id, track_path(track_id) / "compliance.json"), + ] + + return { + "status": "ok", + "agent": AGENT_NAME, + "model": MODEL_ID, + "track_id": track_id, + "session_id": session_id, + "read_from": result["subject"], + "master_is_stale": result.get("master_is_stale", False), + "validation_passed": result["passed"], + "outcome": result.get("outcome"), + "verdict": report["verdict"], + "delivery_qc": result["delivery_qc"], + "similarity": result["similarity"], + "result": markdown, + "artifacts": artifacts, + "workspace_files": list_artifacts(track_id), + "host": host_info(), + } + + except WorkflowError as exc: + logger.warning("workflow error: %s", exc) + return { + "status": "error", + "agent": AGENT_NAME, + "track_id": track_id, + "session_id": session_id, + "error": str(exc), + "validation_passed": False, + "workspace_files": list_artifacts(track_id), + "host": host_info(), + } + + +if __name__ == "__main__": + # Explicit bind and explicit port: see the note in composition_agent.py. + app.run(host="0.0.0.0", port=int(os.environ.get("PORT", "8080"))) diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/composition_agent.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/composition_agent.py new file mode 100644 index 000000000..932f58a2b --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/composition_agent.py @@ -0,0 +1,646 @@ +"""Composition agent. + +Turns a producer's request into an actual audio file, using a generative music +model running on the capacity provider instance's GPU. A Bedrock-hosted model +writes the brief and ACE-Step, running locally on the L4, renders it. + +Be clear about how much of that brief reaches the audio: only ``style_tags`` and +``lyrics`` do. ACE-Step takes no other text input, so ``key``, ``tempo_bpm``, +``time_signature``, ``chord_progression``, ``instrumentation``, ``structure`` and +``title`` are documentation for composition.md and for the downstream agents to +read -- tempo influences the render only because the model writes "124 bpm" into +the tag string. The genuinely hard reasoning here is ``mode=remediate``, which has +to move a rejected piece decisively away from what it resembled. + +That split is the reason this sample needs Runtime Instances rather than +microVMs: there is no GPU on the serverless compute type, so a locally hosted +generative model is not merely slower there, it is impossible. + +The model stack is not in this container image. An AgentCore Runtime image is +capped at 2 GB and a CUDA torch build is 3.13 GB of wheels before weights, so the +stack is built onto the capacity provider's ``models`` volume by ``mode=prepare`` +and invoked as a subprocess. See model_stack/prepare.py. + +Packaged as a container image in Amazon ECR. It also owns creation of each track +directory on the shared volume, because a container agent and a zip agent run as +different Linux identities and whichever creates a directory first can lock the +other out. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import platform +import re +import subprocess +import sys +import time +import uuid +from pathlib import Path + +import audio_dsp as audio +import boto3 +from bedrock_agentcore.runtime import BedrockAgentCoreApp +from pydantic import BaseModel, Field, field_validator +from strands import Agent +from strands.models import BedrockModel +from strands.session import FileSessionManager + +AGENT_NAME = "composition" +PROCESS_ID = uuid.uuid4().hex[:8] + +WORKSPACE = Path(os.environ.get("WORKSPACE_DIR", "/mnt/tracks")) +MODEL_VOLUME = Path(os.environ.get("MODELS_DIR", "/mnt/models")) +# A "global." inference profile on purpose: a "us."-prefixed profile does not +# resolve outside US Regions, and this stack is deliberately Region-portable +# because GPU capacity is what dictates where it lands. +MODEL_ID = os.environ.get("MODEL_ID", "global.anthropic.claude-sonnet-4-6") +REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") +ARTIFACT_BUCKET = os.environ.get("ARTIFACT_BUCKET") + +# Rendering is fast on an L4 (measured: 27 steps in ~2.4 s), so the ceiling here +# is generous only to absorb a cold model load. +RENDER_TIMEOUT_S = int(os.environ.get("RENDER_TIMEOUT_S", "900")) +PREPARE_TIMEOUT_S = int(os.environ.get("PREPARE_TIMEOUT_S", "1500")) + +SYSTEM_PROMPT = """You are an AI music composition specialist. + +You produce briefs that a generative audio model will render, so be concrete and +musical. Give a title, key, tempo, time signature, a section-by-section +arrangement, and the instrumentation. + +`style_tags` is what reaches the audio model. It must be a comma-separated list +of concrete musical descriptors -- genre, instrumentation, mood, tempo -- and +nothing else. Good: "melodic techno, analog bass, warm pads, sidechained kick, +124 bpm". Bad: a sentence, or a section-by-section description. + +When asked to remediate a copyright issue, produce a genuinely different piece: +change the key, change the tempo, and change the melodic and harmonic approach. +Write it as a fresh brief in the same format. Do not name, quote or describe the +work that was infringed, and do not narrate what you changed -- downstream agents +read your output as the composition itself. + +Keep prose under 300 words.""" + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(AGENT_NAME) + +app = BedrockAgentCoreApp() + + +class WorkflowError(Exception): + """A recoverable problem in the pipeline, reported to the caller as data.""" + + +class Section(BaseModel): + name: str = Field(description="Intro, Verse, Chorus, Breakdown, Outro, ...") + bars: int = Field(default=8) + description: str = Field(default="") + + +class CompositionBrief(BaseModel): + title: str = Field(description="A short evocative title.") + key: str = Field(default="A minor") + tempo_bpm: int = Field(default=124) + time_signature: str = Field(default="4/4") + chord_progression: str = Field(default="") + instrumentation: list[str] = Field(default_factory=list) + structure: list[Section] = Field(default_factory=list) + style_tags: str = Field(description="Comma-separated concrete descriptors for the audio model.") + lyrics: str = Field(default="[inst]", description="'[inst]' for an instrumental.") + + @field_validator("tempo_bpm") + @classmethod + def _sane_tempo(cls, v): + return int(min(max(int(v), 50), 200)) + + @field_validator("instrumentation", "structure", mode="before") + @classmethod + def _coerce_list(cls, v): + if v is None: + return [] + if isinstance(v, str): + try: + v = json.loads(v) + except json.JSONDecodeError: + return [s.strip() for s in v.split(",") if s.strip()] + return v if isinstance(v, list) else [] + + @field_validator("style_tags", mode="before") + @classmethod + def _flatten_tags(cls, v): + # Models sometimes return a list here despite the schema saying string. + if isinstance(v, list): + return ", ".join(str(x) for x in v) + return v + + def to_markdown(self, render: dict | None = None) -> str: + lines = [ + f"# {self.title}", + "", + (f"**Key:** {self.key} | **Tempo:** {self.tempo_bpm} BPM | **Time signature:** {self.time_signature}"), + "", + ] + if self.chord_progression: + lines += [f"**Chord progression:** {self.chord_progression}", ""] + if self.instrumentation: + lines += ["**Instrumentation:** " + ", ".join(self.instrumentation), ""] + if self.structure: + lines += ["## Arrangement", ""] + lines += [f"- **{s.name}** ({s.bars} bars) — {s.description}".rstrip(" —") for s in self.structure] + lines.append("") + lines += ["## Style tags given to the audio model", "", f"`{self.style_tags}`", ""] + if render: + a = render.get("audio", {}) + t = render.get("timings", {}) + lines += [ + "## Render", + "", + ( + f"- {a.get('duration_s')} s, {a.get('sample_rate')} Hz, " + f"{a.get('channels')} ch, peak {a.get('peak_dbfs')} dBFS" + ), + ( + f"- generated in {t.get('generate_s')} s on {render.get('device')} " + f"(peak VRAM {t.get('peak_vram_gib')} GiB)" + ), + "", + ] + return "\n".join(lines) + + +# --------------------------------------------------------------------- workspace + + +def track_path(track_id: str) -> Path: + safe = "".join(c for c in track_id if c.isalnum() or c in "-_")[:64] or "track" + return WORKSPACE / safe + + +def ensure_track_dir(track_id: str) -> Path: + """Create this track's directory, writable by the other agents. + + Only ever called during an invocation: the volume is not mounted while the + container is still initialising. Made group-writable with the setgid bit + because a container agent (namespaced root, supplementary group + agentcore-runtime-user) and a zip agent (a real host user in that same group) + are different identities. chmod is best-effort; only the creator can change + the mode. + """ + path = track_path(track_id) + path.mkdir(parents=True, exist_ok=True) + try: + path.chmod(0o2775) + except PermissionError: + logger.debug("could not chmod %s - created by another identity", path) + return path + + +def write_text(track_id: str, name: str, text: str) -> Path: + p = track_path(track_id) / name + p.write_text(text, encoding="utf-8") + try: + p.chmod(0o664) + except PermissionError: + pass + logger.info("wrote %s (%d bytes)", p, len(text)) + return p + + +def read_text(track_id: str, name: str) -> str | None: + p = track_path(track_id) / name + return p.read_text(encoding="utf-8") if p.exists() else None + + +def list_artifacts(track_id: str) -> list[str]: + p = track_path(track_id) + return sorted(x.name for x in p.iterdir() if x.is_file()) if p.is_dir() else [] + + +def publish(track_id: str, path: Path) -> dict: + """Copy an artifact to S3 and hand back a link the caller can open. + + The volume lives inside a managed instance with no shell and is destroyed + with the session, so S3 is the only route by which rendered audio reaches + whoever invoked us. + """ + info: dict = { + "name": path.name, + "bytes": path.stat().st_size, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest()[:16], + } + if not ARTIFACT_BUCKET: + return info + key = f"tracks/{track_id}/{path.name}" + # The regional endpoint is explicit on purpose. Left to itself botocore + # presigned against the global host (bucket.s3.amazonaws.com) while scoping + # the signature to us-east-2, and every URL came back 403 + # SignatureDoesNotMatch. Measured on a live run. + s3 = boto3.client("s3", region_name=REGION, endpoint_url=f"https://s3.{REGION}.amazonaws.com") + s3.upload_file(str(path), ARTIFACT_BUCKET, key) + info["s3_uri"] = f"s3://{ARTIFACT_BUCKET}/{key}" + info["url"] = s3.generate_presigned_url( + "get_object", Params={"Bucket": ARTIFACT_BUCKET, "Key": key}, ExpiresIn=86400 + ) + logger.info("published %s", info["s3_uri"]) + return info + + +def host_info() -> dict: + """Facts that make collocation observable from the response.""" + u = platform.uname() + return {"process_id": PROCESS_ID, "hostname": u.node, "architecture": u.machine, "cpus": os.cpu_count()} + + +# ------------------------------------------------------------------ model stack + + +# AgentCore injects the NVIDIA driver into /usr/lib64, which a Debian-based image +# does not search. Set in the Dockerfile too; repeated here so the subprocess is +# correct even if this agent is run from a differently built image. +DRIVER_LIB_DIR = "/usr/lib64" + + +def gpu_env() -> dict[str, str]: + """Environment for the render subprocess, with the driver on the link path. + + Without /usr/lib64 on LD_LIBRARY_PATH, libcuda.so.1 exists but cannot be + loaded, and torch falls back to the CPU silently rather than failing. Measured + on a live g6.xlarge. + """ + env = dict(os.environ) + existing = env.get("LD_LIBRARY_PATH", "") + parts = [DRIVER_LIB_DIR] + [p for p in existing.split(":") if p and p != DRIVER_LIB_DIR] + env["LD_LIBRARY_PATH"] = ":".join(parts) + return env + + +def model_stack_status() -> dict: + sys.path.insert(0, str(Path(__file__).resolve().parent / "model_stack")) + import prepare as prep + + return prep.status(MODEL_VOLUME) + + +def prepare_model_stack() -> dict: + """Build the venv and download weights onto the models volume. + + Deliberately runs in this process rather than through + InvokeAgentRuntimeCommand: the mount is 2775 root:agentcore-runtime-user and + only the agent process holds that supplementary group. A command shell gets + EACCES here -- measured on a live instance. + """ + if not MODEL_VOLUME.is_dir(): + raise WorkflowError( + f"{MODEL_VOLUME} is not mounted. The composition runtime needs a " + "capacityProviderVolume named 'models' in its filesystemConfigurations." + ) + script = Path(__file__).resolve().parent / "model_stack" / "prepare.py" + logger.info("preparing model stack at %s (several minutes on a cold volume)", MODEL_VOLUME) + + # Streamed rather than captured. subprocess.run(capture_output=True) buffers + # everything until the process exits, so a five-minute build produced no + # CloudWatch output at all and looked indistinguishable from a hang. Lines are + # logged as they arrive and also kept for the response. + deadline = time.monotonic() + PREPARE_TIMEOUT_S + lines: list[str] = [] + proc = subprocess.Popen( + [sys.executable, "-u", str(script), "prepare", str(MODEL_VOLUME)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=gpu_env(), + ) + try: + assert proc.stdout is not None + for line in proc.stdout: + line = line.rstrip() + if line: + logger.info("prepare | %s", line[:500]) + lines.append(line) + if time.monotonic() > deadline: + proc.kill() + raise WorkflowError(f"model stack preparation exceeded {PREPARE_TIMEOUT_S}s") + returncode = proc.wait(timeout=60) + finally: + if proc.poll() is None: + proc.kill() + + tail = "\n".join(lines[-60:]) + if returncode != 0: + raise WorkflowError(f"model stack preparation failed (rc={returncode}):\n{tail}") + logger.info("model stack prepared") + return {**model_stack_status(), "log_tail": tail} + + +def render_audio( + out_path: Path, + brief: CompositionBrief, + seed: int | None = None, + duration_s: float = 30.0, + steps: int = 27, + reference_audio: Path | None = None, + reference_strength: float = 0.5, +) -> dict: + """Run the generator in its own interpreter on the models volume.""" + status = model_stack_status() + if not status.get("ready"): + raise WorkflowError( + "The model stack on the shared volume is not ready " + f"(status: {json.dumps(status)}). Invoke this runtime with " + '{"mode": "prepare"} once per session before composing.' + ) + + cmd = [ + status["python"], + status["runner"], + "--checkpoint-dir", + status["weights_dir"], + "--out", + str(out_path), + "--prompt", + brief.style_tags, + "--lyrics", + brief.lyrics or "[inst]", + "--duration", + str(duration_s), + "--steps", + str(steps), + ] + if seed is not None: + cmd += ["--seed", str(seed)] + if reference_audio is not None: + cmd += ["--reference-audio", str(reference_audio), "--reference-strength", str(reference_strength)] + + logger.info("rendering: %s", " ".join(cmd[2:])) + proc = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=RENDER_TIMEOUT_S, env=gpu_env()) + combined = (proc.stdout or "") + "\n" + (proc.stderr or "") + match = re.search(r"__RENDER_RESULT__ (\{.*\})", combined) + if proc.returncode != 0 or not match: + raise WorkflowError(f"render failed (rc={proc.returncode}). Tail of output:\n{combined[-3000:]}") + result = json.loads(match.group(1)) + if result.get("audio", {}).get("silent"): + raise WorkflowError("the generator produced a silent file") + try: + out_path.chmod(0o664) + except PermissionError: + pass + return result + + +# ------------------------------------------------------------------------ agent + + +def build_agent(session_id: str, track_id: str) -> Agent: + """Construct a fresh Agent for one invocation. + + A module-level Agent would be shared by every concurrent request, and Strands + rejects re-entrant invocation ("Agent is already processing a request"), so a + per-request Agent is a correctness requirement rather than a style choice. + Conversation history is carried on the volume instead, which is what lets a + session resumed days later remember earlier decisions. + """ + kwargs: dict = {"model_id": MODEL_ID} + if REGION: + kwargs["region_name"] = REGION + return Agent( + name=AGENT_NAME, + model=BedrockModel(**kwargs), + system_prompt=SYSTEM_PROMPT, + session_manager=FileSessionManager( + session_id=f"{session_id}-{AGENT_NAME}", + # Per-agent, not shared: FileSessionManager creates its storage + # directory with mode 0700, so a directory shared between agents + # running as different identities locks all but the first one out. + storage_dir=str(track_path(track_id) / f".sessions-{AGENT_NAME}"), + ), + ) + + +def compose_brief(agent: Agent, task: str) -> CompositionBrief: + result = agent(task, structured_output_model=CompositionBrief) + brief = result.structured_output + if brief is None: + raise WorkflowError("the model did not return a composition brief") + return brief + + +@app.entrypoint +def invoke(payload, context): + """AgentCore POSTs the invocation payload here. + + The second parameter must be named exactly ``context`` for the SDK to pass + the request context, which is the only way to read the session id. + """ + session_id = getattr(context, "session_id", None) or "local-session" + track_id = payload.get("track_id", "demo-track") + mode = payload.get("mode", "compose") + prompt = payload.get("prompt") or "Compose an upbeat electronic track." + if not isinstance(prompt, str): + # The payload is arbitrary JSON, and a non-string here can carry toolUse + # content blocks straight into the framework's event loop. + return {"status": "error", "agent": AGENT_NAME, "error": "'prompt' must be a string", "host": host_info()} + + duration_s = float(payload.get("duration_s", 30.0)) + steps = int(payload.get("steps", 27)) + seed = payload.get("seed") + + logger.info("invoke: mode=%s track=%s session=%s model=%s", mode, track_id, session_id, MODEL_ID) + + try: + if mode == "status": + return { + "status": "ok", + "agent": AGENT_NAME, + "mode": mode, + "model_stack": model_stack_status(), + "host": host_info(), + } + + if mode == "prepare": + # Preparation is per-session because the volume is per-session. + stack = prepare_model_stack() + return { + "status": "ok", + "agent": AGENT_NAME, + "mode": mode, + "session_id": session_id, + "model_stack": stack, + "host": host_info(), + } + + # This agent owns creation of the track directory; the others require it. + ensure_track_dir(track_id) + + if mode == "catalogue": + # Render the fictional back-catalogue the compliance agent screens + # against. Generating it with the same model keeps the sample + # self-contained and free of any third-party recording. + cat_dir = track_path(track_id) / "catalogue" + cat_dir.mkdir(exist_ok=True) + try: + cat_dir.chmod(0o2775) + except PermissionError: + pass + agent = build_agent(session_id, track_id) + entries = [] + for i, style in enumerate( + payload.get("styles") + or [ + "melodic techno, analog bass, warm pads, 124 bpm", + "lo-fi hip hop, dusty piano, vinyl crackle, 82 bpm", + ] + ): + brief = compose_brief( + agent, + f"Write a brief for a catalogue reference track in this " + f"style: {style}. Keep style_tags close to that description.", + ) + out = cat_dir / f"catalogue_{i:02d}.wav" + render = render_audio( + out, brief, seed=1000 + i, duration_s=float(payload.get("duration_s", 20.0)), steps=steps + ) + entries.append( + {"file": out.name, "title": brief.title, "style_tags": brief.style_tags, "render": render} + ) + write_text(track_id, "catalogue.json", json.dumps(entries, indent=2)) + return { + "status": "ok", + "agent": AGENT_NAME, + "mode": mode, + "track_id": track_id, + "session_id": session_id, + "catalogue": entries, + "host": host_info(), + } + + agent = build_agent(session_id, track_id) + + if mode == "remediate": + # The most recent attempt, not the original: a second remediation round + # should diverge from what was last rejected, not from where it started. + flagged = read_text(track_id, "composition_remediated.md") or read_text(track_id, "composition.md") + if flagged is None: + raise WorkflowError( + "No composition brief in the shared workspace - run the " + "composition step before requesting remediation." + ) + issue = payload.get("issue") or prompt + avoid = payload.get("avoid") or {} + task = ( + f"A compliance screen flagged this issue:\n\n{issue}\n\n" + f"Here is the brief it applies to:\n\n{flagged}\n\n" + ) + if avoid.get("style_tags"): + # This is the difference between a blind rewrite and an informed + # one. Measured: without these descriptors the replacement moved + # chroma distance only 0.029 -> 0.093, barely past the threshold. + task += ( + "The material it resembles was generated from these descriptors, " + f"so move decisively away from them:\n\n" + f" flagged reference: {avoid.get('title') or avoid['reference']}\n" + f" its style tags: {avoid['style_tags']}\n\n" + ) + others = [o for o in (avoid.get("other_references") or []) if o.get("style_tags")] + if others: + task += ( + "Also stay away from the rest of the catalogue:\n" + + "".join(f" - {o['style_tags']} (distance {o['distance']})\n" for o in others) + + "\n" + ) + task += ( + "Write a complete standalone replacement brief. Change the key, change " + "the tempo by at least 15 BPM, and choose a different genre lineage, " + "different instrumentation and a different harmonic centre. Your " + "style_tags must not reuse the flagged descriptors above -- pick " + "different genre words, different instruments and a different mood. " + "Do not name, quote or describe the flagged work, and do not narrate " + "the changes: downstream agents read your output as the composition." + ) + md_name, wav_name = "composition_remediated.md", "composition_remediated.wav" + else: + task = f"{prompt}\n\nTarget length: about {duration_s:.0f} seconds." + md_name, wav_name = "composition.md", "composition.wav" + + brief = compose_brief(agent, task) + wav = track_path(track_id) / wav_name + + reference = None + if payload.get("imitate_catalogue"): + # Used only to demonstrate detection: conditioning the generator on a + # catalogue track produces genuinely derivative audio, which is a + # far more honest test of the compliance agent than a text file that + # merely claims to be a copy. + reference = track_path(track_id) / "catalogue" / str(payload["imitate_catalogue"]) + if not reference.exists(): + raise WorkflowError(f"reference {reference.name} not found; run mode=catalogue first") + + render = render_audio( + wav, + brief, + seed=seed, + duration_s=duration_s, + steps=steps, + reference_audio=reference, + reference_strength=float(payload.get("reference_strength", 0.75)), + ) + measured = audio.measure(str(wav)).to_dict() + markdown = brief.to_markdown(render) + write_text(track_id, md_name, markdown) + + artifacts = [publish(track_id, wav), publish(track_id, track_path(track_id) / md_name)] + + return { + "status": "ok", + "agent": AGENT_NAME, + "model": MODEL_ID, + "mode": mode, + "track_id": track_id, + "session_id": session_id, + "brief": brief.model_dump(), + "result": markdown, + "render": render, + "measurements": measured, + "artifacts": artifacts, + "workspace_files": list_artifacts(track_id), + "host": host_info(), + } + + except WorkflowError as exc: + # A pipeline-ordering problem is a real outcome, not a crash: report it + # as data. Unexpected exceptions are deliberately left to propagate so + # the service surfaces a 424 and the traceback reaches CloudWatch. + logger.warning("workflow error: %s", exc) + return { + "status": "error", + "agent": AGENT_NAME, + "mode": mode, + "track_id": track_id, + "session_id": session_id, + "error": str(exc), + "workspace_files": list_artifacts(track_id), + "host": host_info(), + } + except subprocess.TimeoutExpired as exc: + logger.warning("subprocess timeout: %s", exc) + return { + "status": "error", + "agent": AGENT_NAME, + "mode": mode, + "track_id": track_id, + "session_id": session_id, + "error": f"timed out after {exc.timeout}s", + "host": host_info(), + } + + +if __name__ == "__main__": + # Bind explicitly. app.run() with no host guesses 0.0.0.0 only when it finds + # /.dockerenv or DOCKER_CONTAINER, neither of which exists under Finch or + # containerd - it would otherwise bind 127.0.0.1 and the runtime could never + # reach port 8080. AgentCore sets PORT, which run() does not consult. + app.run(host="0.0.0.0", port=int(os.environ.get("PORT", "8080"))) diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/mastering_agent.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/mastering_agent.py new file mode 100644 index 000000000..8f1128509 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/mastering_agent.py @@ -0,0 +1,496 @@ +"""Mastering agent. + +Reads the audio the composition agent rendered onto the shared volume, has a +model choose a mastering chain from real measurements of that audio, applies the +chain with real DSP, and measures the result. + +The division of labour is the point. The model decides *what* to do -- which +bands to move, how hard to compress, what to leave alone -- and the DSP in +audio_dsp.py does it deterministically. Nothing here is a plan the model +merely asserts: the output file is measured after processing, and the compliance +agent measures it again independently. + +Packaged as a container image in Amazon ECR. Runs on the same GPU instance as +the composition agent because collocation is what gives it access to the audio, +but it does no GPU work of its own -- mastering is filters and gain, and putting +it on the CPU keeps the GPU free for generation. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import platform +import uuid +from pathlib import Path + +import audio_dsp as audio +import boto3 +from bedrock_agentcore.runtime import BedrockAgentCoreApp +from pydantic import BaseModel, Field, field_validator +from strands import Agent +from strands.models import BedrockModel +from strands.session import FileSessionManager + +AGENT_NAME = "mastering" +PROCESS_ID = uuid.uuid4().hex[:8] + +WORKSPACE = Path(os.environ.get("WORKSPACE_DIR", "/mnt/tracks")) +MODEL_ID = os.environ.get("MODEL_ID", "global.anthropic.claude-sonnet-4-6") +REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") +ARTIFACT_BUCKET = os.environ.get("ARTIFACT_BUCKET") + +# Streaming loudness targets. Platforms normalise on playback, so mastering +# louder than the target buys nothing and costs dynamic range. +PLATFORM_TARGETS = { + "spotify": (-14.0, -1.0), + "apple": (-16.0, -1.0), + "youtube": (-14.0, -1.0), + "amazon": (-14.0, -2.0), + "broadcast": (-23.0, -1.0), +} + +SYSTEM_PROMPT = """You are a mastering engineer. + +You will be given measurements of a rendered mix and a delivery target. Return a +mastering chain as structured data. Be conservative and specific: + +- Only include EQ bands that address something visible in the measurements. + Three or four bands is a normal master; twelve is not. +- Corrective moves are small. Use gains between -4 and +4 dB unless the + measurements justify more. +- If the mix already has healthy dynamics, compress gently or not at all. Say so + in `left_alone` rather than adding processing to look busy. +- Loudness is reached by a single broadband gain after your chain, not by + slamming the limiter. The limiter is there to catch peaks, so expect only a + decibel or two of gain reduction from it. +- `notes` should explain your reasoning in two or three sentences, referring to + the measured numbers you are responding to.""" + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(AGENT_NAME) + +app = BedrockAgentCoreApp() + + +class WorkflowError(Exception): + """A recoverable problem in the pipeline, reported to the caller as data.""" + + +class EqBand(BaseModel): + type: str = Field(description="highpass, lowpass, peaking, lowshelf or highshelf") + freq_hz: float = Field(description="Centre or corner frequency in Hz, 20-20000.") + gain_db: float = Field(default=0.0, description="Cut or boost in dB. Ignored for highpass/lowpass.") + q: float = Field(default=0.707, description="Filter Q, typically 0.5-2.0.") + reason: str = Field(default="", description="What measurement this band addresses.") + + @field_validator("type", mode="before") + @classmethod + def _normalise_type(cls, v): + # Models write these a dozen ways: "high-pass", "High Shelf", "HPF". + if not isinstance(v, str): + return "peaking" + t = v.strip().lower().replace("-", "").replace("_", "").replace(" ", "") + return { + "hpf": "highpass", + "lpf": "lowpass", + "bell": "peaking", + "shelf": "highshelf", + "lowshelving": "lowshelf", + "highshelving": "highshelf", + }.get(t, t) + + @field_validator("freq_hz") + @classmethod + def _clamp_freq(cls, v): + return float(min(max(v, 20.0), 20000.0)) + + @field_validator("gain_db") + @classmethod + def _clamp_gain(cls, v): + # A model occasionally asks for +18 dB. Refuse politely rather than + # destroy the master. + return float(min(max(v, -12.0), 12.0)) + + +class Compressor(BaseModel): + enabled: bool = Field(default=True) + threshold_db: float = Field(default=-18.0) + ratio: float = Field(default=2.0) + attack_ms: float = Field(default=20.0) + release_ms: float = Field(default=200.0) + knee_db: float = Field(default=6.0) + + @field_validator("ratio") + @classmethod + def _clamp_ratio(cls, v): + return float(min(max(v, 1.0), 20.0)) + + +class MasteringPlan(BaseModel): + eq_bands: list[EqBand] = Field(default_factory=list) + compressor: Compressor = Field(default_factory=Compressor) + target_lufs: float = Field(default=-14.0) + target_true_peak_dbtp: float = Field(default=-1.0) + notes: str = Field(default="") + left_alone: str = Field(default="", description="What you deliberately did not touch, and why.") + + @field_validator("eq_bands", mode="before") + @classmethod + def _coerce_bands(cls, v): + if v is None: + return [] + if isinstance(v, str): + try: + v = json.loads(v) + except json.JSONDecodeError: + return [] + return v if isinstance(v, list) else [] + + @field_validator("target_true_peak_dbtp") + @classmethod + def _sane_ceiling(cls, v): + return float(min(max(v, -6.0), -0.1)) + + +# --------------------------------------------------------------------- workspace + + +def track_path(track_id: str) -> Path: + safe = "".join(c for c in track_id if c.isalnum() or c in "-_")[:64] or "track" + return WORKSPACE / safe + + +def require_track_dir(track_id: str) -> Path: + """Fail loudly if the composition agent has not created the track yet. + + Deliberately does not create it: a container agent and a zip agent run as + different identities, so whichever creates a directory first can lock the + other out. Creation belongs to the composition agent alone. + """ + path = track_path(track_id) + if not path.is_dir(): + raise WorkflowError( + f"No workspace for track '{track_id}'. Invoke the composition agent " + "first, with the same runtimeSessionId so both agents land on the " + "same instance and see the same volume." + ) + return path + + +def read_text(track_id: str, name: str) -> str | None: + p = track_path(track_id) / name + return p.read_text(encoding="utf-8") if p.exists() else None + + +def write_text(track_id: str, name: str, text: str) -> Path: + p = track_path(track_id) / name + p.write_text(text, encoding="utf-8") + try: + p.chmod(0o664) + except PermissionError: + pass + logger.info("wrote %s (%d bytes)", p, len(text)) + return p + + +def latest_render(track_id: str) -> tuple[str, Path]: + """Prefer a remediated render over the original. + + Same precedence the compliance agent uses, so a rerun masters the + replacement rather than re-mastering material that has been superseded. + """ + for name in ("composition_remediated.wav", "composition.wav"): + p = track_path(track_id) / name + if p.exists() and p.stat().st_size > 0: + return name, p + raise WorkflowError( + "No rendered audio in the shared workspace. Invoke the composition agent first, with the same runtimeSessionId." + ) + + +def list_artifacts(track_id: str) -> list[str]: + p = track_path(track_id) + return sorted(x.name for x in p.iterdir() if x.is_file()) if p.is_dir() else [] + + +def publish(track_id: str, path: Path) -> dict: + """Copy an artifact to S3 and hand back a link the caller can actually open. + + The volume lives inside a managed instance with no shell and is destroyed + with the session, so S3 is the only route by which a 30 MB WAV reaches + whoever invoked us. + """ + info: dict = { + "name": path.name, + "bytes": path.stat().st_size, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest()[:16], + } + if not ARTIFACT_BUCKET: + return info + key = f"tracks/{track_id}/{path.name}" + # The regional endpoint is explicit on purpose. Left to itself botocore + # presigned against the global host (bucket.s3.amazonaws.com) while scoping + # the signature to us-east-2, and every URL came back 403 + # SignatureDoesNotMatch. Measured on a live run. + s3 = boto3.client("s3", region_name=REGION, endpoint_url=f"https://s3.{REGION}.amazonaws.com") + s3.upload_file(str(path), ARTIFACT_BUCKET, key) + info["s3_uri"] = f"s3://{ARTIFACT_BUCKET}/{key}" + info["url"] = s3.generate_presigned_url( + "get_object", Params={"Bucket": ARTIFACT_BUCKET, "Key": key}, ExpiresIn=86400 + ) + logger.info("published %s", info["s3_uri"]) + return info + + +def host_info() -> dict: + """Facts that make collocation observable from the response.""" + u = platform.uname() + return {"process_id": PROCESS_ID, "hostname": u.node, "architecture": u.machine, "cpus": os.cpu_count()} + + +# ------------------------------------------------------------------------ agent + + +def build_agent(session_id: str, track_id: str) -> Agent: + """Construct a fresh Agent per invocation. + + A module-level Agent is shared by every concurrent request and Strands + rejects re-entrant invocation, so this is a correctness requirement rather + than a style choice. Conversation history lives on the volume instead. + """ + kwargs: dict = {"model_id": MODEL_ID} + if REGION: + kwargs["region_name"] = REGION + return Agent( + name=AGENT_NAME, + model=BedrockModel(**kwargs), + system_prompt=SYSTEM_PROMPT, + session_manager=FileSessionManager( + session_id=f"{session_id}-{AGENT_NAME}", + # Per-agent: FileSessionManager creates its storage directory with + # mode 0700, which locks out an agent running as another identity. + storage_dir=str(track_path(track_id) / f".sessions-{AGENT_NAME}"), + ), + ) + + +def apply_chain(src: Path, dst: Path, plan: MasteringPlan) -> dict: + """Run the model's chain and measure what actually came out. + + Order matters and is fixed here rather than left to the model: tonal shaping, + then dynamics, then loudness, then a true-peak safety limiter last. Reaching + loudness before limiting means the limiter only catches transients instead of + doing the level-setting. + """ + data, rate = audio.read_audio(str(src)) + before = audio.measure(str(src)) + steps: list[dict] = [] + + bands = [b.model_dump() for b in plan.eq_bands] + if bands: + data = audio.apply_filters(data, rate, bands) + steps.append({"stage": "eq", "bands": bands}) + + if plan.compressor.enabled: + data, comp_info = audio.compress( + data, + rate, + threshold_db=plan.compressor.threshold_db, + ratio=plan.compressor.ratio, + attack_ms=plan.compressor.attack_ms, + release_ms=plan.compressor.release_ms, + knee_db=plan.compressor.knee_db, + ) + steps.append({"stage": "compressor", **plan.compressor.model_dump(), **comp_info}) + + data, norm_info = audio.normalise_loudness(data, rate, plan.target_lufs) + steps.append({"stage": "loudness", "target_lufs": plan.target_lufs, **norm_info}) + + data, lim_info = audio.limit(data, rate, ceiling_dbtp=plan.target_true_peak_dbtp) + steps.append({"stage": "limiter", "ceiling_dbtp": plan.target_true_peak_dbtp, **lim_info}) + + audio.write_audio(str(dst), data, rate, subtype="PCM_24") + after = audio.measure(str(dst)) + + # The reason this agent is worth deploying: it checks its own homework. + lufs_err = abs(after.integrated_lufs - plan.target_lufs) if after.integrated_lufs is not None else None + return { + "before": before.to_dict(), + "after": after.to_dict(), + "steps": steps, + "targets": {"lufs": plan.target_lufs, "true_peak_dbtp": plan.target_true_peak_dbtp}, + "hit_loudness_target": bool(lufs_err is not None and lufs_err <= 0.5), + "hit_peak_target": bool( + after.true_peak_dbtp is not None and after.true_peak_dbtp <= plan.target_true_peak_dbtp + 0.05 + ), + "loudness_error_lu": round(lufs_err, 2) if lufs_err is not None else None, + } + + +def render_report(plan: MasteringPlan, result: dict, source_name: str) -> str: + b, a = result["before"], result["after"] + + def row(label: str, key: str, unit: str) -> str: + bv, av = b.get(key), a.get(key) + fmt = lambda v: "-inf" if v is None else f"{v:g}" + return f"| {label} | {fmt(bv)}{unit} | {fmt(av)}{unit} |" + + lines = [ + "# Mastering Report", + "", + (f"**Source:** `{source_name}` | **Target:** {plan.target_lufs} LUFS / {plan.target_true_peak_dbtp} dBTP"), + "", + "## Measured", + "", + "| | before | after |", + "|---|---|---|", + row("Integrated loudness", "integrated_lufs", " LUFS"), + row("Loudness range", "loudness_range_lu", " LU"), + row("True peak", "true_peak_dbtp", " dBTP"), + row("Sample peak", "sample_peak_dbfs", " dBFS"), + row("Stereo correlation", "stereo_correlation", ""), + row("Clipped runs", "clipped_runs", ""), + "", + ( + f"Loudness target {'met' if result['hit_loudness_target'] else 'MISSED'} " + f"(error {result['loudness_error_lu']} LU). " + f"True-peak ceiling {'held' if result['hit_peak_target'] else 'EXCEEDED'}." + ), + "", + "## Chain", + "", + ] + for band in plan.eq_bands: + lines.append(f"- **EQ** {band.type} @ {band.freq_hz:g} Hz, {band.gain_db:+g} dB, Q {band.q:g} — {band.reason}") + c = plan.compressor + lines.append( + f"- **Compressor** {'bypassed' if not c.enabled else f'{c.ratio:g}:1 @ {c.threshold_db:g} dB, attack {c.attack_ms:g} ms, release {c.release_ms:g} ms'}" + ) + for step in result["steps"]: + if step["stage"] == "loudness": + lines.append(f"- **Loudness** {step['applied_gain_db']:+g} dB broadband") + if step["stage"] == "limiter": + lines.append( + f"- **Limiter** ceiling {step['ceiling_dbtp']:g} dBTP, " + f"max {step['max_gain_reduction_db']:g} dB reduction" + ) + lines += [ + "", + "## Engineer's notes", + "", + plan.notes or "_none_", + "", + "## Deliberately left alone", + "", + plan.left_alone or "_nothing noted_", + ] + return "\n".join(lines) + + +@app.entrypoint +def invoke(payload, context): + """AgentCore POSTs the invocation payload here. + + The second parameter must be named exactly ``context`` for the SDK to pass + the request context, which is the only way to read the session id. + """ + session_id = getattr(context, "session_id", None) or "local-session" + track_id = payload.get("track_id", "demo-track") + platform_name = str(payload.get("platform", "spotify")).lower() + prompt = payload.get("prompt") or f"Master this for {platform_name}." + if not isinstance(prompt, str): + # The payload is arbitrary JSON. A non-string here can carry toolUse + # content blocks straight into the framework's event loop. + return {"status": "error", "agent": AGENT_NAME, "error": "'prompt' must be a string", "host": host_info()} + + target_lufs, target_peak = PLATFORM_TARGETS.get(platform_name, PLATFORM_TARGETS["spotify"]) + logger.info("invoke: track=%s session=%s platform=%s model=%s", track_id, session_id, platform_name, MODEL_ID) + + try: + require_track_dir(track_id) + source_name, source = latest_render(track_id) + brief = read_text(track_id, "composition_remediated.md") or read_text(track_id, "composition.md") + source_measurements = audio.measure(str(source)) + logger.info("source %s: %s", source_name, source_measurements.to_dict()) + + task = ( + f"Mix to master (from {source_name}, rendered by the composition agent " + f"on this instance).\n\n" + f"Measured properties of the mix:\n{json.dumps(source_measurements.to_dict(), indent=2)}\n\n" + + (f"Composition brief:\n{brief}\n\n" if brief else "") + + f"Delivery target: {platform_name} at {target_lufs} LUFS integrated, " + f"true peak at or below {target_peak} dBTP.\n\n" + f"Request: {prompt}" + ) + + agent = build_agent(session_id, track_id) + result = agent(task, structured_output_model=MasteringPlan) + plan: MasteringPlan = result.structured_output or MasteringPlan( + target_lufs=target_lufs, target_true_peak_dbtp=target_peak + ) + # The platform target is not the model's to override. + plan.target_lufs = target_lufs + plan.target_true_peak_dbtp = target_peak + + master = track_path(track_id) / "master.wav" + applied = apply_chain(source, master, plan) + try: + master.chmod(0o664) + except PermissionError: + pass + + report = render_report(plan, applied, source_name) + write_text(track_id, "mastering.md", report) + write_text( + track_id, + "mastering.json", + json.dumps( + {"plan": plan.model_dump(), "result": applied, "source": source_name, "platform": platform_name}, + indent=2, + ), + ) + + artifacts = [publish(track_id, master), publish(track_id, track_path(track_id) / "mastering.md")] + + return { + "status": "ok", + "agent": AGENT_NAME, + "model": MODEL_ID, + "track_id": track_id, + "session_id": session_id, + "read_from": source_name, + "platform": platform_name, + "measurements": {"before": applied["before"], "after": applied["after"]}, + "targets_met": {"loudness": applied["hit_loudness_target"], "true_peak": applied["hit_peak_target"]}, + "plan": plan.model_dump(), + "result": report, + "artifacts": artifacts, + "workspace_files": list_artifacts(track_id), + "host": host_info(), + } + + except WorkflowError as exc: + # A pipeline-ordering problem is a real outcome, not a crash: report it + # as data. Unexpected exceptions are left to propagate so the service + # surfaces a 424 and the traceback reaches CloudWatch. + logger.warning("workflow error: %s", exc) + return { + "status": "error", + "agent": AGENT_NAME, + "track_id": track_id, + "session_id": session_id, + "error": str(exc), + "workspace_files": list_artifacts(track_id), + "host": host_info(), + } + + +if __name__ == "__main__": + # Bind explicitly. app.run() with no host guesses 0.0.0.0 only when it finds + # /.dockerenv or DOCKER_CONTAINER, neither of which exists under Finch or + # containerd, and would otherwise bind 127.0.0.1 where the runtime cannot + # reach it. AgentCore sets PORT, which run() does not consult. + app.run(host="0.0.0.0", port=int(os.environ.get("PORT", "8080"))) diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/model_stack/generate.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/model_stack/generate.py new file mode 100644 index 000000000..d6ff4cfbd --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/model_stack/generate.py @@ -0,0 +1,181 @@ +"""Render audio with ACE-Step. Runs in the model venv on the ``models`` volume. + +Invoked as a subprocess by the composition agent, never imported by it. That +isolation is deliberate: + +* The agent's container image is capped at 2 GB and cannot hold a CUDA torch + build, so the whole ML stack lives on a mounted volume instead. +* A capacity provider volume is only mounted at invocation time, not during + container initialisation, so a module-level ``import torch`` in the agent could + not work even if it fitted. +* ACE-Step pins ``transformers==4.50.0`` and pulls in gradio, spacy and + tensorboard. Keeping it in its own interpreter means none of that has to + co-resolve with the agent's Strands dependencies. + +Communicates back over stdout: ACE-Step logs freely, so the machine-readable +result is emitted on one line behind a sentinel. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import time + +RESULT_SENTINEL = "__RENDER_RESULT__" + + +def patch_torchaudio_io() -> str: + """Route torchaudio's file I/O through soundfile, bypassing TorchCodec. + + Recent torchaudio delegates both ``save`` and ``load`` to TorchCodec, which is + not in its dependency tree and not in ACE-Step's either. Measured failures on + a live g6.xlarge: + + ImportError: TorchCodec is required for save_with_torchcodec (writing output) + ImportError: TorchCodec is required for load_with_torchcodec (reading a + reference for + audio2audio) + + soundfile is already installed, so both are redirected to it. Patching rather + than adding torchcodec keeps the dependency set to what ACE-Step already + resolves, and lets us pin the output container to 24-bit PCM. + """ + import numpy as np + import soundfile as sf + import torch + import torchaudio + + def _save(uri, src, sample_rate, **kwargs): + arr = src.detach().cpu().to(torch.float32).numpy() if hasattr(src, "detach") else np.asarray(src) + if arr.ndim == 1: + arr = arr[None, :] + # torchaudio is (channels, samples); soundfile wants (samples, channels). + sf.write(str(uri), arr.T, int(sample_rate), subtype="PCM_24") + + def _load(uri, frame_offset=0, num_frames=-1, normalize=True, channels_first=True, **kwargs): + data, rate = sf.read( + str(uri), + always_2d=True, + dtype="float32", + start=int(frame_offset), + frames=int(num_frames) if num_frames and num_frames > 0 else -1, + ) + tensor = torch.from_numpy(data) # (samples, channels) + if channels_first: + tensor = tensor.transpose(0, 1) # -> (channels, samples) + return tensor.contiguous(), int(rate) + + torchaudio.save = _save + torchaudio.load = _load + return "torchaudio.save and torchaudio.load redirected to soundfile (TorchCodec absent)" + + +def describe(path: str) -> dict: + """Confirm we produced real audio rather than a silent file of the right size.""" + import numpy as np + import soundfile as sf + + data, rate = sf.read(path, always_2d=True, dtype="float64") + peak = float(np.max(np.abs(data))) if data.size else 0.0 + rms = float(np.sqrt(np.mean(data**2))) if data.size else 0.0 + return { + "path": path, + "bytes": os.path.getsize(path), + "sample_rate": int(rate), + "channels": int(data.shape[1]), + "duration_s": round(data.shape[0] / rate, 3), + "peak_dbfs": round(20 * math.log10(peak), 2) if peak > 0 else None, + "rms_dbfs": round(20 * math.log10(rms), 2) if rms > 0 else None, + "silent": peak < 1e-5, + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--checkpoint-dir", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--prompt", required=True, help="Style tags, e.g. 'upbeat electronic, heavy bass, 128 bpm'") + ap.add_argument("--lyrics", default="[inst]", help="'[inst]' renders an instrumental.") + ap.add_argument("--duration", type=float, default=30.0) + ap.add_argument("--steps", type=int, default=27) + ap.add_argument("--guidance", type=float, default=15.0) + ap.add_argument("--seed", type=int, default=None) + # audio2audio is how the sample produces a deliberate near-copy of its own + # back-catalogue, so the compliance agent has something real to detect. + ap.add_argument("--reference-audio", default=None) + ap.add_argument("--reference-strength", type=float, default=0.5) + args = ap.parse_args() + + timings: dict = {} + notes = [patch_torchaudio_io()] + + t0 = time.time() + import torch + from acestep.pipeline_ace_step import ACEStepPipeline + + timings["import_s"] = round(time.time() - t0, 2) + + if not torch.cuda.is_available(): + # Worth failing loudly: on a capacity provider instance the NVIDIA driver + # is injected by AgentCore, so an absent GPU means the runtime is not on + # the fleet we think it is. + print(json.dumps({"error": "CUDA is not available in the model venv"}), file=sys.stderr) + return 2 + device = torch.cuda.get_device_name(0) + + t0 = time.time() + pipe = ACEStepPipeline(checkpoint_dir=args.checkpoint_dir, dtype="bfloat16", torch_compile=False) + timings["pipeline_ctor_s"] = round(time.time() - t0, 2) + + if args.seed is not None: + torch.manual_seed(args.seed) + + call: dict = { + "format": "wav", + "prompt": args.prompt, + "lyrics": args.lyrics or "[inst]", + "audio_duration": float(args.duration), + "infer_step": int(args.steps), + "guidance_scale": float(args.guidance), + "scheduler_type": "euler", + "cfg_type": "apg", + "omega_scale": 10.0, + "save_path": args.out, + } + if args.seed is not None: + call["manual_seeds"] = [int(args.seed)] + if args.reference_audio: + call.update( + audio2audio_enable=True, + ref_audio_input=args.reference_audio, + ref_audio_strength=float(args.reference_strength), + ) + + t0 = time.time() + pipe(**call) + timings["generate_s"] = round(time.time() - t0, 2) + timings["peak_vram_gib"] = round(torch.cuda.max_memory_allocated() / 2**30, 2) + + if not os.path.exists(args.out): + print(json.dumps({"error": f"pipeline did not write {args.out}"}), file=sys.stderr) + return 3 + + result = { + "ok": True, + "device": device, + "torch": torch.__version__, + "timings": timings, + "audio": describe(args.out), + "parameters": {k: v for k, v in call.items() if k != "save_path"}, + "notes": notes, + } + print(f"{RESULT_SENTINEL} {json.dumps(result)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/model_stack/prepare.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/model_stack/prepare.py new file mode 100644 index 000000000..f70dacb71 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/model_stack/prepare.py @@ -0,0 +1,208 @@ +"""Build the generation stack onto the capacity provider's ``models`` volume. + +Runs inside the composition agent's own process, which matters: the volume is +mounted ``2775 root:agentcore-runtime-user`` and the agent process holds that +supplementary group, while a shell started by ``InvokeAgentRuntimeCommand`` does +not and gets EACCES. Measured on a live instance -- the agent is the only thing +that can populate this volume. + +Why the stack is on a volume rather than in the container image: an AgentCore +Runtime container image is capped at 2 GB, and a CUDA build of torch resolves to +3.13 GB of wheels before any model weight is added. The image therefore stays +tiny (agent + SDK) and everything heavy lands here, in a self-contained venv the +agent shells out to. + +Idempotent by design. Each stage writes a stamp, so a resumed session or a +second invocation skips work already done, and a partly-built volume can be +completed rather than restarted. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tarfile +import time +import urllib.request +from pathlib import Path + +# Pinned commit on ace-step/ACE-Step main. Pinned rather than tracking a branch +# because this is the version the sample was tested against, and pinned to a +# GitHub tarball rather than PyPI because the published `ace-step` sdist is +# broken: its setup.py reads a requirements.txt the archive does not contain, so +# `pip install ace-step` fails at metadata generation. +ACESTEP_SHA = os.environ.get("ACESTEP_SHA", "1bee4c9f5b43e30995f8d4d33b3919197ce1bd68") +ACESTEP_TARBALL = f"https://codeload.github.com/ace-step/ACE-Step/tar.gz/{ACESTEP_SHA}" + +# ACE-Step v1 3.5B. Apache-2.0 and ungated, which is why it is the default here: +# no license acceptance and no HuggingFace token, so the sample deploys +# unattended. See the README for the licensing comparison. +WEIGHTS_REPO = os.environ.get("ACESTEP_WEIGHTS_REPO", "ACE-Step/ACE-Step-v1-3.5B") + +STAGES = ("source", "venv", "install", "weights", "runner") + + +def log(msg: str) -> None: + print(f"[prepare] {msg}", flush=True) + + +def run(cmd: list[str], cwd: str | None = None) -> None: + log(f"$ {' '.join(cmd[:6])}{' ...' if len(cmd) > 6 else ''}") + subprocess.run(cmd, check=True, cwd=cwd, stdout=sys.stdout, stderr=sys.stderr) + + +class Stamps: + """Per-stage completion markers, so preparation is resumable.""" + + def __init__(self, root: Path) -> None: + self.dir = root / ".stamps" + self.dir.mkdir(parents=True, exist_ok=True) + + def done(self, stage: str) -> bool: + return (self.dir / stage).exists() + + def mark(self, stage: str, detail: dict) -> None: + (self.dir / stage).write_text(json.dumps(detail, indent=2), encoding="utf-8") + + +def fetch_source(root: Path, stamps: Stamps) -> Path: + """Download and unpack the pinned ACE-Step tree. + + A tarball rather than `git clone` so the container image needs no git, which + keeps it to a zero-RUN-step Dockerfile and therefore buildable for amd64 from + an arm64 machine without emulation. + """ + src_root = root / "src" + expected = src_root / f"ACE-Step-{ACESTEP_SHA}" + if stamps.done("source") and (expected / "setup.py").exists(): + log(f"source already present at {expected}") + return expected + + src_root.mkdir(parents=True, exist_ok=True) + archive = root / "acestep-src.tar.gz" + log(f"downloading {ACESTEP_TARBALL}") + t0 = time.time() + with urllib.request.urlopen(ACESTEP_TARBALL, timeout=180) as r, open(archive, "wb") as fh: + shutil.copyfileobj(r, fh) + log(f"downloaded {archive.stat().st_size / 1e6:.1f} MB in {time.time() - t0:.0f}s") + + with tarfile.open(archive) as tf: + # Refuse any member that would escape the extraction root. + for member in tf.getmembers(): + target = (src_root / member.name).resolve() + if not str(target).startswith(str(src_root.resolve())): + raise RuntimeError(f"unsafe tar member {member.name!r}") + tf.extractall(src_root) + archive.unlink(missing_ok=True) + + if not (expected / "setup.py").exists(): + candidates = [p for p in src_root.iterdir() if p.is_dir() and (p / "setup.py").exists()] + if not candidates: + raise RuntimeError(f"no setup.py found under {src_root}") + expected = candidates[0] + stamps.mark("source", {"sha": ACESTEP_SHA, "path": str(expected)}) + return expected + + +def make_venv(root: Path, stamps: Stamps) -> Path: + venv = root / "venv" + python = venv / "bin" / "python" + if stamps.done("venv") and python.exists(): + log("venv already present") + return python + log(f"creating venv at {venv} from {sys.executable}") + run([sys.executable, "-m", "venv", str(venv)]) + run([str(python), "-m", "pip", "install", "--quiet", "--upgrade", "pip", "wheel", "setuptools"]) + stamps.mark("venv", {"python": str(python)}) + return python + + +def install_stack(python: Path, source: Path, root: Path, stamps: Stamps) -> None: + if stamps.done("install"): + log("stack already installed") + return + log("installing ACE-Step and its dependencies (torch + CUDA, several GB)") + t0 = time.time() + run([str(python), "-m", "pip", "install", "--no-cache-dir", str(source)]) + size = sum(f.stat().st_size for f in (root / "venv").rglob("*") if f.is_file()) + log(f"installed in {time.time() - t0:.0f}s, venv is {size / 1e9:.2f} GB") + stamps.mark("install", {"seconds": round(time.time() - t0), "venv_bytes": size}) + + +def fetch_weights(python: Path, root: Path, stamps: Stamps) -> Path: + target = root / "weights" / WEIGHTS_REPO.split("/")[-1] + if stamps.done("weights") and target.exists(): + log(f"weights already present at {target}") + return target + target.parent.mkdir(parents=True, exist_ok=True) + log(f"downloading weights {WEIGHTS_REPO}") + t0 = time.time() + script = ( + "from huggingface_hub import snapshot_download;" + f"print(snapshot_download({WEIGHTS_REPO!r}, local_dir={str(target)!r}))" + ) + run([str(python), "-c", script]) + size = sum(f.stat().st_size for f in target.rglob("*") if f.is_file()) + log(f"weights downloaded in {time.time() - t0:.0f}s, {size / 1e9:.2f} GB") + stamps.mark("weights", {"repo": WEIGHTS_REPO, "bytes": size, "seconds": round(time.time() - t0)}) + return target + + +def install_runner(root: Path, stamps: Stamps) -> Path: + """Copy the generation entry point next to the venv that runs it.""" + runner = root / "generate.py" + source = Path(__file__).resolve().parent / "generate.py" + shutil.copyfile(source, runner) + try: + runner.chmod(0o664) + except PermissionError: + pass + stamps.mark("runner", {"path": str(runner)}) + return runner + + +def prepare(root: Path) -> dict: + root.mkdir(parents=True, exist_ok=True) + stamps = Stamps(root) + started = time.time() + + source = fetch_source(root, stamps) + python = make_venv(root, stamps) + install_stack(python, source, root, stamps) + weights = fetch_weights(python, root, stamps) + runner = install_runner(root, stamps) + + ready = { + "ready": True, + "acestep_sha": ACESTEP_SHA, + "weights_repo": WEIGHTS_REPO, + "python": str(python), + "weights_dir": str(weights), + "runner": str(runner), + "prepared_in_seconds": round(time.time() - started), + } + (root / "READY.json").write_text(json.dumps(ready, indent=2), encoding="utf-8") + log(f"ready in {ready['prepared_in_seconds']}s") + return ready + + +def status(root: Path) -> dict: + marker = root / "READY.json" + if marker.exists(): + return {**json.loads(marker.read_text()), "stage": "ready"} + stamps = Stamps(root) + completed = [s for s in STAGES if stamps.done(s)] + return { + "ready": False, + "completed_stages": completed, + "next_stage": next((s for s in STAGES if s not in completed), None), + } + + +if __name__ == "__main__": + target_root = Path(sys.argv[2] if len(sys.argv) > 2 else "/mnt/models") + action = sys.argv[1] if len(sys.argv) > 1 else "prepare" + print(json.dumps(status(target_root) if action == "status" else prepare(target_root), indent=2)) diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/requirements.txt b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/requirements.txt new file mode 100644 index 000000000..0ba405807 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/requirements.txt @@ -0,0 +1,20 @@ +# Agent-side dependencies only. These are small on purpose. +# +# An AgentCore Runtime container image is capped at 2 GB and a CUDA build of +# torch resolves to 3.13 GB of wheels before any model weight, so the generative +# stack is NOT here. It is built onto the capacity provider's `models` volume at +# run time by model_stack/prepare.py and invoked as a subprocess. See the README. +# +# Everything here is permissively licensed: Apache-2.0 for the AgentCore SDK and +# Strands, BSD-3 for numpy, scipy and soundfile. Two audio libraries were +# deliberately rejected -- pedalboard is GPL (it links JUCE) and pyloudnorm +# publishes no license metadata -- so the DSP and the ITU-R BS.1770-4 loudness +# measurement are implemented in audio_dsp.py instead. +bedrock-agentcore==1.21.0 +strands-agents==1.52.0 +boto3>=1.43.72 + +# Audio measurement and processing. +numpy==2.5.2 +scipy==1.18.0 +soundfile==0.14.0 diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/cleanup.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/cleanup.py new file mode 100755 index 000000000..2b4e917cf --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/cleanup.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Delete everything deploy.py and invoke.py created. + +Order matters, and several steps are easy to get wrong: + + * Deleting a SESSION is what deprovisions the EC2 instance, its network + interface and its EBS volumes. Stopping an agent runtime does not. + * Runtime versions detach asynchronously, so DeleteCapacityProvider fails for a + while after DeleteAgentRuntime returns. Poll, do not sleep-and-hope. + * AgentCore creates a CloudWatch log group per runtime with no retention + policy, and deleting the runtime does not remove it. + * Capacity provider volumes are EC2 *managed resources*, hidden from + DescribeVolumes unless you pass IncludeManagedResources -- and a volume left + behind by a FAILED placement survives both session and capacity provider + deletion and cannot be deleted by you at all (DeleteVolume returns + UnauthorizedOperation with an explicit deny in a resource-based policy). This + script reports any it finds, because the alternative is silently paying for + them. +""" + +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError + +STATE_FILE = Path(__file__).resolve().parent.parent / "deployment_state.json" + + +def log(msg: str) -> None: + print(f"==> {msg}", flush=True) + + +def warn(msg: str) -> None: + print(f" ! {msg}", flush=True) + + +def main() -> None: + if not STATE_FILE.exists(): + sys.exit(f"No {STATE_FILE.name}; nothing recorded to delete.") + state = json.loads(STATE_FILE.read_text()) + + region = state["region"] + account = state["account"] + cp_id = state["capacity_provider"]["id"] + runtimes = state["runtimes"] + + control = boto3.client("bedrock-agentcore-control", region_name=region) + data = boto3.client("bedrock-agentcore", region_name=region) + + # 1. Delete the sessions. This is the step that stops EC2 and EBS charges. + for session_id in state.get("sessions", []): + if len(session_id) >= 33: + for name, runtime in runtimes.items(): + try: + data.stop_runtime_session(agentRuntimeArn=runtime["arn"], runtimeSessionId=session_id) + except ClientError as exc: + if exc.response["Error"]["Code"] != "ResourceNotFoundException": + warn(f"stop {name}: {exc.response['Error']['Code']}") + try: + data.delete_capacity_provider_session(capacityProviderId=cp_id, sessionId=session_id) + log(f"deleted session {session_id} (instance + volumes deprovisioning)") + except ClientError as exc: + code = exc.response["Error"]["Code"] + if code != "ResourceNotFoundException": + warn(f"delete session {session_id}: {code}") + + # 2. Delete the runtimes, which detaches them from the capacity provider. + for name, runtime in runtimes.items(): + try: + control.delete_agent_runtime(agentRuntimeId=runtime["id"]) + log(f"deleted runtime {name}") + except ClientError as exc: + warn(f"delete runtime {name}: {exc.response['Error']['Code']}") + + # 3. Wait for the versions to actually detach. DeleteAgentRuntime returns + # before this completes, and DeleteCapacityProvider fails until it does. + log("waiting for runtime versions to detach from the capacity provider") + started = time.time() + for _ in range(40): + try: + attached = control.list_agent_runtime_versions_by_capacity_provider(capacityProviderId=cp_id).get( + "agentRuntimes", [] + ) + except ClientError as exc: + if exc.response["Error"]["Code"] == "ResourceNotFoundException": + attached = [] + else: + raise + if not attached: + log(f"versions detached after {time.time() - started:.0f}s") + break + time.sleep(15) + else: + warn("versions still attached; DeleteCapacityProvider may fail - rerun this script") + + # 4. Delete the capacity provider, which also terminates any instance left. + log(f"deleting capacity provider {cp_id}") + try: + control.delete_capacity_provider(capacityProviderId=cp_id) + except ClientError as exc: + warn(f"delete capacity provider: {exc.response['Error']['Code']}") + for _ in range(40): + try: + status = control.get_capacity_provider(capacityProviderId=cp_id)["status"] + except ClientError as exc: + if exc.response["Error"]["Code"] == "ResourceNotFoundException": + log("capacity provider gone") + break + # A throttle is not a success. Only ResourceNotFound means deleted. + warn(f"poll: {exc.response['Error']['Code']}") + time.sleep(15) + continue + if status == "DELETE_FAILED": + # Not terminal: re-issuing the delete has been observed to succeed. + warn("DELETE_FAILED - re-issuing") + try: + control.delete_capacity_provider(capacityProviderId=cp_id) + except ClientError: + pass + time.sleep(15) + else: + warn("capacity provider not confirmed deleted - check the console") + + # 5. Artifacts and roles. + ecr = boto3.client("ecr", region_name=region) + for repo in state.get("ecr_repositories", []): + try: + ecr.delete_repository(repositoryName=repo, force=True) + log(f"deleted ECR repo {repo}") + except ClientError as exc: + warn(f"delete repo {repo}: {exc.response['Error']['Code']}") + + bucket_name = state.get("s3", {}).get("bucket") + if bucket_name: + try: + bucket = boto3.resource("s3", region_name=region).Bucket(bucket_name) + bucket.objects.all().delete() + bucket.delete() + log(f"deleted bucket {bucket_name} (including rendered audio)") + except ClientError as exc: + warn(f"delete bucket: {exc.response['Error']['Code']}") + + iam = boto3.client("iam") + for role in state.get("iam_roles", []): + try: + for policy in iam.list_attached_role_policies(RoleName=role)["AttachedPolicies"]: + iam.detach_role_policy(RoleName=role, PolicyArn=policy["PolicyArn"]) + for policy_name in iam.list_role_policies(RoleName=role)["PolicyNames"]: + iam.delete_role_policy(RoleName=role, PolicyName=policy_name) + iam.delete_role(RoleName=role) + log(f"deleted role {role}") + except ClientError as exc: + warn(f"delete role {role}: {exc.response['Error']['Code']}") + + # 6. CloudWatch log groups. AgentCore creates one per runtime, nothing + # deletes them when the runtime goes, and they are created with no + # retention policy. The names derive from the runtime ids, so this must + # run before the state file is removed. + logs = boto3.client("logs", region_name=region) + deleted = 0 + for runtime in runtimes.values(): + for pattern in ( + "/aws/bedrock-agentcore/runtimes/{rid}-", + "/aws/vendedlogs/bedrock-agentcore/runtime/APPLICATION_LOGS/{rid}", + "/aws/vendedlogs/bedrock-agentcore/runtime/USAGE_LOGS/{rid}", + ): + prefix = pattern.format(rid=runtime["id"]) + try: + for page in logs.get_paginator("describe_log_groups").paginate(logGroupNamePrefix=prefix): + for group in page.get("logGroups", []): + try: + logs.delete_log_group(logGroupName=group["logGroupName"]) + deleted += 1 + except ClientError as exc: + warn(f"delete log group: {exc.response['Error']['Code']}") + except ClientError as exc: + warn(f"list log groups {prefix}: {exc.response['Error']['Code']}") + log(f"deleted {deleted} CloudWatch log group(s)") + + # 7. Report any volume the service did not reclaim. These come from sessions + # whose placement failed partway through: the volume that succeeded is + # orphaned, survives session and capacity provider deletion, and cannot be + # deleted by the account owner. + ec2 = boto3.client("ec2", region_name=region) + + # Deprovisioning is asynchronous, and root volumes are delete-on-termination, + # so checking immediately reports volumes that are merely mid-teardown. Wait + # for the instances to finish terminating first, then only count volumes that + # are actually detached. + log("waiting for instances to finish terminating before checking for orphans") + for _ in range(24): + try: + reservations = ec2.describe_instances( + IncludeManagedResources=True, + Filters=[{"Name": "tag:bedrock-agentcore:capacity-provider-id", "Values": [cp_id]}], + )["Reservations"] + except ClientError: + break + live = [ + i for r in reservations for i in r["Instances"] if i["State"]["Name"] not in ("terminated", "shutting-down") + ] + pending = [i for r in reservations for i in r["Instances"] if i["State"]["Name"] == "shutting-down"] + if not live and not pending: + break + time.sleep(10) + + try: + orphans = [ + v + for v in ec2.describe_volumes( + IncludeManagedResources=True, + Filters=[{"Name": "tag:bedrock-agentcore:capacity-provider-id", "Values": [cp_id]}], + )["Volumes"] + # An attached volume is still being torn down with its instance. + if v["State"] == "available" + ] + except ClientError as exc: + orphans = [] + warn(f"could not check for orphaned volumes: {exc.response['Error']['Code']}") + if orphans: + total = sum(v["Size"] for v in orphans) + warn(f"{len(orphans)} managed volume(s) totalling {total} GiB were NOT reclaimed:") + for v in orphans: + session = next( + (t["Value"] for t in v.get("Tags", []) if t["Key"] == "bedrock-agentcore:runtime-session-id"), "?" + ) + warn(f" {v['VolumeId']} {v['Size']} GiB {v['State']} session={session}") + warn( + "These are EC2 managed resources: DeleteVolume is denied by a " + "resource-based policy even for an administrator. They usually come " + "from a session whose placement failed. Raise a support case if they " + "persist, and watch the EBS line on your bill." + ) + else: + log("no orphaned managed volumes for this capacity provider") + + STATE_FILE.unlink() + log(f"removed {STATE_FILE.name}") + + print( + "\nVerify nothing survived. --include-managed-resources is not optional:\n" + "AgentCore's instances and volumes are EC2 managed resources and are hidden\n" + "from DescribeInstances and DescribeVolumes by default, so a running fleet\n" + "prints as an empty table.\n\n" + f" aws ec2 describe-instances --include-managed-resources --region {region} \\\n" + f" --filters 'Name=tag:bedrock-agentcore:capacity-provider-id,Values={cp_id}' \\\n" + " --query 'Reservations[].Instances[].[InstanceId,State.Name]' --output table\n\n" + f" aws ec2 describe-volumes --include-managed-resources --region {region} \\\n" + f" --filters 'Name=tag:bedrock-agentcore:capacity-provider-id,Values={cp_id}' \\\n" + " --query 'Volumes[].[VolumeId,Size,State]' --output table\n" + f"\n (account {account})" + ) + + +if __name__ == "__main__": + main() diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/deploy.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/deploy.py new file mode 100755 index 000000000..1d0ed3609 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/deploy.py @@ -0,0 +1,757 @@ +#!/usr/bin/env python3 +"""Deploy the music production agents onto an AgentCore Runtime Instances capacity provider. + +Creates, in order: two IAM roles, an S3 bucket for rendered audio, two ECR images, +one zip artifact, one GPU capacity provider with two persistent EBS volumes, and +three agent runtimes bound to it. + +Writes deployment_state.json for invoke.py and cleanup.py. + +Cost warning: this launches a real GPU EC2 instance in your account on the first +invoke, billed while it runs. A g6.xlarge is roughly 13x a general-purpose +instance of the same size. Run cleanup.py when you are finished, and note that +deleting the SESSION is what deprovisions the instance and its volumes -- stopping +a runtime does not. +""" + +from __future__ import annotations + +import base64 +import json +import os +import shutil +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError + +PROJECT = Path(__file__).resolve().parent.parent +STATE_FILE = PROJECT / "deployment_state.json" + +# us-east-2 by default: measured GPU capacity there when us-west-2 had none in +# any Availability Zone for g6.xlarge. +REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") + +# Every AgentCore-supported accelerator family (g4dn, g5, g6, g6e, gr6, g6f, +# gr6f, g7e, inf2) is x86_64, so a GPU capacity provider must be LINUX_X86_64 +# and its images must be amd64. +CP_OS = "LINUX_X86_64" + +# ONE instance type on purpose. Measured behaviour: AgentCore picks the CHEAPEST +# entry in allowedInstanceTypes and only ever attempts that one -- given six GPU +# types it never tried anything but g6.xlarge, and given four CPU types it chose +# c5.large. So a longer list buys no fallback; it just hides which type you get. +# It DOES try every subnet/AZ within a placement attempt. Capacity is handled by +# retrying invoke.py on a fresh session id. +CP_INSTANCE_TYPE = os.environ.get("CP_INSTANCE_TYPE", "g6.xlarge") + +BUILD_PLATFORM = "linux/amd64" +# manylinux_2_28, not manylinux2014: numpy 2.5 and scipy 1.18 no longer publish +# glibc-2.17 wheels, so a 2014 baseline resolves to "no usable wheels" the moment +# --only-binary is enforced. Both runtime targets are newer than 2.28 anyway -- +# the container base (Debian bookworm) is glibc 2.36 and the zip runtime +# (Amazon Linux 2023) is glibc 2.34. +WHEEL_PLATFORM = "x86_64-manylinux_2_28" +PYTHON_RUNTIME = "PYTHON_3_12" +PYTHON_VERSION = "3.12" + +TRACKS_VOLUME, TRACKS_MOUNT = "tracks", "/mnt/tracks" +MODELS_VOLUME, MODELS_MOUNT = "models", "/mnt/models" +TRACKS_SIZE_GIB = int(os.environ.get("TRACKS_SIZE_GIB", "20")) +# The generative stack is ~6.3 GB installed plus ~8.3 GB of weights. 60 GiB +# leaves room for renders and a second model. +MODELS_SIZE_GIB = int(os.environ.get("MODELS_SIZE_GIB", "60")) + +# rootVolume.freeSpaceGiB defaults to 8, which is what produced the "7.6G free" +# a previous version of this sample measured and mistook for a limit. AgentCore +# adds OS overhead on top of whatever is set here. +ROOT_FREE_GIB = int(os.environ.get("ROOT_FREE_GIB", "30")) + +# A GPU instance idling is the most expensive mistake available here, so this is +# deliberately shorter than the service default of 900. +IDLE_INSTANCE_TIMEOUT = int(os.environ.get("IDLE_INSTANCE_TIMEOUT", "600")) +IDLE_SESSION_TIMEOUT = int(os.environ.get("IDLE_SESSION_TIMEOUT", "600")) +# Ceiling on one compute lifecycle; max 1209600 s (14 days) and must be >= the +# idle timeouts. A runtime's maxLifetime must also be <= the capacity provider's. +MAX_LIFETIME = int(os.environ.get("MAX_LIFETIME", "86400")) + +# Optional: an EBS snapshot of an already-prepared models volume. Verified to +# work -- a snapshot-backed volume is NOT reformatted, so its contents survive +# into new sessions and mode=prepare becomes unnecessary. Requires an extra IAM +# statement; see grant_snapshot_restore below. +MODELS_SNAPSHOT_ID = os.environ.get("MODELS_SNAPSHOT_ID") + +# One model for all three agents, which is deliberate after measuring the +# alternative. Cheaper models were tried per agent and the allocation came out +# backwards: the weakest model held the hardest task (remediation -- read a +# rejected brief plus an avoid-list, then diverge in key, tempo, genre and +# harmony), while the most capable held the most constrained one (pick EQ bands +# from a table of numbers). A smaller compliance model also wrote a false claim +# into a delivery report: "0.0892 remains below the automatic fail threshold of +# 0.045", conflating the fail and review thresholds it had been handed by name. +# +# Cost is not the tradeoff it looks like. A full --with-catalogue run is eight +# model calls, which is a rounding error beside a g6.xlarge and 122 GiB of EBS +# running for the length of the session. +# +# The per-agent overrides stay so you can right-size each task yourself -- that +# is a real practice, and this is a good place to measure it rather than a good +# place to assume it. +DEFAULT_MODEL_ID = "global.anthropic.claude-sonnet-4-6" + +MODELS = { + "composition": os.environ.get("COMPOSITION_MODEL_ID", DEFAULT_MODEL_ID), + "mastering": os.environ.get("MASTERING_MODEL_ID", DEFAULT_MODEL_ID), + "compliance": os.environ.get("COMPLIANCE_MODEL_ID", DEFAULT_MODEL_ID), +} + +OPERATOR_ROLE_NAME = "MusicProductionCapacityProviderOperatorRole" +EXECUTION_ROLE_NAME = "MusicProductionRuntimeExecutionRole" +OPERATOR_MANAGED_POLICY = "arn:aws:iam::aws:policy/BedrockAgentCoreRuntimeInstancesOperatorRolePolicy" + + +def log(msg: str) -> None: + print(f"==> {msg}", flush=True) + + +def warn(msg: str) -> None: + print(f" ! {msg}", flush=True) + + +def die(msg: str) -> None: + print(f"ERROR: {msg}", file=sys.stderr) + sys.exit(1) + + +def run(cmd: list[str], **kwargs) -> None: + print(f" $ {' '.join(cmd)}", flush=True) + subprocess.run(cmd, check=True, **kwargs) + + +def preflight() -> tuple[str, str]: + # Refuse to overwrite a recorded deployment. The state file is the only + # record of a session id, and a session's EBS volumes keep billing after its + # instance is gone -- deleting the session is what deprovisions them. + if STATE_FILE.exists(): + die( + f"{STATE_FILE.name} already exists, so a deployment is still recorded.\n" + " Tear it down first: python scripts/cleanup.py\n" + f" Or, if it is already gone: mv {STATE_FILE.name} {STATE_FILE.name}.old" + ) + + if not REGION: + die( + "Set AWS_REGION (or AWS_DEFAULT_REGION). Nothing is defaulted, so that " + "GPU instances are never launched in a Region you did not choose." + ) + + if MAX_LIFETIME < max(IDLE_INSTANCE_TIMEOUT, IDLE_SESSION_TIMEOUT): + die( + f"MAX_LIFETIME ({MAX_LIFETIME}) must be >= IDLE_INSTANCE_TIMEOUT " + f"({IDLE_INSTANCE_TIMEOUT}) and IDLE_SESSION_TIMEOUT ({IDLE_SESSION_TIMEOUT})." + ) + if not 60 <= MAX_LIFETIME <= 1209600: + die(f"MAX_LIFETIME ({MAX_LIFETIME}) must be between 60 and 1209600 seconds.") + + control = boto3.client("bedrock-agentcore-control", region_name=REGION) + if not hasattr(control, "create_capacity_provider"): + die( + "This boto3 has no capacity provider APIs. Upgrade:\n" + " pip install --upgrade 'boto3>=1.43.72' 'botocore>=1.43.72'" + ) + + requested = os.environ.get("CONTAINER_CLI") + if requested: + if not shutil.which(requested): + die(f"CONTAINER_CLI={requested} is not on PATH.") + cli = requested + else: + cli = next((c for c in ("finch", "docker", "nerdctl", "podman") if shutil.which(c)), None) + if not cli: + die("No container CLI found. Install Finch, Docker, nerdctl or Podman.") + + # Prove the engine can build BEFORE creating any AWS resource. Being on PATH + # is not the same as being usable. + probe = subprocess.run([cli, "info"], capture_output=True, text=True, check=False) + if probe.returncode != 0: + hint = { + "finch": "Finch runs containers in a Linux VM. Create it once:\n" + " finch vm init # then: finch vm start", + "docker": "Start Docker Desktop (or the docker daemon) and retry.", + "podman": "podman machine init && podman machine start", + "nerdctl": "Ensure containerd is running and reachable.", + }.get(cli, "Start the container engine and retry.") + detail = (probe.stderr or probe.stdout or "").strip().splitlines() + die( + f"{cli} is installed but not usable yet.\n {hint}\n" + f" Or select another engine: CONTAINER_CLI=docker python scripts/deploy.py\n" + + (f" {cli} said: {detail[-1][:160]}" if detail else "") + ) + + if not shutil.which("uv"): + die( + "uv is required to vendor Linux wheels for the agent artifacts.\n" + " Install: https://docs.astral.sh/uv/getting-started/installation/" + ) + + # A GPU type that the Region does not offer will fail only at first invoke, + # minutes later, so check now. + ec2 = boto3.client("ec2", region_name=REGION) + offered = ec2.describe_instance_type_offerings( + LocationType="availability-zone", Filters=[{"Name": "instance-type", "Values": [CP_INSTANCE_TYPE]}] + )["InstanceTypeOfferings"] + if not offered: + die(f"{CP_INSTANCE_TYPE} is not offered in {REGION}. Set CP_INSTANCE_TYPE.") + log(f"{CP_INSTANCE_TYPE} offered in {len(offered)} AZ(s): {', '.join(sorted(o['Location'] for o in offered))}") + + account = boto3.client("sts", region_name=REGION).get_caller_identity()["Account"] + log(f"region={REGION} account={account} os={CP_OS} instance={CP_INSTANCE_TYPE}") + log(f"container CLI: {cli}") + return account, cli + + +# ------------------------------------------------------------------------- IAM + + +def artifact_bucket(account: str) -> str: + return f"music-production-artifacts-{account}-{REGION}" + + +def ensure_roles(account: str) -> tuple[str, str]: + """Create the operator (infrastructure) role and the runtime execution role. + + Both are assumed by bedrock-agentcore.amazonaws.com, and getting the trust + policy wrong produces a message that reads as if the role does not exist. + """ + iam = boto3.client("iam") + trust = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "bedrock-agentcore.amazonaws.com"}, + "Action": "sts:AssumeRole", + # Confused-deputy guard. + "Condition": {"StringEquals": {"aws:SourceAccount": account}}, + } + ], + } + + def upsert(name: str, description: str) -> str: + try: + arn = iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps(trust), Description=description)[ + "Role" + ]["Arn"] + log(f"created role {name}") + return arn + except iam.exceptions.EntityAlreadyExistsException: + iam.update_assume_role_policy(RoleName=name, PolicyDocument=json.dumps(trust)) + log(f"reusing role {name}") + return iam.get_role(RoleName=name)["Role"]["Arn"] + + operator_arn = upsert(OPERATOR_ROLE_NAME, "AgentCore provisions EC2 for music production capacity providers") + iam.attach_role_policy(RoleName=OPERATOR_ROLE_NAME, PolicyArn=OPERATOR_MANAGED_POLICY) + if MODELS_SNAPSHOT_ID: + grant_snapshot_restore(MODELS_SNAPSHOT_ID) + + execution_arn = upsert(EXECUTION_ROLE_NAME, "Music production agent runtime execution role") + put_execution_policy(account, composition_arn=None) + log("IAM ready - waiting 10s for propagation") + time.sleep(10) + return operator_arn, execution_arn + + +def grant_snapshot_restore(snapshot_id: str) -> None: + """Let the operator role create a volume FROM a snapshot. + + The AWS managed operator policy grants ec2:CreateVolume on `volume/*` only. + Restoring a snapshot is also authorised against the SNAPSHOT resource, so + without this the placement fails with an opaque + "Failed to provision compute resources for the agent", and only CloudTrail + reveals `UnauthorizedOperation ... on resource .../snapshot/snap-...`. + """ + boto3.client("iam").put_role_policy( + RoleName=OPERATOR_ROLE_NAME, + PolicyName="music-production-snapshot-restore", + PolicyDocument=json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "CreateVolumeFromModelSnapshot", + "Effect": "Allow", + "Action": "ec2:CreateVolume", + "Resource": f"arn:aws:ec2:{REGION}::snapshot/{snapshot_id}", + } + ], + } + ), + ) + log(f"granted the operator role CreateVolume on {snapshot_id}") + + +def put_execution_policy(account: str, composition_arn: str | None) -> None: + """Write the runtime execution policy. + + Called twice. A runtime ARN carries a random 10-character suffix, so it + cannot be known before CreateAgentRuntime returns: the first call omits the + cross-agent grant entirely rather than opening it to every runtime in the + account, and the second adds it scoped to the composition runtime. Nothing + invokes anything in between. + """ + bucket = artifact_bucket(account) + statements: list[dict] = [ + { + # An inference profile ARN is not sufficient on its own: the + # foundation model in every Region the profile routes to must also be + # allowed, and `global.` profiles route to a Region-less ARN. + "Sid": "InvokeModels", + "Effect": "Allow", + "Action": ["bedrock:InvokeModel", "bedrock:InvokeModelWithResponseStream"], + "Resource": ["arn:aws:bedrock:*::foundation-model/*", f"arn:aws:bedrock:*:{account}:inference-profile/*"], + }, + { + "Sid": "PullImages", + "Effect": "Allow", + "Action": ["ecr:GetAuthorizationToken", "ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer"], + "Resource": "*", + }, + { + # Read the zip artifact, and write rendered audio. The volume is + # inside a managed instance with no shell and dies with the session, + # so S3 is the only way a WAV reaches the caller. + "Sid": "Artifacts", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:ListBucket"], + "Resource": [f"arn:aws:s3:::{bucket}", f"arn:aws:s3:::{bucket}/*"], + }, + { + "Sid": "Telemetry", + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams", + "logs:DescribeLogGroups", + "cloudwatch:PutMetricData", + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Resource": "*", + }, + ] + if composition_arn: + # Only the compliance agent uses this, and only to reach one runtime. + # The second resource covers the qualifier (endpoint) sub-resource. + statements.append( + { + "Sid": "CrossAgentInvoke", + "Effect": "Allow", + "Action": "bedrock-agentcore:InvokeAgentRuntime", + "Resource": [composition_arn, f"{composition_arn}/*"], + } + ) + + boto3.client("iam").put_role_policy( + RoleName=EXECUTION_ROLE_NAME, + PolicyName="music-production-runtime-access", + PolicyDocument=json.dumps({"Version": "2012-10-17", "Statement": statements}), + ) + + +# ------------------------------------------------------------------- artifacts + + +def vendor_agent_deps() -> Path: + """Vendor the agent-side wheels for the target platform. + + Shared by both container images, which is why it is built once. `--only-binary` + guarantees nothing is compiled on this machine for a different architecture. + """ + target = PROJECT / "build" / "agentdeps" + if target.exists(): + shutil.rmtree(target) + target.mkdir(parents=True) + log(f"vendoring agent dependencies for {WHEEL_PLATFORM} / cp{PYTHON_VERSION}") + run( + [ + "uv", + "pip", + "install", + "--python-platform", + WHEEL_PLATFORM, + "--python-version", + PYTHON_VERSION, + "--target", + str(target), + "--only-binary", + ":all:", + "-r", + str(PROJECT / "requirements.txt"), + ] + ) + size = sum(f.stat().st_size for f in target.rglob("*") if f.is_file()) + log(f"agent dependencies: {size / 1e6:.0f} MB unpacked") + return target + + +def ensure_bucket(account: str) -> str: + bucket = artifact_bucket(account) + s3 = boto3.client("s3", region_name=REGION) + try: + kwargs: dict = {"Bucket": bucket} + if REGION != "us-east-1": + kwargs["CreateBucketConfiguration"] = {"LocationConstraint": REGION} + s3.create_bucket(**kwargs) + log(f"created bucket {bucket}") + except ClientError as exc: + if exc.response["Error"]["Code"] not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + raise + log(f"reusing bucket {bucket}") + return bucket + + +def build_and_push_image(agent: str, account: str, cli: str, tag: str) -> str: + registry = f"{account}.dkr.ecr.{REGION}.amazonaws.com" + repo = f"music-production/{agent}-agent" + uri = f"{registry}/{repo}:{tag}" + + ecr = boto3.client("ecr", region_name=REGION) + try: + ecr.create_repository(repositoryName=repo) + log(f"created ECR repo {repo}") + except ecr.exceptions.RepositoryAlreadyExistsException: + log(f"reusing ECR repo {repo}") + + token = ecr.get_authorization_token()["authorizationData"][0]["authorizationToken"] + password = base64.b64decode(token).decode().split(":", 1)[1] + subprocess.run( + [cli, "login", "--username", "AWS", "--password-stdin", registry], + input=password.encode(), + check=True, + stdout=subprocess.DEVNULL, + ) + + log(f"building {agent} image for {BUILD_PLATFORM}") + run( + [ + cli, + "build", + "--platform", + BUILD_PLATFORM, + "-f", + str(PROJECT / f"Dockerfile.{agent}"), + "-t", + uri, + str(PROJECT), + ] + ) + log(f"pushing {uri}") + run([cli, "push", uri]) + size = subprocess.run( + [cli, "images", "--format", "{{.Size}}", uri], capture_output=True, text=True, check=False + ).stdout.strip() + # The hard cap on an AgentCore Runtime image is 2 GB. + log(f"{agent} image size: {size or ''} (limit 2 GB)") + return uri + + +def build_and_upload_zip(account: str, bucket: str, key: str) -> tuple[str, str]: + """Vendor dependencies for the target platform and upload the compliance zip. + + There is no pip install on the instance: whatever is in the zip is what the + agent gets, so wheels must be built for Linux x86_64 rather than this laptop. + Limits are 250 MB compressed and 750 MB uncompressed, and the uncompressed one + is the easier to hit. + """ + build_dir = PROJECT / "build" / "compliance" + if build_dir.exists(): + shutil.rmtree(build_dir) + build_dir.mkdir(parents=True) + + log(f"vendoring compliance dependencies for {WHEEL_PLATFORM}") + run( + [ + "uv", + "pip", + "install", + "--python-platform", + WHEEL_PLATFORM, + "--python-version", + PYTHON_VERSION, + "--target", + str(build_dir), + "--only-binary", + ":all:", + "-r", + str(PROJECT / "requirements.txt"), + ] + ) + shutil.copy(PROJECT / "compliance_agent.py", build_dir) + shutil.copy(PROJECT / "audio_dsp.py", build_dir) + + archive = PROJECT / "build" / "compliance_agent.zip" + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: + for path in sorted(build_dir.rglob("*")): + if path.is_file() and "__pycache__" not in path.parts: + zf.write(path, path.relative_to(build_dir)) + size_mb = archive.stat().st_size / 1e6 + raw_mb = sum(p.stat().st_size for p in build_dir.rglob("*") if p.is_file()) / 1e6 + log(f"zip built: {size_mb:.1f} MB compressed, {raw_mb:.1f} MB uncompressed (limits 250 MB / 750 MB)") + if size_mb > 250 or raw_mb > 750: + die("compliance zip exceeds the direct-code-deploy limits") + + boto3.client("s3", region_name=REGION).upload_file( + str(archive), bucket, key, ExtraArgs={"ExpectedBucketOwner": account} + ) + log(f"uploaded s3://{bucket}/{key}") + return bucket, key + + +# ------------------------------------------------------- capacity provider + + +def vpc_configuration() -> dict: + """Use explicit subnets/security group if given, else every default-VPC subnet + in an AZ that offers the instance type. + + Pass more than one. AgentCore does try every subnet within a single placement + attempt -- CloudTrail shows four RunInstances calls across four AZs in four + seconds -- but the error it surfaces names only one, which makes a + single-subnet capacity provider look like a service fault instead of an AZ + with no capacity. A capacity provider's configuration cannot be edited after + creation, so getting this wrong means rebuilding it. + + The supported-AZ list in the VPC documentation governs the microVM ENI path, + not capacity providers: a subnet in an AZ absent from that list was accepted + and is where capacity was found. + """ + subnet = os.environ.get("CP_SUBNET_ID") + group = os.environ.get("CP_SECURITY_GROUP_ID") + if subnet and group: + chosen = [s.strip() for s in subnet.split(",") if s.strip()][:16] + if len(chosen) == 1: + warn("one subnet means one AZ; pass several to survive an Insufficient EC2 capacity error in that AZ") + return {"subnets": chosen, "securityGroups": [group]} + if subnet or group: + die("Set CP_SUBNET_ID and CP_SECURITY_GROUP_ID together, or neither.") + + ec2 = boto3.client("ec2", region_name=REGION) + vpcs = ec2.describe_vpcs(Filters=[{"Name": "isDefault", "Values": ["true"]}])["Vpcs"] + if not vpcs: + die("No default VPC. Set CP_SUBNET_ID and CP_SECURITY_GROUP_ID.") + vpc_id = vpcs[0]["VpcId"] + subnets = ec2.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])["Subnets"] + groups = ec2.describe_security_groups( + Filters=[{"Name": "vpc-id", "Values": [vpc_id]}, {"Name": "group-name", "Values": ["default"]}] + )["SecurityGroups"] + if not subnets or not groups: + die(f"Default VPC {vpc_id} has no subnet or no default security group.") + + offered = { + o["Location"] + for o in ec2.describe_instance_type_offerings( + LocationType="availability-zone", Filters=[{"Name": "instance-type", "Values": [CP_INSTANCE_TYPE]}] + )["InstanceTypeOfferings"] + } + usable = [s for s in subnets if s["AvailabilityZone"] in offered][:16] + if not usable: + die(f"No default-VPC subnet is in an AZ offering {CP_INSTANCE_TYPE}.") + zones = sorted({s["AvailabilityZone"] for s in usable}) + log(f"using default VPC {vpc_id}: {len(usable)} subnet(s) across {', '.join(zones)}") + return {"subnets": [s["SubnetId"] for s in usable], "securityGroups": [groups[0]["GroupId"]]} + + +def create_capacity_provider(control, name: str, operator_arn: str) -> tuple[str, str]: + models_ebs: dict = { + "name": MODELS_VOLUME, + "sizeGiB": MODELS_SIZE_GIB, + "volumeType": "gp3", + "encrypted": True, + # The model stack is read-heavy on first load; the gp3 + # default of 125 MiB/s makes a cold start noticeably slower. + "throughput": 500, + } + if MODELS_SNAPSHOT_ID: + # Verified: a snapshot-backed volume is NOT reformatted, so a prepared + # stack survives into every new session and mode=prepare is unnecessary. + models_ebs["snapshotId"] = MODELS_SNAPSHOT_ID + log(f"models volume will be restored from {MODELS_SNAPSHOT_ID}") + + log(f"creating capacity provider {name}") + resp = control.create_capacity_provider( + name=name, + description="Music production agent fleet (GPU)", + permissionsConfiguration={"capacityProviderOperatorRoleArn": operator_arn}, + computeConfiguration={ + "ec2Configuration": { + "launchTemplateSource": { + "launchParameters": { + "operatingSystem": CP_OS, + "instanceRequirements": {"allowedInstanceTypes": [CP_INSTANCE_TYPE]}, + } + }, + "vpcConfiguration": vpc_configuration(), + "volumes": [ + # The shared workspace, mounted by all three agents. + { + "ebsConfiguration": { + "name": TRACKS_VOLUME, + "sizeGiB": TRACKS_SIZE_GIB, + "volumeType": "gp3", + "encrypted": True, + } + }, + # The generative stack, mounted only by the composition agent. + {"ebsConfiguration": models_ebs}, + ], + "rootVolume": {"freeSpaceGiB": ROOT_FREE_GIB, "volumeType": "gp3"}, + "lifecycleConfiguration": {"idleInstanceTimeout": IDLE_INSTANCE_TIMEOUT, "maxLifetime": MAX_LIFETIME}, + } + }, + ) + cp_id, cp_arn = resp["capacityProviderId"], resp["capacityProviderArn"] + + # The terminal state is READY (the API enum has no ACTIVE, whatever the + # documentation prose says), and there is no waiter, so poll. This launches + # no instances -- the fleet is a declaration until the first invoke. + log(f"waiting for {cp_id} to become READY") + while True: + got = control.get_capacity_provider(capacityProviderId=cp_id) + status = got["status"] + if status == "READY": + break + if "FAILED" in status: + die(f"{status}: {got.get('statusReason')} ({got.get('statusCode')})") + time.sleep(5) + log(f"capacity provider READY: {cp_arn}") + return cp_id, cp_arn + + +def create_runtime( + control, name: str, artifact: dict, execution_arn: str, cp_arn: str, env: dict, volumes: list[tuple[str, str]] +) -> dict: + log(f"creating runtime {name} (mounts {', '.join(m for _, m in volumes)})") + resp = control.create_agent_runtime( + agentRuntimeName=name, + roleArn=execution_arn, + agentRuntimeArtifact=artifact, + protocolConfiguration={"serverProtocol": "HTTP"}, + # Binds the runtime to the fleet. Mutually exclusive with + # networkConfiguration: the VPC belongs to the capacity provider. + capacityProviderConfiguration={"capacityProviderArn": cp_arn}, + # Only the volumes a runtime declares are mounted for it, so the mastering + # and compliance agents never see the model stack. + filesystemConfigurations=[{"capacityProviderVolume": {"volumeName": v, "mountPath": m}} for v, m in volumes], + lifecycleConfiguration={"idleRuntimeSessionTimeout": IDLE_SESSION_TIMEOUT, "maxLifetime": MAX_LIFETIME}, + environmentVariables=env, + ) + runtime_id, arn = resp["agentRuntimeId"], resp["agentRuntimeArn"] + while True: + status = control.get_agent_runtime(agentRuntimeId=runtime_id)["status"] + if status == "READY": + break + if "FAILED" in status: + die(f"runtime {name} is {status}") + time.sleep(5) + log(f"runtime READY: {arn}") + return {"name": name, "id": runtime_id, "arn": arn} + + +def main() -> None: + account, cli = preflight() + suffix = str(int(time.time())) + control = boto3.client("bedrock-agentcore-control", region_name=REGION) + + operator_arn, execution_arn = ensure_roles(account) + bucket = ensure_bucket(account) + vendor_agent_deps() + + composition_image = build_and_push_image("composition", account, cli, f"v1-{suffix}") + mastering_image = build_and_push_image("mastering", account, cli, f"v1-{suffix}") + _, key = build_and_upload_zip(account, bucket, f"compliance/{suffix}/compliance_agent.zip") + + cp_id, cp_arn = create_capacity_provider(control, f"music_production_capacity_{suffix}", operator_arn) + + base_env = {"AWS_REGION": REGION, "WORKSPACE_DIR": TRACKS_MOUNT, "ARTIFACT_BUCKET": bucket} + + # Composition first: the compliance agent needs its ARN, which cannot be + # constructed by hand because of the random 10-character suffix. + composition = create_runtime( + control, + f"music_production_composition_{suffix}", + {"containerConfiguration": {"containerUri": composition_image}}, + execution_arn, + cp_arn, + {**base_env, "MODEL_ID": MODELS["composition"], "MODELS_DIR": MODELS_MOUNT}, + [(TRACKS_VOLUME, TRACKS_MOUNT), (MODELS_VOLUME, MODELS_MOUNT)], + ) + + # Now that the composition runtime exists, scope the cross-agent grant to + # that one ARN instead of leaving it open to every runtime in the account. + put_execution_policy(account, composition_arn=composition["arn"]) + log(f"scoped CrossAgentInvoke to {composition['arn']}") + + mastering = create_runtime( + control, + f"music_production_mastering_{suffix}", + {"containerConfiguration": {"containerUri": mastering_image}}, + execution_arn, + cp_arn, + {**base_env, "MODEL_ID": MODELS["mastering"]}, + [(TRACKS_VOLUME, TRACKS_MOUNT)], + ) + + compliance = create_runtime( + control, + f"music_production_compliance_{suffix}", + { + "codeConfiguration": { + "code": {"s3": {"bucket": bucket, "prefix": key}}, + "runtime": PYTHON_RUNTIME, + "entryPoint": ["compliance_agent.py"], + } + }, + execution_arn, + cp_arn, + { + **base_env, + "MODEL_ID": MODELS["compliance"], + "COMPOSITION_RUNTIME_ARN": composition["arn"], + "COMPOSITION_QUALIFIER": "DEFAULT", + }, + [(TRACKS_VOLUME, TRACKS_MOUNT)], + ) + + state = { + "region": REGION, + "account": account, + "suffix": suffix, + "instance_type": CP_INSTANCE_TYPE, + "capacity_provider": {"id": cp_id, "arn": cp_arn}, + "runtimes": {"composition": composition, "mastering": mastering, "compliance": compliance}, + "ecr_repositories": ["music-production/composition-agent", "music-production/mastering-agent"], + "s3": {"bucket": bucket, "key": key}, + "iam_roles": [OPERATOR_ROLE_NAME, EXECUTION_ROLE_NAME], + "volumes": {TRACKS_VOLUME: TRACKS_MOUNT, MODELS_VOLUME: MODELS_MOUNT}, + "models_snapshot_id": MODELS_SNAPSHOT_ID, + "sessions": [], + } + STATE_FILE.write_text(json.dumps(state, indent=2)) + log(f"wrote {STATE_FILE}") + print("\nDeployed. Next: python scripts/invoke.py (then scripts/cleanup.py)") + print(f"Note: no EC2 instance exists yet - the first invoke provisions a {CP_INSTANCE_TYPE}.") + if not MODELS_SNAPSHOT_ID: + print( + "The first invoke also builds the generative stack onto the models " + "volume (a few minutes, once per session)." + ) + + +if __name__ == "__main__": + main() diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/invoke.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/invoke.py new file mode 100755 index 000000000..f89732a1a --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/invoke.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +"""Run the music production workflow against the deployed runtimes. + +All three agents are invoked with the same runtimeSessionId on the same capacity +provider, which is what places them on one EC2 instance sharing one volume. Each +response reports the host that served it, so collocation is observable. + + python scripts/invoke.py # prepare, compose, master, comply + python scripts/invoke.py --with-catalogue # also render a back-catalogue and a + # deliberate near-copy of it, so the + # similarity screen has something real + # to catch + python scripts/invoke.py --resume # stop the previous session, resume it, + # and re-run only the compliance step + python scripts/invoke.py --track my-song --session <33-100 chars> + +Rendered audio is downloaded from S3 into runs//. The agents write to the +capacity provider's EBS volume inside a managed instance with no shell, and that +volume is destroyed when the session is deleted, so the S3 copy is the only one +that outlives the fleet. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.request +import uuid +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +STATE_FILE = Path(__file__).resolve().parent.parent / "deployment_state.json" + +# A capacity failure is fast (~13 s) and is bound to the placement AgentCore +# chose, so retrying on a fresh session id is what buys another attempt. +CAPACITY_ATTEMPTS = 8 + + +def load_state() -> dict: + if not STATE_FILE.exists(): + sys.exit(f"No {STATE_FILE.name}. Run scripts/deploy.py first.") + return json.loads(STATE_FILE.read_text()) + + +def save_state(state: dict) -> None: + STATE_FILE.write_text(json.dumps(state, indent=2)) + + +def data_client(region: str): + return boto3.client( + "bedrock-agentcore", + region_name=region, + # A cold start provisions a GPU instance and the first invoke also builds + # the model stack, so the default 60 s read timeout is far too short. + # 900 s is the service's own synchronous request ceiling. Retries are + # limited because InvokeAgentRuntime is not idempotent. + config=Config(read_timeout=900, retries={"max_attempts": 2, "mode": "standard"}), + ) + + +def record_session(state: dict, session_id: str) -> None: + """Record every session id before it is used. + + Even a session whose placement FAILED exists as a session record, and its + half-created EBS volume survives both DeleteCapacityProviderSession and + DeleteCapacityProvider. Recording the id is the only way cleanup can try. + """ + if session_id not in state.setdefault("sessions", []): + state["sessions"].append(session_id) + save_state(state) + + +def invoke(client, state: dict, runtime: dict, session_id: str, payload: dict, label: str) -> tuple[dict, str]: + """Invoke one runtime, retrying capacity failures on a fresh session id.""" + for attempt in range(1, CAPACITY_ATTEMPTS + 1): + sid = session_id if attempt == 1 else f"{session_id[:-3]}{attempt:03d}" + record_session(state, sid) + started = time.time() + try: + response = client.invoke_agent_runtime( + agentRuntimeArn=runtime["arn"], + qualifier="DEFAULT", + runtimeSessionId=sid, + payload=json.dumps(payload).encode(), + ) + # The response body member is named "response", not "body". + body = json.loads(response["response"].read()) + print(f" {label}: {time.time() - started:.1f}s") + return body, sid + except ClientError as exc: + code = exc.response["Error"]["Code"] + message = exc.response["Error"]["Message"] + elapsed = time.time() - started + if "Insufficient EC2 capacity" in message: + az = message.split("requested (")[-1].split(")")[0] if "requested (" in message else "?" + print( + f" {label}: no {state['instance_type']} capacity in {az} " + f"after {elapsed:.0f}s - retry {attempt}/{CAPACITY_ATTEMPTS} " + f"on a fresh session" + ) + time.sleep(5) + continue + if code in ("InternalServerException", "RetryableConflictException", "ThrottlingException"): + print(f" {label}: {code} after {elapsed:.1f}s - retry {attempt}/{CAPACITY_ATTEMPTS}") + time.sleep(5 * attempt) + continue + raise + sys.exit( + f"{label}: no capacity after {CAPACITY_ATTEMPTS} attempts. GPU capacity " + f"for {state['instance_type']} is exhausted in every AZ of this capacity " + f"provider. Try again later, choose another Region, or reserve capacity " + f"with an On-Demand Capacity Reservation." + ) + + +def download_artifacts(body: dict, out_dir: Path, region: str) -> list[Path]: + """Fetch what the agent produced. + + Prefers the caller's own credentials over the presigned URL the agent + returned. The URL is genuinely useful for handing a render to someone without + AWS access, but depending on it here would make this script fail for reasons + that have nothing to do with the agents -- a presigned URL is only valid while + the credentials that signed it are, and AgentCore rotates the execution role + credentials it vends to the agent. + """ + saved: list[Path] = [] + s3 = boto3.client("s3", region_name=region, endpoint_url=f"https://s3.{region}.amazonaws.com") + for art in body.get("artifacts") or []: + uri = art.get("s3_uri") + out_dir.mkdir(parents=True, exist_ok=True) + dest = out_dir / art["name"] + if uri: + bucket, _, key = uri[len("s3://") :].partition("/") + try: + s3.download_file(bucket, key, str(dest)) + saved.append(dest) + continue + except ClientError as exc: + print( + f" ! s3 download of {art['name']} failed: " + f"{exc.response['Error']['Code']}; trying the presigned URL" + ) + url = art.get("url") + if not url: + continue + try: + with urllib.request.urlopen(url, timeout=300) as r, open(dest, "wb") as fh: + fh.write(r.read()) + saved.append(dest) + except Exception as exc: # noqa: BLE001 + print(f" ! could not download {art['name']}: {exc}") + return saved + + +def show_measurements(label: str, m: dict | None) -> None: + if not m: + return + print( + f" {label:9s} {m.get('duration_s')}s {m.get('sample_rate')}Hz " + f"{m.get('channels')}ch {m.get('integrated_lufs')} LUFS " + f"peak {m.get('true_peak_dbtp')} dBTP LRA {m.get('loudness_range_lu')} LU" + ) + + +def report(step: str, body: dict, out_dir: Path | None = None, region: str = "us-east-2") -> dict: + host = body.get("host") or {} + print(f" {step}") + print(f" status : {body.get('status')}") + if body.get("status") != "ok": + print(f" error : {body.get('error')}") + return host + print(f" host : {host.get('hostname')} (process {host.get('process_id')}, {host.get('architecture')})") + if body.get("read_from"): + print(f" read : {body['read_from']} <- written by another agent") + + render = body.get("render") or {} + if render: + t = render.get("timings", {}) + print( + f" rendered : {render.get('device')} in {t.get('generate_s')}s " + f"(load {t.get('pipeline_ctor_s')}s, peak VRAM {t.get('peak_vram_gib')} GiB)" + ) + if body.get("measurements") and "before" not in body["measurements"]: + show_measurements("audio", body["measurements"]) + if body.get("measurements") and "before" in body["measurements"]: + show_measurements("before", body["measurements"]["before"]) + show_measurements("after", body["measurements"]["after"]) + if body.get("targets_met"): + tm = body["targets_met"] + print( + f" targets : loudness {'met' if tm['loudness'] else 'MISSED'}, " + f"true peak {'held' if tm['true_peak'] else 'EXCEEDED'}" + ) + + if "validation_passed" in body: + label = {"cleared": "CLEARED", "review_required": "REVIEW REQUIRED", "not_cleared": "NOT CLEARED"}.get( + body.get("outcome"), "CLEARED" if body["validation_passed"] else "NOT CLEARED" + ) + print(f" verdict : {label}") + qc = body.get("delivery_qc") or {} + for c in qc.get("checks", []): + if not c["pass"]: + print(f" FAIL {c['check']}: {c['detail']}") + sim = body.get("similarity") or {} + if sim.get("references"): + closest = sim.get("closest") or {} + print( + f" screen : {sim['references']} reference(s), closest " + f"{closest.get('reference')} distance {closest.get('distance')} " + f"({closest.get('verdict')})" + ) + if sim.get("standout_ratio") is not None: + print( + f" standout : {sim['standout_ratio']}x the next-closest " + f"(flag below {sim['thresholds']['standout_ratio_below']})" + ) + if sim.get("flag_reason"): + print(f" flagged : {sim['flag_reason']}") + elif sim.get("note"): + print(f" screen : {sim['note']}") + if body.get("master_is_stale"): + print(" stale : master.wav predates the screened render - re-run mastering") + if (body.get("verdict") or {}).get("remediation_requested"): + print(" remediation was requested from the composition agent") + + if body.get("model_stack"): + ms = body["model_stack"] + print( + f" stack : ready={ms.get('ready')} " + f"{'in ' + str(ms.get('prepared_in_seconds')) + 's' if ms.get('prepared_in_seconds') else ''}" + ) + + print(f" files : {body.get('workspace_files')}") + if out_dir: + for p in download_artifacts(body, out_dir, region): + print(f" saved : {p} ({p.stat().st_size / 1e6:.2f} MB)") + return host + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--track", default=None, help="defaults to a fresh track id") + parser.add_argument("--session", default=None, help="33-100 characters") + parser.add_argument( + "--with-catalogue", + action="store_true", + help="render a back-catalogue, then a deliberate near-copy of " + "it, so the similarity screen has real material to flag", + ) + parser.add_argument( + "--resume", + action="store_true", + help="stop the session recorded by the previous run and resume it, running only the compliance step", + ) + parser.add_argument("--duration", type=float, default=30.0, help="seconds of audio to render") + args = parser.parse_args() + + state = load_state() + region = state["region"] + runtimes = state["runtimes"] + client = data_client(region) + + if args.resume: + last = state.get("last_run") or {} + session_id = args.session or last.get("session_id") + track = args.track or last.get("track") + if not (session_id and track): + sys.exit( + "--resume needs a previous run. Run 'python scripts/invoke.py' " + "first, or pass --session and --track explicitly." + ) + else: + session_id = args.session or f"music-production-{uuid.uuid4()}" + track = args.track or f"track-{int(time.time())}" + + # InvokeAgentRuntime accepts 33-256 characters, but DeleteCapacityProviderSession + # caps sessionId at 100 - a longer id can be invoked and then never deleted, + # stranding its EBS volumes. Hold the whole tool to the deletable range. + if not 33 <= len(session_id) <= 100: + sys.exit(f"--session must be 33-100 characters (got {len(session_id)}).") + + state["last_run"] = {"session_id": session_id, "track": track} + record_session(state, session_id) + + print(f"track : {track}") + print(f"session : {session_id}") + print(f"instance: {state['instance_type']} region: {region}") + if args.resume: + print("Resuming the session above - only the compliance step runs.\n") + else: + print( + f"The first invoke provisions a {state['instance_type']} and builds the " + "generative stack on the models volume, so expect several minutes.\n" + ) + + run_dir = STATE_FILE.parent / "runs" / track + hosts: dict[str, dict] = {} + active = session_id + + if args.resume: + print(" -- stopping every agent in the session --") + for name, runtime in runtimes.items(): + try: + client.stop_runtime_session(agentRuntimeArn=runtime["arn"], runtimeSessionId=session_id) + print(f" stopped {name}") + except ClientError as exc: + code = exc.response["Error"]["Code"] + # Normal, not a failure: that runtime was never invoked in this + # session, so there is no session of its own to stop. + print(f" {name}: {'nothing to stop' if code == 'ResourceNotFoundException' else code}") + # StopRuntimeSession stops one agent in a session; the instance goes away + # only once every agent on it has been idle for idleInstanceTimeout. + print(" waiting 30s before resuming\n") + time.sleep(30) + else: + # 1. Build the generative stack on the models volume. Per-session, + # because the volume is per-session, unless MODELS_SNAPSHOT_ID was set + # at deploy time. + body, active = invoke(client, state, runtimes["composition"], active, {"mode": "prepare"}, "prepare") + hosts["prepare"] = report("1. prepare model stack (GPU instance + torch + weights)", body) + if body.get("status") != "ok": + sys.exit("model stack preparation failed; see the error above") + + step = 2 + if args.with_catalogue: + body, active = invoke( + client, + state, + runtimes["composition"], + active, + {"mode": "catalogue", "track_id": track, "duration_s": min(args.duration, 20.0)}, + "catalogue", + ) + hosts["catalogue"] = report(f"{step}. render back-catalogue", body, run_dir, region) + step += 1 + + # 2. Compose. With --with-catalogue this deliberately imitates a + # catalogue entry so the compliance screen has something real to find. + compose_payload: dict = { + "mode": "compose", + "track_id": track, + "prompt": "Create an upbeat electronic track with heavy bass and synth melodies.", + "duration_s": args.duration, + "seed": 42, + } + if args.with_catalogue: + compose_payload.update( + imitate_catalogue="catalogue_00.wav", + reference_strength=0.85, + prompt="Create a melodic techno track with analog bass and warm pads, close to our catalogue sound.", + ) + body, active = invoke(client, state, runtimes["composition"], active, compose_payload, "compose") + hosts["composition"] = report(f"{step}. compose (renders audio on the GPU)", body, run_dir, region) + step += 1 + + # 3. Master, reading the audio the composition agent rendered. + body, active = invoke( + client, + state, + runtimes["mastering"], + active, + {"track_id": track, "platform": "spotify", "prompt": "Master this for streaming."}, + "master", + ) + hosts["mastering"] = report(f"{step}. master (real DSP, verified by measurement)", body, run_dir, region) + step += 1 + + body, active = invoke( + client, + state, + runtimes["compliance"], + active, + {"track_id": track, "prompt": "Screen this master for release."}, + "compliance", + ) + hosts["compliance"] = report( + "compliance (resumed session)" if args.resume else "5. compliance screen", body, run_dir, region + ) + + # Close the loop. A remediation replaces the composition, which leaves the + # master describing audio that no longer exists -- the compliance agent says so + # via master_is_stale. Without this the run ends holding a master of the + # rejected material, which is the one artifact a producer would actually ship. + if body.get("status") == "ok" and body.get("master_is_stale"): + print("\n -- remediation happened, so the master is stale: re-mastering --") + body, active = invoke( + client, + state, + runtimes["mastering"], + active, + {"track_id": track, "platform": "spotify", "prompt": "Master the replacement for streaming."}, + "re-master", + ) + hosts["re-master"] = report("6. re-master the replacement", body, run_dir, region) + + body, active = invoke( + client, + state, + runtimes["compliance"], + active, + { + "track_id": track, + # The replacement has already been screened once; this + # pass is about the new master, so do not remediate again. + "auto_remediate": False, + "prompt": "Screen the re-mastered replacement.", + }, + "re-screen", + ) + hosts["re-screen"] = report("7. re-screen the new master", body, run_dir, region) + + state["last_run"] = {"session_id": active, "track": track} + save_state(state) + + print("\n -- collocation --") + names = [h.get("hostname") for h in hosts.values() if h.get("hostname")] + for agent, host in hosts.items(): + print(f" {agent:12} {host.get('hostname')}") + if len(names) < 2: + print(" Only one step ran. Collocation is visible on a full pipeline.") + elif len(set(names)) == 1: + print(f" All {len(names)} steps were served by one instance, as intended.") + else: + print(" Steps landed on different hosts - the shared volume still carried") + print(" the artifacts, but a capacity retry started a new session.") + + if run_dir.exists(): + produced = sorted(p for p in run_dir.iterdir() if p.is_file()) + print("\n -- what the agents produced (downloaded from S3) --") + for p in produced: + print(f" {p} ({p.stat().st_size / 1e6:.2f} MB)") + print(" The originals are on the instance's volume and go away with the") + print(" session; these copies and the S3 objects are what you keep.") + + print("\nDone. Delete the session and the fleet with: python scripts/cleanup.py") + print("Deleting the SESSION is what stops EC2 and EBS charges.") + + +if __name__ == "__main__": + main() diff --git a/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/update.py b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/update.py new file mode 100755 index 000000000..efb5ad331 --- /dev/null +++ b/02-use-cases/02-workflow-automation-agents/gpu-music-production-agent/scripts/update.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Ship a new version of one agent, without touching the others. + +This is the independent-deployment story made real: each agent has its own +artifact and its own runtime version, so one team can release while the other two +keep serving. The capacity provider, the volumes and the session are untouched. + + python scripts/update.py composition # rebuild + update one agent + python scripts/update.py composition mastering # or several + python scripts/update.py --all + python scripts/update.py composition --restart-session + +Two behaviours worth knowing: + +* ``UpdateAgentRuntime`` is a replace, not a merge. Omitting a member drops it, so + the current configuration is read back with ``GetAgentRuntime`` and re-sent with + only the artifact changed. +* A new version does NOT reach a session that is already running. AgentCore keeps + serving the code the session started with, with no error anywhere, so a fix can + look deployed and have no effect. ``--restart-session`` stops the recorded + sessions so the next invoke picks up the new version. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import shutil +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError + +PROJECT = Path(__file__).resolve().parent.parent +STATE_FILE = PROJECT / "deployment_state.json" + +BUILD_PLATFORM = "linux/amd64" +WHEEL_PLATFORM = "x86_64-manylinux_2_28" +PYTHON_VERSION = "3.12" +CONTAINER_AGENTS = ("composition", "mastering") +ALL_AGENTS = ("composition", "mastering", "compliance") + + +def log(msg: str) -> None: + print(f"==> {msg}", flush=True) + + +def warn(msg: str) -> None: + print(f" ! {msg}", flush=True) + + +def die(msg: str) -> None: + sys.exit(f"ERROR: {msg}") + + +def run(cmd: list[str], **kwargs) -> None: + print(f" $ {' '.join(cmd)}", flush=True) + subprocess.run(cmd, check=True, **kwargs) + + +def container_cli() -> str: + cli = next((c for c in ("finch", "docker", "nerdctl", "podman") if shutil.which(c)), None) + if not cli: + die("no container CLI found") + if subprocess.run([cli, "info"], capture_output=True, check=False).returncode != 0: + die(f"{cli} is installed but not usable (start its VM or daemon)") + return cli + + +def vendor_agent_deps() -> Path: + target = PROJECT / "build" / "agentdeps" + if target.exists(): + shutil.rmtree(target) + target.mkdir(parents=True) + log("vendoring agent dependencies") + run( + [ + "uv", + "pip", + "install", + "--python-platform", + WHEEL_PLATFORM, + "--python-version", + PYTHON_VERSION, + "--target", + str(target), + "--only-binary", + ":all:", + "-r", + str(PROJECT / "requirements.txt"), + ] + ) + return target + + +def push_image(agent: str, account: str, region: str, cli: str, tag: str) -> str: + registry = f"{account}.dkr.ecr.{region}.amazonaws.com" + uri = f"{registry}/music-production/{agent}-agent:{tag}" + ecr = boto3.client("ecr", region_name=region) + token = ecr.get_authorization_token()["authorizationData"][0]["authorizationToken"] + password = base64.b64decode(token).decode().split(":", 1)[1] + subprocess.run( + [cli, "login", "--username", "AWS", "--password-stdin", registry], + input=password.encode(), + check=True, + stdout=subprocess.DEVNULL, + ) + log(f"building {agent} for {BUILD_PLATFORM}") + run( + [ + cli, + "build", + "--platform", + BUILD_PLATFORM, + "-f", + str(PROJECT / f"Dockerfile.{agent}"), + "-t", + uri, + str(PROJECT), + ] + ) + run([cli, "push", uri]) + size = subprocess.run( + [cli, "images", "--format", "{{.Size}}", uri], capture_output=True, text=True, check=False + ).stdout.strip() + log(f"{agent} image {size or ''} (limit 2 GB)") + return uri + + +def push_zip(state: dict, tag: str) -> str: + bucket = state["s3"]["bucket"] + key = f"compliance/{tag}/compliance_agent.zip" + build_dir = PROJECT / "build" / "compliance" + if build_dir.exists(): + shutil.rmtree(build_dir) + build_dir.mkdir(parents=True) + log("vendoring compliance dependencies") + run( + [ + "uv", + "pip", + "install", + "--python-platform", + WHEEL_PLATFORM, + "--python-version", + PYTHON_VERSION, + "--target", + str(build_dir), + "--only-binary", + ":all:", + "-r", + str(PROJECT / "requirements.txt"), + ] + ) + shutil.copy(PROJECT / "compliance_agent.py", build_dir) + shutil.copy(PROJECT / "audio_dsp.py", build_dir) + archive = PROJECT / "build" / "compliance_agent.zip" + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: + for p in sorted(build_dir.rglob("*")): + if p.is_file() and "__pycache__" not in p.parts: + zf.write(p, p.relative_to(build_dir)) + mb = archive.stat().st_size / 1e6 + raw = sum(p.stat().st_size for p in build_dir.rglob("*") if p.is_file()) / 1e6 + log(f"zip {mb:.1f} MB compressed, {raw:.1f} MB uncompressed (limits 250 / 750)") + if mb > 250 or raw > 750: + die("compliance zip exceeds the direct-code-deploy limits") + boto3.client("s3", region_name=state["region"]).upload_file(str(archive), bucket, key) + log(f"uploaded s3://{bucket}/{key}") + state["s3"]["key"] = key + return key + + +def update_runtime(region: str, runtime_id: str, artifact: dict) -> str: + """Re-send the whole configuration with only the artifact changed.""" + control = boto3.client("bedrock-agentcore-control", region_name=region) + cur = control.get_agent_runtime(agentRuntimeId=runtime_id) + kwargs: dict = { + "agentRuntimeId": runtime_id, + "agentRuntimeArtifact": artifact, + "roleArn": cur["roleArn"], + "protocolConfiguration": cur.get("protocolConfiguration") or {"serverProtocol": "HTTP"}, + "capacityProviderConfiguration": cur["capacityProviderConfiguration"], + "filesystemConfigurations": cur["filesystemConfigurations"], + "environmentVariables": cur.get("environmentVariables") or {}, + } + if cur.get("lifecycleConfiguration"): + kwargs["lifecycleConfiguration"] = cur["lifecycleConfiguration"] + control.update_agent_runtime(**kwargs) + while True: + got = control.get_agent_runtime(agentRuntimeId=runtime_id) + if got["status"] == "READY": + return got.get("agentRuntimeVersion", "?") + if "FAILED" in got["status"]: + die(f"update left runtime {runtime_id} in {got['status']}") + time.sleep(5) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("agents", nargs="*", choices=ALL_AGENTS, default=[]) + ap.add_argument("--all", action="store_true") + ap.add_argument( + "--restart-session", + action="store_true", + help="stop recorded sessions so the next invoke picks up the new " + "version; without this a running session keeps serving the old code", + ) + args = ap.parse_args() + + agents = list(ALL_AGENTS) if args.all else args.agents + if not agents: + ap.error("name at least one agent, or pass --all") + + if not STATE_FILE.exists(): + die(f"no {STATE_FILE.name}; run scripts/deploy.py first") + state = json.loads(STATE_FILE.read_text()) + region, account = state["region"], state["account"] + tag = f"v2-{int(time.time())}" + + if any(a in CONTAINER_AGENTS for a in agents): + vendor_agent_deps() + cli = container_cli() + + for agent in agents: + runtime = state["runtimes"][agent] + if agent in CONTAINER_AGENTS: + uri = push_image(agent, account, region, cli, tag) + artifact = {"containerConfiguration": {"containerUri": uri}} + else: + key = push_zip(state, tag) + artifact = { + "codeConfiguration": { + "code": {"s3": {"bucket": state["s3"]["bucket"], "prefix": key}}, + "runtime": "PYTHON_3_12", + "entryPoint": ["compliance_agent.py"], + } + } + version = update_runtime(region, runtime["id"], artifact) + log(f"{agent} runtime is now version {version}") + + STATE_FILE.write_text(json.dumps(state, indent=2)) + + if args.restart_session: + data = boto3.client("bedrock-agentcore", region_name=region) + for session_id in state.get("sessions", []): + if len(session_id) < 33: + continue + for name in agents: + try: + data.stop_runtime_session( + agentRuntimeArn=state["runtimes"][name]["arn"], runtimeSessionId=session_id + ) + log(f"stopped {name} in {session_id}") + except ClientError as exc: + code = exc.response["Error"]["Code"] + if code != "ResourceNotFoundException": + warn(f"stop {name}: {code}") + print( + "\nSessions stopped. The next invoke provisions fresh compute on the same " + "session and picks up the new version. Volumes are retained, so a prepared " + "model stack survives." + ) + else: + print( + "\nUpdated. NOTE: a session that is already running keeps serving the " + "previous version with no error. Re-run with --restart-session, or use a " + "new session id, to actually exercise the new code." + ) + + +if __name__ == "__main__": + main() diff --git a/02-use-cases/README.md b/02-use-cases/README.md index 1c197a12d..a380ab12e 100644 --- a/02-use-cases/README.md +++ b/02-use-cases/README.md @@ -48,6 +48,7 @@ Agents that run without a user in the loop. They are triggered by events such as | [enterprise-web-intelligence-agent](./02-workflow-automation-agents/enterprise-web-intelligence-agent/) | Market Intelligence | Runtime, Browser | | [intelligent-event-agent](./02-workflow-automation-agents/intelligent-event-agent/) | General / Events | Runtime, Memory, Gateway *(in development)* | | [multi-isv-orchestration](./02-workflow-automation-agents/multi-isv-orchestration/) | Enterprise CRM + ERP | Gateway (multi-target), Identity (Cognito + CustomOauth2) | +| [gpu-music-production-agent](./02-workflow-automation-agents/gpu-music-production-agent/) | Media & Entertainment | Runtime (EC2 capacity provider, GPU), Memory; local model inference, collocated agents on a shared volume | ### [03-coding-assistants](./03-coding-assistants/)