feat(android): repack and sign APKs in TypeScript, and one repack entry for the Studio's Worker

.github/workflows/android-repack.yml

@@ -1,10 +1,10 @@
 name: Android runtime and repack
 on:
   pull_request:
-    paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"]
+    paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "tests/repack-worker.test.ts", "tests/helpers/android-fixture.ts", "tests/helpers/apk-test-signer.ts", "tests/fixtures/repack-android/**", "tools/android-link.ts", "tools/community-key.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"]
   push:
     branches: [main]
-    paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"]
+    paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "tests/repack-worker.test.ts", "tests/helpers/android-fixture.ts", "tests/helpers/apk-test-signer.ts", "tests/fixtures/repack-android/**", "tools/android-link.ts", "tools/community-key.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"]
 permissions:
   contents: read
 jobs:
@@ -17,6 +17,7 @@ jobs:
         with:
           bun-version: 1.3.14
       - run: bun install --frozen-lockfile
-      # The native half (aapt2, zipalign, apksigner) is skipped where the
-      # pinned Android SDK is absent, as it is on this runner.
-      - run: bun test tests/repack-android.test.ts tests/repack-shared.test.ts
+      # The aapt2 + zipalign + apksigner comparison is skipped where the
+      # pinned Android SDK is absent, as it is on this runner; a TypeScript
+      # verifier checks the v1 and v2 signatures.
+      - run: bun test tests/repack-android.test.ts tests/repack-worker.test.ts tests/repack-shared.test.ts

.github/workflows/repack.yml

@@ -3,7 +3,7 @@ name: Runtime repack
 # The repack step (tools/repack/): a prebuilt PocketJS runtime and a game's
 # .pocket into a device's installation package, in TypeScript with no native
 # tool. These tests hold each target's package structure, its determinism and
-# its refusals.
+# its refusals, and that every repack bundles for a Worker.
 on:
   pull_request:
     paths:
@@ -16,6 +16,10 @@ on:
       - "tools/png.ts"
       - "contracts/spec/pocket-package.ts"
       - "tests/repack-*.test.ts"
+      - "tests/helpers/android-fixture.ts"
+      - "tests/helpers/apk-test-signer.ts"
+      - "tools/android-link.ts"
+      - "tools/community-key.ts"
       - "tests/fixtures/packages/**"
       - "tests/fixtures/repack-*/**"
       - "tools/vita-package.ts"
@@ -32,6 +36,10 @@ on:
       - "tools/png.ts"
       - "contracts/spec/pocket-package.ts"
       - "tests/repack-*.test.ts"
+      - "tests/helpers/android-fixture.ts"
+      - "tests/helpers/apk-test-signer.ts"
+      - "tools/android-link.ts"
+      - "tools/community-key.ts"
       - "tests/fixtures/packages/**"
       - "tests/fixtures/repack-*/**"
       - "tools/vita-package.ts"

docs/ANDROID.md

@@ -135,50 +135,94 @@ output.
 **The native library takes four values from the profile and none from the
 app** (target id, host ABI, logical viewport, raster density), so every app
 of a profile links the same `libpocketjs.so` and `classes.dex`. The runtime
-build makes them once, with no guest; the repack adds a game to them.
+build compiles them with no guest and links them into one template APK;
+the repack adds a game to it in TypeScript, with no SDK tool.
 
 ```sh
 bun tools/runtime.ts android        # dist/runtime/android/ with runtime.json
+bun tools/community-key.ts          # the community key as two DER files
 bun tools/pocket-pack.ts build --manifest <pocket.json> --project-root <dir> \
   --target redmi-1s-dev -o game.pocket
 bun tools/repack.ts --target android --runtime dist/runtime/android --pocket game.pocket \
   --id <id> --title <title> --author <author> --version <x.y.z> [--icon <square.png>] -o game.apk
 ```
 
-- **The runtime directory** holds `lib/armeabi-v7a/libpocketjs.so`,
-  `lib/arm64-v8a/libpocketjs.so`, `classes.dex`, the manifest and string
-  templates, the default icon at the five densities, and `runtime.json`:
-  target `redmi-1s-dev`, host ABI 9, the PocketJS commit, the SDK levels and
-  the size and SHA-256 of each file. **Its libraries and `classes.dex` are
-  the bytes `build-app` links** at the same commit.
+- **The runtime directory holds `template.apk`, the two templates it was
+  linked from (`AndroidManifest.xml`, `res/values/strings.xml`) and
+  `runtime.json`**: target `redmi-1s-dev`, host ABI 9, the PocketJS commit,
+  the SDK levels, the ABIs and the size and SHA-256 of each file.
+  `template.apk` is aapt2's link of the runtime with the identity
+  `dev.pocket-nexus.runtime.template` (`ANDROID_TEMPLATE_IDENTITY`), the
+  default icon at five densities, empty `assets/app.js` and `assets/app.pak`,
+  `classes.dex` and `lib/armeabi-v7a/libpocketjs.so`,
+  `lib/arm64-v8a/libpocketjs.so`; it is unsigned (1.76 MB for `redmi-1s`).
 - **`tools/repack/android.ts` checks before it writes**
   (`tools/repack/shared/runtime.ts`): the runtime's files against
   `runtime.json`, the `.pocket` footer, a `redmi-1s-dev` variant at host ABI
-  9 whose app id is `--id`, and the variant's viewport against the runtime's.
-  Any mismatch stops the repack with the reason.
-- It writes the variant's bundle (without the section's NUL terminator) and
-  pack as `assets/app.js` and `assets/app.pak`, the two files
-  `PocketActivity` reads; renders the manifest (package name = the id with
-  each `-` as `_`, the version code as above, `android:debuggable="false"`)
-  and the label; and scales `--icon`, a square PNG of any size, to the five
-  densities. Without `--icon` the runtime's default icon goes in.
-- **The native steps are one function**, `AndroidNativeSteps`
-  (`tools/repack/android-native.ts`): aapt2 compile and link, the merge with
-  `classes.dex` and the libraries, `zipalign -p 4`, `apksigner sign` (v1 + v2)
-  and `apksigner verify`. The steps before it are TypeScript with no file
-  system, so a container can run the native function for a caller that has
-  no SDK.
+  9 whose app id is `--id`, the variant's viewport against the runtime's, and
+  the template's SDK levels. Any mismatch stops the repack with the reason.
+- It keeps the template's entries in their order and replaces five things.
+  **The compiled `AndroidManifest.xml` gets the package name, versionCode
+  and versionName; `resources.arsc` gets the package name and
+  `string/app_name`** (`tools/repack/shared/android-resources.ts`). Each
+  string pool is written as aapt2 writes one: aapt2 sorts a pool (the
+  manifest's attribute names with resource ids first, then every other string
+  by its UTF-8 bytes), so a changed string moves to its sorted place and every
+  reference is renumbered; the table's UTF-8 pool writes a character past
+  U+FFFF as two 3-byte surrogates. The variant's bundle (without the
+  section's NUL terminator) and pack become `assets/app.js` and
+  `assets/app.pak`, the files `PocketActivity` reads. `--icon`, a square PNG
+  of any size, is scaled to the five densities; without it the template's
+  icon stays. Stored entries are 4-byte aligned and stored `.so` files
+  4096-byte aligned, as `zipalign -p 4` places them.
+- **The names:** the package name is the id with each `-` as `_`, and a
+  dot-separated segment that then starts with a digit gets `x` in front
+  (`dev.pocket-nexus.studio.2pac.8-ball` →
+  `dev.pocket_nexus.studio.x2pac.x8_ball`); each segment is a letter, then
+  letters, digits and `_`, and the name holds at most 127 characters (a
+  resource table's limit). Two ids map to one package name when one has a
+  segment `x<digit>…` where the other has `<digit>…`. The versionCode is
+  major × 1 000 000 + minor × 1 000 + patch. The label is the title with
+  leading and trailing ASCII whitespace removed and each inner run of it as
+  one space, which is what aapt2 makes of a string resource; a control
+  character is refused.
+- **Signing is v1 + v2 with Web Crypto** (`tools/repack/shared/apk-sign.ts`):
+  `META-INF/MANIFEST.MF`, `CERT.SF` (`X-Android-APK-Signed: 2`) and
+  `CERT.RSA` (PKCS #7, RSA PKCS #1 v1.5, SHA-256), then the APK Signing Block
+  with the v2 signature before the central directory. Android 4.3 reads the
+  v1 signature and not the v2 block. The repack verifies the v2 signature with the certificate's public
+  key before it returns, so a key that is not the certificate's is refused.
 - **A repacked game is signed with the Pocket Studio community key, never
-  the Pocket Nexus release key.** `POCKET_STUDIO_COMMUNITY_KEY` names the
-  PKCS #12 keystore, default
-  `~/.config/pocket-nexus/signing/pocket-studio-community.p12`, alias
-  `pocket-studio-community`; the password is the first line of the
-  `.password` file beside it, read by apksigner. The certificate's SHA-256 is
-  `POCKET_STUDIO_COMMUNITY_SIGNER` in `tools/repack/android.ts`, and the
-  native step refuses an APK that verifies under another certificate.
-- **The same inputs give the same APK bytes**: every entry the merge writes is
-  dated 1980-01-01, the entries keep a fixed order, and v1 + v2 signatures
-  with an RSA key carry no time.
+  the Pocket Nexus release key.** The repack takes
+  `signer: { privateKey, certificate }`: a PKCS #8 DER key (or a `CryptoKey`
+  imported for RSASSA-PKCS1-v1_5 with SHA-256) and an X.509 DER certificate.
+  **It refuses a certificate whose SHA-256 is not
+  `POCKET_STUDIO_COMMUNITY_SIGNER`** unless the caller names another
+  certificate in `options.certificateSha256`, which the tests do for a
+  throwaway key. `bun tools/community-key.ts` turns the keystore
+  `~/.config/pocket-nexus/signing/pocket-studio-community.p12`
+  (`POCKET_STUDIO_COMMUNITY_KEY` names another) into
+  `pocket-studio-community.pkcs8.der` and `pocket-studio-community.cert.der`
+  beside it, mode 600. OpenSSL reads the password from the `.password` file
+  (`-passin file:`), and the tool writes nothing unless the certificate is
+  the community certificate and the key signs what it verifies. A Worker
+  holds the two files' bytes as secrets.
+- **The same inputs give the same APK bytes on one JavaScript runtime**: every
+  entry is dated 1980-01-01, the entries keep the template's order, and RSA
+  PKCS #1 v1.5 signatures carry no time. The deflate streams (zip entries and
+  the icons' PNG data) come from the runtime's `CompressionStream`, so Bun and
+  workerd write different compressed bytes for the same uncompressed entries
+  and pixels.
+- **aapt2, zipalign and apksigner are the test oracle.** Where the Android SDK
+  is installed, `tests/repack-android.test.ts` links each game of a corpus
+  (hyphens, digit segments, quotes, a backslash, Japanese, an emoji label,
+  whitespace runs) with aapt2, aligns it with zipalign and signs it with
+  apksigner, and requires the TypeScript APK to hold the same entries in the
+  same order with the same bytes outside `META-INF/`, to pass
+  `apksigner verify` (v1 and v2) and `zipalign -c -p 4`. It runs on the test
+  fixture and on `dist/runtime/android` when one is built. Without the SDK
+  (CI) those tests skip, and a verifier in `tests/helpers/apk-test-signer.ts`
+  checks both signatures.
 
 ## On the device
 

docs/PLATFORM.md

@@ -142,6 +142,21 @@ shared pieces use Web-platform APIs only (a Worker runs them): `zip.ts`
 | `3ds-dev` | `romfs:/app.pocket` | `.3dsx` (hosts/3ds/README.md) |
 | `psp` | `app.pocket` in the EBOOT's folder (the module's argv[0] directory) into one kernel block before the arena reserves the rest | `.zip`: `PSP/GAME/Studio<Name>/EBOOT.PBP` (PARAM.SFO with the game's `TITLE` and `MEMSIZE = 1`, ICON0 144×80 from the icon, the runtime's `DATA.PSP` unchanged) and `app.pocket` beside it; a plan that asks for `io.offload` is refused |
 | `vita` | `app0:app.pocket` into one 16-byte-aligned block; no compiled app id | `.vpk`: `sce_sys/param.sfo` (byte for byte what `vita-mksfoex` writes for `tools/vita.ts`: the game's title, title id `P` + 8 hex of SHA-256(id)), the runtime's `eboot.bin` unchanged, `app.pocket`, `sce_sys/icon0.png` 128×128 indexed PNG-8 from the icon, the runtime's LiveArea files |
+| `ipodtouch4-dev` | `app.pocket` in the app bundle | `.ipa`: the runtime's executable under the game's bundle name, an Info.plist with the game's bundle id (the app id), name and version, 57 and 114 px icons, `app.pocket` (docs/IPODTOUCH4.md) |
+| `redmi-1s-dev`, `moto-g-play-dev` | `assets/app.js` and `assets/app.pak` | `.apk`: the runtime's `template.apk` with the package name, versionCode, label, icons and the variant's bundle and pack rewritten, signed v1 + v2 with the Pocket Studio community key in Web Crypto (docs/ANDROID.md) |
+
+**`tools/repack/index.ts` is the entry Pocket Studio's Worker imports.**
+`REPACK_TARGETS` has one row per Studio target (`psp`, `vita`, `3ds`,
+`ipod-touch`, `android`): `repack(input)`, `runtime` (the
+`tools/runtime.ts` name: `psp`, `vita`, `3ds`, `ipod`, `android`),
+`variantTargets` (the `.pocket` targets that target's runtimes boot),
+`extension` (`.zip`, `.vpk`, `.3dsx`, `.ipa`, `.apk`), `contentType` and
+`signed` (Android alone takes `input.signer`). `tools/repack.ts` reads the
+files and calls the same rows. **`tests/repack-worker.test.ts` bundles the
+entry and each target's module with `Bun.build({ target: "browser" })` and
+fails on any Node or Bun built-in module and on any reference to `Bun`,
+`process`, `require`, `__dirname` or `import.meta.dir`**; it also runs the
+bundled Android module and compares its APK with the source module's.
 
 On the PSP the game file takes the place of the embedded build's rodata
 copy: with twenty48 (723 KB file) the arena's free block measured

tests/helpers/android-fixture.ts

@@ -0,0 +1,104 @@
+// The Android repack's test fixture: a runtime directory around
+// tests/fixtures/repack-android/template.apk and a `.pocket` of Clear's
+// Redmi 1S plan under a test id. Shared by tests/repack-android.test.ts and
+// tests/repack-worker.test.ts.
+
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { encodePocketPackage, type PocketPackageVariant } from "../../contracts/spec/pocket-package.ts";
+import { canonicalJson } from "../../framework/src/manifest/plan.ts";
+import { linkAndroidApk } from "../../tools/android-link.ts";
+import { makeVariant } from "../../tools/pocket-pack.ts";
+import { REDMI_1S_CONTRACTS, REDMI_1S_TARGET, resolveRedmi1sBuildPlan } from "../../tools/redmi-1s-profile.ts";
+import { ANDROID_ICON_DENSITIES, ANDROID_TEMPLATE, androidTemplateJob } from "../../tools/repack/android.ts";
+import { encodePng } from "../../tools/repack/shared/png.ts";
+
+const ROOT = join(import.meta.dir, "../..");
+const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex");
+const bytesOf = (text: string) => new TextEncoder().encode(text);
+
+export const HOST = join(ROOT, "hosts/android/app");
+/**
+ * aapt2's link of hosts/android/app's templates with ANDROID_TEMPLATE_IDENTITY,
+ * stand-in classes.dex and libraries and a plain icon. Rewrite it with
+ * `POCKETJS_WRITE_ANDROID_FIXTURE=1 bun test tests/repack-android.test.ts`
+ * (Android SDK required); tests/repack-android.test.ts checks it against aapt2.
+ */
+export const FIXTURE = join(ROOT, "tests/fixtures/repack-android/template.apk");
+export const ABIS = ["armeabi-v7a", "arm64-v8a"] as const;
+export const SDK_LEVELS = { apiLevel: 34, buildToolsVersion: "35.0.0", minSdkVersion: 18, targetSdkVersion: 34 } as const;
+export const HOST_ABI = REDMI_1S_CONTRACTS.targets[REDMI_1S_TARGET].hostAbi;
+
+export function square(side: number, paint: (x: number, y: number) => [number, number, number, number]): Uint8Array {
+  const rgba = new Uint8Array(side * side * 4);
+  for (let y = 0; y < side; y++) for (let x = 0; x < side; x++) rgba.set(paint(x, y), (y * side + x) * 4);
+  return rgba;
+}
+
+/** The runtime build's files the fixture template is linked from. */
+export async function fixtureParts(): Promise<Map<string, Uint8Array>> {
+  const files = new Map<string, Uint8Array>([
+    ["AndroidManifest.xml", new Uint8Array(readFileSync(join(HOST, "AndroidManifest.xml")))],
+    ["res/values/strings.xml", new Uint8Array(readFileSync(join(HOST, "res/values/strings.xml")))],
+    ["classes.dex", bytesOf("dex\n035\0 stand-in for PocketActivity")],
+    ...ABIS.map((abi) => [`lib/${abi}/libpocketjs.so`, bytesOf(`\x7fELF stand-in ${abi}`)] as [string, Uint8Array]),
+  ]);
+  for (const [density, side] of ANDROID_ICON_DENSITIES) {
+    files.set(`res/mipmap-${density}/icon.png`, await encodePng({ width: side, height: side, rgba: square(side, () => [23, 18, 38, 255]) }));
+  }
+  return files;
+}
+
+if (process.env.POCKETJS_WRITE_ANDROID_FIXTURE === "1") {
+  mkdirSync(join(FIXTURE, ".."), { recursive: true });
+  writeFileSync(FIXTURE, await linkAndroidApk(androidTemplateJob(await fixtureParts(), SDK_LEVELS, ABIS)));
+}
+export const TEMPLATE = new Uint8Array(readFileSync(FIXTURE));
+
+/** A runtime directory around a template: the sources from hosts/android, runtime.json describing them. */
+export function fixtureRuntime(overrides: Record<string, unknown> = {}, template: Uint8Array = TEMPLATE): Map<string, Uint8Array> {
+  const files = new Map<string, Uint8Array>([
+    [ANDROID_TEMPLATE, template],
+    ["AndroidManifest.xml", new Uint8Array(readFileSync(join(HOST, "AndroidManifest.xml")))],
+    ["res/values/strings.xml", new Uint8Array(readFileSync(join(HOST, "res/values/strings.xml")))],
+  ]);
+  const manifest = {
+    target: REDMI_1S_TARGET,
+    hostAbi: HOST_ABI,
+    pocketjs: "0".repeat(40),
+    profile: "redmi-1s",
+    viewport: { logical: [360, 640], rasterDensity: 2 },
+    android: { ...SDK_LEVELS, abis: [...ABIS] },
+    files: Object.fromEntries([...files].map(([path, bytes]) => [path, { bytes: bytes.length, sha256: sha256(bytes) }])),
+    ...overrides,
+  };
+  files.set("runtime.json", bytesOf(JSON.stringify(manifest)));
+  return files;
+}
+
+export const IDENTITY = { id: "dev.pocket-nexus.studio.test-game", title: "Tom's \"Test\" & <Game>", author: "A Creator", version: "1.2.3" };
+/** Clear's Redmi 1S manifest under an id: the repack refuses an identity that is not the .pocket's app. */
+export const clear = (id = IDENTITY.id) => ({ ...JSON.parse(readFileSync(join(ROOT, "apps/clear/pocket.redmi-1s.json"), "utf8")), id });
+export const GAME_JS = bytesOf("globalThis.game = 1;\n".repeat(64));
+export const GAME_PAK = Uint8Array.from({ length: 4096 }, (_, i) => (i * 7) & 0xff);
+
+export function variantFor(plan: ReturnType<typeof resolveRedmi1sBuildPlan>): PocketPackageVariant {
+  return makeVariant({
+    target: plan.target.id,
+    hostAbi: plan.target.hostAbi,
+    planJson: canonicalJson(plan),
+    identity: { output: plan.app.output, id: plan.app.id, title: plan.app.title },
+    js: GAME_JS,
+    pak: GAME_PAK,
+  });
+}
+
+export function fixturePocket(id = IDENTITY.id, variants?: PocketPackageVariant[]): Uint8Array {
+  const manifest = clear(id);
+  return encodePocketPackage({
+    manifest: bytesOf(JSON.stringify(manifest)),
+    variants: variants ?? [variantFor(resolveRedmi1sBuildPlan(manifest))],
+  });
+}
+