diff --git a/.github/workflows/android-repack.yml b/.github/workflows/android-repack.yml index e355d1c86..58a566a49 100644 --- a/.github/workflows/android-repack.yml +++ b/.github/workflows/android-repack.yml @@ -1,10 +1,10 @@ name: Android runtime and repack on: pull_request: - paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"] + paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "tests/repack-worker.test.ts", "tests/helpers/android-fixture.ts", "tests/helpers/apk-test-signer.ts", "tests/fixtures/repack-android/**", "tools/android-link.ts", "tools/community-key.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"] push: branches: [main] - paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"] + paths: [".github/workflows/android-repack.yml", "tools/repack.ts", "tools/repack/**", "tools/runtime.ts", "tools/runtime/**", "tests/repack-android.test.ts", "tests/repack-worker.test.ts", "tests/helpers/android-fixture.ts", "tests/helpers/apk-test-signer.ts", "tests/fixtures/repack-android/**", "tools/android-link.ts", "tools/community-key.ts", "contracts/spec/pocket-package.ts", "tools/pocket-pack.ts", "tools/android.ts", "tools/android-icon.ts", "hosts/android/**"] permissions: contents: read jobs: @@ -17,6 +17,7 @@ jobs: with: bun-version: 1.3.14 - run: bun install --frozen-lockfile - # The native half (aapt2, zipalign, apksigner) is skipped where the - # pinned Android SDK is absent, as it is on this runner. - - run: bun test tests/repack-android.test.ts tests/repack-shared.test.ts + # The aapt2 + zipalign + apksigner comparison is skipped where the + # pinned Android SDK is absent, as it is on this runner; a TypeScript + # verifier checks the v1 and v2 signatures. + - run: bun test tests/repack-android.test.ts tests/repack-worker.test.ts tests/repack-shared.test.ts diff --git a/.github/workflows/repack.yml b/.github/workflows/repack.yml index 01b61e6f8..56a407a63 100644 --- a/.github/workflows/repack.yml +++ b/.github/workflows/repack.yml @@ -3,7 +3,7 @@ name: Runtime repack # The repack step (tools/repack/): a prebuilt PocketJS runtime and a game's # .pocket into a device's installation package, in TypeScript with no native # tool. These tests hold each target's package structure, its determinism and -# its refusals. +# its refusals, and that every repack bundles for a Worker. on: pull_request: paths: @@ -16,6 +16,10 @@ on: - "tools/png.ts" - "contracts/spec/pocket-package.ts" - "tests/repack-*.test.ts" + - "tests/helpers/android-fixture.ts" + - "tests/helpers/apk-test-signer.ts" + - "tools/android-link.ts" + - "tools/community-key.ts" - "tests/fixtures/packages/**" - "tests/fixtures/repack-*/**" - "tools/vita-package.ts" @@ -32,6 +36,10 @@ on: - "tools/png.ts" - "contracts/spec/pocket-package.ts" - "tests/repack-*.test.ts" + - "tests/helpers/android-fixture.ts" + - "tests/helpers/apk-test-signer.ts" + - "tools/android-link.ts" + - "tools/community-key.ts" - "tests/fixtures/packages/**" - "tests/fixtures/repack-*/**" - "tools/vita-package.ts" diff --git a/docs/ANDROID.md b/docs/ANDROID.md index f73fcfc93..5283c8ee5 100644 --- a/docs/ANDROID.md +++ b/docs/ANDROID.md @@ -135,50 +135,94 @@ output. **The native library takes four values from the profile and none from the app** (target id, host ABI, logical viewport, raster density), so every app of a profile links the same `libpocketjs.so` and `classes.dex`. The runtime -build makes them once, with no guest; the repack adds a game to them. +build compiles them with no guest and links them into one template APK; +the repack adds a game to it in TypeScript, with no SDK tool. ```sh bun tools/runtime.ts android # dist/runtime/android/ with runtime.json +bun tools/community-key.ts # the community key as two DER files bun tools/pocket-pack.ts build --manifest --project-root \ --target redmi-1s-dev -o game.pocket bun tools/repack.ts --target android --runtime dist/runtime/android --pocket game.pocket \ --id --title --author <author> --version <x.y.z> [--icon <square.png>] -o game.apk ``` -- **The runtime directory** holds `lib/armeabi-v7a/libpocketjs.so`, - `lib/arm64-v8a/libpocketjs.so`, `classes.dex`, the manifest and string - templates, the default icon at the five densities, and `runtime.json`: - target `redmi-1s-dev`, host ABI 9, the PocketJS commit, the SDK levels and - the size and SHA-256 of each file. **Its libraries and `classes.dex` are - the bytes `build-app` links** at the same commit. +- **The runtime directory holds `template.apk`, the two templates it was + linked from (`AndroidManifest.xml`, `res/values/strings.xml`) and + `runtime.json`**: target `redmi-1s-dev`, host ABI 9, the PocketJS commit, + the SDK levels, the ABIs and the size and SHA-256 of each file. + `template.apk` is aapt2's link of the runtime with the identity + `dev.pocket-nexus.runtime.template` (`ANDROID_TEMPLATE_IDENTITY`), the + default icon at five densities, empty `assets/app.js` and `assets/app.pak`, + `classes.dex` and `lib/armeabi-v7a/libpocketjs.so`, + `lib/arm64-v8a/libpocketjs.so`; it is unsigned (1.76 MB for `redmi-1s`). - **`tools/repack/android.ts` checks before it writes** (`tools/repack/shared/runtime.ts`): the runtime's files against `runtime.json`, the `.pocket` footer, a `redmi-1s-dev` variant at host ABI - 9 whose app id is `--id`, and the variant's viewport against the runtime's. - Any mismatch stops the repack with the reason. -- It writes the variant's bundle (without the section's NUL terminator) and - pack as `assets/app.js` and `assets/app.pak`, the two files - `PocketActivity` reads; renders the manifest (package name = the id with - each `-` as `_`, the version code as above, `android:debuggable="false"`) - and the label; and scales `--icon`, a square PNG of any size, to the five - densities. Without `--icon` the runtime's default icon goes in. -- **The native steps are one function**, `AndroidNativeSteps` - (`tools/repack/android-native.ts`): aapt2 compile and link, the merge with - `classes.dex` and the libraries, `zipalign -p 4`, `apksigner sign` (v1 + v2) - and `apksigner verify`. The steps before it are TypeScript with no file - system, so a container can run the native function for a caller that has - no SDK. + 9 whose app id is `--id`, the variant's viewport against the runtime's, and + the template's SDK levels. Any mismatch stops the repack with the reason. +- It keeps the template's entries in their order and replaces five things. + **The compiled `AndroidManifest.xml` gets the package name, versionCode + and versionName; `resources.arsc` gets the package name and + `string/app_name`** (`tools/repack/shared/android-resources.ts`). Each + string pool is written as aapt2 writes one: aapt2 sorts a pool (the + manifest's attribute names with resource ids first, then every other string + by its UTF-8 bytes), so a changed string moves to its sorted place and every + reference is renumbered; the table's UTF-8 pool writes a character past + U+FFFF as two 3-byte surrogates. The variant's bundle (without the + section's NUL terminator) and pack become `assets/app.js` and + `assets/app.pak`, the files `PocketActivity` reads. `--icon`, a square PNG + of any size, is scaled to the five densities; without it the template's + icon stays. Stored entries are 4-byte aligned and stored `.so` files + 4096-byte aligned, as `zipalign -p 4` places them. +- **The names:** the package name is the id with each `-` as `_`, and a + dot-separated segment that then starts with a digit gets `x` in front + (`dev.pocket-nexus.studio.2pac.8-ball` → + `dev.pocket_nexus.studio.x2pac.x8_ball`); each segment is a letter, then + letters, digits and `_`, and the name holds at most 127 characters (a + resource table's limit). Two ids map to one package name when one has a + segment `x<digit>…` where the other has `<digit>…`. The versionCode is + major × 1 000 000 + minor × 1 000 + patch. The label is the title with + leading and trailing ASCII whitespace removed and each inner run of it as + one space, which is what aapt2 makes of a string resource; a control + character is refused. +- **Signing is v1 + v2 with Web Crypto** (`tools/repack/shared/apk-sign.ts`): + `META-INF/MANIFEST.MF`, `CERT.SF` (`X-Android-APK-Signed: 2`) and + `CERT.RSA` (PKCS #7, RSA PKCS #1 v1.5, SHA-256), then the APK Signing Block + with the v2 signature before the central directory. Android 4.3 reads the + v1 signature and not the v2 block. The repack verifies the v2 signature with the certificate's public + key before it returns, so a key that is not the certificate's is refused. - **A repacked game is signed with the Pocket Studio community key, never - the Pocket Nexus release key.** `POCKET_STUDIO_COMMUNITY_KEY` names the - PKCS #12 keystore, default - `~/.config/pocket-nexus/signing/pocket-studio-community.p12`, alias - `pocket-studio-community`; the password is the first line of the - `.password` file beside it, read by apksigner. The certificate's SHA-256 is - `POCKET_STUDIO_COMMUNITY_SIGNER` in `tools/repack/android.ts`, and the - native step refuses an APK that verifies under another certificate. -- **The same inputs give the same APK bytes**: every entry the merge writes is - dated 1980-01-01, the entries keep a fixed order, and v1 + v2 signatures - with an RSA key carry no time. + the Pocket Nexus release key.** The repack takes + `signer: { privateKey, certificate }`: a PKCS #8 DER key (or a `CryptoKey` + imported for RSASSA-PKCS1-v1_5 with SHA-256) and an X.509 DER certificate. + **It refuses a certificate whose SHA-256 is not + `POCKET_STUDIO_COMMUNITY_SIGNER`** unless the caller names another + certificate in `options.certificateSha256`, which the tests do for a + throwaway key. `bun tools/community-key.ts` turns the keystore + `~/.config/pocket-nexus/signing/pocket-studio-community.p12` + (`POCKET_STUDIO_COMMUNITY_KEY` names another) into + `pocket-studio-community.pkcs8.der` and `pocket-studio-community.cert.der` + beside it, mode 600. OpenSSL reads the password from the `.password` file + (`-passin file:`), and the tool writes nothing unless the certificate is + the community certificate and the key signs what it verifies. A Worker + holds the two files' bytes as secrets. +- **The same inputs give the same APK bytes on one JavaScript runtime**: every + entry is dated 1980-01-01, the entries keep the template's order, and RSA + PKCS #1 v1.5 signatures carry no time. The deflate streams (zip entries and + the icons' PNG data) come from the runtime's `CompressionStream`, so Bun and + workerd write different compressed bytes for the same uncompressed entries + and pixels. +- **aapt2, zipalign and apksigner are the test oracle.** Where the Android SDK + is installed, `tests/repack-android.test.ts` links each game of a corpus + (hyphens, digit segments, quotes, a backslash, Japanese, an emoji label, + whitespace runs) with aapt2, aligns it with zipalign and signs it with + apksigner, and requires the TypeScript APK to hold the same entries in the + same order with the same bytes outside `META-INF/`, to pass + `apksigner verify` (v1 and v2) and `zipalign -c -p 4`. It runs on the test + fixture and on `dist/runtime/android` when one is built. Without the SDK + (CI) those tests skip, and a verifier in `tests/helpers/apk-test-signer.ts` + checks both signatures. ## On the device diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index a02615504..af820f661 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -142,6 +142,21 @@ shared pieces use Web-platform APIs only (a Worker runs them): `zip.ts` | `3ds-dev` | `romfs:/app.pocket` | `.3dsx` (hosts/3ds/README.md) | | `psp` | `app.pocket` in the EBOOT's folder (the module's argv[0] directory) into one kernel block before the arena reserves the rest | `.zip`: `PSP/GAME/Studio<Name>/EBOOT.PBP` (PARAM.SFO with the game's `TITLE` and `MEMSIZE = 1`, ICON0 144×80 from the icon, the runtime's `DATA.PSP` unchanged) and `app.pocket` beside it; a plan that asks for `io.offload` is refused | | `vita` | `app0:app.pocket` into one 16-byte-aligned block; no compiled app id | `.vpk`: `sce_sys/param.sfo` (byte for byte what `vita-mksfoex` writes for `tools/vita.ts`: the game's title, title id `P` + 8 hex of SHA-256(id)), the runtime's `eboot.bin` unchanged, `app.pocket`, `sce_sys/icon0.png` 128×128 indexed PNG-8 from the icon, the runtime's LiveArea files | +| `ipodtouch4-dev` | `app.pocket` in the app bundle | `.ipa`: the runtime's executable under the game's bundle name, an Info.plist with the game's bundle id (the app id), name and version, 57 and 114 px icons, `app.pocket` (docs/IPODTOUCH4.md) | +| `redmi-1s-dev`, `moto-g-play-dev` | `assets/app.js` and `assets/app.pak` | `.apk`: the runtime's `template.apk` with the package name, versionCode, label, icons and the variant's bundle and pack rewritten, signed v1 + v2 with the Pocket Studio community key in Web Crypto (docs/ANDROID.md) | + +**`tools/repack/index.ts` is the entry Pocket Studio's Worker imports.** +`REPACK_TARGETS` has one row per Studio target (`psp`, `vita`, `3ds`, +`ipod-touch`, `android`): `repack(input)`, `runtime` (the +`tools/runtime.ts` name: `psp`, `vita`, `3ds`, `ipod`, `android`), +`variantTargets` (the `.pocket` targets that target's runtimes boot), +`extension` (`.zip`, `.vpk`, `.3dsx`, `.ipa`, `.apk`), `contentType` and +`signed` (Android alone takes `input.signer`). `tools/repack.ts` reads the +files and calls the same rows. **`tests/repack-worker.test.ts` bundles the +entry and each target's module with `Bun.build({ target: "browser" })` and +fails on any Node or Bun built-in module and on any reference to `Bun`, +`process`, `require`, `__dirname` or `import.meta.dir`**; it also runs the +bundled Android module and compares its APK with the source module's. On the PSP the game file takes the place of the embedded build's rodata copy: with twenty48 (723 KB file) the arena's free block measured diff --git a/tests/fixtures/repack-android/template.apk b/tests/fixtures/repack-android/template.apk new file mode 100644 index 000000000..ffcfb8d59 Binary files /dev/null and b/tests/fixtures/repack-android/template.apk differ diff --git a/tests/helpers/android-fixture.ts b/tests/helpers/android-fixture.ts new file mode 100644 index 000000000..4cb0c3847 --- /dev/null +++ b/tests/helpers/android-fixture.ts @@ -0,0 +1,104 @@ +// The Android repack's test fixture: a runtime directory around +// tests/fixtures/repack-android/template.apk and a `.pocket` of Clear's +// Redmi 1S plan under a test id. Shared by tests/repack-android.test.ts and +// tests/repack-worker.test.ts. + +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { encodePocketPackage, type PocketPackageVariant } from "../../contracts/spec/pocket-package.ts"; +import { canonicalJson } from "../../framework/src/manifest/plan.ts"; +import { linkAndroidApk } from "../../tools/android-link.ts"; +import { makeVariant } from "../../tools/pocket-pack.ts"; +import { REDMI_1S_CONTRACTS, REDMI_1S_TARGET, resolveRedmi1sBuildPlan } from "../../tools/redmi-1s-profile.ts"; +import { ANDROID_ICON_DENSITIES, ANDROID_TEMPLATE, androidTemplateJob } from "../../tools/repack/android.ts"; +import { encodePng } from "../../tools/repack/shared/png.ts"; + +const ROOT = join(import.meta.dir, "../.."); +const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); +const bytesOf = (text: string) => new TextEncoder().encode(text); + +export const HOST = join(ROOT, "hosts/android/app"); +/** + * aapt2's link of hosts/android/app's templates with ANDROID_TEMPLATE_IDENTITY, + * stand-in classes.dex and libraries and a plain icon. Rewrite it with + * `POCKETJS_WRITE_ANDROID_FIXTURE=1 bun test tests/repack-android.test.ts` + * (Android SDK required); tests/repack-android.test.ts checks it against aapt2. + */ +export const FIXTURE = join(ROOT, "tests/fixtures/repack-android/template.apk"); +export const ABIS = ["armeabi-v7a", "arm64-v8a"] as const; +export const SDK_LEVELS = { apiLevel: 34, buildToolsVersion: "35.0.0", minSdkVersion: 18, targetSdkVersion: 34 } as const; +export const HOST_ABI = REDMI_1S_CONTRACTS.targets[REDMI_1S_TARGET].hostAbi; + +export function square(side: number, paint: (x: number, y: number) => [number, number, number, number]): Uint8Array { + const rgba = new Uint8Array(side * side * 4); + for (let y = 0; y < side; y++) for (let x = 0; x < side; x++) rgba.set(paint(x, y), (y * side + x) * 4); + return rgba; +} + +/** The runtime build's files the fixture template is linked from. */ +export async function fixtureParts(): Promise<Map<string, Uint8Array>> { + const files = new Map<string, Uint8Array>([ + ["AndroidManifest.xml", new Uint8Array(readFileSync(join(HOST, "AndroidManifest.xml")))], + ["res/values/strings.xml", new Uint8Array(readFileSync(join(HOST, "res/values/strings.xml")))], + ["classes.dex", bytesOf("dex\n035\0 stand-in for PocketActivity")], + ...ABIS.map((abi) => [`lib/${abi}/libpocketjs.so`, bytesOf(`\x7fELF stand-in ${abi}`)] as [string, Uint8Array]), + ]); + for (const [density, side] of ANDROID_ICON_DENSITIES) { + files.set(`res/mipmap-${density}/icon.png`, await encodePng({ width: side, height: side, rgba: square(side, () => [23, 18, 38, 255]) })); + } + return files; +} + +if (process.env.POCKETJS_WRITE_ANDROID_FIXTURE === "1") { + mkdirSync(join(FIXTURE, ".."), { recursive: true }); + writeFileSync(FIXTURE, await linkAndroidApk(androidTemplateJob(await fixtureParts(), SDK_LEVELS, ABIS))); +} +export const TEMPLATE = new Uint8Array(readFileSync(FIXTURE)); + +/** A runtime directory around a template: the sources from hosts/android, runtime.json describing them. */ +export function fixtureRuntime(overrides: Record<string, unknown> = {}, template: Uint8Array = TEMPLATE): Map<string, Uint8Array> { + const files = new Map<string, Uint8Array>([ + [ANDROID_TEMPLATE, template], + ["AndroidManifest.xml", new Uint8Array(readFileSync(join(HOST, "AndroidManifest.xml")))], + ["res/values/strings.xml", new Uint8Array(readFileSync(join(HOST, "res/values/strings.xml")))], + ]); + const manifest = { + target: REDMI_1S_TARGET, + hostAbi: HOST_ABI, + pocketjs: "0".repeat(40), + profile: "redmi-1s", + viewport: { logical: [360, 640], rasterDensity: 2 }, + android: { ...SDK_LEVELS, abis: [...ABIS] }, + files: Object.fromEntries([...files].map(([path, bytes]) => [path, { bytes: bytes.length, sha256: sha256(bytes) }])), + ...overrides, + }; + files.set("runtime.json", bytesOf(JSON.stringify(manifest))); + return files; +} + +export const IDENTITY = { id: "dev.pocket-nexus.studio.test-game", title: "Tom's \"Test\" & <Game>", author: "A Creator", version: "1.2.3" }; +/** Clear's Redmi 1S manifest under an id: the repack refuses an identity that is not the .pocket's app. */ +export const clear = (id = IDENTITY.id) => ({ ...JSON.parse(readFileSync(join(ROOT, "apps/clear/pocket.redmi-1s.json"), "utf8")), id }); +export const GAME_JS = bytesOf("globalThis.game = 1;\n".repeat(64)); +export const GAME_PAK = Uint8Array.from({ length: 4096 }, (_, i) => (i * 7) & 0xff); + +export function variantFor(plan: ReturnType<typeof resolveRedmi1sBuildPlan>): PocketPackageVariant { + return makeVariant({ + target: plan.target.id, + hostAbi: plan.target.hostAbi, + planJson: canonicalJson(plan), + identity: { output: plan.app.output, id: plan.app.id, title: plan.app.title }, + js: GAME_JS, + pak: GAME_PAK, + }); +} + +export function fixturePocket(id = IDENTITY.id, variants?: PocketPackageVariant[]): Uint8Array { + const manifest = clear(id); + return encodePocketPackage({ + manifest: bytesOf(JSON.stringify(manifest)), + variants: variants ?? [variantFor(resolveRedmi1sBuildPlan(manifest))], + }); +} + diff --git a/tests/helpers/apk-test-signer.ts b/tests/helpers/apk-test-signer.ts new file mode 100644 index 000000000..38a6bae5a --- /dev/null +++ b/tests/helpers/apk-test-signer.ts @@ -0,0 +1,192 @@ +// A throwaway RSA key and a self-signed X.509 certificate for it, made with +// Web Crypto: what the Android repack tests sign with. Never the community +// key, never the release key. + +import { concat, der, derObjectId, DER_NULL } from "../../tools/repack/shared/apk-sign.ts"; + +export interface TestSigner { + readonly privateKey: Uint8Array; + readonly cryptoKey: CryptoKey; + readonly certificate: Uint8Array; + readonly certificateSha256: string; +} + +const SHA256_WITH_RSA = "1.2.840.113549.1.1.11"; + +function name(commonName: string): Uint8Array { + return der(0x30, der(0x31, der(0x30, derObjectId("2.5.4.3"), der(0x0c, new TextEncoder().encode(commonName))))); +} + +export async function makeTestSigner(commonName = "PocketJS Repack Test", modulusLength = 2048): Promise<TestSigner> { + const pair = await crypto.subtle.generateKey( + { name: "RSASSA-PKCS1-v1_5", modulusLength, publicExponent: Uint8Array.of(1, 0, 1), hash: "SHA-256" }, + true, + ["sign", "verify"], + ); + const spki = new Uint8Array(await crypto.subtle.exportKey("spki", pair.publicKey)); + const algorithm = der(0x30, derObjectId(SHA256_WITH_RSA), DER_NULL); + const time = (text: string) => der(0x17, new TextEncoder().encode(text)); + const tbs = der( + 0x30, + der(0xa0, der(0x02, Uint8Array.of(2))), + der(0x02, Uint8Array.of(0x01, 0x23, 0x45, 0x67)), + algorithm, + name(commonName), + der(0x30, time("260101000000Z"), time("360101000000Z")), + name(commonName), + spki, + ); + const signature = new Uint8Array(await crypto.subtle.sign("RSASSA-PKCS1-v1_5", pair.privateKey, tbs as Uint8Array<ArrayBuffer>)); + const certificate = der(0x30, tbs, algorithm, der(0x03, concat(Uint8Array.of(0), signature))); + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", certificate as Uint8Array<ArrayBuffer>)); + return { + privateKey: new Uint8Array(await crypto.subtle.exportKey("pkcs8", pair.privateKey)), + cryptoKey: pair.privateKey, + certificate, + certificateSha256: Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""), + }; +} + +// ------------------------------------------------------------ verification + +export interface ApkSignatureReport { + /** The v2 signer's certificate, DER. */ + readonly v2Certificate: Uint8Array; + /** The v1 PKCS #7 block's certificate, DER. */ + readonly v1Certificate: Uint8Array; + /** Entry names MANIFEST.MF lists. */ + readonly v1Entries: readonly string[]; +} + +const base64 = (bytes: Uint8Array) => btoa(Array.from(bytes, (byte) => String.fromCharCode(byte)).join("")); +const sha256 = async (bytes: Uint8Array) => new Uint8Array(await crypto.subtle.digest("SHA-256", bytes as Uint8Array<ArrayBuffer>)); +const equal = (a: Uint8Array, b: Uint8Array) => a.length === b.length && a.every((byte, index) => byte === b[index]); + +async function rsaVerifies(spki: Uint8Array, data: Uint8Array, signature: Uint8Array): Promise<boolean> { + const key = await crypto.subtle.importKey("spki", spki as Uint8Array<ArrayBuffer>, { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, false, ["verify"]); + return crypto.subtle.verify("RSASSA-PKCS1-v1_5", key, signature as Uint8Array<ArrayBuffer>, data as Uint8Array<ArrayBuffer>); +} + +/** Reads v2's length-prefixed values one after another. */ +class Prefixed { + private at: number; + constructor(private readonly bytes: Uint8Array, start = 0, private readonly end = bytes.length) { + this.at = start; + } + get done(): boolean { + return this.at >= this.end; + } + u32(): number { + const value = new DataView(this.bytes.buffer, this.bytes.byteOffset).getUint32(this.at, true); + this.at += 4; + return value; + } + next(): Uint8Array { + const length = this.u32(); + if (this.at + length > this.end) throw new Error("v2: a length-prefixed value runs past its parent"); + const value = this.bytes.subarray(this.at, this.at + length); + this.at += length; + return value; + } +} + +/** + * Checks an APK's v1 and v2 signatures the way a verifier reads them, with + * code apart from the signer's: v2's block, content digest and signature; + * v1's entry digests, CERT.SF and the PKCS #7 signature over it. Throws on + * the first failure. + */ +export async function verifyApkSignatures(apk: Uint8Array): Promise<ApkSignatureReport> { + const { readZip, unzipEntry } = await import("../../tools/repack/shared/zip.ts"); + const { derChildren, certificateFields } = await import("../../tools/repack/shared/apk-sign.ts"); + const data = new DataView(apk.buffer, apk.byteOffset, apk.byteLength); + const end = apk.length - 22; + if (data.getUint32(end, true) !== 0x06054b50) throw new Error("no end-of-central-directory record at the end"); + const centralDirectory = data.getUint32(end + 16, true); + if (new TextDecoder().decode(apk.subarray(centralDirectory - 16, centralDirectory)) !== "APK Sig Block 42") throw new Error("v2: no APK Signing Block"); + const blockSize = Number(data.getBigUint64(centralDirectory - 24, true)); + const blockStart = centralDirectory - blockSize - 8; + if (Number(data.getBigUint64(blockStart, true)) !== blockSize) throw new Error("v2: the block's two sizes differ"); + let v2: Uint8Array | undefined; + for (let at = blockStart + 8; at < centralDirectory - 24;) { + const length = Number(data.getBigUint64(at, true)); + if (data.getUint32(at + 8, true) === 0x7109871a) v2 = apk.subarray(at + 12, at + 8 + length); + at += 8 + length; + } + if (!v2) throw new Error("v2: no APK Signature Scheme v2 pair"); + const signers = new Prefixed(new Prefixed(v2).next()); + const signer = new Prefixed(signers.next()); + if (!signers.done) throw new Error("v2: more than one signer"); + const signedData = signer.next(); + const signatures = new Prefixed(signer.next()); + const publicKey = signer.next(); + const signature = new Prefixed(signatures.next()); + if (signature.u32() !== 0x0103) throw new Error("v2: the signature is not RSASSA-PKCS1-v1_5 SHA-256"); + if (!(await rsaVerifies(publicKey, signedData, signature.next()))) throw new Error("v2: the signature does not verify"); + const fields = new Prefixed(signedData); + const digest = new Prefixed(new Prefixed(fields.next()).next()); + if (digest.u32() !== 0x0103) throw new Error("v2: the digest is not SHA-256"); + const expected = digest.next(); + const v2Certificate = new Prefixed(fields.next()).next(); + if (!equal(certificateFields(v2Certificate).publicKey, publicKey)) throw new Error("v2: the public key is not the certificate's"); + const tail = new Uint8Array(apk.subarray(end)); + new DataView(tail.buffer).setUint32(16, blockStart, true); + const chunks: Uint8Array[] = []; + for (const section of [apk.subarray(0, blockStart), apk.subarray(centralDirectory, end), tail]) { + for (let at = 0; at < section.length; at += 1 << 20) { + const chunk = section.subarray(at, Math.min(section.length, at + (1 << 20))); + const prefix = new Uint8Array(5); + prefix[0] = 0xa5; + new DataView(prefix.buffer).setUint32(1, chunk.length, true); + chunks.push(await sha256(concat(prefix, chunk))); + } + } + const top = new Uint8Array(5 + chunks.length * 32); + top[0] = 0x5a; + new DataView(top.buffer).setUint32(1, chunks.length, true); + chunks.forEach((chunk, index) => top.set(chunk, 5 + index * 32)); + if (!equal(await sha256(top), expected)) throw new Error("v2: the content digest does not match the APK"); + + const entries = readZip(apk); + const file = async (name: string) => unzipEntry(entries.find((entry) => entry.name === name) ?? (() => { throw new Error(`v1: no ${name}`); })()); + const manifest = await file("META-INF/MANIFEST.MF"); + const signatureFile = await file("META-INF/CERT.SF"); + const sections = (text: string) => text.split("\r\n\r\n").filter(Boolean).map((section) => { + const lines = section.replace(/\r\n /g, "").split("\r\n"); + return { text: `${section}\r\n\r\n`, fields: new Map(lines.map((line) => [line.slice(0, line.indexOf(": ")), line.slice(line.indexOf(": ") + 2)])) }; + }); + const manifestSections = sections(new TextDecoder().decode(manifest)); + const digests = new Map(manifestSections.slice(1).map((section) => [section.fields.get("Name")!, section])); + const signed = entries.filter((entry) => !entry.name.startsWith("META-INF/") && !entry.name.endsWith("/")); + if (digests.size !== signed.length) throw new Error(`v1: MANIFEST.MF lists ${digests.size} entries, the APK holds ${signed.length}`); + for (const entry of signed) { + if (digests.get(entry.name)?.fields.get("SHA-256-Digest") !== base64(await sha256(await unzipEntry(entry)))) { + throw new Error(`v1: ${entry.name} does not match its digest`); + } + } + const signatureSections = sections(new TextDecoder().decode(signatureFile)); + const header = signatureSections[0]!.fields; + if (header.get("SHA-256-Digest-Manifest") !== base64(await sha256(manifest))) throw new Error("v1: CERT.SF's manifest digest is wrong"); + if (header.get("X-Android-APK-Signed") !== "2") throw new Error("v1: CERT.SF does not say the APK is v2-signed"); + for (const section of signatureSections.slice(1)) { + const name = section.fields.get("Name")!; + if (section.fields.get("SHA-256-Digest") !== base64(await sha256(new TextEncoder().encode(digests.get(name)!.text)))) { + throw new Error(`v1: CERT.SF's digest of ${name}'s section is wrong`); + } + } + const block = await file("META-INF/CERT.RSA"); + const [contentInfo] = derChildren(block, 0, block.length); + const [, explicit] = derChildren(block, contentInfo!.body, contentInfo!.end); + const [signedDataNode] = derChildren(block, explicit!.body, explicit!.end); + const parts = derChildren(block, signedDataNode!.body, signedDataNode!.end); + const certificates = parts.find((part) => part.tag === 0xa0)!; + const [certificateNode] = derChildren(block, certificates.body, certificates.end); + const v1Certificate = block.subarray(certificateNode!.start, certificateNode!.end); + const signerInfos = parts.at(-1)!; + const [signerInfo] = derChildren(block, signerInfos.body, signerInfos.end); + const encrypted = derChildren(block, signerInfo!.body, signerInfo!.end).at(-1)!; + if (!(await rsaVerifies(certificateFields(v1Certificate).publicKey, signatureFile, block.subarray(encrypted.body, encrypted.end)))) { + throw new Error("v1: the PKCS #7 signature over CERT.SF does not verify"); + } + return { v2Certificate, v1Certificate, v1Entries: [...digests.keys()] }; +} diff --git a/tests/repack-android.test.ts b/tests/repack-android.test.ts index e2e5086a3..bb8788d4d 100644 --- a/tests/repack-android.test.ts +++ b/tests/repack-android.test.ts @@ -3,27 +3,53 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { encodePocketPackage, type PocketPackageVariant } from "../contracts/spec/pocket-package.ts"; -import { canonicalJson } from "../framework/src/manifest/plan.ts"; +import { encodePocketPackage } from "../contracts/spec/pocket-package.ts"; import { ANDROID_LAUNCHER_DENSITIES } from "../tools/android-icon.ts"; +import { buildTool, linkAndroidApk, localAndroidSdk, runTool } from "../tools/android-link.ts"; import { MOTO_G_PLAY_TARGET, resolveMotoGPlayBuildPlan } from "../tools/moto-g-play-profile.ts"; -import { makeVariant } from "../tools/pocket-pack.ts"; import { REDMI_1S_TARGET, resolveRedmi1sBuildPlan } from "../tools/redmi-1s-profile.ts"; +import { readRuntimeDirectory } from "../tools/repack.ts"; import { ANDROID_ICON_DENSITIES, ANDROID_RUNTIME_TARGETS, + ANDROID_TEMPLATE, + ANDROID_TEMPLATE_IDENTITY, POCKET_STUDIO_COMMUNITY_SIGNER, androidIdentity, + androidLabel, + androidLinkJob, + androidPackageName, androidString, - expectedApkEntries, - prepareAndroidJob, + androidTemplateJob, repackAndroid, - type AndroidNativeJob, } from "../tools/repack/android.ts"; -import { androidNativeSteps, localAndroidSdk } from "../tools/repack/android-native.ts"; -import type { RepackInput } from "../tools/repack/shared/runtime.ts"; +import { + patchManifest, + patchResourceTable, + readManifest, + readResourceTable, + readStringPool, + writeStringPool, +} from "../tools/repack/shared/android-resources.ts"; import { decodePng, encodePng } from "../tools/repack/shared/png.ts"; -import { readZip, unzipEntry, writeZip } from "../tools/repack/shared/zip.ts"; +import type { RepackInput } from "../tools/repack/shared/runtime.ts"; +import { readZip, unzipEntry, writeZip, ZIP_STORED, type ZipEntry } from "../tools/repack/shared/zip.ts"; +import { + ABIS, + clear, + fixtureParts, + fixturePocket, + fixtureRuntime, + GAME_JS, + GAME_PAK, + HOST_ABI, + IDENTITY, + SDK_LEVELS, + square, + TEMPLATE, + variantFor, +} from "./helpers/android-fixture.ts"; +import { makeTestSigner, verifyApkSignatures } from "./helpers/apk-test-signer.ts"; const ROOT = join(import.meta.dir, ".."); const work = join(tmpdir(), `pocketjs-repack-android-${process.pid}`); @@ -32,87 +58,54 @@ afterAll(() => rmSync(work, { recursive: true, force: true })); const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); const bytesOf = (text: string) => new TextEncoder().encode(text); +const signer = await makeTestSigner(); +const options = { certificateSha256: signer.certificateSha256 }; -function square(side: number, paint: (x: number, y: number) => [number, number, number, number]): Uint8Array { - const rgba = new Uint8Array(side * side * 4); - for (let y = 0; y < side; y++) for (let x = 0; x < side; x++) rgba.set(paint(x, y), (y * side + x) * 4); - return rgba; -} - -/** A runtime with stand-in native files: what runtime.json records, the templates from hosts/android. */ -async function fixtureRuntime(overrides: Record<string, unknown> = {}): Promise<Map<string, Uint8Array>> { - const files = new Map<string, Uint8Array>([ - ["AndroidManifest.xml", new Uint8Array(readFileSync(join(ROOT, "hosts/android/app/AndroidManifest.xml")))], - ["res/values/strings.xml", new Uint8Array(readFileSync(join(ROOT, "hosts/android/app/res/values/strings.xml")))], - ["classes.dex", bytesOf("dex\n035\0 stand-in for PocketActivity")], - ["lib/armeabi-v7a/libpocketjs.so", bytesOf("\x7fELF stand-in armeabi-v7a")], - ["lib/arm64-v8a/libpocketjs.so", bytesOf("\x7fELF stand-in arm64-v8a")], - ]); - for (const [density, side] of ANDROID_ICON_DENSITIES) { - files.set(`res/mipmap-${density}/icon.png`, await encodePng({ width: side, height: side, rgba: square(side, () => [23, 18, 38, 255]) })); - } - const manifest = { - schema: 1, - target: REDMI_1S_TARGET, - hostAbi: 9, - pocketjs: "0".repeat(40), - profile: "redmi-1s", - viewport: { logical: [360, 640], rasterDensity: 2 }, - android: { apiLevel: 34, buildToolsVersion: "35.0.0", minSdkVersion: 18, targetSdkVersion: 34, abis: ["armeabi-v7a", "arm64-v8a"] }, - files: Object.fromEntries([...files].map(([path, bytes]) => [path, { bytes: bytes.length, sha256: sha256(bytes) }])), +function input(overrides: Partial<RepackInput> = {}): RepackInput { + return { + runtime: fixtureRuntime(), + pocket: fixturePocket(overrides.identity?.id), + identity: IDENTITY, + signer: { privateKey: signer.privateKey, certificate: signer.certificate }, ...overrides, }; - files.set("runtime.json", bytesOf(JSON.stringify(manifest))); - return files; } -const IDENTITY = { id: "dev.pocket-nexus.studio.test-game", title: "Tom's \"Test\" & <Game>", author: "A Creator", version: "1.2.3" }; -/** Clear's Redmi 1S manifest under the identity's id: the repack refuses an identity that is not the .pocket's app. */ -const clear = (id = IDENTITY.id) => ({ ...JSON.parse(readFileSync(join(ROOT, "apps/clear/pocket.redmi-1s.json"), "utf8")), id }); -const GAME_JS = bytesOf("globalThis.game = 1;"); -const GAME_PAK = Uint8Array.from({ length: 4096 }, (_, i) => (i * 7) & 0xff); - -function variantFor(plan: ReturnType<typeof resolveRedmi1sBuildPlan>): PocketPackageVariant { - return makeVariant({ - target: plan.target.id, - hostAbi: plan.target.hostAbi, - planJson: canonicalJson(plan), - identity: { output: plan.app.output, id: plan.app.id, title: plan.app.title }, - js: GAME_JS, - pak: GAME_PAK, - }); -} +const ICON = await encodePng({ width: 300, height: 300, rgba: square(300, (x) => (x < 150 ? [255, 210, 63, 255] : [169, 139, 255, 255])) }); +const templateEntry = async (name: string) => unzipEntry(readZip(TEMPLATE).find((entry) => entry.name === name)!); +const entryOf = (entries: readonly ZipEntry[], name: string) => entries.find((entry) => entry.name === name)!; -function fixturePocket(variants?: PocketPackageVariant[]): Uint8Array { - const manifest = clear(); - return encodePocketPackage({ - manifest: bytesOf(JSON.stringify(manifest)), - variants: variants ?? [variantFor(resolveRedmi1sBuildPlan(manifest))], +describe("Android identity", () => { + test("the package name is the id with `-` as `_`, and `x` before a segment that starts with a digit", () => { + expect(androidPackageName("dev.pocket-nexus.studio.test-game")).toBe("dev.pocket_nexus.studio.test_game"); + expect(androidPackageName("dev.pocket-nexus.studio.2pac.8-ball")).toBe("dev.pocket_nexus.studio.x2pac.x8_ball"); + expect(androidPackageName("dev.pocket-nexus.2048")).toBe("dev.pocket_nexus.x2048"); + expect(androidPackageName("dev.pocket-nexus.studio.twenty48")).toBe("dev.pocket_nexus.studio.twenty48"); + expect(() => androidPackageName("single")).toThrow(/Android package name/); + expect(() => androidPackageName("dev..game")).toThrow(/Android package name/); + expect(() => androidPackageName("dev.-game")).toThrow(/Android package name \(dev\._game\)/); + expect(() => androidPackageName(`dev.${"a".repeat(124)}`)).toThrow(/longer than 127 characters/); + expect(androidPackageName(`dev.${"a".repeat(123)}`)).toHaveLength(127); }); -} - -async function input(overrides: Partial<RepackInput> = {}): Promise<RepackInput> { - return { runtime: await fixtureRuntime(), pocket: fixturePocket(), identity: IDENTITY, ...overrides }; -} - -/** A job as bytes, for byte-for-byte comparison. */ -function serialize(job: AndroidNativeJob): string { - const files = (map: ReadonlyMap<string, Uint8Array>) => [...map].map(([path, bytes]) => `${path}:${sha256(bytes)}`); - return JSON.stringify({ manifest: job.manifest, resources: files(job.resources), assets: files(job.assets), entries: files(job.entries), sdk: job.sdk }); -} -describe("Android identity", () => { - test("the package name is the id with hyphens as underscores; versionCode comes from major.minor.patch", () => { + test("versionCode comes from major.minor.patch", () => { expect(androidIdentity(IDENTITY)).toEqual({ packageName: "dev.pocket_nexus.studio.test_game", versionCode: 1_002_003, versionName: "1.2.3", label: IDENTITY.title, }); - expect(() => androidIdentity({ ...IDENTITY, id: "single" })).toThrow(/Android package name/); - expect(() => androidIdentity({ ...IDENTITY, id: "dev.pocket-nexus.2048" })).toThrow(/Android package name/); + expect(androidIdentity({ ...IDENTITY, version: "1999.999.999" }).versionCode).toBe(1_999_999_999); expect(() => androidIdentity({ ...IDENTITY, version: "1.2" })).toThrow(/major\.minor\.patch/); expect(() => androidIdentity({ ...IDENTITY, version: "1.1000.0" })).toThrow(/versionCode range/); + expect(() => androidIdentity({ ...IDENTITY, version: "0.0.0" })).toThrow(/versionCode 0/); + }); + + test("the label is the title with its ASCII whitespace trimmed and collapsed", () => { + expect(androidLabel(" Space \t Game\n ")).toBe("Space Game"); + expect(androidLabel("ゲーム\u3000タイトル")).toBe("ゲーム\u3000タイトル"); + expect(() => androidIdentity({ ...IDENTITY, title: " \n " })).toThrow(/title is empty/); + expect(() => androidIdentity({ ...IDENTITY, title: "bell\u0007" })).toThrow(/control character/); }); test("a label is escaped for aapt2 and XML", () => { @@ -136,166 +129,317 @@ describe("Android identity", () => { }); }); -describe("the TypeScript half lays out the APK", () => { - test("manifest, label, icons, assets and runtime entries", async () => { - const icon = await encodePng({ width: 300, height: 300, rgba: square(300, (x) => (x < 150 ? [255, 210, 63, 255] : [169, 139, 255, 255])) }); - const job = await prepareAndroidJob(await input({ identity: { ...IDENTITY, icon } })); - expect(job.manifest).toContain('package="dev.pocket_nexus.studio.test_game"'); - expect(job.manifest).toContain('android:versionCode="1002003"'); - expect(job.manifest).toContain('android:versionName="1.2.3"'); - expect(job.manifest).toContain('android:minSdkVersion="18" android:targetSdkVersion="34"'); - expect(job.manifest).toContain('android:debuggable="false"'); - expect(job.manifest).not.toContain("@POCKET_"); - expect(new TextDecoder().decode(job.resources.get("values/strings.xml"))).toContain( - '<string name="app_name">Tom\\\'s \\"Test\\" & <Game></string>', - ); +describe("binary manifest and resource table", () => { + const IDENTITIES = [ + { packageName: "dev.pocket_nexus.studio.x2pac.x8_ball", versionCode: 1000, versionName: "0.1.0", label: "8 Ball" }, + { packageName: "app.a", versionCode: 10_020_030, versionName: "10.20.30", label: "日本語のゲーム" }, + { packageName: "zz.zz", versionCode: 1_999_999_999, versionName: "1999.999.999", label: "zebra 🦓 game" }, + { packageName: "com.x.y", versionCode: 1, versionName: "0.0.1", label: "@home" }, + ]; + + test("the template reads back as the template identity, SDK 18 to 34, not debuggable", async () => { + expect(readManifest(await templateEntry("AndroidManifest.xml"))).toMatchObject({ + packageName: androidPackageName(ANDROID_TEMPLATE_IDENTITY.id), + versionCode: 1, + versionName: ANDROID_TEMPLATE_IDENTITY.version, + minSdkVersion: 18, + targetSdkVersion: 34, + debuggable: false, + }); + expect(readResourceTable(await templateEntry("resources.arsc"))).toMatchObject({ + packageId: 0x7f, + packageName: androidPackageName(ANDROID_TEMPLATE_IDENTITY.id), + label: ANDROID_TEMPLATE_IDENTITY.title, + }); + }); + + test("a rewrite reads back, and depends on the identity alone, not on the one before it", async () => { + const manifest = await templateEntry("AndroidManifest.xml"); + const table = await templateEntry("resources.arsc"); + for (const identity of IDENTITIES) { + const once = patchManifest(manifest, identity); + expect(readManifest(once)).toMatchObject({ packageName: identity.packageName, versionCode: identity.versionCode, versionName: identity.versionName }); + const onceTable = patchResourceTable(table, identity); + expect(readResourceTable(onceTable)).toMatchObject({ packageName: identity.packageName, label: identity.label }); + for (const before of IDENTITIES) { + expect(patchManifest(patchManifest(manifest, before), identity)).toEqual(once); + expect(patchResourceTable(patchResourceTable(table, before), identity)).toEqual(onceTable); + } + } + }); + + test("the table's pool stays sorted by UTF-8 bytes, and a label equal to a path shares its entry", async () => { + const table = await templateEntry("resources.arsc"); + for (const identity of IDENTITIES) { + const pool = readStringPool(patchResourceTable(table, identity), 12).strings; + const bytes = pool.map((text) => Buffer.from(text, "utf8")); + for (let index = 1; index < bytes.length; index++) expect(Buffer.compare(bytes[index - 1]!, bytes[index]!)).toBe(-1); + } + const shared = patchResourceTable(table, { packageName: "a.b", label: "res/mipmap-hdpi-v4/icon.png" }); + expect(readStringPool(shared, 12).strings).toHaveLength(5); + const strings = new Map(readResourceTable(shared).strings); + expect(strings.get("string/app_name")).toBe("res/mipmap-hdpi-v4/icon.png"); + expect(readResourceTable(shared).strings.filter(([key]) => key === "mipmap/icon").map(([, path]) => path).sort()) + .toEqual(ANDROID_ICON_DENSITIES.map(([density]) => `res/mipmap-${density}-v4/icon.png`).sort()); + }); + + test("a UTF-8 pool writes a character past U+FFFF as two 3-byte surrogates, as aapt2 does", () => { + const pool = writeStringPool(["🦓", "é"], true); + expect([...pool.subarray(28 + 8, 28 + 8 + 9)]).toEqual([2, 6, 0xed, 0xa0, 0xbe, 0xed, 0xb6, 0x93, 0]); + expect(readStringPool(pool, 0).strings).toEqual(["🦓", "é"]); + expect(readStringPool(writeStringPool(["🦓", "ab"], false), 0).strings).toEqual(["🦓", "ab"]); + }); + + test("a rewrite refuses what a table or manifest cannot hold", async () => { + await expect(async () => patchResourceTable(await templateEntry("resources.arsc"), { packageName: `a.${"b".repeat(126)}`, label: "x" })) + .toThrow(/at most 127 characters/); + await expect(async () => patchManifest(await templateEntry("AndroidManifest.xml"), { packageName: "a.b", versionCode: 0, versionName: "0" })) + .toThrow(/positive 31-bit integer/); + }); +}); + +describe("the repack", () => { + test("the template's entries in order, the game's files and identity, then the signature files", async () => { + const apk = await repackAndroid(input({ identity: { ...IDENTITY, icon: ICON } }), options); + const entries = readZip(apk); + expect(entries.map((entry) => entry.name)).toEqual([ + ...readZip(TEMPLATE).map((entry) => entry.name), + "META-INF/MANIFEST.MF", + "META-INF/CERT.SF", + "META-INF/CERT.RSA", + ]); + expect(readManifest(await unzipEntry(entryOf(entries, "AndroidManifest.xml")))).toMatchObject({ + packageName: "dev.pocket_nexus.studio.test_game", + versionCode: 1_002_003, + versionName: "1.2.3", + minSdkVersion: 18, + targetSdkVersion: 34, + debuggable: false, + }); + expect(readResourceTable(await unzipEntry(entryOf(entries, "resources.arsc")))).toMatchObject({ + packageName: "dev.pocket_nexus.studio.test_game", + label: IDENTITY.title, + }); + expect(await unzipEntry(entryOf(entries, "assets/app.js"))).toEqual(GAME_JS); + expect(await unzipEntry(entryOf(entries, "assets/app.pak"))).toEqual(GAME_PAK); for (const [density, side] of ANDROID_ICON_DENSITIES) { - const png = await decodePng(job.resources.get(`mipmap-${density}/icon.png`)!); + const png = await decodePng(await unzipEntry(entryOf(entries, `res/mipmap-${density}-v4/icon.png`))); expect([png.width, png.height]).toEqual([side, side]); expect([...png.rgba.subarray(0, 4)]).toEqual([255, 210, 63, 255]); expect([...png.rgba.subarray(png.rgba.length - 4)]).toEqual([169, 139, 255, 255]); } - expect([...job.assets.keys()]).toEqual(["app.js", "app.pak"]); - expect(job.assets.get("app.js")).toEqual(GAME_JS); - expect(job.assets.get("app.pak")).toEqual(GAME_PAK); - expect([...job.entries.keys()]).toEqual(["classes.dex", "lib/arm64-v8a/libpocketjs.so", "lib/armeabi-v7a/libpocketjs.so"]); - expect(job.sdk).toEqual({ apiLevel: 34, buildToolsVersion: "35.0.0", minSdkVersion: 18, targetSdkVersion: 34 }); + for (const name of ["classes.dex", ...ABIS.map((abi) => `lib/${abi}/libpocketjs.so`)]) { + expect(await unzipEntry(entryOf(entries, name))).toEqual(await templateEntry(name)); + } }); - test("without an icon the runtime's default icon goes in", async () => { - const runtime = await fixtureRuntime(); - const job = await prepareAndroidJob(await input({ runtime })); - for (const [density] of ANDROID_ICON_DENSITIES) { - expect(job.resources.get(`mipmap-${density}/icon.png`)).toEqual(runtime.get(`res/mipmap-${density}/icon.png`)!); + test("stored entries sit where zipalign -p 4 puts them; the table is stored", async () => { + const apk = await repackAndroid(input(), options); + const entries = readZip(apk); + expect(entryOf(entries, "resources.arsc").method).toBe(ZIP_STORED); + const stored = entries.filter((entry) => entry.method === ZIP_STORED && !entry.name.startsWith("META-INF/")); + expect(stored.some((entry) => entry.name.endsWith(".so"))).toBe(true); + for (const entry of stored) { + expect((entry.stored.byteOffset - apk.byteOffset) % (entry.name.endsWith(".so") ? 4096 : 4)).toBe(0); } }); - test("the same inputs give the same files", async () => { - const icon = await encodePng({ width: 512, height: 512, rgba: square(512, (x, y) => [x & 255, y & 255, (x ^ y) & 255, 255]) }); - const a = serialize(await prepareAndroidJob(await input({ identity: { ...IDENTITY, icon } }))); - const b = serialize(await prepareAndroidJob(await input({ identity: { ...IDENTITY, icon } }))); - expect(a).toBe(b); + test("v1 and v2 signatures verify under the signer's certificate", async () => { + const apk = await repackAndroid(input({ identity: { ...IDENTITY, icon: ICON } }), options); + const report = await verifyApkSignatures(apk); + expect(report.v1Certificate).toEqual(signer.certificate); + expect(report.v2Certificate).toEqual(signer.certificate); + expect([...report.v1Entries].sort()).toEqual(readZip(TEMPLATE).map((entry) => entry.name).sort()); + }); + + test("the same inputs give the same bytes, from PKCS #8 bytes or a CryptoKey", async () => { + const apk = await repackAndroid(input({ identity: { ...IDENTITY, icon: ICON } }), options); + expect(await repackAndroid(input({ identity: { ...IDENTITY, icon: ICON } }), options)).toEqual(apk); + const withKey = input({ identity: { ...IDENTITY, icon: ICON }, signer: { privateKey: signer.cryptoKey, certificate: signer.certificate } }); + expect(await repackAndroid(withKey, options)).toEqual(apk); + }); + + test("without an icon the template's default icon stays", async () => { + const entries = readZip(await repackAndroid(input(), options)); + for (const [density] of ANDROID_ICON_DENSITIES) { + const name = `res/mipmap-${density}-v4/icon.png`; + expect(await unzipEntry(entryOf(entries, name))).toEqual(await templateEntry(name)); + } }); }); describe("the repack refuses", () => { + test("no signer", async () => { + await expect(repackAndroid({ ...input(), signer: undefined })).rejects.toThrow(/no signer/); + }); + + test("a certificate other than the community certificate, unless the caller names it", async () => { + await expect(repackAndroid(input())).rejects.toThrow( + new RegExp(`certificate is ${signer.certificateSha256}, not the Pocket Studio community certificate ${POCKET_STUDIO_COMMUNITY_SIGNER}`), + ); + await expect(repackAndroid(input(), { certificateSha256: "0".repeat(64) })).rejects.toThrow(/not 0{64}/); + }); + + test("a key that is not the certificate's", async () => { + const other = await makeTestSigner("Another Key"); + await expect(repackAndroid(input({ signer: { privateKey: other.privateKey, certificate: signer.certificate } }), options)) + .rejects.toThrow(/not the certificate's key/); + await expect(repackAndroid(input({ signer: { privateKey: bytesOf("not a key"), certificate: signer.certificate } }), options)) + .rejects.toThrow(/not an RSA PKCS #8 key/); + }); + + test("a runtime without template.apk, or with a signed one", async () => { + const runtime = fixtureRuntime(); + const manifest = JSON.parse(new TextDecoder().decode(runtime.get("runtime.json"))); + delete manifest.files[ANDROID_TEMPLATE]; + runtime.set("runtime.json", bytesOf(JSON.stringify(manifest))); + await expect(repackAndroid(input({ runtime }), options)).rejects.toThrow(/lacks template\.apk/); + const signed = await writeZip([...readZip(TEMPLATE).map((entry) => ({ name: entry.name, entry })), { name: "META-INF/MANIFEST.MF", data: bytesOf("x") }]); + await expect(repackAndroid(input({ runtime: fixtureRuntime({}, signed) }), options)).rejects.toThrow(/template\.apk is signed/); + }); + test("a .pocket without the runtime's target", async () => { const moto = resolveMotoGPlayBuildPlan({ ...JSON.parse(readFileSync(join(ROOT, "apps/clear/pocket.android.json"), "utf8")), id: IDENTITY.id }); - const pocket = fixturePocket([variantFor(moto)]); - await expect(prepareAndroidJob(await input({ pocket }))).rejects.toThrow(/no redmi-1s-dev variant \(it has moto-g-play-dev\)/); + const pocket = fixturePocket(IDENTITY.id, [variantFor(moto)]); + await expect(repackAndroid(input({ pocket }), options)).rejects.toThrow(/no redmi-1s-dev variant \(it has moto-g-play-dev\)/); }); test("a variant for another host ABI", async () => { const variant = variantFor(resolveRedmi1sBuildPlan(clear())); - const pocket = fixturePocket([{ ...variant, hostAbi: 8 }]); - await expect(prepareAndroidJob(await input({ pocket }))).rejects.toThrow(/host ABI 8; this runtime is ABI 9/); + const pocket = fixturePocket(IDENTITY.id, [{ ...variant, hostAbi: HOST_ABI - 1 }]); + await expect(repackAndroid(input({ pocket }), options)).rejects.toThrow(new RegExp(`host ABI ${HOST_ABI - 1}; this runtime is ABI ${HOST_ABI}`)); }); test("a runtime for another host ABI", async () => { - const runtime = await fixtureRuntime({ hostAbi: 10 }); - await expect(prepareAndroidJob(await input({ runtime }))).rejects.toThrow(/host ABI 9; this runtime is ABI 10/); + await expect(repackAndroid(input({ runtime: fixtureRuntime({ hostAbi: HOST_ABI + 1 }) }), options)) + .rejects.toThrow(new RegExp(`host ABI ${HOST_ABI}; this runtime is ABI ${HOST_ABI + 1}`)); }); test("a broken footer", async () => { const pocket = fixturePocket(); pocket[pocket.length - 20] ^= 1; - await expect(prepareAndroidJob(await input({ pocket }))).rejects.toThrow(/\.pocket is damaged: pocket package: hash mismatch/); + await expect(repackAndroid(input({ pocket }), options)).rejects.toThrow(/\.pocket is damaged: pocket package: hash mismatch/); }); test("a runtime file that differs from runtime.json", async () => { - const runtime = await fixtureRuntime(); - runtime.set("classes.dex", bytesOf("another dex")); - await expect(prepareAndroidJob(await input({ runtime }))).rejects.toThrow(/classes\.dex is not the file runtime\.json describes/); + const runtime = fixtureRuntime(); + runtime.set(ANDROID_TEMPLATE, bytesOf("another template")); + await expect(repackAndroid(input({ runtime }), options)).rejects.toThrow(/template\.apk is not the file runtime\.json describes/); }); test("a runtime drawing another viewport than the game's", async () => { - const runtime = await fixtureRuntime({ viewport: { logical: [360, 800], rasterDensity: 2 } }); - await expect(prepareAndroidJob(await input({ runtime }))).rejects.toThrow(/laid out at 360x640.*runtime draws 360x800/); + const runtime = fixtureRuntime({ viewport: { logical: [360, 800], rasterDensity: 2 } }); + await expect(repackAndroid(input({ runtime }), options)).rejects.toThrow(/laid out at 360x640.*runtime draws 360x800/); }); test("an icon that is not square", async () => { const icon = await encodePng({ width: 200, height: 100, rgba: new Uint8Array(200 * 100 * 4) }); - await expect(prepareAndroidJob(await input({ identity: { ...IDENTITY, icon } }))).rejects.toThrow(/200x100 is not square/); + await expect(repackAndroid(input({ identity: { ...IDENTITY, icon } }), options)).rejects.toThrow(/200x100 is not square/); }); test("an identity that is not the .pocket's app", async () => { - await expect(prepareAndroidJob(await input({ identity: { ...IDENTITY, id: "dev.pocket-nexus.studio.other" } }))) + await expect(repackAndroid({ ...input(), identity: { ...IDENTITY, id: "dev.pocket-nexus.studio.other" } }, options)) .rejects.toThrow(/the identity says "dev\.pocket-nexus\.studio\.other"/); }); test("a runtime for another device", async () => { - const runtime = await fixtureRuntime({ target: "3ds-dev" }); - await expect(prepareAndroidJob(await input({ runtime }))).rejects.toThrow(/this runtime is for "3ds-dev", not "redmi-1s-dev"/); + await expect(repackAndroid(input({ runtime: fixtureRuntime({ target: "3ds-dev" }) }), options)) + .rejects.toThrow(/this runtime is for "3ds-dev", not "redmi-1s-dev"/); }); }); describe("zip", () => { - test("zip entries round-trip, stored entries align, and the bytes are the same twice", async () => { + test("entries round-trip, stored entries align, and the bytes are the same twice", async () => { const inputs = [ - { name: "a.txt", data: bytesOf("hello ".repeat(100)) }, + { name: "a.txt", data: bytesOf("hello ".repeat(100)), align: 4 }, { name: "lib/x.so", data: Uint8Array.from({ length: 999 }, (_, i) => i & 255), compress: false, align: 4096 }, ]; const zip = await writeZip(inputs); expect(await writeZip(inputs)).toEqual(zip); const entries = readZip(zip); - expect(entries.map((e) => e.name)).toEqual(["a.txt", "lib/x.so"]); - expect(await unzipEntry(entries[0])).toEqual(inputs[0].data); - expect(entries[1].stored.byteOffset % 4096).toBe(0); - expect(await unzipEntry(entries[1])).toEqual(inputs[1].data); + expect(entries.map((e) => [e.name, e.method])).toEqual([["a.txt", 8], ["lib/x.so", 0]]); + expect(await unzipEntry(entries[0]!)).toEqual(inputs[0]!.data); + expect(entries[1]!.stored.byteOffset % 4096).toBe(0); + expect(await unzipEntry(entries[1]!)).toEqual(inputs[1]!.data); }); }); -// The native half needs the pinned build-tools, the platform and Java 17 -// (`bun tools/android.ts --profile=redmi-1s setup`). It signs with a key made -// for this test, never the community key. +// Where the Android SDK is installed (`bun tools/android.ts --profile=redmi-1s +// setup`): aapt2 links each game of a corpus, zipalign aligns it and +// apksigner signs it with the test key, and the TypeScript repack gives the +// same entries in the same order with the same bytes, apart from META-INF/; +// apksigner verifies the TypeScript APK (v1 + v2) and zipalign -c accepts it. const sdk = localAndroidSdk(); -const nativeReady = ["build-tools/35.0.0/aapt2", "build-tools/35.0.0/apksigner", "build-tools/35.0.0/zipalign", "platforms/android-34/android.jar"] - .every((path) => existsSync(join(sdk.root, path))) && existsSync(join(sdk.javaHome, "bin/keytool")); - -describe.skipIf(!nativeReady)("the native half", () => { - function testSigner() { - const keystore = join(work, "test.p12"), passwordFile = join(work, "test.password"), certificate = join(work, "test.der"); - if (!existsSync(keystore)) { - writeFileSync(passwordFile, "repack-android-test\n", { mode: 0o600 }); - const keytool = join(sdk.javaHome, "bin/keytool"); - const store = ["-storetype", "PKCS12", "-keystore", keystore, "-storepass:file", passwordFile, "-alias", "test"]; - for (const args of [ - ["-genkeypair", "-noprompt", ...store, "-keyalg", "RSA", "-keysize", "2048", "-validity", "2", "-dname", "CN=Repack Test"], - ["-exportcert", ...store, "-file", certificate], - ]) { - const done = Bun.spawnSync([keytool, ...args], { stdout: "pipe", stderr: "pipe" }); - if (done.exitCode !== 0) throw new Error(done.stderr.toString()); - } - } - return { keystore, passwordFile, alias: "test", certificateSha256: sha256(new Uint8Array(readFileSync(certificate))) }; +const sdkReady = ["build-tools/35.0.0/aapt2", "build-tools/35.0.0/apksigner", "build-tools/35.0.0/zipalign", "platforms/android-34/android.jar"] + .every((path) => existsSync(join(sdk.root, path))) && existsSync(join(sdk.javaHome, "bin/java")); + +describe.skipIf(!sdkReady)("aapt2, zipalign and apksigner agree", () => { + const javaEnv = { ...process.env, JAVA_HOME: sdk.javaHome, PATH: `${join(sdk.javaHome, "bin")}:${process.env.PATH ?? ""}` }; + const tool = (name: string) => buildTool(sdk, SDK_LEVELS.buildToolsVersion, name); + const key = join(work, "test.pk8"); + const certificate = join(work, "test.der"); + writeFileSync(key, signer.privateKey); + writeFileSync(certificate, signer.certificate); + + async function aapt2Apk(game: RepackInput, name: string): Promise<Uint8Array> { + const unsigned = join(work, `${name}-unsigned.apk`), aligned = join(work, `${name}-aligned.apk`), signed = join(work, `${name}-signed.apk`); + writeFileSync(unsigned, await linkAndroidApk(await androidLinkJob(game), sdk)); + runTool(tool("zipalign"), ["-f", "-p", "4", unsigned, aligned]); + runTool(tool("apksigner"), [ + "sign", "--key", key, "--cert", certificate, "--min-sdk-version", "18", + "--v1-signing-enabled", "true", "--v2-signing-enabled", "true", "--v3-signing-enabled", "false", "--v4-signing-enabled", "false", + "--out", signed, aligned, + ], javaEnv); + return new Uint8Array(readFileSync(signed)); } - test("links, aligns and signs v1 + v2; the same inputs give the same APK", async () => { - const native = androidNativeSteps({ sdk, signer: testSigner() }); - const job = await prepareAndroidJob(await input()); - const apk = await repackAndroid(await input(), { native }); - expect(await repackAndroid(await input(), { native })).toEqual(apk); - const entries = readZip(apk); - const names = entries.map((entry) => entry.name); - for (const name of [...expectedApkEntries(job), ...ANDROID_ICON_DENSITIES.map(([d]) => `res/mipmap-${d}-v4/icon.png`)]) { - expect(names).toContain(name); - } - expect(names.some((name) => /^META-INF\/[^/]+\.RSA$/.test(name))).toBe(true); - const entry = (name: string) => entries.find((e) => e.name === name)!; - expect(await unzipEntry(entry("assets/app.js"))).toEqual(GAME_JS); - expect(await unzipEntry(entry("lib/arm64-v8a/libpocketjs.so"))).toEqual(job.entries.get("lib/arm64-v8a/libpocketjs.so")!); - const path = join(work, "game.apk"); - writeFileSync(path, apk); - const badging = Bun.spawnSync([join(sdk.root, "build-tools/35.0.0/aapt2"), "dump", "badging", path], { stdout: "pipe" }).stdout.toString(); - expect(badging).toContain("package: name='dev.pocket_nexus.studio.test_game' versionCode='1002003' versionName='1.2.3'"); - expect(badging).toContain("minSdkVersion:'18'"); - expect(badging).toMatch(/application-label:'Tom's \\?"Test\\?" & <Game>'/); - expect(badging).toContain("native-code: 'arm64-v8a' 'armeabi-v7a'"); - expect(badging).not.toContain("application-debuggable"); + test("the fixture template is aapt2's link of hosts/android/app with the stand-in files", async () => { + const linked = await linkAndroidApk(androidTemplateJob(await fixtureParts(), SDK_LEVELS, ABIS), sdk); + // A change to hosts/android/app's templates needs POCKETJS_WRITE_ANDROID_FIXTURE=1 (see FIXTURE). + expect(sha256(linked)).toBe(sha256(TEMPLATE)); }, 60_000); - test("a signer whose certificate is not the expected one is refused after signing", async () => { - const native = androidNativeSteps({ sdk, signer: { ...testSigner(), certificateSha256: POCKET_STUDIO_COMMUNITY_SIGNER } }); - await expect(repackAndroid(await input(), { native })).rejects.toThrow(/not 54c2abe6/); - }, 60_000); + const CORPUS = [ + { id: "dev.pocket-nexus.studio.test-game", title: `Tom's "Test" & <Game> 100%`, version: "1.2.3", icon: true }, + { id: "dev.pocket-nexus.studio.2pac.8-ball", title: "8 Ball", version: "0.1.0" }, + { id: "app.a", title: "日本語のゲーム\u3000タイトル", version: "10.20.30" }, + { id: "zz.zz", title: "Zebra 🦓 game", version: "1999.999.999" }, + { id: "dev.pocket-nexus.studio.twenty48-remix", title: " Space \t Game ", version: "0.0.1" }, + { id: "com.x.y", title: "@home \\ ?", version: "1.0.0" }, + ]; + /** The runtimes compared: the fixture, and dist/runtime/android when a build of this tree left one. */ + const runtimes: Array<[string, Map<string, Uint8Array>]> = [["fixture", fixtureRuntime()]]; + if (existsSync(join(ROOT, "dist/runtime/android", ANDROID_TEMPLATE))) runtimes.push(["dist/runtime/android", readRuntimeDirectory(join(ROOT, "dist/runtime/android"))]); + + for (const [label, runtime] of runtimes) { + test(`a corpus repacked on ${label} matches aapt2's, and apksigner verifies it`, async () => { + for (const game of CORPUS) { + const manifest = clear(game.id); + const plan = resolveRedmi1sBuildPlan(manifest); + if (label !== "fixture" && plan.target.hostAbi !== JSON.parse(new TextDecoder().decode(runtime.get("runtime.json"))).hostAbi) continue; + const repackInput = input({ + runtime, + pocket: encodePocketPackage({ manifest: bytesOf(JSON.stringify(manifest)), variants: [variantFor(plan)] }), + identity: { id: game.id, title: game.title, author: "A Creator", version: game.version, ...(game.icon ? { icon: ICON } : {}) }, + }); + const typescript = await repackAndroid(repackInput, options); + const oracle = await aapt2Apk(repackInput, "oracle"); + const files = async (apk: Uint8Array) => + Promise.all(readZip(apk).filter((entry) => !entry.name.startsWith("META-INF/")).map(async (entry) => [entry.name, sha256(await unzipEntry(entry))])); + expect(await files(typescript)).toEqual(await files(oracle)); + + const path = join(work, "typescript.apk"); + writeFileSync(path, typescript); + const report = runTool(tool("apksigner"), ["verify", "--verbose", "--print-certs", "--min-sdk-version", "18", path], javaEnv); + expect(report).toContain("Verified using v1 scheme (JAR signing): true"); + expect(report).toContain("Verified using v2 scheme (APK Signature Scheme v2): true"); + expect(report).toContain(`Signer #1 certificate SHA-256 digest: ${signer.certificateSha256}`); + expect(Bun.spawnSync([tool("zipalign"), "-c", "-p", "4", path]).exitCode).toBe(0); + const badging = runTool(tool("aapt2"), ["dump", "badging", path]); + expect(badging).toContain(`package: name='${androidPackageName(game.id)}'`); + // aapt2 dump badging prints a backslash and a double quote escaped. + expect(badging).toContain(`application-label:'${androidLabel(game.title).replace(/[\\"]/g, "\\$&")}'`); + } + }, 240_000); + } }); diff --git a/tests/repack-worker.test.ts b/tests/repack-worker.test.ts new file mode 100644 index 000000000..371e073f0 --- /dev/null +++ b/tests/repack-worker.test.ts @@ -0,0 +1,136 @@ +// Every repack runs where a Cloudflare Worker runs: each target's module and +// the Studio's entry (tools/repack/index.ts) bundle for the browser with no +// Node or Bun built-in module, and the bundles name no Bun, process, require +// or file path API. The Android bundle repacks the same APK as the source. + +import { describe, expect, test } from "bun:test"; +import { builtinModules } from "node:module"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { REDMI_1S_TARGET } from "../tools/redmi-1s-profile.ts"; +import { ANDROID_RUNTIME_TARGETS, repackAndroid } from "../tools/repack/android.ts"; +import { THREE_DS_RUNTIME_TARGET } from "../tools/repack/3ds.ts"; +import { REPACK_TARGETS, repackTarget } from "../tools/repack/index.ts"; +import { IPOD_RUNTIME_TARGET } from "../tools/repack/ipod.ts"; +import { PSP_RUNTIME_TARGET } from "../tools/repack/psp.ts"; +import { VITA_RUNTIME_TARGET } from "../tools/repack/vita.ts"; +import { fixturePocket, fixtureRuntime, IDENTITY } from "./helpers/android-fixture.ts"; +import { makeTestSigner } from "./helpers/apk-test-signer.ts"; + +const ROOT = join(import.meta.dir, ".."); +const ENTRIES = ["index.ts", "psp.ts", "vita.ts", "3ds.ts", "ipod.ts", "android.ts"] as const; +const escape = (name: string) => name.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); +const BUILTIN = new RegExp(`^(node:.*|bun:.*|bun|${builtinModules.map(escape).join("|")})(/.*)?$`); +/** What a Worker lacks: Bun's and Node's globals, CommonJS, and the module's own path. */ +const HOST_APIS = /\bBun\.|\bprocess\.|\brequire\(|__dirname|__filename|import\.meta\.(dir|path|file|main)\b|\bDeno\./g; + +async function bundle(entry: string): Promise<{ text: string; builtins: string[] }> { + const builtins: string[] = []; + const result = await Bun.build({ + entrypoints: [join(ROOT, "tools/repack", entry)], + target: "browser", + format: "esm", + plugins: [{ + name: "node-builtins", + setup(build) { + build.onResolve({ filter: BUILTIN }, (args) => { + builtins.push(`${args.path} (from ${args.importer.replace(`${ROOT}/`, "")})`); + return { path: args.path, external: true }; + }); + }, + }], + }); + if (!result.success) throw new Error(result.logs.map((log) => log.message).join("\n")); + return { text: await result.outputs[0]!.text(), builtins }; +} + +describe("every repack bundles for a Worker", () => { + for (const entry of ENTRIES) { + test(`tools/repack/${entry} imports no Node or Bun module and calls no host API`, async () => { + const { text, builtins } = await bundle(entry); + expect(builtins).toEqual([]); + expect(text.match(HOST_APIS) ?? []).toEqual([]); + }); + } + + test("the check sees a Node import and a Bun call", async () => { + const directory = mkdtempSync(join(tmpdir(), "pocketjs-repack-worker-")); + try { + writeFileSync(join(directory, "bad.ts"), 'import { readFileSync } from "node:fs";\nexport const read = () => [readFileSync, Bun.file];\n'); + const builtins: string[] = []; + const result = await Bun.build({ + entrypoints: [join(directory, "bad.ts")], + target: "browser", + plugins: [{ + name: "node-builtins", + setup(build) { + build.onResolve({ filter: BUILTIN }, (args) => { + builtins.push(args.path); + return { path: args.path, external: true }; + }); + }, + }], + }); + expect(builtins).toEqual(["node:fs"]); + expect((await result.outputs[0]!.text()).match(HOST_APIS)).toEqual(["Bun."]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + test("the bundled modules repack what the source modules repack", async () => { + const directory = mkdtempSync(join(tmpdir(), "pocketjs-repack-worker-")); + try { + const load = async (entry: string) => { + const path = join(directory, entry.replace(/\.ts$/, ".js")); + writeFileSync(path, (await bundle(entry)).text); + return import(path); + }; + const index = (await load("index.ts")) as typeof import("../tools/repack/index.ts"); + const android = (await load("android.ts")) as typeof import("../tools/repack/android.ts"); + const signer = await makeTestSigner(); + const input = { + runtime: fixtureRuntime(), + pocket: fixturePocket(), + identity: IDENTITY, + signer: { privateKey: signer.privateKey, certificate: signer.certificate }, + }; + const options = { certificateSha256: signer.certificateSha256 }; + expect(await android.repackAndroid(input, options)).toEqual(await repackAndroid(input, options)); + // The Studio's row signs with the community key alone. + await expect(index.REPACK_TARGETS.android.repack(input)).rejects.toThrow(/not the Pocket Studio community certificate/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); + +describe("the Studio's table", () => { + test("one row per Studio target, each naming its runtime, variants, file and type", () => { + const rows = Object.fromEntries(Object.entries(REPACK_TARGETS).map(([name, row]) => [name, { ...row, repack: typeof row.repack }])); + expect(rows).toEqual({ + psp: { repack: "function", runtime: "psp", variantTargets: [PSP_RUNTIME_TARGET], extension: ".zip", contentType: "application/zip", signed: false }, + vita: { repack: "function", runtime: "vita", variantTargets: [VITA_RUNTIME_TARGET], extension: ".vpk", contentType: "application/octet-stream", signed: false }, + "3ds": { repack: "function", runtime: "3ds", variantTargets: [THREE_DS_RUNTIME_TARGET], extension: ".3dsx", contentType: "application/octet-stream", signed: false }, + "ipod-touch": { repack: "function", runtime: "ipod", variantTargets: [IPOD_RUNTIME_TARGET], extension: ".ipa", contentType: "application/octet-stream", signed: false }, + android: { + repack: "function", + runtime: "android", + variantTargets: [...ANDROID_RUNTIME_TARGETS], + extension: ".apk", + contentType: "application/vnd.android.package-archive", + signed: true, + }, + }); + expect(REPACK_TARGETS.android.variantTargets[0]).toBe(REDMI_1S_TARGET); + }); + + test("a row by Studio target or runtime name", () => { + expect(repackTarget("ipod")).toBe(REPACK_TARGETS["ipod-touch"]); + expect(repackTarget("ipod-touch")).toBe(REPACK_TARGETS["ipod-touch"]); + expect(repackTarget("android")).toBe(REPACK_TARGETS.android); + expect(repackTarget("toString")).toBeUndefined(); + expect(repackTarget("gba")).toBeUndefined(); + }); +}); diff --git a/tools/android-link.ts b/tools/android-link.ts new file mode 100644 index 000000000..88b5ed61d --- /dev/null +++ b/tools/android-link.ts @@ -0,0 +1,86 @@ +// aapt2 link of an AndroidLinkJob (tools/repack/android.ts) on this machine's +// Android SDK: compile the resources, link the manifest, resources and +// assets, and add classes.dex and the libraries. The result is unsigned and +// unaligned. +// +// Two callers: tools/runtime/android.ts links the runtime's template.apk, and +// tests/repack-android.test.ts links a game to compare with the TypeScript +// repack. A repack itself runs no SDK tool. + +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import type { AndroidLinkJob } from "./repack/android.ts"; +import { readZip, writeZip } from "./repack/shared/zip.ts"; + +export interface AndroidSdk { + /** The SDK root holding build-tools/<version> and platforms/android-<level>. */ + readonly root: string; + readonly javaHome: string; +} + +/** The SDK tools/android.ts uses: POCKETJS_ANDROID_SDK_ROOT, else the Homebrew SDK on macOS, else the toolchain cache. */ +export function localAndroidSdk(env: NodeJS.ProcessEnv = process.env): AndroidSdk { + return { + root: env.POCKETJS_ANDROID_SDK_ROOT ?? + (process.platform === "darwin" ? "/opt/homebrew/share/android-commandlinetools" : join(homedir(), ".cache/pocket-nexus/android/sdk")), + javaHome: env.JAVA_HOME ?? "/opt/homebrew/opt/openjdk@17", + }; +} + +/** build-tools/<version>/<tool>. */ +export function buildTool(sdk: AndroidSdk, version: string, tool: string): string { + return join(sdk.root, "build-tools", version, tool); +} + +export function runTool(program: string, args: readonly string[], env: NodeJS.ProcessEnv = process.env): string { + const done = Bun.spawnSync([program, ...args], { stdout: "pipe", stderr: "pipe", env }); + if (done.exitCode !== 0) { + throw new Error(`${program.split("/").pop()} exited ${done.exitCode}\n${done.stderr.toString().trim() || done.stdout.toString().trim()}`); + } + return done.stdout.toString(); +} + +function writeTree(root: string, files: ReadonlyMap<string, Uint8Array>): void { + for (const [path, bytes] of files) { + if (path.startsWith("/") || path.split("/").includes("..")) throw new Error(`android link: ${path} leaves its directory`); + const file = join(root, path); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, bytes); + } +} + +/** aapt2 compile --no-crunch + aapt2 link, then the job's entries after aapt2's, dated 1980-01-01. */ +export async function linkAndroidApk(job: AndroidLinkJob, sdk: AndroidSdk = localAndroidSdk()): Promise<Uint8Array> { + const aapt2 = buildTool(sdk, job.sdk.buildToolsVersion, "aapt2"); + const androidJar = join(sdk.root, "platforms", `android-${job.sdk.apiLevel}`, "android.jar"); + for (const path of [aapt2, androidJar]) { + if (!existsSync(path)) throw new Error(`android link: ${path} is absent (run \`bun tools/android.ts --profile=redmi-1s setup\`)`); + } + const work = mkdtempSync(join(tmpdir(), "pocket-android-link-")); + try { + writeFileSync(join(work, "AndroidManifest.xml"), job.manifest); + writeTree(join(work, "res"), job.resources); + mkdirSync(join(work, "assets"), { recursive: true }); + writeTree(join(work, "assets"), job.assets); + const flat = join(work, "res.zip"); + // `--no-crunch`: the launcher icons go in byte for byte. + runTool(aapt2, ["compile", "--no-crunch", "--dir", join(work, "res"), "-o", flat]); + const linked = join(work, "linked.apk"); + runTool(aapt2, [ + "link", "-o", linked, + "--manifest", join(work, "AndroidManifest.xml"), + "-I", androidJar, + "-A", join(work, "assets"), + "--min-sdk-version", String(job.sdk.minSdkVersion), + "--target-sdk-version", String(job.sdk.targetSdkVersion), + flat, + ]); + return await writeZip([ + ...readZip(new Uint8Array(readFileSync(linked))).map((entry) => ({ name: entry.name, entry })), + ...[...job.entries].map(([name, data]) => ({ name, data })), + ]); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} diff --git a/tools/community-key.ts b/tools/community-key.ts new file mode 100644 index 000000000..8e56ffced --- /dev/null +++ b/tools/community-key.ts @@ -0,0 +1,131 @@ +// bun tools/community-key.ts +// +// The Pocket Studio community key in the form the Android repack and a +// Worker secret take: the PKCS #12 keystore +// `~/.config/pocket-nexus/signing/pocket-studio-community.p12` +// (POCKET_STUDIO_COMMUNITY_KEY names another path) becomes two files beside +// it, mode 600: +// +// pocket-studio-community.pkcs8.der the RSA private key, PKCS #8 DER +// pocket-studio-community.cert.der the X.509 certificate, DER +// +// OpenSSL reads the keystore's password from the `.password` file beside it +// (`-passin file:`), so the password is on no command line and in no output; +// the key goes from OpenSSL's stdout to the file without being printed. The +// tool writes nothing unless the certificate's SHA-256 is +// POCKET_STUDIO_COMMUNITY_SIGNER and the key signs what the certificate +// verifies. It never reads the Pocket Nexus release key. +// +// tools/repack.ts reads the two files with readCommunitySigner(); a Worker +// takes their bytes from its secrets (base64) as the repack's `signer`. + +import { chmodSync, existsSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, join } from "node:path"; +import { POCKET_STUDIO_COMMUNITY_SIGNER } from "./repack/android.ts"; +import { verifiesUnder, type ApkSigner } from "./repack/shared/apk-sign.ts"; +import { sha256Hex } from "./repack/shared/runtime.ts"; + +export interface CommunityKeyPaths { + readonly keystore: string; + readonly password: string; + readonly privateKey: string; + readonly certificate: string; +} + +/** The keystore, its password file and the two DER files, from POCKET_STUDIO_COMMUNITY_KEY or the default. */ +export function communityKeyPaths(env: NodeJS.ProcessEnv = process.env): CommunityKeyPaths { + const keystore = env.POCKET_STUDIO_COMMUNITY_KEY || join(env.HOME ?? homedir(), ".config/pocket-nexus/signing/pocket-studio-community.p12"); + if (!keystore.endsWith(".p12")) throw new Error(`community key: ${keystore} is not a .p12 keystore`); + if (basename(keystore).includes("release")) throw new Error("community key: that is a release keystore; this tool converts the Pocket Studio community key only"); + const stem = keystore.slice(0, -".p12".length); + return { keystore, password: `${stem}.password`, privateKey: `${stem}.pkcs8.der`, certificate: `${stem}.cert.der` }; +} + +/** The community key's two DER files as the Android repack's signer. */ +export function readCommunitySigner(env: NodeJS.ProcessEnv = process.env): ApkSigner { + const paths = communityKeyPaths(env); + for (const file of [paths.privateKey, paths.certificate]) { + if (!existsSync(file)) throw new Error(`community key: ${file} is absent; run \`bun tools/community-key.ts\``); + } + return { privateKey: new Uint8Array(readFileSync(paths.privateKey)), certificate: new Uint8Array(readFileSync(paths.certificate)) }; +} + +function openssl(): string { + const homebrew = "/opt/homebrew/opt/openssl@3/bin/openssl"; + return existsSync(homebrew) ? homebrew : "openssl"; +} + +/** One PEM block's bytes from OpenSSL's output; the output itself is never printed. */ +function pemBlock(output: Uint8Array, label: string): Uint8Array { + const text = new TextDecoder().decode(output); + const match = new RegExp(`-----BEGIN ${label}-----([A-Za-z0-9+/=\\s]+)-----END ${label}-----`).exec(text); + if (!match) throw new Error(`community key: openssl printed no ${label} block`); + const binary = atob(match[1]!.replace(/\s+/g, "")); + return Uint8Array.from(binary, (character) => character.charCodeAt(0)); +} + +/** `openssl pkcs12` over the keystore with the password from its file; stdout comes back, stderr goes into the error. */ +function pkcs12(paths: CommunityKeyPaths, args: readonly string[]): Uint8Array { + const done = Bun.spawnSync([openssl(), "pkcs12", "-in", paths.keystore, "-passin", `file:${paths.password}`, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + if (done.exitCode !== 0) throw new Error(`community key: openssl pkcs12 exited ${done.exitCode}\n${done.stderr.toString().trim()}`); + return new Uint8Array(done.stdout); +} + +function writePrivate(path: string, bytes: Uint8Array): void { + const staging = `${path}.${process.pid}.tmp`; + writeFileSync(staging, bytes, { mode: 0o600 }); + chmodSync(staging, 0o600); + renameSync(staging, path); + chmodSync(path, 0o600); +} + +export async function convertCommunityKey(env: NodeJS.ProcessEnv = process.env): Promise<CommunityKeyPaths> { + const paths = communityKeyPaths(env); + for (const file of [paths.keystore, paths.password]) { + if (!existsSync(file)) throw new Error(`community key: ${file} is absent`); + } + const keyOutput = pkcs12(paths, ["-nocerts", "-nodes"]); + const privateKey = pemBlock(keyOutput, "PRIVATE KEY"); + keyOutput.fill(0); + const certificate = pemBlock(pkcs12(paths, ["-nokeys", "-clcerts"]), "CERTIFICATE"); + + const digest = await sha256Hex(certificate); + if (digest !== POCKET_STUDIO_COMMUNITY_SIGNER) { + privateKey.fill(0); + throw new Error(`community key: the keystore's certificate is ${digest}, not the Pocket Studio community certificate; nothing written`); + } + let key: CryptoKey; + try { + key = await crypto.subtle.importKey("pkcs8", privateKey as Uint8Array<ArrayBuffer>, { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, false, ["sign"]); + } catch { + privateKey.fill(0); + throw new Error("community key: the keystore's key is not an RSA PKCS #8 key; nothing written"); + } + const probe = new TextEncoder().encode("pocket studio community key check"); + const signature = new Uint8Array(await crypto.subtle.sign("RSASSA-PKCS1-v1_5", key, probe)); + if (!(await verifiesUnder(certificate, probe, signature))) { + privateKey.fill(0); + throw new Error("community key: the keystore's key does not match its certificate; nothing written"); + } + writePrivate(paths.privateKey, privateKey); + writePrivate(paths.certificate, certificate); + privateKey.fill(0); + return paths; +} + +if (import.meta.main) { + try { + const paths = await convertCommunityKey(); + console.log(`private key: ${paths.privateKey} (PKCS #8 DER, mode 600)`); + console.log(`certificate: ${paths.certificate} (X.509 DER, mode 600)`); + console.log(`certificate SHA-256: ${POCKET_STUDIO_COMMUNITY_SIGNER} (the Pocket Studio community certificate); the key signs what it verifies`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/tools/repack.ts b/tools/repack.ts index 5ad064b4d..a9342720f 100644 --- a/tools/repack.ts +++ b/tools/repack.ts @@ -4,22 +4,18 @@ // // A game's installation package from a prebuilt runtime (tools/runtime.ts) // and the game's `.pocket`, with no native tools. This file only reads the -// files; tools/repack/<target>.ts does the work on bytes, so a Worker can call -// it the same way. +// files; tools/repack/index.ts names each target's repack, which works on +// bytes, so a Worker calls it the same way. `--target` is a runtime name +// (psp, vita, 3ds, ipod, android) or a Pocket Studio target (ipod-touch). +// +// Android signs with the Pocket Studio community key: the two DER files +// `bun tools/community-key.ts` writes beside the keystore +// (POCKET_STUDIO_COMMUNITY_KEY names another keystore). import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join, relative } from "node:path"; -import type { RepackInput } from "./repack/shared/runtime.ts"; - -type Repack = (input: RepackInput) => Promise<Uint8Array>; - -const TARGETS: Record<string, () => Promise<Repack>> = { - "3ds": async () => (await import("./repack/3ds.ts")).repack3ds, - android: async () => (await import("./repack/android.ts")).repackAndroid, - psp: async () => (await import("./repack/psp.ts")).repackPsp, - vita: async () => (await import("./repack/vita.ts")).repackVita, - ipod: async () => (await import("./repack/ipod.ts")).repackIPod, -}; +import { readCommunitySigner } from "./community-key.ts"; +import { REPACK_TARGETS, repackTarget } from "./repack/index.ts"; /** Every file below `directory`, by "/"-separated relative path. */ export function readRuntimeDirectory(directory: string): Map<string, Uint8Array> { @@ -35,8 +31,10 @@ export function readRuntimeDirectory(directory: string): Map<string, Uint8Array> return files; } +const TARGET_NAMES = [...new Set(Object.values(REPACK_TARGETS).map((target) => target.runtime))]; + const USAGE = - `usage: bun tools/repack.ts --target <${Object.keys(TARGETS).join("|")}> --runtime <dir> --pocket <file> ` + + `usage: bun tools/repack.ts --target <${TARGET_NAMES.join("|")}> --runtime <dir> --pocket <file> ` + "--id <id> --title <title> --author <author> --version <version> [--icon <png>] -o <out>"; export async function repackMain(argv: readonly string[]): Promise<string> { @@ -55,12 +53,12 @@ export async function repackMain(argv: readonly string[]): Promise<string> { if (missing.length || unknown.length) { throw new Error(`${USAGE}\n${[...missing.map((name) => `missing --${name}`), ...unknown.map((name) => `unknown --${name}`)].join(", ")}`); } - const load = TARGETS[options.get("target")!]; - if (!load) throw new Error(USAGE); - const repack = await load(); + const target = repackTarget(options.get("target")!); + if (!target) throw new Error(USAGE); const icon = options.get("icon"); + const signer = target.signed ? readCommunitySigner() : undefined; const started = performance.now(); - const bytes = await repack({ + const bytes = await target.repack({ runtime: readRuntimeDirectory(options.get("runtime")!), pocket: new Uint8Array(readFileSync(options.get("pocket")!)), identity: { @@ -70,6 +68,7 @@ export async function repackMain(argv: readonly string[]): Promise<string> { version: options.get("version")!, ...(icon ? { icon: new Uint8Array(readFileSync(icon)) } : {}), }, + ...(signer ? { signer } : {}), }); const elapsed = performance.now() - started; const out = options.get("out")!; diff --git a/tools/repack/android-native.ts b/tools/repack/android-native.ts deleted file mode 100644 index 9c39b6620..000000000 --- a/tools/repack/android-native.ts +++ /dev/null @@ -1,151 +0,0 @@ -// The native half of the Android repack (tools/repack/android.ts): the one -// step of any repack that runs SDK tools. It takes an AndroidNativeJob (the -// APK's files laid out) and returns the signed APK: -// -// aapt2 compile --no-crunch res/ (strings + launcher icons) into flat files -// aapt2 link binary manifest + resources.arsc + res/ + assets/ -// zip merge (TypeScript) + classes.dex + lib/<abi>/libpocketjs.so, dated 1980-01-01 -// zipalign -p 4 stored entries on 4-byte boundaries -// apksigner sign v1 + v2 with the signer's key (v3/v4 off) -// apksigner verify both schemes hold and the certificate is the expected one -// -// It needs build-tools and the platform the job names, and Java 17. The -// signer's password is the first line of a file apksigner reads itself -// (`--ks-pass file:`), so it is on no command line and in no log. - -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { homedir, tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { readZip, writeZip } from "./shared/zip.ts"; -import { POCKET_STUDIO_COMMUNITY_SIGNER, type AndroidNativeJob, type AndroidNativeSteps } from "./android.ts"; - -export interface AndroidSigner { - /** A PKCS #12 keystore. */ - readonly keystore: string; - /** The file whose first line is the keystore's password. */ - readonly passwordFile: string; - readonly alias: string; - /** SHA-256 of the certificate the APK must verify under; checked after signing. */ - readonly certificateSha256: string; -} - -export interface AndroidSdk { - /** The SDK root holding build-tools/<version> and platforms/android-<level>. */ - readonly root: string; - readonly javaHome: string; -} - -/** The SDK tools/android.ts uses: POCKETJS_ANDROID_SDK_ROOT, else the Homebrew SDK on macOS, else the toolchain cache. */ -export function localAndroidSdk(env: NodeJS.ProcessEnv = process.env): AndroidSdk { - return { - root: env.POCKETJS_ANDROID_SDK_ROOT ?? - (process.platform === "darwin" ? "/opt/homebrew/share/android-commandlinetools" : join(homedir(), ".cache/pocket-nexus/android/sdk")), - javaHome: env.JAVA_HOME ?? "/opt/homebrew/opt/openjdk@17", - }; -} - -/** - * The Pocket Studio community key: the PKCS #12 keystore POCKET_STUDIO_COMMUNITY_KEY - * names, default `~/.config/pocket-nexus/signing/pocket-studio-community.p12` - * (alias `pocket-studio-community`), its password the first line of the - * `.password` file beside it. - */ -export function communitySigner(env: NodeJS.ProcessEnv = process.env): AndroidSigner { - const keystore = env.POCKET_STUDIO_COMMUNITY_KEY || - join(env.HOME ?? homedir(), ".config/pocket-nexus/signing/pocket-studio-community.p12"); - const passwordFile = keystore.replace(/\.[^./]*$/, "") + ".password"; - for (const file of [keystore, passwordFile]) { - if (!existsSync(file)) { - throw new Error( - `repack android: a game is signed with the Pocket Studio community key, and ${file} is absent ` + - "(POCKET_STUDIO_COMMUNITY_KEY names the keystore; its password is the first line of the .password file beside it)", - ); - } - } - return { keystore, passwordFile, alias: "pocket-studio-community", certificateSha256: POCKET_STUDIO_COMMUNITY_SIGNER }; -} - -function run(program: string, args: readonly string[], env: NodeJS.ProcessEnv = process.env): string { - const done = Bun.spawnSync([program, ...args], { stdout: "pipe", stderr: "pipe", env }); - if (done.exitCode !== 0) { - throw new Error(`repack android: ${program.split("/").pop()} exited ${done.exitCode}\n${done.stderr.toString().trim() || done.stdout.toString().trim()}`); - } - return done.stdout.toString(); -} - -function writeTree(root: string, files: ReadonlyMap<string, Uint8Array>): void { - for (const [path, bytes] of files) { - if (path.startsWith("/") || path.split("/").includes("..")) throw new Error(`repack android: ${path} leaves its directory`); - const file = join(root, path); - mkdirSync(dirname(file), { recursive: true }); - writeFileSync(file, bytes); - } -} - -/** The native steps on this machine's SDK, signing with `signer` (default: the community key). */ -export function androidNativeSteps(options: { sdk?: AndroidSdk; signer?: AndroidSigner } = {}): AndroidNativeSteps { - return async (job: AndroidNativeJob) => { - const sdk = options.sdk ?? localAndroidSdk(); - const signer = options.signer ?? communitySigner(); - const buildTools = join(sdk.root, "build-tools", job.sdk.buildToolsVersion); - const androidJar = join(sdk.root, "platforms", `android-${job.sdk.apiLevel}`, "android.jar"); - for (const path of [join(buildTools, "aapt2"), join(buildTools, "zipalign"), join(buildTools, "apksigner"), androidJar]) { - if (!existsSync(path)) throw new Error(`repack android: ${path} is absent (run \`bun tools/android.ts --profile=redmi-1s setup\`)`); - } - const javaEnv = { ...process.env, JAVA_HOME: sdk.javaHome, PATH: `${join(sdk.javaHome, "bin")}:${process.env.PATH ?? ""}` }; - const work = mkdtempSync(join(tmpdir(), "pocket-repack-android-")); - try { - writeFileSync(join(work, "AndroidManifest.xml"), job.manifest); - writeTree(join(work, "res"), job.resources); - writeTree(join(work, "assets"), job.assets); - const flat = join(work, "res.zip"); - // `--no-crunch`: the launcher icons go in as the TypeScript half wrote them, byte for byte. - run(join(buildTools, "aapt2"), ["compile", "--no-crunch", "--dir", join(work, "res"), "-o", flat]); - const linked = join(work, "linked.apk"); - run(join(buildTools, "aapt2"), [ - "link", "-o", linked, - "--manifest", join(work, "AndroidManifest.xml"), - "-I", androidJar, - "-A", join(work, "assets"), - "--min-sdk-version", String(job.sdk.minSdkVersion), - "--target-sdk-version", String(job.sdk.targetSdkVersion), - flat, - ]); - const merged = await writeZip([ - ...readZip(new Uint8Array(readFileSync(linked))).map((entry) => ({ name: entry.name, entry })), - ...[...job.entries].map(([name, data]) => ({ name, data })), - ]); - const unsigned = join(work, "unsigned.apk"); - writeFileSync(unsigned, merged); - const aligned = join(work, "aligned.apk"); - run(join(buildTools, "zipalign"), ["-f", "-p", "4", unsigned, aligned]); - const signed = join(work, "signed.apk"); - const apksigner = join(buildTools, "apksigner"); - run(apksigner, [ - "sign", - "--ks", signer.keystore, - "--ks-type", "PKCS12", - "--ks-pass", `file:${signer.passwordFile}`, - "--ks-key-alias", signer.alias, - "--min-sdk-version", String(job.sdk.minSdkVersion), - "--v1-signing-enabled", "true", - "--v2-signing-enabled", "true", - "--v3-signing-enabled", "false", - "--v4-signing-enabled", "false", - "--out", signed, - aligned, - ], javaEnv); - const report = run(apksigner, ["verify", "--verbose", "--print-certs", "--min-sdk-version", String(job.sdk.minSdkVersion), signed], javaEnv); - for (const scheme of ["v1 scheme (JAR signing)", "v2 scheme (APK Signature Scheme v2)"]) { - if (!report.includes(`Verified using ${scheme}: true`)) throw new Error(`repack android: the APK does not verify under the ${scheme}`); - } - const certificate = /^Signer #1 certificate SHA-256 digest: ([0-9a-f]{64})$/m.exec(report)?.[1]; - if (certificate !== signer.certificateSha256) { - throw new Error(`repack android: the APK is signed by certificate ${certificate}, not ${signer.certificateSha256}`); - } - return new Uint8Array(readFileSync(signed)); - } finally { - rmSync(work, { recursive: true, force: true }); - } - }; -} diff --git a/tools/repack/android.ts b/tools/repack/android.ts index 05b9fb0fd..a8c1e8d78 100644 --- a/tools/repack/android.ts +++ b/tools/repack/android.ts @@ -1,29 +1,50 @@ -// Repack: one Android APK from the generic Android runtime -// (tools/runtime/android.ts), a game's `.pocket` and its identity. +// Repack: one signed Android APK from the generic Android runtime +// (tools/runtime/android.ts), a game's `.pocket`, its identity and a signer. +// TypeScript and Web APIs only (crypto.subtle, CompressionStream): no SDK, +// no file system, so a Cloudflare Worker runs it as the CLI does. // -// The TypeScript half (this file, no file system and no native tools) checks -// the inputs and lays out every file the APK holds: +// The runtime carries `template.apk`: aapt2's link of the runtime with +// ANDROID_TEMPLATE_IDENTITY, the default icon, empty assets, classes.dex and +// lib/<abi>/libpocketjs.so, unsigned. The repack keeps its entries in their +// order and replaces five things: // -// AndroidManifest.xml the runtime's template with the package name -// (the id with `-` as `_`), versionCode -// (major * 1 000 000 + minor * 1 000 + patch), -// versionName, the SDK levels, not debuggable -// res/values/strings.xml the label (the identity's title) -// res/mipmap-*/icon.png the identity's icon scaled to 48 … 192 pixels, -// or the runtime's default icon -// assets/app.js, assets/app.pak the `.pocket` variant's bundle (without -// the section's NUL) and pack, the two files -// PocketActivity reads -// classes.dex, lib/<abi>/libpocketjs.so from the runtime, byte for byte +// AndroidManifest.xml package, android:versionCode and android:versionName +// rewritten in the compiled XML (shared/android-resources.ts) +// resources.arsc the package name and string/app_name (the label) +// res/mipmap-*/icon.png the identity's icon scaled to 48 … 192 pixels, or +// the template's default icon +// assets/app.js, assets/app.pak the `.pocket` variant's bundle (without the +// section's NUL) and pack, the files PocketActivity reads +// META-INF/ v1 (JAR) and v2 signatures (shared/apk-sign.ts) // -// The native half is one function, `AndroidNativeSteps`, that takes those -// files and returns the signed APK: aapt2 compile + link, the zip merge, -// zipalign and apksigner (v1 + v2; Android 4.3 reads v1 only). The default -// runs the local Android SDK and signs with the Pocket Studio community key -// (tools/repack/android-native.ts); a container can run the same function. +// Stored entries are 4-byte aligned and stored .so files 4096-byte aligned, +// as `zipalign -p 4` places them. Where the Android SDK is installed, +// tests/repack-android.test.ts compares the APK with aapt2 + zipalign + +// apksigner's output for the same identity: the same entries in the same +// order and the same bytes in each, apart from the signature files. +// +// Names, from the identity: +// +// package name the id with each `-` as `_`; a dot-separated segment that +// then starts with a digit gets `x` in front +// (dev.pocket-nexus.studio.2pac.8-ball → +// dev.pocket_nexus.studio.x2pac.x8_ball). Each segment is a +// letter, then letters, digits and `_`; at most 127 characters. +// versionCode major * 1 000 000 + minor * 1 000 + patch +// versionName the version as given +// label the title with leading and trailing ASCII whitespace +// removed and each inner run of it as one space (what aapt2 +// makes of a string resource); a control character is refused +// +// The signer is the Pocket Studio community key: the repack refuses a +// certificate whose SHA-256 is not POCKET_STUDIO_COMMUNITY_SIGNER, unless the +// caller names another certificate in `options.certificateSha256` (a test's +// throwaway key). The Pocket Nexus release key signs Pocket Nexus's own apps +// only. import { POCKET_SECTION, findSection } from "../../contracts/spec/pocket-package.ts"; -import { readZip } from "./shared/zip.ts"; +import { signApk } from "./shared/apk-sign.ts"; +import { MAX_PACKAGE_NAME, patchManifest, patchResourceTable, readManifest, readResourceTable } from "./shared/android-resources.ts"; import { decodePng, encodePng } from "./shared/png.ts"; import { squareIcon } from "./shared/scale.ts"; import { @@ -31,9 +52,12 @@ import { checkIdentity, readRuntime, RUNTIME_MANIFEST, + sha256Hex, + type RepackIdentity, type RepackInput, type RuntimeManifest, } from "./shared/runtime.ts"; +import { readZip, unzipEntry, ZIP_STORED, type ZipEntry, type ZipInput } from "./shared/zip.ts"; /** * SHA-256 of the certificate of the Pocket Studio community key @@ -56,6 +80,18 @@ export const ANDROID_ICON_DENSITIES = [ /** The `.pocket` targets an Android runtime boots: the redmi-1s and moto-g-play profiles (tools/runtime/android.ts). */ export const ANDROID_RUNTIME_TARGETS = ["redmi-1s-dev", "moto-g-play-dev"] as const; +/** The runtime's unsigned APK, which every repack starts from. */ +export const ANDROID_TEMPLATE = "template.apk"; +/** The identity template.apk is linked with. */ +export const ANDROID_TEMPLATE_IDENTITY: RepackIdentity = { + id: "dev.pocket-nexus.runtime.template", + title: "PocketJS", + author: "Pocket Nexus", + version: "0.0.1", +}; +/** The text template.apk was linked from, kept in the runtime for the aapt2 comparison. */ +export const ANDROID_TEMPLATE_SOURCES = ["AndroidManifest.xml", "res/values/strings.xml"] as const; + /** runtime.json of an Android runtime. */ export interface AndroidRuntimeManifest extends RuntimeManifest { readonly viewport: { readonly logical: readonly [number, number]; readonly rasterDensity: number }; @@ -68,15 +104,19 @@ export interface AndroidRuntimeManifest extends RuntimeManifest { }; } -/** Everything the native steps need: the APK's files laid out, and the SDK levels to link against. */ -export interface AndroidNativeJob { +/** + * What aapt2 links for an APK: the manifest and resources as source, the + * assets, and the entries added to the linked APK as they are. The runtime + * build links template.apk from one; the aapt2 comparison links each game. + */ +export interface AndroidLinkJob { /** AndroidManifest.xml, as text. */ readonly manifest: string; /** Resource files by path under `res/`. */ readonly resources: ReadonlyMap<string, Uint8Array>; /** Asset files by path under `assets/`. */ readonly assets: ReadonlyMap<string, Uint8Array>; - /** Entries added to the linked APK as they are: `classes.dex`, `lib/<abi>/libpocketjs.so`. */ + /** `classes.dex`, `lib/<abi>/libpocketjs.so`. */ readonly entries: ReadonlyMap<string, Uint8Array>; readonly sdk: { /** The platform whose android.jar the manifest links against. */ @@ -87,9 +127,6 @@ export interface AndroidNativeJob { }; } -/** aapt2 compile + link, zip merge, zipalign, apksigner: the job's files in, a signed APK out. */ -export type AndroidNativeSteps = (job: AndroidNativeJob) => Promise<Uint8Array>; - export interface AndroidPackageIdentity { readonly packageName: string; readonly versionCode: number; @@ -97,21 +134,50 @@ export interface AndroidPackageIdentity { readonly label: string; } -/** The Android names of an identity: package = id with `-` as `_`; versionCode from major.minor.patch. */ -export function androidIdentity(identity: RepackInput["identity"]): AndroidPackageIdentity { - const packageName = identity.id.replaceAll("-", "_"); - if (!/^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z][A-Za-z0-9_]*)+$/.test(packageName)) { - throw new Error(`repack android: ${identity.id} does not make an Android package name (${packageName})`); +export interface AndroidRepackOptions { + /** The certificate SHA-256 (hex) the signer must have; default POCKET_STUDIO_COMMUNITY_SIGNER. */ + readonly certificateSha256?: string; +} + +function fail(message: string): never { + throw new Error(`repack android: ${message}`); +} + +/** + * The Android package name of an app id: each `-` as `_`, and `x` in front + * of a dot-separated segment that then starts with a digit. + */ +export function androidPackageName(id: string): string { + const segments = id.split(".").map((segment) => { + const name = segment.replaceAll("-", "_"); + return /^[0-9]/.test(name) ? `x${name}` : name; + }); + const packageName = segments.join("."); + if (segments.length < 2 || !segments.every((segment) => /^[A-Za-z][A-Za-z0-9_]*$/.test(segment))) { + fail(`${id} does not make an Android package name (${packageName})`); } + if (packageName.length > MAX_PACKAGE_NAME) fail(`the package name of ${id} is longer than ${MAX_PACKAGE_NAME} characters`); + return packageName; +} + +/** The launcher label of a title: ASCII whitespace trimmed, each inner run of it one space. */ +export function androidLabel(title: string): string { + return title.replace(/^[ \t\n\r\f\v]+|[ \t\n\r\f\v]+$/g, "").replace(/[ \t\n\r\f\v]+/g, " "); +} + +/** The Android names of an identity. */ +export function androidIdentity(identity: Pick<RepackIdentity, "id" | "title" | "version">): AndroidPackageIdentity { + const packageName = androidPackageName(identity.id); const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(identity.version); - if (!match) throw new Error(`repack android: version ${identity.version} is not major.minor.patch`); - const [major, minor, patch] = match.slice(1, 4).map(Number); - if (major >= 2000 || minor >= 1000 || patch >= 1000) { - throw new Error(`repack android: version ${identity.version} is past the versionCode range`); - } - const label = identity.title.trim(); - if (!label) throw new Error("repack android: the title is empty"); - return { packageName, versionCode: major * 1_000_000 + minor * 1_000 + patch, versionName: identity.version, label }; + if (!match) fail(`version ${identity.version} is not major.minor.patch`); + const [major, minor, patch] = match.slice(1, 4).map(Number) as [number, number, number]; + if (major >= 2000 || minor >= 1000 || patch >= 1000) fail(`version ${identity.version} is past the versionCode range`); + const versionCode = major * 1_000_000 + minor * 1_000 + patch; + if (versionCode === 0) fail("version 0.0.0 gives versionCode 0, which Android refuses"); + const label = androidLabel(identity.title); + if (!label) fail("the title is empty"); + if (/[\u0000-\u001f\u007f]/.test(label)) fail("the title holds a control character"); + return { packageName, versionCode, versionName: identity.version, label }; } function xmlEscape(text: string): string { @@ -119,8 +185,9 @@ function xmlEscape(text: string): string { } /** - * A string resource's text: XML-escaped, with the characters aapt2 reads as - * syntax (backslash, both quotes, a leading `@` or `?`) escaped by a backslash. + * A string resource's source text for aapt2: XML-escaped, with the characters + * aapt2 reads as syntax (backslash, both quotes, a leading `@` or `?`) + * escaped by a backslash. */ export function androidString(text: string): string { const escaped = text.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/"/g, '\\"').replace(/^([@?])/, "\\$1"); @@ -129,13 +196,32 @@ export function androidString(text: string): string { function render(template: string, values: Readonly<Record<string, string | number>>): string { return template.replace(/@POCKET_([A-Z_]+)@/g, (token, name: string) => { - if (!(name in values)) throw new Error(`repack android: the runtime's template has no value for ${token}`); + if (!(name in values)) fail(`the runtime's template has no value for ${token}`); return String(values[name]); }); } +/** The manifest and strings sources rendered for an identity: what aapt2 links. */ +export function renderAndroidSources( + sources: { readonly manifest: string; readonly strings: string }, + names: AndroidPackageIdentity, + sdk: { readonly minSdkVersion: number; readonly targetSdkVersion: number }, +): { manifest: string; strings: string } { + return { + manifest: render(sources.manifest, { + PACKAGE: names.packageName, + VERSION_CODE: names.versionCode, + VERSION_NAME: xmlEscape(names.versionName), + MIN_SDK: sdk.minSdkVersion, + TARGET_SDK: sdk.targetSdkVersion, + DEBUGGABLE: "false", + }), + strings: render(sources.strings, { TITLE: androidString(names.label) }), + }; +} + /** The runtime's runtime.json, checked as an Android runtime. */ -export async function readAndroidRuntime(runtime: Map<string, Uint8Array>): Promise<AndroidRuntimeManifest> { +export async function readAndroidRuntime(runtime: ReadonlyMap<string, Uint8Array>): Promise<AndroidRuntimeManifest> { let target: unknown; try { target = JSON.parse(new TextDecoder().decode(runtime.get(RUNTIME_MANIFEST) ?? new Uint8Array())).target; @@ -144,38 +230,54 @@ export async function readAndroidRuntime(runtime: Map<string, Uint8Array>): Prom } // readRuntime refuses a missing or broken runtime.json with the reason; a known target goes to it as is. const expected = (ANDROID_RUNTIME_TARGETS as readonly unknown[]).includes(target) ? String(target) : ANDROID_RUNTIME_TARGETS[0]; - const manifest = (await readRuntime(runtime, expected)) as AndroidRuntimeManifest; + const manifest = (await readRuntime(runtime as Map<string, Uint8Array>, expected)) as AndroidRuntimeManifest; const android = manifest.android; if (!android || !Array.isArray(android.abis) || !android.abis.length || !Number.isInteger(android.minSdkVersion)) { - throw new Error("repack android: runtime.json has no android section; build it with tools/runtime/android.ts"); + fail("runtime.json has no android section; build it with tools/runtime/android.ts"); } - for (const path of ["AndroidManifest.xml", "res/values/strings.xml", "classes.dex", ...android.abis.map((abi) => `lib/${abi}/libpocketjs.so`)]) { - if (!manifest.files[path]) throw new Error(`repack android: the runtime lacks ${path}`); + for (const path of [ANDROID_TEMPLATE, ...ANDROID_TEMPLATE_SOURCES]) { + if (!manifest.files[path]) fail(`the runtime lacks ${path}; build it with tools/runtime/android.ts`); } return manifest; } -/** The launcher icon files, by path under `res/`: the identity's icon scaled, or the runtime's default. */ -async function launcherIcons(runtime: Map<string, Uint8Array>, icon: Uint8Array | undefined): Promise<Map<string, Uint8Array>> { - const files = new Map<string, Uint8Array>(); - if (!icon) { - for (const [density] of ANDROID_ICON_DENSITIES) { - const path = `mipmap-${density}/icon.png`; - const bytes = runtime.get(`res/${path}`); - if (!bytes) throw new Error(`repack android: no icon given and the runtime has no res/${path}`); - files.set(path, bytes); - } - return files; +interface Template { + readonly entries: readonly ZipEntry[]; + readonly entry: (name: string) => ZipEntry; + /** The launcher icon's entry name per density (`res/mipmap-<density>-v4/icon.png`). */ + readonly icons: ReadonlyMap<string, string>; +} + +/** template.apk's entries, checked: the files the repack replaces and the runtime's program are there, no signature. */ +function readTemplate(bytes: Uint8Array, runtime: AndroidRuntimeManifest): Template { + const entries = readZip(bytes); + const byName = new Map(entries.map((entry) => [entry.name, entry])); + const entry = (name: string) => byName.get(name) ?? fail(`${ANDROID_TEMPLATE} has no ${name}`); + for (const name of ["AndroidManifest.xml", "resources.arsc", "assets/app.js", "assets/app.pak", "classes.dex", ...runtime.android.abis.map((abi) => `lib/${abi}/libpocketjs.so`)]) { + entry(name); } - const image = await decodePng(icon); - for (const [density, side] of ANDROID_ICON_DENSITIES) { - files.set(`mipmap-${density}/icon.png`, await encodePng(squareIcon(image, side))); + if (entries.some((candidate) => candidate.name.startsWith("META-INF/"))) fail(`${ANDROID_TEMPLATE} is signed; the runtime's template is unsigned`); + const icons = new Map<string, string>(); + for (const [density] of ANDROID_ICON_DENSITIES) { + const found = entries.filter((candidate) => new RegExp(`^res/mipmap-${density}(-v\\d+)?/icon\\.png$`).test(candidate.name)); + if (found.length !== 1) fail(`${ANDROID_TEMPLATE} has ${found.length} ${density} launcher icons`); + icons.set(density, found[0]!.name); } - return files; + return { entries, entry, icons }; +} + +interface AdmittedGame { + readonly names: AndroidPackageIdentity; + readonly runtime: AndroidRuntimeManifest; + readonly template: Template; + readonly js: Uint8Array; + readonly pak: Uint8Array; + /** The identity's icon at each density, by density; none without an icon. */ + readonly icons: ReadonlyMap<string, Uint8Array> | null; } -/** Checks the inputs and lays out the APK's files: the TypeScript half of the repack. */ -export async function prepareAndroidJob(input: RepackInput): Promise<AndroidNativeJob> { +/** The checks every Android repack makes, and the game's files. */ +async function admitAndroidGame(input: RepackInput): Promise<AdmittedGame> { checkIdentity(input.identity); const names = androidIdentity(input.identity); const runtime = await readAndroidRuntime(input.runtime); @@ -183,70 +285,145 @@ export async function prepareAndroidJob(input: RepackInput): Promise<AndroidNati const viewport = game.plan.viewport as { logical?: readonly number[]; rasterDensity?: number } | undefined; const [width, height] = runtime.viewport.logical; if (viewport?.logical?.[0] !== width || viewport.logical[1] !== height || viewport.rasterDensity !== runtime.viewport.rasterDensity) { - throw new Error( - `repack android: the game is laid out at ${viewport?.logical?.join("x")} density ${viewport?.rasterDensity}, ` + + fail( + `the game is laid out at ${viewport?.logical?.join("x")} density ${viewport?.rasterDensity}, ` + `and the runtime draws ${width}x${height} density ${runtime.viewport.rasterDensity}`, ); } const js = findSection(game.variant, POCKET_SECTION.js)!; const pak = findSection(game.variant, POCKET_SECTION.pak); - if (js[js.length - 1] !== 0) throw new Error(`repack android: the ${runtime.target} js section is not NUL-terminated`); - if (!pak) throw new Error(`repack android: the .pocket's ${runtime.target} variant has no pak`); - const { android } = runtime; - const decoder = new TextDecoder(); - const manifest = render(decoder.decode(input.runtime.get("AndroidManifest.xml")!), { - PACKAGE: names.packageName, - VERSION_CODE: names.versionCode, - VERSION_NAME: xmlEscape(names.versionName), - MIN_SDK: android.minSdkVersion, - TARGET_SDK: android.targetSdkVersion, - DEBUGGABLE: "false", + if (js[js.length - 1] !== 0) fail(`the ${runtime.target} js section is not NUL-terminated`); + if (!pak) fail(`the .pocket's ${runtime.target} variant has no pak`); + const template = readTemplate(input.runtime.get(ANDROID_TEMPLATE)!, runtime); + let icons: Map<string, Uint8Array> | null = null; + if (input.identity.icon) { + const image = await decodePng(input.identity.icon); + icons = new Map(); + for (const [density, side] of ANDROID_ICON_DENSITIES) icons.set(density, await encodePng(squareIcon(image, side))); + } + // PocketActivity hands app.js to QuickJS with its length: the section's NUL stays out. + return { names, runtime, template, js: js.subarray(0, js.length - 1), pak, icons }; +} + +/** Where zipalign -p 4 puts a stored entry's data: .so files on 4096 bytes, the rest on 4. */ +function alignment(name: string): number { + return name.endsWith(".so") ? 4096 : 4; +} + +/** The signer's certificate, checked against the one the caller expects (default: the community key's). */ +async function checkSigner(input: RepackInput, options: AndroidRepackOptions): Promise<NonNullable<RepackInput["signer"]>> { + const signer = input.signer; + if (!signer?.privateKey || !(signer.certificate instanceof Uint8Array)) { + fail("no signer: an APK is signed with the Pocket Studio community key ({ privateKey, certificate })"); + } + const expected = options.certificateSha256 ?? POCKET_STUDIO_COMMUNITY_SIGNER; + const actual = await sha256Hex(signer.certificate); + if (actual !== expected) { + fail( + expected === POCKET_STUDIO_COMMUNITY_SIGNER + ? `the signer's certificate is ${actual}, not the Pocket Studio community certificate ${POCKET_STUDIO_COMMUNITY_SIGNER}` + : `the signer's certificate is ${actual}, not ${expected}`, + ); + } + return signer; +} + +/** The signed APK of a game. */ +export async function repackAndroid(input: RepackInput, options: AndroidRepackOptions = {}): Promise<Uint8Array> { + const signer = await checkSigner(input, options); + const { names, runtime, template, js, pak, icons } = await admitAndroidGame(input); + + const templateManifestBytes = await unzipEntry(template.entry("AndroidManifest.xml")); + const templateManifest = readManifest(templateManifestBytes); + if ( + templateManifest.minSdkVersion !== runtime.android.minSdkVersion || + templateManifest.targetSdkVersion !== runtime.android.targetSdkVersion || + templateManifest.debuggable + ) { + fail(`${ANDROID_TEMPLATE} is not linked for SDK ${runtime.android.minSdkVersion} to ${runtime.android.targetSdkVersion}, not debuggable`); + } + const manifest = patchManifest(templateManifestBytes, names); + const table = patchResourceTable(await unzipEntry(template.entry("resources.arsc")), names); + const check = readManifest(manifest); + if (check.packageName !== names.packageName || check.versionCode !== names.versionCode || check.versionName !== names.versionName) { + fail("the rewritten manifest does not read back as the identity"); + } + if (readResourceTable(table).label !== names.label) fail("the rewritten resource table does not read back the label"); + + const replaced = new Map<string, { readonly data: Uint8Array; readonly compress: boolean }>([ + // aapt2 deflates the manifest and stores the table, which Android maps from the APK. + ["AndroidManifest.xml", { data: manifest, compress: true }], + ["resources.arsc", { data: table, compress: false }], + ["assets/app.js", { data: js, compress: true }], + ["assets/app.pak", { data: pak, compress: true }], + ]); + if (icons) { + // aapt2 stores PNG files as they are. + for (const [density, name] of template.icons) replaced.set(name, { data: icons.get(density)!, compress: false }); + } + const entries: ZipInput[] = template.entries.map((entry) => { + const change = replaced.get(entry.name); + if (change) return { name: entry.name, data: change.data, compress: change.compress, align: alignment(entry.name) }; + return { name: entry.name, entry, ...(entry.method === ZIP_STORED ? { align: alignment(entry.name) } : {}) }; }); - const resources = new Map<string, Uint8Array>(); - resources.set("values/strings.xml", new TextEncoder().encode(render(decoder.decode(input.runtime.get("res/values/strings.xml")!), { - TITLE: androidString(names.label), - }))); - for (const [path, bytes] of await launcherIcons(input.runtime, input.identity.icon)) resources.set(path, bytes); + return signApk(entries, signer); +} + +/** + * The files aapt2 links for the same game as repackAndroid: the rendered + * sources, the scaled or default icons, the assets, and classes.dex and the + * libraries from template.apk. tests/repack-android.test.ts links it with + * aapt2 to compare. + */ +export async function androidLinkJob(input: RepackInput): Promise<AndroidLinkJob> { + const { names, runtime, template, js, pak, icons } = await admitAndroidGame(input); + const decoder = new TextDecoder(); + const sources = renderAndroidSources( + { manifest: decoder.decode(input.runtime.get(ANDROID_TEMPLATE_SOURCES[0])), strings: decoder.decode(input.runtime.get(ANDROID_TEMPLATE_SOURCES[1])) }, + names, + runtime.android, + ); + const resources = new Map<string, Uint8Array>([["values/strings.xml", new TextEncoder().encode(sources.strings)]]); + for (const [density, name] of template.icons) { + resources.set(`mipmap-${density}/icon.png`, icons?.get(density) ?? (await unzipEntry(template.entry(name)))); + } const entries = new Map<string, Uint8Array>(); - entries.set("classes.dex", input.runtime.get("classes.dex")!); - for (const abi of [...android.abis].sort()) entries.set(`lib/${abi}/libpocketjs.so`, input.runtime.get(`lib/${abi}/libpocketjs.so`)!); + entries.set("classes.dex", await unzipEntry(template.entry("classes.dex"))); + for (const abi of [...runtime.android.abis].sort()) { + entries.set(`lib/${abi}/libpocketjs.so`, await unzipEntry(template.entry(`lib/${abi}/libpocketjs.so`))); + } return { - manifest, + manifest: sources.manifest, resources, - // PocketActivity hands app.js to QuickJS with its length: the section's NUL stays out. - assets: new Map([["app.js", js.subarray(0, js.length - 1)], ["app.pak", pak]]), + assets: new Map([["app.js", js], ["app.pak", pak]]), entries, sdk: { - apiLevel: android.apiLevel, - buildToolsVersion: android.buildToolsVersion, - minSdkVersion: android.minSdkVersion, - targetSdkVersion: android.targetSdkVersion, + apiLevel: runtime.android.apiLevel, + buildToolsVersion: runtime.android.buildToolsVersion, + minSdkVersion: runtime.android.minSdkVersion, + targetSdkVersion: runtime.android.targetSdkVersion, }, }; } -/** The entries every repacked APK holds, besides `res/` files. */ -export function expectedApkEntries(job: AndroidNativeJob): string[] { - return [ - "AndroidManifest.xml", - "resources.arsc", - ...[...job.assets.keys()].map((path) => `assets/${path}`), - ...job.entries.keys(), - "META-INF/MANIFEST.MF", - ]; -} - /** - * The APK of a game: `input` checked and laid out here, then linked, aligned - * and signed by `options.native` (default: the local Android SDK and the - * Pocket Studio community key). + * What aapt2 links into template.apk: the sources rendered with + * ANDROID_TEMPLATE_IDENTITY, the default icon, empty assets, and the + * runtime's classes.dex and libraries. `files` holds the runtime build's + * files by path: AndroidManifest.xml, res/values/strings.xml, + * res/mipmap-<density>/icon.png, classes.dex, lib/<abi>/libpocketjs.so. */ -export async function repackAndroid(input: RepackInput, options: { native?: AndroidNativeSteps } = {}): Promise<Uint8Array> { - const job = await prepareAndroidJob(input); - const native = options.native ?? (await import("./android-native.ts")).androidNativeSteps(); - const apk = await native(job); - const names = new Set(readZip(apk).map((entry) => entry.name)); - const missing = expectedApkEntries(job).filter((name) => !names.has(name)); - if (missing.length) throw new Error(`repack android: the native steps left out ${missing.join(", ")}`); - return apk; +export function androidTemplateJob(files: ReadonlyMap<string, Uint8Array>, sdk: AndroidLinkJob["sdk"], abis: readonly string[]): AndroidLinkJob { + const file = (path: string) => files.get(path) ?? fail(`the runtime build has no ${path}`); + const decoder = new TextDecoder(); + const sources = renderAndroidSources( + { manifest: decoder.decode(file(ANDROID_TEMPLATE_SOURCES[0])), strings: decoder.decode(file(ANDROID_TEMPLATE_SOURCES[1])) }, + androidIdentity(ANDROID_TEMPLATE_IDENTITY), + sdk, + ); + const resources = new Map<string, Uint8Array>([["values/strings.xml", new TextEncoder().encode(sources.strings)]]); + for (const [density] of ANDROID_ICON_DENSITIES) resources.set(`mipmap-${density}/icon.png`, file(`res/mipmap-${density}/icon.png`)); + const entries = new Map<string, Uint8Array>([["classes.dex", file("classes.dex")]]); + for (const abi of [...abis].sort()) entries.set(`lib/${abi}/libpocketjs.so`, file(`lib/${abi}/libpocketjs.so`)); + return { manifest: sources.manifest, resources, assets: new Map([["app.js", new Uint8Array()], ["app.pak", new Uint8Array()]]), entries, sdk }; } diff --git a/tools/repack/index.ts b/tools/repack/index.ts new file mode 100644 index 000000000..eb1a34773 --- /dev/null +++ b/tools/repack/index.ts @@ -0,0 +1,93 @@ +// The repack entry a Pocket Studio Worker imports: one row per Studio device +// target, each a function from bytes to bytes with the facts the Studio +// needs to store and serve its result. No file system, no network, no +// Node or Bun API in this module or anything it imports +// (tests/repack-worker.test.ts bundles it for the browser and checks). +// +// // Pocket Studio holds PocketJS as the submodule vendor/pocketjs. +// import { REPACK_TARGETS } from "../vendor/pocketjs/tools/repack/index.ts"; +// const target = REPACK_TARGETS.android; +// const apk = await target.repack({ runtime, pocket, identity, signer }); +// +// `runtime` is the files of `bun tools/runtime.ts <target.runtime>`'s output +// directory by relative path, runtime.json included; `pocket` holds a +// variant for one of `target.variantTargets`, the one the runtime's +// runtime.json names. Android alone takes `signer`. + +import { ANDROID_RUNTIME_TARGETS, repackAndroid } from "./android.ts"; +import { THREE_DS_RUNTIME_TARGET, repack3ds } from "./3ds.ts"; +import { IPOD_RUNTIME_TARGET, repackIPod } from "./ipod.ts"; +import { PSP_RUNTIME_TARGET, repackPsp } from "./psp.ts"; +import type { RepackInput } from "./shared/runtime.ts"; +import { VITA_RUNTIME_TARGET, repackVita } from "./vita.ts"; + +export type { ApkSigner } from "./shared/apk-sign.ts"; +export type { RepackIdentity, RepackInput, RuntimeManifest } from "./shared/runtime.ts"; +export { POCKET_STUDIO_COMMUNITY_SIGNER, androidPackageName } from "./android.ts"; + +/** The device targets of Pocket Studio, as its catalog names them. */ +export type StudioTarget = "psp" | "vita" | "3ds" | "ipod-touch" | "android"; + +export interface RepackTarget { + /** A game's package from a runtime directory and the game's `.pocket`. */ + readonly repack: (input: RepackInput) => Promise<Uint8Array>; + /** `bun tools/runtime.ts <runtime>`: the runtime's name and its directory's name. */ + readonly runtime: "psp" | "vita" | "3ds" | "ipod" | "android"; + /** The `.pocket` targets this target's runtimes boot; the runtime's runtime.json names the one it was built for. */ + readonly variantTargets: readonly string[]; + /** The package file's extension, dot included. */ + readonly extension: ".zip" | ".vpk" | ".3dsx" | ".ipa" | ".apk"; + /** The Content-Type a download of the package is served with. */ + readonly contentType: string; + /** Whether `input.signer` is required. */ + readonly signed: boolean; +} + +export const REPACK_TARGETS: Readonly<Record<StudioTarget, RepackTarget>> = { + psp: { + repack: repackPsp, + runtime: "psp", + variantTargets: [PSP_RUNTIME_TARGET], + extension: ".zip", + contentType: "application/zip", + signed: false, + }, + vita: { + repack: repackVita, + runtime: "vita", + variantTargets: [VITA_RUNTIME_TARGET], + extension: ".vpk", + contentType: "application/octet-stream", + signed: false, + }, + "3ds": { + repack: repack3ds, + runtime: "3ds", + variantTargets: [THREE_DS_RUNTIME_TARGET], + extension: ".3dsx", + contentType: "application/octet-stream", + signed: false, + }, + "ipod-touch": { + repack: repackIPod, + runtime: "ipod", + variantTargets: [IPOD_RUNTIME_TARGET], + extension: ".ipa", + contentType: "application/octet-stream", + signed: false, + }, + android: { + repack: (input) => repackAndroid(input), + runtime: "android", + variantTargets: [...ANDROID_RUNTIME_TARGETS], + extension: ".apk", + contentType: "application/vnd.android.package-archive", + signed: true, + }, +}; + +/** A row by Studio target or by runtime name (`ipod` is `ipod-touch`); undefined for anything else. */ +export function repackTarget(name: string): RepackTarget | undefined { + if (Object.hasOwn(REPACK_TARGETS, name)) return REPACK_TARGETS[name as StudioTarget]; + return Object.values(REPACK_TARGETS).find((target) => target.runtime === name); +} diff --git a/tools/repack/shared/android-resources.ts b/tools/repack/shared/android-resources.ts new file mode 100644 index 000000000..00defd37d --- /dev/null +++ b/tools/repack/shared/android-resources.ts @@ -0,0 +1,588 @@ +// Binary Android resources as aapt2 writes them: string pools, the compiled +// XML of AndroidManifest.xml and the resource table (resources.arsc). The +// Android repack (tools/repack/android.ts) changes the identity of a template +// APK that aapt2 linked when the runtime was built; this file rewrites the +// two binary files the way aapt2 would have written them for that identity. +// +// - A string pool is a 28-byte header, one u32 offset per string, the +// strings (UTF-16 in the manifest, modified UTF-8 in the table: a +// character past U+FFFF is two 3-byte surrogates; length-prefixed and +// NUL-terminated) and zero padding to 4 bytes. No styles. +// - aapt2 sorts every pool: the manifest's attribute names that carry a +// resource id come first, in the resource map's order, then every other +// string by its UTF-8 bytes; the table's global pool by UTF-8 bytes. One +// entry per distinct string, and only strings something refers to. +// - A changed string goes to its sorted place, and every reference to the +// pool (element and attribute names, raw values, typed string values, +// namespaces, a table entry's string value) is renumbered with it. +// +// tests/repack-android.test.ts compares the result with aapt2's own link of +// the same identity, byte for byte, where the Android SDK is installed. + +const RES_STRING_POOL = 0x0001; +const RES_TABLE = 0x0002; +const RES_XML = 0x0003; +const RES_XML_START_NAMESPACE = 0x0100; +const RES_XML_END_NAMESPACE = 0x0101; +const RES_XML_START_ELEMENT = 0x0102; +const RES_XML_END_ELEMENT = 0x0103; +const RES_XML_CDATA = 0x0104; +const RES_XML_RESOURCE_MAP = 0x0180; +const RES_TABLE_PACKAGE = 0x0200; +const RES_TABLE_TYPE = 0x0201; +const RES_TABLE_TYPE_SPEC = 0x0202; + +const UTF8_FLAG = 1 << 8; +const NO_REFERENCE = 0xffffffff; +const TYPE_STRING = 0x03; +const TYPE_INT_DEC = 0x10; +const TYPE_INT_BOOLEAN = 0x12; +const ENTRY_FLAG_COMPLEX = 0x0001; +const ENTRY_FLAG_COMPACT = 0x0008; + +/** Resource ids of the manifest attributes the repack reads or writes (android.R.attr). */ +export const ANDROID_ATTR = { + label: 0x01010001, + debuggable: 0x0101000f, + minSdkVersion: 0x0101020c, + versionCode: 0x0101021b, + versionName: 0x0101021c, + targetSdkVersion: 0x01010270, +} as const; + +/** The longest package name a resource table holds: 128 UTF-16 units with the NUL. */ +export const MAX_PACKAGE_NAME = 127; + +function view(bytes: Uint8Array): DataView { + return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); +} + +function fail(message: string): never { + throw new Error(`android resources: ${message}`); +} + +interface Chunk { + readonly type: number; + readonly headerSize: number; + readonly size: number; + readonly at: number; +} + +/** The chunks laid end to end from `start` to `end`. */ +function chunks(bytes: Uint8Array, start: number, end: number): Chunk[] { + const data = view(bytes); + const out: Chunk[] = []; + let at = start; + while (at < end) { + if (at + 8 > end) fail(`a chunk header at ${at} runs past its parent`); + const type = data.getUint16(at, true); + const headerSize = data.getUint16(at + 2, true); + const size = data.getUint32(at + 4, true); + if (headerSize < 8 || size < headerSize || at + size > end) fail(`chunk 0x${type.toString(16)} at ${at} has a bad size`); + out.push({ type, headerSize, size, at }); + at += size; + } + return out; +} + +// ------------------------------------------------------------ string pools + +export interface StringPool { + readonly strings: string[]; + readonly utf8: boolean; + /** The pool chunk's size in bytes. */ + readonly size: number; +} + +const utf16Decoder = new TextDecoder("utf-16le", { fatal: true }); +const utf8Encoder = new TextEncoder(); + +/** Modified UTF-8, as aapt2 writes a UTF-8 pool: UTF-16 units encoded one by one, so a surrogate pair is two 3-byte sequences. */ +function encodeModifiedUtf8(text: string): Uint8Array { + const out: number[] = []; + for (let index = 0; index < text.length; index++) { + const unit = text.charCodeAt(index); + if (unit < 0x80) out.push(unit); + else if (unit < 0x800) out.push(0xc0 | (unit >> 6), 0x80 | (unit & 0x3f)); + else out.push(0xe0 | (unit >> 12), 0x80 | ((unit >> 6) & 0x3f), 0x80 | (unit & 0x3f)); + } + return Uint8Array.from(out); +} + +/** UTF-8 or modified UTF-8 bytes as a string. */ +function decodeModifiedUtf8(bytes: Uint8Array): string { + const units: number[] = []; + for (let at = 0; at < bytes.length;) { + const lead = bytes[at]!; + const next = (offset: number) => { + const byte = bytes[at + offset]; + if (byte === undefined || (byte & 0xc0) !== 0x80) fail("a UTF-8 pool string is malformed"); + return byte & 0x3f; + }; + if (lead < 0x80) { + units.push(lead); + at += 1; + } else if ((lead & 0xe0) === 0xc0) { + units.push(((lead & 0x1f) << 6) | next(1)); + at += 2; + } else if ((lead & 0xf0) === 0xe0) { + units.push(((lead & 0x0f) << 12) | (next(1) << 6) | next(2)); + at += 3; + } else if ((lead & 0xf8) === 0xf0) { + const point = (((lead & 0x07) << 18) | (next(1) << 12) | (next(2) << 6) | next(3)) - 0x10000; + units.push(0xd800 | (point >> 10), 0xdc00 | (point & 0x3ff)); + at += 4; + } else { + fail("a UTF-8 pool string is malformed"); + } + } + let text = ""; + for (let index = 0; index < units.length; index += 0x8000) text += String.fromCharCode(...units.slice(index, index + 0x8000)); + return text; +} + +/** The string pool chunk at `at`. Refuses a pool with styles. */ +export function readStringPool(bytes: Uint8Array, at: number): StringPool { + const data = view(bytes); + if (data.getUint16(at, true) !== RES_STRING_POOL) fail(`no string pool at ${at}`); + const headerSize = data.getUint16(at + 2, true); + const size = data.getUint32(at + 4, true); + const count = data.getUint32(at + 8, true); + const styleCount = data.getUint32(at + 12, true); + const flags = data.getUint32(at + 16, true); + const stringsStart = data.getUint32(at + 20, true); + if (styleCount !== 0) fail("a pool with styled strings is not read here"); + if (at + size > bytes.length || headerSize + count * 4 > size) fail(`the string pool at ${at} runs past its chunk`); + const utf8 = (flags & UTF8_FLAG) !== 0; + const strings: string[] = []; + for (let index = 0; index < count; index++) { + let cursor = at + stringsStart + data.getUint32(at + headerSize + index * 4, true); + if (utf8) { + if (bytes[cursor++]! & 0x80) cursor++; // the UTF-16 length, unused here + let length = bytes[cursor++]!; + if (length & 0x80) length = ((length & 0x7f) << 8) | bytes[cursor++]!; + strings.push(decodeModifiedUtf8(bytes.subarray(cursor, cursor + length))); + } else { + let length = data.getUint16(cursor, true); + cursor += 2; + if (length & 0x8000) { + length = ((length & 0x7fff) << 16) | data.getUint16(cursor, true); + cursor += 2; + } + strings.push(utf16Decoder.decode(bytes.subarray(cursor, cursor + length * 2))); + } + } + return { strings, utf8, size }; +} + +/** A string pool chunk as aapt2 writes one: the strings in order, no styles, padded to 4 bytes. */ +export function writeStringPool(strings: readonly string[], utf8: boolean): Uint8Array { + const parts: number[] = []; + const offsets: number[] = []; + for (const text of strings) { + offsets.push(parts.length); + if (utf8) { + const encoded = encodeModifiedUtf8(text); + if (text.length > 0x7fff || encoded.length > 0x7fff) fail(`a string of ${encoded.length} bytes is past the pool's length field`); + for (const length of [text.length, encoded.length]) { + if (length > 0x7f) parts.push(0x80 | (length >> 8), length & 0xff); + else parts.push(length); + } + for (const byte of encoded) parts.push(byte); + parts.push(0); + } else { + if (text.length > 0x7fff) fail(`a string of ${text.length} UTF-16 units is past the pool's length field`); + parts.push(text.length & 0xff, text.length >> 8); + for (let index = 0; index < text.length; index++) { + const unit = text.charCodeAt(index); + parts.push(unit & 0xff, unit >> 8); + } + parts.push(0, 0); + } + } + while (parts.length % 4) parts.push(0); + const headerSize = 28; + const stringsStart = headerSize + offsets.length * 4; + const out = new Uint8Array(stringsStart + parts.length); + const data = view(out); + data.setUint16(0, RES_STRING_POOL, true); + data.setUint16(2, headerSize, true); + data.setUint32(4, out.length, true); + data.setUint32(8, strings.length, true); + data.setUint32(12, 0, true); + data.setUint32(16, utf8 ? UTF8_FLAG : 0, true); + data.setUint32(20, strings.length ? stringsStart : 0, true); + data.setUint32(24, 0, true); + offsets.forEach((offset, index) => data.setUint32(headerSize + index * 4, offset, true)); + out.set(parts, stringsStart); + return out; +} + +function compareUtf8(a: string, b: string): number { + const x = utf8Encoder.encode(a); + const y = utf8Encoder.encode(b); + const length = Math.min(x.length, y.length); + for (let index = 0; index < length; index++) { + if (x[index] !== y[index]) return x[index]! - y[index]!; + } + return x.length - y.length; +} + +/** + * The pool aapt2 writes for the referenced strings: the first `fixed` + * entries stay where they are (the manifest's attribute names with resource + * ids), every other referenced string once, sorted by its UTF-8 bytes. + * `remap[old index]` is the new index of a referenced entry. + */ +function sortPool(strings: readonly string[], referenced: ReadonlySet<number>, fixed: number): { strings: string[]; remap: Map<number, number> } { + const head = strings.slice(0, fixed); + const rest = [...new Set([...referenced].filter((index) => index >= fixed).map((index) => strings[index]!))].sort(compareUtf8); + const place = new Map(rest.map((text, index) => [text, fixed + index])); + const remap = new Map<number, number>(); + for (const index of referenced) remap.set(index, index < fixed ? index : place.get(strings[index]!)!); + return { strings: [...head, ...rest], remap }; +} + +function splice(bytes: Uint8Array, at: number, removed: number, inserted: Uint8Array): Uint8Array { + const out = new Uint8Array(bytes.length - removed + inserted.length); + out.set(bytes.subarray(0, at)); + out.set(inserted, at); + out.set(bytes.subarray(at + removed), at + inserted.length); + return out; +} + +// ------------------------------------------------------------ compiled XML + +/** A place in the file that holds a string pool index. */ +type Reference = number; + +interface XmlAttribute { + readonly at: number; + readonly namespace: number; + readonly name: number; + readonly rawValue: number; + readonly dataType: number; + readonly data: number; +} + +interface XmlElement { + readonly name: string; + readonly attributes: readonly XmlAttribute[]; +} + +interface XmlDocument { + readonly pool: StringPool; + readonly resourceIds: readonly number[]; + readonly elements: readonly XmlElement[]; + /** Every place that holds a pool index. */ + readonly references: readonly Reference[]; +} + +function readXml(bytes: Uint8Array): XmlDocument { + const data = view(bytes); + if (bytes.length < 8 || data.getUint16(0, true) !== RES_XML || data.getUint32(4, true) !== bytes.length) { + fail("AndroidManifest.xml is not compiled XML"); + } + const parts = chunks(bytes, data.getUint16(2, true), bytes.length); + if (parts[0]?.type !== RES_STRING_POOL) fail("the compiled XML does not start with its string pool"); + const pool = readStringPool(bytes, parts[0].at); + const resourceIds: number[] = []; + const elements: XmlElement[] = []; + const references: Reference[] = []; + const reference = (at: number) => { + const index = data.getUint32(at, true); + if (index === NO_REFERENCE) return; + if (index >= pool.strings.length) fail(`a reference at ${at} is past the string pool`); + references.push(at); + }; + for (const chunk of parts.slice(1)) { + const { at, headerSize, size } = chunk; + if (chunk.type === RES_XML_RESOURCE_MAP) { + for (let cursor = at + headerSize; cursor < at + size; cursor += 4) resourceIds.push(data.getUint32(cursor, true)); + continue; + } + if (chunk.type < RES_XML_START_NAMESPACE || chunk.type > RES_XML_CDATA) fail(`compiled XML chunk 0x${chunk.type.toString(16)} is not read here`); + reference(at + 12); // the node's comment + const body = at + headerSize; + switch (chunk.type) { + case RES_XML_START_NAMESPACE: + case RES_XML_END_NAMESPACE: + case RES_XML_END_ELEMENT: + reference(body); + reference(body + 4); + break; + case RES_XML_CDATA: + reference(body); + if (bytes[body + 7] === TYPE_STRING) reference(body + 8); + break; + case RES_XML_START_ELEMENT: { + reference(body); + reference(body + 4); + const attributeStart = data.getUint16(body + 8, true); + const attributeSize = data.getUint16(body + 10, true); + const count = data.getUint16(body + 12, true); + if (attributeSize < 20) fail("an attribute record is shorter than 20 bytes"); + const attributes: XmlAttribute[] = []; + for (let index = 0; index < count; index++) { + const record = body + attributeStart + index * attributeSize; + if (record + 20 > at + size) fail("an attribute runs past its element"); + reference(record); + reference(record + 4); + reference(record + 8); + const dataType = bytes[record + 15]!; + if (dataType === TYPE_STRING) reference(record + 16); + attributes.push({ + at: record, + namespace: data.getUint32(record, true), + name: data.getUint32(record + 4, true), + rawValue: data.getUint32(record + 8, true), + dataType, + data: data.getUint32(record + 16, true), + }); + } + const name = data.getUint32(body + 4, true); + elements.push({ name: pool.strings[name] ?? "", attributes }); + break; + } + } + } + return { pool, resourceIds, elements, references }; +} + +function attributeById(document: XmlDocument, element: XmlElement, id: number): XmlAttribute | undefined { + return element.attributes.find((attribute) => document.resourceIds[attribute.name] === id); +} + +function attributeByName(document: XmlDocument, element: XmlElement, name: string): XmlAttribute | undefined { + return element.attributes.find((attribute) => + attribute.namespace === NO_REFERENCE && document.resourceIds[attribute.name] === undefined && document.pool.strings[attribute.name] === name + ); +} + +function element(document: XmlDocument, name: string): XmlElement { + const found = document.elements.filter((candidate) => candidate.name === name); + if (found.length !== 1) fail(`the manifest has ${found.length} <${name}> elements`); + return found[0]!; +} + +function stringValue(document: XmlDocument, attribute: XmlAttribute | undefined, what: string): string { + if (!attribute || attribute.dataType !== TYPE_STRING || attribute.rawValue !== attribute.data) fail(`the manifest's ${what} is not a string`); + return document.pool.strings[attribute.data]!; +} + +function intValue(attribute: XmlAttribute | undefined, what: string): number { + if (!attribute || attribute.dataType !== TYPE_INT_DEC) fail(`the manifest's ${what} is not a decimal integer`); + return attribute.data; +} + +/** What the repack reads back from a compiled AndroidManifest.xml. */ +export interface ManifestSummary { + readonly packageName: string; + readonly versionCode: number; + readonly versionName: string; + readonly minSdkVersion: number; + readonly targetSdkVersion: number; + /** `android:debuggable` on <application>; false when absent. */ + readonly debuggable: boolean; + /** The resource id `android:label` on <application> refers to. */ + readonly labelResource: number; +} + +export function readManifest(axml: Uint8Array): ManifestSummary { + const document = readXml(axml); + const manifest = element(document, "manifest"); + const usesSdk = element(document, "uses-sdk"); + const application = element(document, "application"); + const debuggable = attributeById(document, application, ANDROID_ATTR.debuggable); + if (debuggable && debuggable.dataType !== TYPE_INT_BOOLEAN) fail("android:debuggable is not a boolean"); + const label = attributeById(document, application, ANDROID_ATTR.label); + if (!label || label.dataType !== 0x01) fail("<application> has no android:label reference"); + return { + packageName: stringValue(document, attributeByName(document, manifest, "package"), "package"), + versionCode: intValue(attributeById(document, manifest, ANDROID_ATTR.versionCode), "android:versionCode"), + versionName: stringValue(document, attributeById(document, manifest, ANDROID_ATTR.versionName), "android:versionName"), + minSdkVersion: intValue(attributeById(document, usesSdk, ANDROID_ATTR.minSdkVersion), "android:minSdkVersion"), + targetSdkVersion: intValue(attributeById(document, usesSdk, ANDROID_ATTR.targetSdkVersion), "android:targetSdkVersion"), + debuggable: debuggable ? debuggable.data !== 0 : false, + labelResource: label.data, + }; +} + +/** + * The compiled manifest with `package`, `android:versionCode` and + * `android:versionName` on <manifest> replaced, its pool sorted as aapt2 sorts + * it and every reference renumbered. + */ +export function patchManifest( + axml: Uint8Array, + identity: { readonly packageName: string; readonly versionCode: number; readonly versionName: string }, +): Uint8Array { + if (!Number.isInteger(identity.versionCode) || identity.versionCode < 1 || identity.versionCode > 0x7fffffff) { + fail(`versionCode ${identity.versionCode} is not a positive 31-bit integer`); + } + const document = readXml(axml); + const manifest = element(document, "manifest"); + const packageAttribute = attributeByName(document, manifest, "package"); + const versionName = attributeById(document, manifest, ANDROID_ATTR.versionName); + const versionCode = attributeById(document, manifest, ANDROID_ATTR.versionCode); + stringValue(document, packageAttribute, "package"); + stringValue(document, versionName, "android:versionName"); + intValue(versionCode, "android:versionCode"); + + const out = new Uint8Array(axml); + const data = view(out); + // The new strings go past the end of the pool, the two attributes point at them, and the sort places them. + const strings = [...document.pool.strings, identity.packageName, identity.versionName]; + const packageIndex = strings.length - 2; + const versionNameIndex = strings.length - 1; + for (const [attribute, index] of [[packageAttribute!, packageIndex], [versionName!, versionNameIndex]] as const) { + data.setUint32(attribute.at + 8, index, true); + data.setUint32(attribute.at + 16, index, true); + } + data.setUint32(versionCode!.at + 16, identity.versionCode, true); + + const referenced = new Set<number>(); + for (const at of document.references) referenced.add(data.getUint32(at, true)); + const fixed = document.resourceIds.length; + for (let index = 0; index < fixed; index++) referenced.add(index); + const sorted = sortPool(strings, referenced, fixed); + for (const at of document.references) data.setUint32(at, sorted.remap.get(data.getUint32(at, true))!, true); + + const poolAt = data.getUint16(2, true); + const result = splice(out, poolAt, document.pool.size, writeStringPool(sorted.strings, document.pool.utf8)); + view(result).setUint32(4, result.length, true); + return result; +} + +// ---------------------------------------------------------- resource table + +interface TableValue { + /** Where the Res_value's data field is. */ + readonly at: number; + readonly type: string; + readonly key: string; + readonly defaultConfig: boolean; + readonly dataType: number; + readonly data: number; +} + +interface Table { + readonly pool: StringPool; + readonly poolAt: number; + readonly packageAt: number; + readonly packageId: number; + readonly packageName: string; + readonly values: readonly TableValue[]; +} + +function readTable(arsc: Uint8Array): Table { + const data = view(arsc); + if (arsc.length < 12 || data.getUint16(0, true) !== RES_TABLE || data.getUint32(4, true) !== arsc.length) { + fail("resources.arsc is not a resource table"); + } + if (data.getUint32(8, true) !== 1) fail("the resource table holds more than one package"); + const parts = chunks(arsc, data.getUint16(2, true), arsc.length); + if (parts.length !== 2 || parts[0]!.type !== RES_STRING_POOL || parts[1]!.type !== RES_TABLE_PACKAGE) { + fail("the resource table is not one string pool and one package"); + } + const pool = readStringPool(arsc, parts[0]!.at); + const packageChunk = parts[1]!; + const packageAt = packageChunk.at; + const packageId = data.getUint32(packageAt + 8, true); + const nameUnits: number[] = []; + for (let index = 0; index < 128; index++) { + const unit = data.getUint16(packageAt + 12 + index * 2, true); + if (unit === 0) break; + nameUnits.push(unit); + } + const typeStrings = readStringPool(arsc, packageAt + data.getUint32(packageAt + 268, true)).strings; + const keyStrings = readStringPool(arsc, packageAt + data.getUint32(packageAt + 276, true)).strings; + const values: TableValue[] = []; + for (const chunk of chunks(arsc, packageAt + packageChunk.headerSize, packageAt + packageChunk.size)) { + if (chunk.type === RES_STRING_POOL || chunk.type === RES_TABLE_TYPE_SPEC) continue; + if (chunk.type !== RES_TABLE_TYPE) fail(`resource table chunk 0x${chunk.type.toString(16)} is not read here`); + const { at } = chunk; + const typeName = typeStrings[arsc[at + 8]! - 1]; + if (typeName === undefined) fail(`a type chunk names type ${arsc[at + 8]}, which the package lacks`); + if (arsc[at + 9] !== 0) fail(`the ${typeName} type chunk is sparse or uses 16-bit offsets, which is not read here`); + const entryCount = data.getUint32(at + 12, true); + const entriesStart = data.getUint32(at + 16, true); + const configSize = data.getUint32(at + 20, true); + let defaultConfig = true; + for (let cursor = at + 24; cursor < at + 20 + configSize; cursor++) if (arsc[cursor] !== 0) defaultConfig = false; + for (let index = 0; index < entryCount; index++) { + const offset = data.getUint32(at + chunk.headerSize + index * 4, true); + if (offset === NO_REFERENCE) continue; + const entry = at + entriesStart + offset; + const size = data.getUint16(entry, true); + const flags = data.getUint16(entry + 2, true); + if (flags & ENTRY_FLAG_COMPACT) fail("compact table entries are not read here"); + const key = keyStrings[data.getUint32(entry + 4, true)] ?? fail("a table entry's key is past the key pool"); + const value = (valueAt: number) => { + values.push({ at: valueAt + 4, type: typeName, key, defaultConfig, dataType: arsc[valueAt + 3]!, data: data.getUint32(valueAt + 4, true) }); + }; + if (flags & ENTRY_FLAG_COMPLEX) { + const count = data.getUint32(entry + 12, true); + for (let item = 0; item < count; item++) value(entry + size + item * 12 + 4); + } else { + value(entry + size); + } + } + } + for (const value of values) { + if (value.dataType === TYPE_STRING && value.data >= pool.strings.length) fail(`the ${value.type}/${value.key} value is past the string pool`); + } + return { pool, poolAt: parts[0]!.at, packageAt, packageId, packageName: String.fromCharCode(...nameUnits), values }; +} + +function labelValue(table: Table): TableValue { + const found = table.values.filter((value) => value.type === "string" && value.key === "app_name"); + if (found.length !== 1 || !found[0]!.defaultConfig || found[0]!.dataType !== TYPE_STRING) { + fail("the resource table does not hold one string/app_name in the default configuration"); + } + return found[0]!; +} + +/** What the repack reads back from a resource table. */ +export interface TableSummary { + readonly packageId: number; + readonly packageName: string; + /** string/app_name, the launcher label. */ + readonly label: string; + /** Every string value by `type/key`, in table order (a key with several configurations repeats). */ + readonly strings: ReadonlyArray<readonly [string, string]>; +} + +export function readResourceTable(arsc: Uint8Array): TableSummary { + const table = readTable(arsc); + return { + packageId: table.packageId, + packageName: table.packageName, + label: table.pool.strings[labelValue(table).data]!, + strings: table.values.filter((value) => value.dataType === TYPE_STRING).map((value) => [`${value.type}/${value.key}`, table.pool.strings[value.data]!]), + }; +} + +/** + * The resource table with its package's name and string/app_name replaced, + * the global pool sorted as aapt2 sorts it and every string value renumbered. + */ +export function patchResourceTable(arsc: Uint8Array, identity: { readonly packageName: string; readonly label: string }): Uint8Array { + if (identity.packageName.length > MAX_PACKAGE_NAME) fail(`a package name holds at most ${MAX_PACKAGE_NAME} characters`); + const table = readTable(arsc); + const label = labelValue(table); + const out = new Uint8Array(arsc); + const data = view(out); + const strings = [...table.pool.strings, identity.label]; + data.setUint32(label.at, strings.length - 1, true); + const references = table.values.filter((value) => value.dataType === TYPE_STRING).map((value) => value.at); + const referenced = new Set(references.map((at) => data.getUint32(at, true))); + const sorted = sortPool(strings, referenced, 0); + for (const at of references) data.setUint32(at, sorted.remap.get(data.getUint32(at, true))!, true); + for (let index = 0; index < 128; index++) { + data.setUint16(table.packageAt + 12 + index * 2, index < identity.packageName.length ? identity.packageName.charCodeAt(index) : 0, true); + } + const result = splice(out, table.poolAt, table.pool.size, writeStringPool(sorted.strings, table.pool.utf8)); + view(result).setUint32(4, result.length, true); + return result; +} diff --git a/tools/repack/shared/apk-sign.ts b/tools/repack/shared/apk-sign.ts new file mode 100644 index 000000000..1e0579364 --- /dev/null +++ b/tools/repack/shared/apk-sign.ts @@ -0,0 +1,310 @@ +// APK signing with Web Crypto: JAR signing (v1) and APK Signature Scheme v2, +// RSA PKCS #1 v1.5 with SHA-256, the two schemes apksigner writes for a +// minSdkVersion of 18 to 23. Android 4.3 reads v1 only; Android 7 and later +// read v2 and refuse an APK whose v1 signature says v2 was stripped. +// +// v1 META-INF/MANIFEST.MF SHA-256 of every entry's uncompressed bytes +// META-INF/CERT.SF SHA-256 of the manifest and of each of its sections, +// `X-Android-APK-Signed: 2` +// META-INF/CERT.RSA PKCS #7 SignedData: the certificate and the +// signature of CERT.SF, no signed attributes +// v2 an APK Signing Block before the central directory: SHA-256 over 1 MiB +// chunks of the entries, the central directory and the end record, the +// certificate, and the signature of that signed data +// +// The signature of RSA PKCS #1 v1.5 carries no time and no randomness: the +// same key and the same entries give the same bytes. The key is checked +// against the certificate by verifying the v2 signature with the +// certificate's public key before the APK is returned. + +import { unzipEntry, writeZip, type ZipInput } from "./zip.ts"; + +/** The key and certificate an APK is signed with. */ +export interface ApkSigner { + /** An RSA private key: PKCS #8 DER bytes, or a CryptoKey imported for RSASSA-PKCS1-v1_5 with SHA-256 and usage "sign". */ + readonly privateKey: CryptoKey | Uint8Array; + /** The X.509 certificate, DER. */ + readonly certificate: Uint8Array; +} + +const RSA_PKCS1_SHA256 = { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" } as const; +/** APK Signature Scheme v2's id for RSASSA-PKCS1-v1_5 with SHA-256. */ +const V2_RSA_PKCS1_SHA256 = 0x0103; +const V2_BLOCK_ID = 0x7109871a; +const V1_NAME = "CERT"; +const encoder = new TextEncoder(); + +function fail(message: string): never { + throw new Error(`apk signing: ${message}`); +} + +async function sha256(bytes: Uint8Array): Promise<Uint8Array> { + return new Uint8Array(await crypto.subtle.digest("SHA-256", bytes as Uint8Array<ArrayBuffer>)); +} + +function base64(bytes: Uint8Array): string { + let text = ""; + for (let index = 0; index < bytes.length; index += 0x8000) text += String.fromCharCode(...bytes.subarray(index, index + 0x8000)); + return btoa(text); +} + +export function concat(...parts: readonly Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +} + +function u32(value: number): Uint8Array { + const out = new Uint8Array(4); + new DataView(out.buffer).setUint32(0, value, true); + return out; +} + +function u64(value: number): Uint8Array { + const out = new Uint8Array(8); + new DataView(out.buffer).setBigUint64(0, BigInt(value), true); + return out; +} + +/** A v2 length-prefixed value: u32 length, then the parts. */ +function prefixed(...parts: readonly Uint8Array[]): Uint8Array { + const body = concat(...parts); + return concat(u32(body.length), body); +} + +// ------------------------------------------------------------------ DER + +function derLength(length: number): Uint8Array { + if (length < 0x80) return Uint8Array.of(length); + const bytes: number[] = []; + for (let rest = length; rest > 0; rest = Math.floor(rest / 256)) bytes.unshift(rest & 0xff); + return Uint8Array.of(0x80 | bytes.length, ...bytes); +} + +/** A DER value: tag, length, contents. */ +export function der(tag: number, ...contents: readonly Uint8Array[]): Uint8Array { + const body = concat(...contents); + return concat(Uint8Array.of(tag), derLength(body.length), body); +} + +export function derObjectId(dotted: string): Uint8Array { + const [first, second, ...rest] = dotted.split(".").map(Number); + const bytes = [40 * first! + second!]; + for (const arc of rest) { + const groups: number[] = []; + let value = arc; + do { + groups.unshift(value & 0x7f); + value = Math.floor(value / 128); + } while (value > 0); + groups.forEach((group, index) => bytes.push(index < groups.length - 1 ? group | 0x80 : group)); + } + return der(0x06, Uint8Array.from(bytes)); +} + +export const DER_NULL = Uint8Array.of(0x05, 0x00); +const SEQUENCE = 0x30; +const SET = 0x31; +const OID_SHA256 = "2.16.840.1.101.3.4.2.1"; +const OID_RSA = "1.2.840.113549.1.1.1"; +const OID_PKCS7_DATA = "1.2.840.113549.1.7.1"; +const OID_PKCS7_SIGNED = "1.2.840.113549.1.7.2"; + +interface DerNode { + readonly tag: number; + /** The whole value, tag and length included. */ + readonly start: number; + readonly body: number; + readonly end: number; +} + +/** The DER values laid end to end from `start` to `end`. */ +export function derChildren(bytes: Uint8Array, start: number, end: number): DerNode[] { + const out: DerNode[] = []; + let at = start; + while (at < end) { + const tag = bytes[at]!; + let length = bytes[at + 1]!; + let header = 2; + if (length & 0x80) { + const count = length & 0x7f; + if (count === 0 || count > 4) fail("a certificate field has an indefinite or oversized length"); + length = 0; + for (let index = 0; index < count; index++) length = length * 256 + bytes[at + 2 + index]!; + header = 2 + count; + } + if (at + header + length > end) fail("a certificate field runs past its parent"); + out.push({ tag, start: at, body: at + header, end: at + header + length }); + at += header + length; + } + return out; +} + +/** The fields of an X.509 certificate a signature names: serial, issuer and public key. */ +export function certificateFields(certificate: Uint8Array): { serial: Uint8Array; issuer: Uint8Array; publicKey: Uint8Array } { + const [outer] = derChildren(certificate, 0, certificate.length); + if (!outer || outer.tag !== SEQUENCE || outer.end !== certificate.length) fail("the certificate is not one DER sequence"); + const [tbs] = derChildren(certificate, outer.body, outer.end); + if (!tbs || tbs.tag !== SEQUENCE) fail("the certificate has no TBSCertificate"); + const fields = derChildren(certificate, tbs.body, tbs.end); + const offset = fields[0]?.tag === 0xa0 ? 1 : 0; // [0] version + const pick = (index: number) => { + const field = fields[offset + index] ?? fail("the certificate lacks a TBSCertificate field"); + return certificate.subarray(field.start, field.end); + }; + const publicKey = pick(5); + const [algorithm] = derChildren(publicKey, derChildren(publicKey, 0, publicKey.length)[0]!.body, publicKey.length); + const [oid] = derChildren(publicKey, algorithm!.body, algorithm!.end); + const rsa = derObjectId(OID_RSA); + if (!oid || publicKey.subarray(oid.start, oid.end).join() !== rsa.join()) fail("the certificate's key is not RSA"); + return { serial: pick(0), issuer: pick(2), publicKey }; +} + +// ------------------------------------------------------------------ keys + +async function signingKey(privateKey: CryptoKey | Uint8Array): Promise<CryptoKey> { + if (privateKey instanceof Uint8Array) { + try { + return await crypto.subtle.importKey("pkcs8", privateKey as Uint8Array<ArrayBuffer>, RSA_PKCS1_SHA256, false, ["sign"]); + } catch { + fail("the private key is not an RSA PKCS #8 key"); + } + } + const algorithm = privateKey.algorithm as RsaHashedKeyAlgorithm; + if (privateKey.type !== "private" || algorithm.name !== "RSASSA-PKCS1-v1_5" || algorithm.hash?.name !== "SHA-256" || !privateKey.usages.includes("sign")) { + fail("the private key is not an RSASSA-PKCS1-v1_5 SHA-256 signing key"); + } + return privateKey; +} + +async function sign(key: CryptoKey, data: Uint8Array): Promise<Uint8Array> { + return new Uint8Array(await crypto.subtle.sign(RSA_PKCS1_SHA256, key, data as Uint8Array<ArrayBuffer>)); +} + +/** Whether `signature` over `data` verifies under the certificate's public key. */ +export async function verifiesUnder(certificate: Uint8Array, data: Uint8Array, signature: Uint8Array): Promise<boolean> { + const key = await crypto.subtle.importKey("spki", certificateFields(certificate).publicKey as Uint8Array<ArrayBuffer>, RSA_PKCS1_SHA256, false, ["verify"]); + return crypto.subtle.verify(RSA_PKCS1_SHA256, key, signature as Uint8Array<ArrayBuffer>, data as Uint8Array<ArrayBuffer>); +} + +// ------------------------------------------------------------------ v1 + +/** A manifest line: 70 bytes, continued on lines of a space and 69 bytes (apksigner's widths). */ +function manifestLine(text: string): string { + if (!/^[\x20-\x7e]*$/.test(text)) fail(`${JSON.stringify(text)} is not printable ASCII`); + let out = `${text.slice(0, 70)}\r\n`; + for (let at = 70; at < text.length; at += 69) out += ` ${text.slice(at, at + 69)}\r\n`; + return out; +} + +async function v1Entries(files: ReadonlyArray<readonly [string, Uint8Array]>, key: CryptoKey, certificate: Uint8Array): Promise<ZipInput[]> { + const sorted = [...files].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + let manifest = "Manifest-Version: 1.0\r\nCreated-By: 1.0 (PocketJS repack)\r\n\r\n"; + let sections = ""; + for (const [name, bytes] of sorted) { + const section = `${manifestLine(`Name: ${name}`)}SHA-256-Digest: ${base64(await sha256(bytes))}\r\n\r\n`; + manifest += section; + sections += `${manifestLine(`Name: ${name}`)}SHA-256-Digest: ${base64(await sha256(encoder.encode(section)))}\r\n\r\n`; + } + const manifestBytes = encoder.encode(manifest); + const signatureFile = encoder.encode( + "Signature-Version: 1.0\r\nCreated-By: 1.0 (PocketJS repack)\r\n" + + `SHA-256-Digest-Manifest: ${base64(await sha256(manifestBytes))}\r\nX-Android-APK-Signed: 2\r\n\r\n${sections}`, + ); + const signature = await sign(key, signatureFile); + const { serial, issuer } = certificateFields(certificate); + const sha256Algorithm = der(SEQUENCE, derObjectId(OID_SHA256), DER_NULL); + const signerInfo = der( + SEQUENCE, + der(0x02, Uint8Array.of(1)), + der(SEQUENCE, issuer, serial), + sha256Algorithm, + der(SEQUENCE, derObjectId(OID_RSA), DER_NULL), + der(0x04, signature), + ); + const signedData = der( + SEQUENCE, + der(0x02, Uint8Array.of(1)), + der(SET, sha256Algorithm), + der(SEQUENCE, derObjectId(OID_PKCS7_DATA)), + der(0xa0, certificate), + der(SET, signerInfo), + ); + return [ + { name: "META-INF/MANIFEST.MF", data: manifestBytes }, + { name: `META-INF/${V1_NAME}.SF`, data: signatureFile }, + { name: `META-INF/${V1_NAME}.RSA`, data: der(SEQUENCE, derObjectId(OID_PKCS7_SIGNED), der(0xa0, signedData)) }, + ]; +} + +// ------------------------------------------------------------------ v2 + +/** The end-of-central-directory record's offset; the archive has no comment. */ +function endOfCentralDirectory(apk: Uint8Array): number { + const data = new DataView(apk.buffer, apk.byteOffset, apk.byteLength); + const at = apk.length - 22; + if (at < 0 || data.getUint32(at, true) !== 0x06054b50) fail("the archive does not end with its end-of-central-directory record"); + return at; +} + +/** v2's content digest: SHA-256 over 1 MiB chunks of each section, then over the chunk digests. */ +async function contentDigest(sections: readonly Uint8Array[]): Promise<Uint8Array> { + const digests: Uint8Array[] = []; + for (const section of sections) { + for (let at = 0; at < section.length; at += 1 << 20) { + const chunk = section.subarray(at, Math.min(section.length, at + (1 << 20))); + digests.push(await sha256(concat(Uint8Array.of(0xa5), u32(chunk.length), chunk))); + } + } + return sha256(concat(Uint8Array.of(0x5a), u32(digests.length), ...digests)); +} + +async function v2Sign(apk: Uint8Array, key: CryptoKey, certificate: Uint8Array): Promise<Uint8Array> { + const end = endOfCentralDirectory(apk); + const centralDirectory = new DataView(apk.buffer, apk.byteOffset, apk.byteLength).getUint32(end + 16, true); + const digest = await contentDigest([apk.subarray(0, centralDirectory), apk.subarray(centralDirectory, end), apk.subarray(end)]); + const signedData = concat( + prefixed(prefixed(u32(V2_RSA_PKCS1_SHA256), prefixed(digest))), + prefixed(prefixed(certificate)), + prefixed(), + ); + const signature = await sign(key, signedData); + if (!(await verifiesUnder(certificate, signedData, signature))) fail("the private key is not the certificate's key"); + const signer = concat( + prefixed(signedData), + prefixed(prefixed(u32(V2_RSA_PKCS1_SHA256), prefixed(signature))), + prefixed(certificateFields(certificate).publicKey), + ); + const value = prefixed(prefixed(signer)); + const pair = concat(u64(4 + value.length), u32(V2_BLOCK_ID), value); + const blockSize = pair.length + 8 + 16; + const block = concat(u64(blockSize), pair, u64(blockSize), encoder.encode("APK Sig Block 42")); + const tail = new Uint8Array(apk.subarray(end)); + new DataView(tail.buffer).setUint32(16, centralDirectory + block.length, true); + return concat(apk.subarray(0, centralDirectory), block, apk.subarray(centralDirectory, end), tail); +} + +// ------------------------------------------------------------------ APK + +/** + * The signed APK of `entries`, in their order, with the v1 files after them + * and the v2 block before the central directory. Entries carry their own + * alignment (`align` on stored entries, as zipalign places them); none may be + * under META-INF/. + */ +export async function signApk(entries: readonly ZipInput[], signer: ApkSigner): Promise<Uint8Array> { + const key = await signingKey(signer.privateKey); + certificateFields(signer.certificate); + const files: Array<readonly [string, Uint8Array]> = []; + for (const input of entries) { + if (input.name.startsWith("META-INF/")) fail(`${input.name} is a signature file; the entries go in unsigned`); + if (input.name.endsWith("/")) continue; + files.push([input.name, "entry" in input ? await unzipEntry(input.entry) : input.data]); + } + const withV1 = await writeZip([...entries, ...(await v1Entries(files, key, signer.certificate))]); + return v2Sign(withV1, key, signer.certificate); +} diff --git a/tools/repack/shared/runtime.ts b/tools/repack/shared/runtime.ts index 1f84e0106..f46583883 100644 --- a/tools/repack/shared/runtime.ts +++ b/tools/repack/shared/runtime.ts @@ -17,6 +17,7 @@ import { type PocketPackageIdentity, type PocketPackageVariant, } from "../../../contracts/spec/pocket-package.ts"; +import type { ApkSigner } from "./apk-sign.ts"; export interface RepackIdentity { /** Reverse-DNS application id; must equal the `.pocket` variant's app id. */ @@ -34,6 +35,9 @@ export interface RepackInput { /** A `.pocket` holding this target's variant. */ readonly pocket: Uint8Array; readonly identity: RepackIdentity; + /** The key and certificate an Android APK is signed with: the Pocket Studio + * community key. The other targets take none. */ + readonly signer?: ApkSigner; } /** dist/runtime/<target>/runtime.json */ @@ -95,7 +99,7 @@ export async function readRuntime( export interface AdmittedPocket { /** The package to ship: the input bytes when they hold only this target's - * variant, otherwise the same package thinned to it. */ + * variant, otherwise the same package thinned to it (encoded on first read). */ readonly bytes: Uint8Array; readonly variant: PocketPackageVariant; readonly identity: PocketPackageIdentity; @@ -151,10 +155,16 @@ export function admitPocket( if (target?.id !== runtime.target || target.hostAbi !== runtime.hostAbi) { throw new Error(`repack: the .pocket's ${runtime.target} plan resolves another target or ABI`); } - const bytes = decoded.variants.length === 1 - ? pocket - : encodePocketPackage(thinPocketPackage(decoded, [runtime.target])); - return { bytes, variant, identity: packageIdentity, plan }; + let thinned: Uint8Array | undefined; + return { + get bytes() { + thinned ??= decoded.variants.length === 1 ? pocket : encodePocketPackage(thinPocketPackage(decoded, [runtime.target])); + return thinned; + }, + variant, + identity: packageIdentity, + plan, + }; } /** Identity fields every package needs, checked once for every target. */ diff --git a/tools/repack/shared/zip.ts b/tools/repack/shared/zip.ts index ad03061bc..ed3f9525b 100644 --- a/tools/repack/shared/zip.ts +++ b/tools/repack/shared/zip.ts @@ -41,9 +41,9 @@ export type ZipInput = | { readonly name: string; readonly data: Uint8Array; - /** Deflate the data (default true); stored entries can be aligned. */ + /** Deflate the data when that makes it smaller (default true); false stores it. */ readonly compress?: boolean; - /** Align a stored entry's data to this many bytes within the archive. */ + /** Align the data to this many bytes within the archive when the entry is stored. */ readonly align?: number; readonly unixMode?: number; } @@ -130,7 +130,7 @@ export async function zipEntry(input: ZipInput): Promise<ZipEntry> { if ("entry" in input) return { ...input.entry, name: input.name }; const crc = crc32(input.data); const base = { name: input.name, crc32: crc, size: input.data.length, ...(input.unixMode !== undefined ? { unixMode: input.unixMode } : {}) }; - if (input.compress === false || input.align) return { ...base, method: ZIP_STORED, stored: input.data }; + if (input.compress === false) return { ...base, method: ZIP_STORED, stored: input.data }; const deflated = await deflateRaw(input.data); return deflated.length < input.data.length ? { ...base, method: ZIP_DEFLATED, stored: deflated } diff --git a/tools/runtime/android.ts b/tools/runtime/android.ts index 5ade3bb5d..121a36e2e 100644 --- a/tools/runtime/android.ts +++ b/tools/runtime/android.ts @@ -1,22 +1,31 @@ // The generic Android runtime: what tools/android.ts build-app compiles for -// every game of a profile, built once, with no game in it. +// every game of a profile, built once, with no game in it, and linked by +// aapt2 into one unsigned template APK. // // bun tools/runtime.ts android [--profile=redmi-1s] [--out=dist/runtime/android] // // Writes into the output directory: -// lib/armeabi-v7a/libpocketjs.so, lib/arm64-v8a/libpocketjs.so -// Rust core + QuickJS + host, compiled with the -// profile's target id, host ABI and viewport only -// classes.dex PocketActivity (reads assets/app.js and app.pak) -// AndroidManifest.xml the manifest template (@POCKET_*@ placeholders) -// res/values/strings.xml the label template -// res/mipmap-*/icon.png the default launcher icon (PocketJS's mark) +// template.apk aapt2's link of the runtime with ANDROID_TEMPLATE_IDENTITY: +// the compiled manifest and resources.arsc, the default +// launcher icon (PocketJS's mark) at five densities, +// empty assets/app.js and assets/app.pak, classes.dex +// (PocketActivity) and lib/armeabi-v7a/libpocketjs.so, +// lib/arm64-v8a/libpocketjs.so (Rust core + QuickJS + +// host, compiled with the profile's target id, host ABI +// and viewport only); unsigned +// AndroidManifest.xml, res/values/strings.xml +// the templates it was linked from (@POCKET_*@ placeholders) // runtime.json target, hostAbi, pocketjs, profile, viewport, android, files // -// tools/repack/android.ts turns it and a `.pocket` into an APK. +// tools/repack/android.ts turns it and a `.pocket` into a signed APK with no +// SDK tool. -import { readdirSync, readFileSync, statSync } from "node:fs"; -import { join, relative, resolve } from "node:path"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { linkAndroidApk } from "../android-link.ts"; +import { ANDROID_TEMPLATE, ANDROID_TEMPLATE_SOURCES, androidTemplateJob } from "../repack/android.ts"; +import { RUNTIME_MANIFEST, type RuntimeManifest } from "../repack/shared/runtime.ts"; import { MOTO_G_PLAY_CONTRACTS, MOTO_G_PLAY_TARGET } from "../moto-g-play-profile.ts"; import { REDMI_1S_CONTRACTS, REDMI_1S_TARGET } from "../redmi-1s-profile.ts"; import { writeRuntimeManifest } from "./manifest.ts"; @@ -53,39 +62,62 @@ export async function buildRuntimeAndroid(argv: readonly string[] = []): Promise if (!chosen) throw new Error(`runtime android: no profile ${profile}; use ${Object.keys(ANDROID_RUNTIME_PROFILES).join(" or ")}`); out ??= join(repository, profile === "redmi-1s" ? "dist/runtime/android" : `dist/runtime/android-${profile}`); + // The directory is replaced: refuse one that holds anything but an earlier runtime. + if (existsSync(out) && readdirSync(out).length && !existsSync(join(out, RUNTIME_MANIFEST))) { + throw new Error(`runtime android: ${out} is not empty and holds no earlier runtime`); + } + const started = performance.now(); - const build = Bun.spawnSync(["bun", join(repository, "tools/android.ts"), `--profile=${profile}`, "runtime", `--out=${out}`], { - cwd: repository, - stdout: "inherit", - stderr: "inherit", - }); - if (build.exitCode !== 0) throw new Error(`runtime android: tools/android.ts runtime exited ${build.exitCode}`); + const parts = mkdtempSync(join(tmpdir(), "pocket-runtime-android-")); + let manifest: RuntimeManifest; + try { + const build = Bun.spawnSync(["bun", join(repository, "tools/android.ts"), `--profile=${profile}`, "runtime", `--out=${parts}`], { + cwd: repository, + stdout: "inherit", + stderr: "inherit", + }); + if (build.exitCode !== 0) throw new Error(`runtime android: tools/android.ts runtime exited ${build.exitCode}`); + const toolchain = JSON.parse(readFileSync(join(repository, `tools/cli/${profile}-toolchain.json`), "utf8")); + const sdk = { + apiLevel: toolchain.android.apiLevel, + buildToolsVersion: toolchain.android.buildToolsVersion, + minSdkVersion: toolchain.android.minSdkVersion, + targetSdkVersion: toolchain.android.targetSdkVersion, + }; + const abis: string[] = toolchain.android.abis.map((abi: { abi: string }) => abi.abi); + const files = new Map(walk(parts).map((path) => [relative(parts, path).split("\\").join("/"), new Uint8Array(readFileSync(path))])); + const template = await linkAndroidApk(androidTemplateJob(files, sdk, abis)); - const toolchain = JSON.parse(readFileSync(join(repository, `tools/cli/${profile}-toolchain.json`), "utf8")); - const files = walk(out).map((path) => relative(out!, path).split("\\").join("/")).filter((name) => name !== "runtime.json").sort(); - const manifest = await writeRuntimeManifest(out, { - target: chosen.target, - hostAbi: chosen.contract.hostAbi, - profile, - files, - extra: { - viewport: { logical: chosen.contract.display.logicalViewports[0], rasterDensity: chosen.contract.display.rasterDensity }, - android: { - toolchain: toolchain.toolchainVersion, - apiLevel: toolchain.android.apiLevel, - buildToolsVersion: toolchain.android.buildToolsVersion, - ndkVersion: toolchain.android.ndkVersion, - minSdkVersion: toolchain.android.minSdkVersion, - targetSdkVersion: toolchain.android.targetSdkVersion, - abis: toolchain.android.abis.map((abi: { abi: string }) => abi.abi), - quickjs: toolchain.quickjs.revision, - rust: toolchain.rust.toolchain, + rmSync(out, { recursive: true, force: true }); + mkdirSync(out, { recursive: true }); + writeFileSync(join(out, ANDROID_TEMPLATE), template); + for (const source of ANDROID_TEMPLATE_SOURCES) { + mkdirSync(dirname(join(out, source)), { recursive: true }); + copyFileSync(join(parts, source), join(out, source)); + } + manifest = await writeRuntimeManifest(out, { + target: chosen.target, + hostAbi: chosen.contract.hostAbi, + profile, + files: [ANDROID_TEMPLATE, ...ANDROID_TEMPLATE_SOURCES].sort(), + extra: { + viewport: { logical: chosen.contract.display.logicalViewports[0], rasterDensity: chosen.contract.display.rasterDensity }, + android: { + toolchain: toolchain.toolchainVersion, + ...sdk, + ndkVersion: toolchain.android.ndkVersion, + abis, + quickjs: toolchain.quickjs.revision, + rust: toolchain.rust.toolchain, + }, }, - }, - }); + }); + } finally { + rmSync(parts, { recursive: true, force: true }); + } const total = Object.values(manifest.files).reduce((sum, file) => sum + file.bytes, 0); console.log( - `output: ${out} (${files.length} files, ${total} bytes, PocketJS ${manifest.pocketjs}, ` + + `output: ${out} (${Object.keys(manifest.files).length} files, ${total} bytes, PocketJS ${manifest.pocketjs}, ` + `${((performance.now() - started) / 1000).toFixed(1)} s)`, ); return out; diff --git a/tools/test.ts b/tools/test.ts index 3ab678f53..427feb946 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -138,6 +138,7 @@ const SUITE: readonly Stage[] = [ "tests/redmi-1s-profile.test.ts", "tests/android-icon.test.ts", "tests/repack-android.test.ts", + "tests/repack-worker.test.ts", "tests/offload-provider.test.ts", "tests/companion-session.test.ts", "tests/resource-cache.test.ts",