From dace8c974ae3ad276b61ea096061b89993699958 Mon Sep 17 00:00:00 2001 From: "david.rzepa" Date: Mon, 15 Jun 2026 19:33:55 -0600 Subject: [PATCH 1/2] feat: add lz4 compression mode --- README.md | 14 ++++++ __tests__/cacheUtils.test.ts | 91 ++++++++++++++++++++++++++++++++++++ __tests__/tar.test.ts | 67 ++++++++++++++++++++++++++ action.yml | 4 ++ dist/cache/cacheUtils.d.ts | 1 + dist/cache/cacheUtils.js | 57 ++++++++++++++++++---- dist/cache/constants.d.ts | 4 +- dist/cache/constants.js | 2 + dist/cache/index.js | 3 ++ dist/cache/tar.js | 46 ++++++++++++------ dist/constants.d.ts | 3 +- dist/constants.js | 3 +- dist/utils/testUtils.d.ts | 1 + dist/utils/testUtils.js | 4 ++ restore/action.yml | 6 ++- save/action.yml | 6 ++- src/cache/cacheUtils.ts | 57 ++++++++++++++++++++-- src/cache/constants.ts | 4 +- src/cache/index.ts | 13 ++++++ src/cache/tar.ts | 46 ++++++++++++------ src/constants.ts | 3 +- src/utils/testUtils.ts | 5 ++ 22 files changed, 391 insertions(+), 49 deletions(-) create mode 100644 __tests__/cacheUtils.test.ts create mode 100644 __tests__/tar.test.ts diff --git a/README.md b/README.md index 12ba68b0..c3a692b0 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ The created cache tarballs are placed in `$RUNNER_TOOL_CACHE/$GITHUB_REPOSITORY` `GITHUB_REPOSITORY` will be set to the name (`org/repo`) of the repository from where the action is executed. `RUNNER_TOOL_CACHE` defaults to `/opt/hostedtoolcache`. This should point to a folder on a persistent storage. +By default, `compression-mode` is `auto`, which keeps the existing behavior of using zstd when available and gzip otherwise. You can set it to `gzip`, `zstd`, or `lz4` to force a specific mode. Changing this mode changes the archive format, so existing caches created with another compression mode will not be restored. Archive filenames use `cache.tgz` for gzip, `cache.tzst` for zstd, and `cache.tlz4` for lz4. When using `lz4`, make sure the `lz4` command is installed on the runner. + ### Example cache workflow ````yaml @@ -40,6 +42,18 @@ jobs: ```` +### Compression mode + +The action defaults to `compression-mode: auto`. To force lz4 compression on runners where `lz4` is installed: + +````yaml +- uses: maxnowack/local-cache@v2 + with: + path: prime-numbers + key: ${{ runner.os }}-primes + compression-mode: lz4 +```` + ## Contributing Contributions to the project are welcome. Feel free to fork and improve. I do my best accept pull requests in a timely manor, especially when tests and updated docs are included. diff --git a/__tests__/cacheUtils.test.ts b/__tests__/cacheUtils.test.ts new file mode 100644 index 00000000..fae1d9fc --- /dev/null +++ b/__tests__/cacheUtils.test.ts @@ -0,0 +1,91 @@ +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { + getCacheFileName, + getCacheSizeInBytes, + getCompressionMethod, +} from '../src/cache/cacheUtils' +import { + CacheFilename, + CompressionMethod, +} from '../src/cache/constants' +import { Inputs } from '../src/constants' +import * as testUtils from '../src/utils/testUtils' + +jest.mock('@actions/exec') + +const execMock = exec.exec as jest.MockedFunction +let originalWorkspace: string | undefined + +function mockVersionOutput(output: string): void { + execMock.mockImplementation(async (commandLine, args, options) => { + options?.listeners?.stdout?.(Buffer.from(output)) + return 0 + }) +} + +afterEach(() => { + process.env.GITHUB_WORKSPACE = originalWorkspace + testUtils.clearInputs() + jest.clearAllMocks() +}) + +test('getCompressionMethod returns lz4 when selected', async () => { + testUtils.setInput(Inputs.CompressionMode, 'lz4') + + await expect(getCompressionMethod()).resolves.toBe(CompressionMethod.Lz4) + expect(execMock).not.toHaveBeenCalled() +}) + +test('getCompressionMethod returns zstd when selected', async () => { + testUtils.setInput(Inputs.CompressionMode, 'zstd') + + await expect(getCompressionMethod()).resolves.toBe( + CompressionMethod.ZstdWithoutLong, + ) + expect(execMock).not.toHaveBeenCalled() +}) + +test('getCompressionMethod auto-selects zstd when available', async () => { + mockVersionOutput('1.5.5') + + await expect(getCompressionMethod()).resolves.toBe( + CompressionMethod.ZstdWithoutLong, + ) +}) + +test('getCompressionMethod auto-selects gzip when zstd is unavailable', async () => { + mockVersionOutput('') + + await expect(getCompressionMethod()).resolves.toBe(CompressionMethod.Gzip) +}) + +test('getCompressionMethod rejects unsupported compression modes', async () => { + testUtils.setInput(Inputs.CompressionMode, 'snappy') + + await expect(getCompressionMethod()).rejects.toThrow( + 'Unsupported compression mode: snappy', + ) +}) + +test('getCacheFileName returns lz4 archive filename', () => { + expect(getCacheFileName(CompressionMethod.Lz4)).toBe(CacheFilename.Lz4) +}) + +test('getCacheSizeInBytes returns recursive source size', () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'local-cache-size-')) + originalWorkspace = process.env.GITHUB_WORKSPACE + process.env.GITHUB_WORKSPACE = workspace + + try { + fs.mkdirSync(path.join(workspace, 'cache-dir')) + fs.writeFileSync(path.join(workspace, 'cache-dir', 'first.txt'), 'hello') + fs.writeFileSync(path.join(workspace, 'second.txt'), 'world!') + + expect(getCacheSizeInBytes(['cache-dir', 'second.txt'])).toBe(11) + } finally { + fs.rmSync(workspace, { recursive: true, force: true }) + } +}) diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts new file mode 100644 index 00000000..3f721076 --- /dev/null +++ b/__tests__/tar.test.ts @@ -0,0 +1,67 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import * as exec from '@actions/exec' +import * as io from '@actions/io' +import * as cacheUtils from '../src/cache/cacheUtils' +import { CompressionMethod } from '../src/cache/constants' +import { + createTar, + extractTar, + listTar, +} from '../src/cache/tar' + +jest.mock('@actions/exec') +jest.mock('@actions/io') + +const execMock = exec.exec as jest.MockedFunction +const mkdirPMock = io.mkdirP as jest.MockedFunction +const whichMock = io.which as jest.MockedFunction + +let archiveFolder = '' +let originalWorkspace: string | undefined + +beforeEach(() => { + archiveFolder = fs.mkdtempSync(path.join(os.tmpdir(), 'local-cache-tar-')) + originalWorkspace = process.env.GITHUB_WORKSPACE + process.env.GITHUB_WORKSPACE = process.cwd() + + execMock.mockResolvedValue(0) + mkdirPMock.mockResolvedValue() + whichMock.mockResolvedValue('tar') + jest.spyOn(cacheUtils, 'getGnuTarPathOnWindows').mockResolvedValue('tar') +}) + +afterEach(() => { + process.env.GITHUB_WORKSPACE = originalWorkspace + fs.rmSync(archiveFolder, { recursive: true, force: true }) + jest.restoreAllMocks() +}) + +test('createTar uses lz4 compression program', async () => { + await createTar(archiveFolder, ['node_modules'], CompressionMethod.Lz4) + + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock.mock.calls[0][0]).toContain('--use-compress-program lz4') + expect(execMock.mock.calls[0][0]).toContain('cache.tlz4') + expect(execMock.mock.calls[0][0]).toContain('--files-from manifest.txt') + expect(execMock.mock.calls[0][2]?.cwd).toBe(archiveFolder) +}) + +test('extractTar uses lz4 decompression program', async () => { + await extractTar(path.join(archiveFolder, 'cache.tlz4'), CompressionMethod.Lz4) + + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock.mock.calls[0][0]).toContain('--use-compress-program') + expect(execMock.mock.calls[0][0]).toContain('lz4 -d') + expect(execMock.mock.calls[0][0]).toContain('cache.tlz4') +}) + +test('listTar uses lz4 decompression program', async () => { + await listTar(path.join(archiveFolder, 'cache.tlz4'), CompressionMethod.Lz4) + + expect(execMock).toHaveBeenCalledTimes(1) + expect(execMock.mock.calls[0][0]).toContain('--use-compress-program') + expect(execMock.mock.calls[0][0]).toContain('lz4 -d') + expect(execMock.mock.calls[0][0]).toContain('cache.tlz4') +}) diff --git a/action.yml b/action.yml index c76f1848..65cc3733 100644 --- a/action.yml +++ b/action.yml @@ -18,6 +18,10 @@ inputs: description: 'Check if a cache entry exists for the given input(s) (key, restore-keys) without downloading the cache' default: 'false' required: false + compression-mode: + description: 'Compression mode to use for cache archives. Supported values are auto, gzip, zstd, and lz4.' + default: 'auto' + required: false outputs: cache-hit: description: 'A boolean value to indicate if cache was found and restored' diff --git a/dist/cache/cacheUtils.d.ts b/dist/cache/cacheUtils.d.ts index 4e3041fb..e0f174d2 100644 --- a/dist/cache/cacheUtils.d.ts +++ b/dist/cache/cacheUtils.d.ts @@ -3,6 +3,7 @@ import * as fs from 'fs'; import { CompressionMethod } from './constants'; export declare function createTempDirectory(): Promise; export declare function getArchiveFileSizeInBytes(filePath: string): number; +export declare function getCacheSizeInBytes(paths: string[]): number; export declare function resolvePaths(patterns: string[]): Promise; export declare function unlinkFile(filePath: fs.PathLike): Promise; export declare function getCompressionMethod(): Promise; diff --git a/dist/cache/cacheUtils.js b/dist/cache/cacheUtils.js index 4debd832..a0ba9c6b 100644 --- a/dist/cache/cacheUtils.js +++ b/dist/cache/cacheUtils.js @@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.isGhes = exports.assertDefined = exports.getGnuTarPathOnWindows = exports.getCacheFileName = exports.getCompressionMethod = exports.unlinkFile = exports.resolvePaths = exports.getArchiveFileSizeInBytes = exports.createTempDirectory = void 0; +exports.isGhes = exports.assertDefined = exports.getGnuTarPathOnWindows = exports.getCacheFileName = exports.getCompressionMethod = exports.unlinkFile = exports.resolvePaths = exports.getCacheSizeInBytes = exports.getArchiveFileSizeInBytes = exports.createTempDirectory = void 0; const fs = __importStar(require("fs")); const path = __importStar(require("path")); const util = __importStar(require("util")); @@ -33,7 +33,8 @@ const glob = __importStar(require("@actions/glob")); const io = __importStar(require("@actions/io")); const semver = __importStar(require("semver")); const uuid_1 = require("uuid"); -const constants_1 = require("./constants"); +const constants_1 = require("../constants"); +const constants_2 = require("./constants"); // From https://github.com/actions/toolkit/blob/main/packages/tool-cache/src/tool-cache.ts#L23 async function createTempDirectory() { const IS_WINDOWS = process.platform === 'win32'; @@ -61,6 +62,26 @@ function getArchiveFileSizeInBytes(filePath) { return fs.statSync(filePath).size; } exports.getArchiveFileSizeInBytes = getArchiveFileSizeInBytes; +function getPathSizeInBytes(filePath) { + const stats = fs.lstatSync(filePath); + if (!stats.isDirectory()) { + return stats.size; + } + return fs.readdirSync(filePath).reduce((total, entry) => total + getPathSizeInBytes(path.join(filePath, entry)), 0); +} +function getCacheSizeInBytes(paths) { + const workspace = process.env.GITHUB_WORKSPACE ?? process.cwd(); + return paths.reduce((total, cachePath) => { + const filePath = path.isAbsolute(cachePath) + ? cachePath + : path.join(workspace, cachePath); + if (!fs.existsSync(filePath)) { + return total; + } + return total + getPathSizeInBytes(filePath); + }, 0); +} +exports.getCacheSizeInBytes = getCacheSizeInBytes; async function resolvePaths(patterns) { const paths = []; const workspace = process.env.GITHUB_WORKSPACE ?? process.cwd(); @@ -118,24 +139,42 @@ async function getVersion(app, additionalArgs = []) { } // Use zstandard if possible to maximize cache performance async function getCompressionMethod() { + const compressionMode = core.getInput(constants_1.Inputs.CompressionMode) || 'auto'; + switch (compressionMode.toLowerCase()) { + case 'auto': + break; + case constants_2.CompressionMethod.Gzip: + return constants_2.CompressionMethod.Gzip; + case constants_2.CompressionMethod.Lz4: + return constants_2.CompressionMethod.Lz4; + case constants_2.CompressionMethod.Zstd: + return constants_2.CompressionMethod.ZstdWithoutLong; + default: + throw new Error(`Unsupported compression mode: ${compressionMode}. Supported modes are: auto, gzip, zstd, lz4.`); + } const versionOutput = await getVersion('zstd', ['--quiet']); const version = semver.clean(versionOutput); core.debug(`zstd version: ${version}`); if (versionOutput === '') { - return constants_1.CompressionMethod.Gzip; + return constants_2.CompressionMethod.Gzip; } - return constants_1.CompressionMethod.ZstdWithoutLong; + return constants_2.CompressionMethod.ZstdWithoutLong; } exports.getCompressionMethod = getCompressionMethod; function getCacheFileName(compressionMethod) { - return compressionMethod === constants_1.CompressionMethod.Gzip - ? constants_1.CacheFilename.Gzip - : constants_1.CacheFilename.Zstd; + switch (compressionMethod) { + case constants_2.CompressionMethod.Gzip: + return constants_2.CacheFilename.Gzip; + case constants_2.CompressionMethod.Lz4: + return constants_2.CacheFilename.Lz4; + default: + return constants_2.CacheFilename.Zstd; + } } exports.getCacheFileName = getCacheFileName; async function getGnuTarPathOnWindows() { - if (fs.existsSync(constants_1.GnuTarPathOnWindows)) { - return constants_1.GnuTarPathOnWindows; + if (fs.existsSync(constants_2.GnuTarPathOnWindows)) { + return constants_2.GnuTarPathOnWindows; } const versionOutput = await getVersion('tar'); return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''; diff --git a/dist/cache/constants.d.ts b/dist/cache/constants.d.ts index 1d8de719..115659c5 100644 --- a/dist/cache/constants.d.ts +++ b/dist/cache/constants.d.ts @@ -1,9 +1,11 @@ export declare enum CacheFilename { Gzip = "cache.tgz", - Zstd = "cache.tzst" + Zstd = "cache.tzst", + Lz4 = "cache.tlz4" } export declare enum CompressionMethod { Gzip = "gzip", + Lz4 = "lz4", ZstdWithoutLong = "zstd-without-long", Zstd = "zstd" } diff --git a/dist/cache/constants.js b/dist/cache/constants.js index 16fa7339..6fabea11 100644 --- a/dist/cache/constants.js +++ b/dist/cache/constants.js @@ -5,10 +5,12 @@ var CacheFilename; (function (CacheFilename) { CacheFilename["Gzip"] = "cache.tgz"; CacheFilename["Zstd"] = "cache.tzst"; + CacheFilename["Lz4"] = "cache.tlz4"; })(CacheFilename = exports.CacheFilename || (exports.CacheFilename = {})); var CompressionMethod; (function (CompressionMethod) { CompressionMethod["Gzip"] = "gzip"; + CompressionMethod["Lz4"] = "lz4"; // Long range mode was added to zstd in v1.3.2. // This enum is for earlier version of zstd that does not have --long support CompressionMethod["ZstdWithoutLong"] = "zstd-without-long"; diff --git a/dist/cache/index.js b/dist/cache/index.js index 35dc7d1d..c058e7b5 100644 --- a/dist/cache/index.js +++ b/dist/cache/index.js @@ -140,12 +140,15 @@ async function saveCache(paths, key) { const archivePath = path.join(archiveFolder, utils.getCacheFileName(compressionMethod)); core.debug(`Archive Path: ${archivePath}`); try { + const cacheSizeBeforeCompression = utils.getCacheSizeInBytes(cachePaths); + core.info(`Cache Size before compression: ~${Math.round(cacheSizeBeforeCompression / (1024 * 1024))} MB (${cacheSizeBeforeCompression} B)`); await (0, tar_1.createTar)(archiveFolder, cachePaths, compressionMethod); if (core.isDebug()) { await (0, tar_1.listTar)(archivePath, compressionMethod); } const fileSizeLimit = 10 * 1024 * 1024 * 1024; // 10GB per repo limit const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath); + core.info(`Cache Size after compression: ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B)`); core.debug(`File Size: ${archiveFileSize}`); // For GHES, this check will take place in ReserveCache API with enterprise file size limit if (archiveFileSize > fileSizeLimit && !utils.isGhes()) { diff --git a/dist/cache/tar.js b/dist/cache/tar.js index 73de8a86..6ca50bd2 100644 --- a/dist/cache/tar.js +++ b/dist/cache/tar.js @@ -68,10 +68,10 @@ async function getTarPath() { } async function getCacheFileName(compressionMethod, resolveTarPath = getTarPath()) { const tarPath = await resolveTarPath; - const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === constants_1.ArchiveToolType.BSD && compressionMethod !== constants_1.CompressionMethod.Gzip && IS_WINDOWS; - return BSD_TAR_ZSTD + return BSD_TAR_EXTERNAL_COMPRESSION ? 'cache.tar' : utils.getCacheFileName(compressionMethod); } @@ -82,8 +82,8 @@ async function getTarArgs(tarPath, compressionMethod, type, archivePath = '') { const cacheFileName = path.join(archivePath, await getCacheFileName(compressionMethod, Promise.resolve(tarPath))); const tarFile = 'cache.tar'; const workingDirectory = getWorkingDirectory(); - // Speficic args for BSD tar on windows for workaround - const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD + // Specific args for BSD tar on windows for workaround + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === constants_1.ArchiveToolType.BSD && compressionMethod !== constants_1.CompressionMethod.Gzip && IS_WINDOWS; // Method specific args @@ -92,12 +92,12 @@ async function getTarArgs(tarPath, compressionMethod, type, archivePath = '') { args.push('--posix', '-cf', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', constants_1.ManifestFilename); break; case 'extract': - args.push('-xf', BSD_TAR_ZSTD + args.push('-xf', BSD_TAR_EXTERNAL_COMPRESSION ? tarFile : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')); break; case 'list': - args.push('-tf', BSD_TAR_ZSTD + args.push('-tf', BSD_TAR_EXTERNAL_COMPRESSION ? tarFile : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P'); break; @@ -125,16 +125,16 @@ async function getCommands(compressionMethod, type, archivePath = '') { const compressionArgs = type !== 'create' ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) : await getCompressionProgram(tarPath, compressionMethod); - const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === constants_1.ArchiveToolType.BSD && compressionMethod !== constants_1.CompressionMethod.Gzip && IS_WINDOWS; - if (BSD_TAR_ZSTD && type !== 'create') { + if (BSD_TAR_EXTERNAL_COMPRESSION && type !== 'create') { args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')]; } else { args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')]; } - if (BSD_TAR_ZSTD) { + if (BSD_TAR_EXTERNAL_COMPRESSION) { return args; } return [args.join(' ')]; @@ -148,12 +148,20 @@ function getDecompressionProgram(tarPath, compressionMethod, archivePath) { // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === constants_1.ArchiveToolType.BSD && compressionMethod !== constants_1.CompressionMethod.Gzip && IS_WINDOWS; switch (compressionMethod) { + case constants_1.CompressionMethod.Lz4: + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION + ? [ + 'lz4 -d --force', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + constants_1.TarFilename, + ] + : ['--use-compress-program', IS_WINDOWS ? '"lz4 -d"' : 'lz4 -d']); case constants_1.CompressionMethod.Zstd: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -d --long=30 --force -o', constants_1.TarFilename, @@ -164,7 +172,7 @@ function getDecompressionProgram(tarPath, compressionMethod, archivePath) { IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30', ]); case constants_1.CompressionMethod.ZstdWithoutLong: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -d --force -o', constants_1.TarFilename, @@ -183,12 +191,20 @@ function getDecompressionProgram(tarPath, compressionMethod, archivePath) { // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. function getCompressionProgram(tarPath, compressionMethod) { const cacheFileName = utils.getCacheFileName(compressionMethod); - const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === constants_1.ArchiveToolType.BSD && compressionMethod !== constants_1.CompressionMethod.Gzip && IS_WINDOWS; switch (compressionMethod) { + case constants_1.CompressionMethod.Lz4: + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION + ? [ + 'lz4 --force', + constants_1.TarFilename, + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + ] + : ['--use-compress-program', 'lz4']); case constants_1.CompressionMethod.Zstd: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -T0 --long=30 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), @@ -199,7 +215,7 @@ function getCompressionProgram(tarPath, compressionMethod) { IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30', ]); case constants_1.CompressionMethod.ZstdWithoutLong: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -T0 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), diff --git a/dist/constants.d.ts b/dist/constants.d.ts index 4d3000b8..58d86825 100644 --- a/dist/constants.d.ts +++ b/dist/constants.d.ts @@ -3,7 +3,8 @@ export declare enum Inputs { Path = "path", RestoreKeys = "restore-keys", FailOnCacheMiss = "fail-on-cache-miss", - LookupOnly = "lookup-only" + LookupOnly = "lookup-only", + CompressionMode = "compression-mode" } export declare enum Outputs { CacheHit = "cache-hit", diff --git a/dist/constants.js b/dist/constants.js index 0840fcb2..3e913f9b 100644 --- a/dist/constants.js +++ b/dist/constants.js @@ -7,7 +7,8 @@ var Inputs; Inputs["Path"] = "path"; Inputs["RestoreKeys"] = "restore-keys"; Inputs["FailOnCacheMiss"] = "fail-on-cache-miss"; - Inputs["LookupOnly"] = "lookup-only"; // Input for cache, restore action + Inputs["LookupOnly"] = "lookup-only"; + Inputs["CompressionMode"] = "compression-mode"; // Input for cache, restore, save action })(Inputs = exports.Inputs || (exports.Inputs = {})); var Outputs; (function (Outputs) { diff --git a/dist/utils/testUtils.d.ts b/dist/utils/testUtils.d.ts index 1ba08679..af8893ba 100644 --- a/dist/utils/testUtils.d.ts +++ b/dist/utils/testUtils.d.ts @@ -6,6 +6,7 @@ interface CacheInput { enableCrossOsArchive?: boolean; failOnCacheMiss?: boolean; lookupOnly?: boolean; + compressionMode?: string; } export declare function setInputs(input: CacheInput): void; export declare function clearInputs(): void; diff --git a/dist/utils/testUtils.js b/dist/utils/testUtils.js index e3ecf04e..f454b458 100644 --- a/dist/utils/testUtils.js +++ b/dist/utils/testUtils.js @@ -21,6 +21,9 @@ function setInputs(input) { if (input.lookupOnly !== undefined) { setInput(constants_1.Inputs.LookupOnly, input.lookupOnly.toString()); } + if (input.compressionMode !== undefined) { + setInput(constants_1.Inputs.CompressionMode, input.compressionMode); + } } exports.setInputs = setInputs; function clearInputs() { @@ -29,5 +32,6 @@ function clearInputs() { delete process.env[getInputName(constants_1.Inputs.RestoreKeys)]; delete process.env[getInputName(constants_1.Inputs.FailOnCacheMiss)]; delete process.env[getInputName(constants_1.Inputs.LookupOnly)]; + delete process.env[getInputName(constants_1.Inputs.CompressionMode)]; } exports.clearInputs = clearInputs; diff --git a/restore/action.yml b/restore/action.yml index 28456f81..2462ce60 100644 --- a/restore/action.yml +++ b/restore/action.yml @@ -18,6 +18,10 @@ inputs: description: 'Check if a cache entry exists for the given input(s) (key, restore-keys) without downloading the cache' default: 'false' required: false + compression-mode: + description: 'Compression mode to use for cache archives. Supported values are auto, gzip, zstd, and lz4.' + default: 'auto' + required: false outputs: cache-hit: description: 'A boolean value to indicate an exact match was found for the primary key' @@ -30,4 +34,4 @@ runs: main: '../dist/restoreOnly.js' branding: icon: 'archive' - color: 'gray-dark' + color: 'gray-dark' \ No newline at end of file diff --git a/save/action.yml b/save/action.yml index 493955fe..b82f02ca 100644 --- a/save/action.yml +++ b/save/action.yml @@ -7,9 +7,13 @@ inputs: key: description: 'An explicit key for saving the cache' required: true + compression-mode: + description: 'Compression mode to use for cache archives. Supported values are auto, gzip, zstd, and lz4.' + default: 'auto' + required: false runs: using: 'node20' main: '../dist/saveOnly.js' branding: icon: 'archive' - color: 'gray-dark' + color: 'gray-dark' \ No newline at end of file diff --git a/src/cache/cacheUtils.ts b/src/cache/cacheUtils.ts index 61d6882c..44b35637 100644 --- a/src/cache/cacheUtils.ts +++ b/src/cache/cacheUtils.ts @@ -7,6 +7,7 @@ import * as glob from '@actions/glob' import * as io from '@actions/io' import * as semver from 'semver' import { v4 as uuidV4 } from 'uuid' +import { Inputs } from '../constants' import { CacheFilename, CompressionMethod, @@ -41,6 +42,35 @@ export function getArchiveFileSizeInBytes(filePath: string): number { return fs.statSync(filePath).size } +function getPathSizeInBytes(filePath: string): number { + const stats = fs.lstatSync(filePath) + + if (!stats.isDirectory()) { + return stats.size + } + + return fs.readdirSync(filePath).reduce( + (total, entry) => total + getPathSizeInBytes(path.join(filePath, entry)), + 0, + ) +} + +export function getCacheSizeInBytes(paths: string[]): number { + const workspace = process.env.GITHUB_WORKSPACE ?? process.cwd() + + return paths.reduce((total, cachePath) => { + const filePath = path.isAbsolute(cachePath) + ? cachePath + : path.join(workspace, cachePath) + + if (!fs.existsSync(filePath)) { + return total + } + + return total + getPathSizeInBytes(filePath) + }, 0) +} + export async function resolvePaths(patterns: string[]): Promise { const paths: string[] = [] const workspace = process.env.GITHUB_WORKSPACE ?? process.cwd() @@ -103,6 +133,22 @@ async function getVersion( // Use zstandard if possible to maximize cache performance export async function getCompressionMethod(): Promise { + const compressionMode = core.getInput(Inputs.CompressionMode) || 'auto' + switch (compressionMode.toLowerCase()) { + case 'auto': + break + case CompressionMethod.Gzip: + return CompressionMethod.Gzip + case CompressionMethod.Lz4: + return CompressionMethod.Lz4 + case CompressionMethod.Zstd: + return CompressionMethod.ZstdWithoutLong + default: + throw new Error( + `Unsupported compression mode: ${compressionMode}. Supported modes are: auto, gzip, zstd, lz4.`, + ) + } + const versionOutput = await getVersion('zstd', ['--quiet']) const version = semver.clean(versionOutput) core.debug(`zstd version: ${version}`) @@ -114,9 +160,14 @@ export async function getCompressionMethod(): Promise { } export function getCacheFileName(compressionMethod: CompressionMethod): string { - return compressionMethod === CompressionMethod.Gzip - ? CacheFilename.Gzip - : CacheFilename.Zstd + switch (compressionMethod) { + case CompressionMethod.Gzip: + return CacheFilename.Gzip + case CompressionMethod.Lz4: + return CacheFilename.Lz4 + default: + return CacheFilename.Zstd + } } export async function getGnuTarPathOnWindows(): Promise { diff --git a/src/cache/constants.ts b/src/cache/constants.ts index 9e1a3308..572352b8 100644 --- a/src/cache/constants.ts +++ b/src/cache/constants.ts @@ -1,10 +1,12 @@ export enum CacheFilename { Gzip = 'cache.tgz', - Zstd = 'cache.tzst' + Zstd = 'cache.tzst', + Lz4 = 'cache.tlz4' } export enum CompressionMethod { Gzip = 'gzip', + Lz4 = 'lz4', // Long range mode was added to zstd in v1.3.2. // This enum is for earlier version of zstd that does not have --long support ZstdWithoutLong = 'zstd-without-long', diff --git a/src/cache/index.ts b/src/cache/index.ts index dac5278e..aef65c69 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -156,12 +156,25 @@ export async function saveCache( core.debug(`Archive Path: ${archivePath}`) try { + const cacheSizeBeforeCompression = utils.getCacheSizeInBytes(cachePaths) + core.info( + `Cache Size before compression: ~${Math.round( + cacheSizeBeforeCompression / (1024 * 1024), + )} MB (${cacheSizeBeforeCompression} B)`, + ) + await createTar(archiveFolder, cachePaths, compressionMethod) + if (core.isDebug()) { await listTar(archivePath, compressionMethod) } const fileSizeLimit = 10 * 1024 * 1024 * 1024 // 10GB per repo limit const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) + core.info( + `Cache Size after compression: ~${Math.round( + archiveFileSize / (1024 * 1024), + )} MB (${archiveFileSize} B)`, + ) core.debug(`File Size: ${archiveFileSize}`) // For GHES, this check will take place in ReserveCache API with enterprise file size limit diff --git a/src/cache/tar.ts b/src/cache/tar.ts index 0a093c47..8c12bb9c 100644 --- a/src/cache/tar.ts +++ b/src/cache/tar.ts @@ -54,10 +54,10 @@ export async function getCacheFileName( resolveTarPath = getTarPath(), ) { const tarPath = await resolveTarPath - const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && IS_WINDOWS - return BSD_TAR_ZSTD + return BSD_TAR_EXTERNAL_COMPRESSION ? 'cache.tar' : utils.getCacheFileName(compressionMethod) } @@ -76,8 +76,8 @@ async function getTarArgs( )) const tarFile = 'cache.tar' const workingDirectory = getWorkingDirectory() - // Speficic args for BSD tar on windows for workaround - const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD + // Specific args for BSD tar on windows for workaround + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && IS_WINDOWS @@ -98,7 +98,7 @@ async function getTarArgs( case 'extract': args.push( '-xf', - BSD_TAR_ZSTD + BSD_TAR_EXTERNAL_COMPRESSION ? tarFile : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -109,7 +109,7 @@ async function getTarArgs( case 'list': args.push( '-tf', - BSD_TAR_ZSTD + BSD_TAR_EXTERNAL_COMPRESSION ? tarFile : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -152,17 +152,17 @@ async function getCommands( const compressionArgs = type !== 'create' ? await getDecompressionProgram(tarPath, compressionMethod, archivePath) : await getCompressionProgram(tarPath, compressionMethod) - const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && IS_WINDOWS - if (BSD_TAR_ZSTD && type !== 'create') { + if (BSD_TAR_EXTERNAL_COMPRESSION && type !== 'create') { args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')] } else { args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')] } - if (BSD_TAR_ZSTD) { + if (BSD_TAR_EXTERNAL_COMPRESSION) { return args } @@ -183,12 +183,20 @@ function getDecompressionProgram( // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. - const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && IS_WINDOWS switch (compressionMethod) { + case CompressionMethod.Lz4: + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION + ? [ + 'lz4 -d --force', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + TarFilename, + ] + : ['--use-compress-program', IS_WINDOWS ? '"lz4 -d"' : 'lz4 -d']) case CompressionMethod.Zstd: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -d --long=30 --force -o', TarFilename, @@ -199,7 +207,7 @@ function getDecompressionProgram( IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30', ]) case CompressionMethod.ZstdWithoutLong: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -d --force -o', TarFilename, @@ -222,12 +230,20 @@ function getCompressionProgram( compressionMethod: CompressionMethod, ): Promise { const cacheFileName = utils.getCacheFileName(compressionMethod) - const BSD_TAR_ZSTD = tarPath.type === ArchiveToolType.BSD + const BSD_TAR_EXTERNAL_COMPRESSION = tarPath.type === ArchiveToolType.BSD && compressionMethod !== CompressionMethod.Gzip && IS_WINDOWS switch (compressionMethod) { + case CompressionMethod.Lz4: + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION + ? [ + 'lz4 --force', + TarFilename, + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + ] + : ['--use-compress-program', 'lz4']) case CompressionMethod.Zstd: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -T0 --long=30 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), @@ -238,7 +254,7 @@ function getCompressionProgram( IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30', ]) case CompressionMethod.ZstdWithoutLong: - return Promise.resolve(BSD_TAR_ZSTD + return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ 'zstd -T0 --force -o', cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), diff --git a/src/constants.ts b/src/constants.ts index c8414aa5..05958e4c 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,7 +3,8 @@ export enum Inputs { Path = 'path', // Input for cache, restore, save action RestoreKeys = 'restore-keys', // Input for cache, restore action FailOnCacheMiss = 'fail-on-cache-miss', // Input for cache, restore action - LookupOnly = 'lookup-only' // Input for cache, restore action + LookupOnly = 'lookup-only', // Input for cache, restore action + CompressionMode = 'compression-mode' // Input for cache, restore, save action } export enum Outputs { diff --git a/src/utils/testUtils.ts b/src/utils/testUtils.ts index 16566133..8fbec33c 100644 --- a/src/utils/testUtils.ts +++ b/src/utils/testUtils.ts @@ -16,6 +16,7 @@ interface CacheInput { enableCrossOsArchive?: boolean, failOnCacheMiss?: boolean, lookupOnly?: boolean, + compressionMode?: string, } export function setInputs(input: CacheInput): void { @@ -28,6 +29,9 @@ export function setInputs(input: CacheInput): void { if (input.lookupOnly !== undefined) { setInput(Inputs.LookupOnly, input.lookupOnly.toString()) } + if (input.compressionMode !== undefined) { + setInput(Inputs.CompressionMode, input.compressionMode) + } } export function clearInputs(): void { @@ -36,4 +40,5 @@ export function clearInputs(): void { delete process.env[getInputName(Inputs.RestoreKeys)] delete process.env[getInputName(Inputs.FailOnCacheMiss)] delete process.env[getInputName(Inputs.LookupOnly)] + delete process.env[getInputName(Inputs.CompressionMode)] } From 2aceab712f23f0841d5452d7db54ef0309a5b3bb Mon Sep 17 00:00:00 2001 From: "david.rzepa" Date: Tue, 16 Jun 2026 08:46:40 -0600 Subject: [PATCH 2/2] feat: add compression args and cache size logs --- README.md | 3 ++ __tests__/tar.test.ts | 61 +++++++++++++++++++++++++++++++++++++- action.yml | 3 ++ dist/cache/cacheUtils.d.ts | 1 + dist/cache/cacheUtils.js | 6 +++- dist/cache/index.js | 5 +++- dist/cache/tar.js | 48 ++++++++++++++++++++++-------- dist/constants.d.ts | 3 +- dist/constants.js | 3 +- dist/utils/testUtils.d.ts | 1 + dist/utils/testUtils.js | 4 +++ restore/action.yml | 5 +++- save/action.yml | 5 +++- src/cache/cacheUtils.ts | 4 +++ src/cache/index.ts | 9 +++++- src/cache/tar.ts | 54 +++++++++++++++++++++++++-------- src/constants.ts | 3 +- src/utils/testUtils.ts | 5 ++++ 18 files changed, 190 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index c3a692b0..44db30de 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,11 @@ The action defaults to `compression-mode: auto`. To force lz4 compression on run path: prime-numbers key: ${{ runner.os }}-primes compression-mode: lz4 + compression-args: --fast=2 ```` +`compression-args` are passed to the active compression program. In a save step they tune compression; in a restore step they tune decompression. + ## Contributing Contributions to the project are welcome. Feel free to fork and improve. I do my best accept pull requests in a timely manor, especially when tests and updated docs are included. diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts index 3f721076..9f062afe 100644 --- a/__tests__/tar.test.ts +++ b/__tests__/tar.test.ts @@ -10,6 +10,8 @@ import { extractTar, listTar, } from '../src/cache/tar' +import { Inputs } from '../src/constants' +import * as testUtils from '../src/utils/testUtils' jest.mock('@actions/exec') jest.mock('@actions/io') @@ -34,6 +36,7 @@ beforeEach(() => { afterEach(() => { process.env.GITHUB_WORKSPACE = originalWorkspace + testUtils.clearInputs() fs.rmSync(archiveFolder, { recursive: true, force: true }) jest.restoreAllMocks() }) @@ -42,7 +45,7 @@ test('createTar uses lz4 compression program', async () => { await createTar(archiveFolder, ['node_modules'], CompressionMethod.Lz4) expect(execMock).toHaveBeenCalledTimes(1) - expect(execMock.mock.calls[0][0]).toContain('--use-compress-program lz4') + expect(execMock.mock.calls[0][0]).toContain('--use-compress-program "lz4"') expect(execMock.mock.calls[0][0]).toContain('cache.tlz4') expect(execMock.mock.calls[0][0]).toContain('--files-from manifest.txt') expect(execMock.mock.calls[0][2]?.cwd).toBe(archiveFolder) @@ -65,3 +68,59 @@ test('listTar uses lz4 decompression program', async () => { expect(execMock.mock.calls[0][0]).toContain('lz4 -d') expect(execMock.mock.calls[0][0]).toContain('cache.tlz4') }) + +test('createTar forwards compression args to lz4', async () => { + testUtils.setInput(Inputs.CompressionArgs, '--fast=2') + + await createTar(archiveFolder, ['node_modules'], CompressionMethod.Lz4) + + expect(execMock.mock.calls[0][0]).toContain( + '--use-compress-program "lz4 --fast=2"', + ) +}) + +test('createTar forwards compression args to zstd', async () => { + testUtils.setInput(Inputs.CompressionArgs, '--adapt') + + await createTar(archiveFolder, ['node_modules'], CompressionMethod.Zstd) + + expect(execMock.mock.calls[0][0]).toMatch(/zstd(mt)?/) + expect(execMock.mock.calls[0][0]).toContain('--adapt') +}) + +test('createTar forwards compression args to gzip', async () => { + testUtils.setInput(Inputs.CompressionArgs, '-1') + + await createTar(archiveFolder, ['node_modules'], CompressionMethod.Gzip) + + expect(execMock.mock.calls[0][0]).toContain( + '--use-compress-program "gzip -1"', + ) +}) + +test('extractTar forwards compression args to lz4 decompression', async () => { + testUtils.setInput(Inputs.CompressionArgs, '--no-sparse') + + await extractTar(path.join(archiveFolder, 'cache.tlz4'), CompressionMethod.Lz4) + + expect(execMock.mock.calls[0][0]).toContain('lz4 -d --no-sparse') +}) + +test('extractTar forwards compression args to zstd decompression', async () => { + testUtils.setInput(Inputs.CompressionArgs, '--long=31') + + await extractTar(path.join(archiveFolder, 'cache.tzst'), CompressionMethod.Zstd) + + expect(execMock.mock.calls[0][0]).toMatch(/unzstd|zstd -d/) + expect(execMock.mock.calls[0][0]).toContain('--long=31') +}) + +test('extractTar forwards compression args to gzip decompression', async () => { + testUtils.setInput(Inputs.CompressionArgs, '--verbose') + + await extractTar(path.join(archiveFolder, 'cache.tgz'), CompressionMethod.Gzip) + + expect(execMock.mock.calls[0][0]).toContain( + '--use-compress-program "gzip -d --verbose"', + ) +}) diff --git a/action.yml b/action.yml index 65cc3733..d177b2ce 100644 --- a/action.yml +++ b/action.yml @@ -22,6 +22,9 @@ inputs: description: 'Compression mode to use for cache archives. Supported values are auto, gzip, zstd, and lz4.' default: 'auto' required: false + compression-args: + description: 'Additional arguments to pass to the compression program when saving or restoring a cache archive.' + required: false outputs: cache-hit: description: 'A boolean value to indicate if cache was found and restored' diff --git a/dist/cache/cacheUtils.d.ts b/dist/cache/cacheUtils.d.ts index e0f174d2..a9cce894 100644 --- a/dist/cache/cacheUtils.d.ts +++ b/dist/cache/cacheUtils.d.ts @@ -8,6 +8,7 @@ export declare function resolvePaths(patterns: string[]): Promise; export declare function unlinkFile(filePath: fs.PathLike): Promise; export declare function getCompressionMethod(): Promise; export declare function getCacheFileName(compressionMethod: CompressionMethod): string; +export declare function getCompressionArgs(): string; export declare function getGnuTarPathOnWindows(): Promise; export declare function assertDefined(name: string, value?: T): T; export declare function isGhes(): boolean; diff --git a/dist/cache/cacheUtils.js b/dist/cache/cacheUtils.js index a0ba9c6b..97bbe1c2 100644 --- a/dist/cache/cacheUtils.js +++ b/dist/cache/cacheUtils.js @@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.isGhes = exports.assertDefined = exports.getGnuTarPathOnWindows = exports.getCacheFileName = exports.getCompressionMethod = exports.unlinkFile = exports.resolvePaths = exports.getCacheSizeInBytes = exports.getArchiveFileSizeInBytes = exports.createTempDirectory = void 0; +exports.isGhes = exports.assertDefined = exports.getGnuTarPathOnWindows = exports.getCompressionArgs = exports.getCacheFileName = exports.getCompressionMethod = exports.unlinkFile = exports.resolvePaths = exports.getCacheSizeInBytes = exports.getArchiveFileSizeInBytes = exports.createTempDirectory = void 0; const fs = __importStar(require("fs")); const path = __importStar(require("path")); const util = __importStar(require("util")); @@ -172,6 +172,10 @@ function getCacheFileName(compressionMethod) { } } exports.getCacheFileName = getCacheFileName; +function getCompressionArgs() { + return core.getInput(constants_1.Inputs.CompressionArgs).trim(); +} +exports.getCompressionArgs = getCompressionArgs; async function getGnuTarPathOnWindows() { if (fs.existsSync(constants_2.GnuTarPathOnWindows)) { return constants_2.GnuTarPathOnWindows; diff --git a/dist/cache/index.js b/dist/cache/index.js index c058e7b5..655c4bb9 100644 --- a/dist/cache/index.js +++ b/dist/cache/index.js @@ -96,8 +96,11 @@ async function restoreCache(paths, primaryKey, restoreKeys, lookupOnly) { await (0, tar_1.listTar)(archivePath, compressionMethod); } const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath); - core.info(`Cache Size: ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B)`); + core.info(`Cache Size before decompression: ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B)`); await (0, tar_1.extractTar)(archivePath, compressionMethod); + const cachePaths = await utils.resolvePaths(paths); + const cacheSizeAfterDecompression = utils.getCacheSizeInBytes(cachePaths); + core.info(`Cache Size after decompression: ~${Math.round(cacheSizeAfterDecompression / (1024 * 1024))} MB (${cacheSizeAfterDecompression} B)`); core.info('Cache restored successfully'); return cacheEntry.cacheKey; } diff --git a/dist/cache/tar.js b/dist/cache/tar.js index 6ca50bd2..187b93de 100644 --- a/dist/cache/tar.js +++ b/dist/cache/tar.js @@ -142,6 +142,13 @@ async function getCommands(compressionMethod, type, archivePath = '') { function getWorkingDirectory() { return process.env.GITHUB_WORKSPACE ?? process.cwd(); } +function appendCompressionArgs(command) { + const compressionArgs = utils.getCompressionArgs(); + return compressionArgs ? `${command} ${compressionArgs}` : command; +} +function getUseCompressProgram(command) { + return `"${appendCompressionArgs(command)}"`; +} // Common function for extractTar and listTar to get the compression method function getDecompressionProgram(tarPath, compressionMethod, archivePath) { // -d: Decompress. @@ -155,30 +162,40 @@ function getDecompressionProgram(tarPath, compressionMethod, archivePath) { case constants_1.CompressionMethod.Lz4: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'lz4 -d --force', + appendCompressionArgs('lz4 -d --force'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), constants_1.TarFilename, ] - : ['--use-compress-program', IS_WINDOWS ? '"lz4 -d"' : 'lz4 -d']); + : [ + '--use-compress-program', + getUseCompressProgram('lz4 -d'), + ]); case constants_1.CompressionMethod.Zstd: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -d --long=30 --force -o', + appendCompressionArgs('zstd -d --long=30 --force -o'), constants_1.TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), ] : [ '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30', + getUseCompressProgram(IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'), ]); case constants_1.CompressionMethod.ZstdWithoutLong: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -d --force -o', + appendCompressionArgs('zstd -d --force -o'), constants_1.TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']); + : [ + '--use-compress-program', + getUseCompressProgram(IS_WINDOWS ? 'zstd -d' : 'unzstd'), + ]); + case constants_1.CompressionMethod.Gzip: + return Promise.resolve(utils.getCompressionArgs() + ? ['--use-compress-program', getUseCompressProgram('gzip -d')] + : ['-z']); default: return Promise.resolve(['-z']); } @@ -198,30 +215,37 @@ function getCompressionProgram(tarPath, compressionMethod) { case constants_1.CompressionMethod.Lz4: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'lz4 --force', + appendCompressionArgs('lz4 --force'), constants_1.TarFilename, cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), ] - : ['--use-compress-program', 'lz4']); + : ['--use-compress-program', getUseCompressProgram('lz4')]); case constants_1.CompressionMethod.Zstd: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -T0 --long=30 --force -o', + appendCompressionArgs('zstd -T0 --long=30 --force -o'), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), constants_1.TarFilename, ] : [ '--use-compress-program', - IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30', + getUseCompressProgram(IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'), ]); case constants_1.CompressionMethod.ZstdWithoutLong: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -T0 --force -o', + appendCompressionArgs('zstd -T0 --force -o'), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), constants_1.TarFilename, ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']); + : [ + '--use-compress-program', + getUseCompressProgram(IS_WINDOWS ? 'zstd -T0' : 'zstdmt'), + ]); + case constants_1.CompressionMethod.Gzip: + return Promise.resolve(utils.getCompressionArgs() + ? ['--use-compress-program', getUseCompressProgram('gzip')] + : ['-z']); default: return Promise.resolve(['-z']); } diff --git a/dist/constants.d.ts b/dist/constants.d.ts index 58d86825..3b35080a 100644 --- a/dist/constants.d.ts +++ b/dist/constants.d.ts @@ -4,7 +4,8 @@ export declare enum Inputs { RestoreKeys = "restore-keys", FailOnCacheMiss = "fail-on-cache-miss", LookupOnly = "lookup-only", - CompressionMode = "compression-mode" + CompressionMode = "compression-mode", + CompressionArgs = "compression-args" } export declare enum Outputs { CacheHit = "cache-hit", diff --git a/dist/constants.js b/dist/constants.js index 3e913f9b..b1522a7d 100644 --- a/dist/constants.js +++ b/dist/constants.js @@ -8,7 +8,8 @@ var Inputs; Inputs["RestoreKeys"] = "restore-keys"; Inputs["FailOnCacheMiss"] = "fail-on-cache-miss"; Inputs["LookupOnly"] = "lookup-only"; - Inputs["CompressionMode"] = "compression-mode"; // Input for cache, restore, save action + Inputs["CompressionMode"] = "compression-mode"; + Inputs["CompressionArgs"] = "compression-args"; // Input for cache, restore, save action })(Inputs = exports.Inputs || (exports.Inputs = {})); var Outputs; (function (Outputs) { diff --git a/dist/utils/testUtils.d.ts b/dist/utils/testUtils.d.ts index af8893ba..d1172483 100644 --- a/dist/utils/testUtils.d.ts +++ b/dist/utils/testUtils.d.ts @@ -7,6 +7,7 @@ interface CacheInput { failOnCacheMiss?: boolean; lookupOnly?: boolean; compressionMode?: string; + compressionArgs?: string; } export declare function setInputs(input: CacheInput): void; export declare function clearInputs(): void; diff --git a/dist/utils/testUtils.js b/dist/utils/testUtils.js index f454b458..963b4446 100644 --- a/dist/utils/testUtils.js +++ b/dist/utils/testUtils.js @@ -24,6 +24,9 @@ function setInputs(input) { if (input.compressionMode !== undefined) { setInput(constants_1.Inputs.CompressionMode, input.compressionMode); } + if (input.compressionArgs !== undefined) { + setInput(constants_1.Inputs.CompressionArgs, input.compressionArgs); + } } exports.setInputs = setInputs; function clearInputs() { @@ -33,5 +36,6 @@ function clearInputs() { delete process.env[getInputName(constants_1.Inputs.FailOnCacheMiss)]; delete process.env[getInputName(constants_1.Inputs.LookupOnly)]; delete process.env[getInputName(constants_1.Inputs.CompressionMode)]; + delete process.env[getInputName(constants_1.Inputs.CompressionArgs)]; } exports.clearInputs = clearInputs; diff --git a/restore/action.yml b/restore/action.yml index 2462ce60..04633533 100644 --- a/restore/action.yml +++ b/restore/action.yml @@ -22,6 +22,9 @@ inputs: description: 'Compression mode to use for cache archives. Supported values are auto, gzip, zstd, and lz4.' default: 'auto' required: false + compression-args: + description: 'Additional arguments to pass to the compression program when restoring a cache archive.' + required: false outputs: cache-hit: description: 'A boolean value to indicate an exact match was found for the primary key' @@ -34,4 +37,4 @@ runs: main: '../dist/restoreOnly.js' branding: icon: 'archive' - color: 'gray-dark' \ No newline at end of file + color: 'gray-dark' diff --git a/save/action.yml b/save/action.yml index b82f02ca..dbc5d176 100644 --- a/save/action.yml +++ b/save/action.yml @@ -11,9 +11,12 @@ inputs: description: 'Compression mode to use for cache archives. Supported values are auto, gzip, zstd, and lz4.' default: 'auto' required: false + compression-args: + description: 'Additional arguments to pass to the compression program when saving a cache archive.' + required: false runs: using: 'node20' main: '../dist/saveOnly.js' branding: icon: 'archive' - color: 'gray-dark' \ No newline at end of file + color: 'gray-dark' diff --git a/src/cache/cacheUtils.ts b/src/cache/cacheUtils.ts index 44b35637..5307d2b7 100644 --- a/src/cache/cacheUtils.ts +++ b/src/cache/cacheUtils.ts @@ -170,6 +170,10 @@ export function getCacheFileName(compressionMethod: CompressionMethod): string { } } +export function getCompressionArgs(): string { + return core.getInput(Inputs.CompressionArgs).trim() +} + export async function getGnuTarPathOnWindows(): Promise { if (fs.existsSync(GnuTarPathOnWindows)) { return GnuTarPathOnWindows diff --git a/src/cache/index.ts b/src/cache/index.ts index aef65c69..300f6792 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -96,12 +96,19 @@ export async function restoreCache( const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath) core.info( - `Cache Size: ~${Math.round( + `Cache Size before decompression: ~${Math.round( archiveFileSize / (1024 * 1024), )} MB (${archiveFileSize} B)`, ) await extractTar(archivePath, compressionMethod) + const cachePaths = await utils.resolvePaths(paths) + const cacheSizeAfterDecompression = utils.getCacheSizeInBytes(cachePaths) + core.info( + `Cache Size after decompression: ~${Math.round( + cacheSizeAfterDecompression / (1024 * 1024), + )} MB (${cacheSizeAfterDecompression} B)`, + ) core.info('Cache restored successfully') return cacheEntry.cacheKey diff --git a/src/cache/tar.ts b/src/cache/tar.ts index 8c12bb9c..fdc2fd09 100644 --- a/src/cache/tar.ts +++ b/src/cache/tar.ts @@ -173,6 +173,15 @@ function getWorkingDirectory(): string { return process.env.GITHUB_WORKSPACE ?? process.cwd() } +function appendCompressionArgs(command: string): string { + const compressionArgs = utils.getCompressionArgs() + return compressionArgs ? `${command} ${compressionArgs}` : command +} + +function getUseCompressProgram(command: string): string { + return `"${appendCompressionArgs(command)}"` +} + // Common function for extractTar and listTar to get the compression method function getDecompressionProgram( tarPath: ArchiveTool, @@ -190,30 +199,42 @@ function getDecompressionProgram( case CompressionMethod.Lz4: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'lz4 -d --force', + appendCompressionArgs('lz4 -d --force'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename, ] - : ['--use-compress-program', IS_WINDOWS ? '"lz4 -d"' : 'lz4 -d']) + : [ + '--use-compress-program', + getUseCompressProgram('lz4 -d'), + ]) case CompressionMethod.Zstd: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -d --long=30 --force -o', + appendCompressionArgs('zstd -d --long=30 --force -o'), TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), ] : [ '--use-compress-program', - IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30', + getUseCompressProgram( + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + ), ]) case CompressionMethod.ZstdWithoutLong: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -d --force -o', + appendCompressionArgs('zstd -d --force -o'), TarFilename, archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']) + : [ + '--use-compress-program', + getUseCompressProgram(IS_WINDOWS ? 'zstd -d' : 'unzstd'), + ]) + case CompressionMethod.Gzip: + return Promise.resolve(utils.getCompressionArgs() + ? ['--use-compress-program', getUseCompressProgram('gzip -d')] + : ['-z']) default: return Promise.resolve(['-z']) } @@ -237,30 +258,39 @@ function getCompressionProgram( case CompressionMethod.Lz4: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'lz4 --force', + appendCompressionArgs('lz4 --force'), TarFilename, cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), ] - : ['--use-compress-program', 'lz4']) + : ['--use-compress-program', getUseCompressProgram('lz4')]) case CompressionMethod.Zstd: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -T0 --long=30 --force -o', + appendCompressionArgs('zstd -T0 --long=30 --force -o'), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename, ] : [ '--use-compress-program', - IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30', + getUseCompressProgram( + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', + ), ]) case CompressionMethod.ZstdWithoutLong: return Promise.resolve(BSD_TAR_EXTERNAL_COMPRESSION ? [ - 'zstd -T0 --force -o', + appendCompressionArgs('zstd -T0 --force -o'), cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), TarFilename, ] - : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']) + : [ + '--use-compress-program', + getUseCompressProgram(IS_WINDOWS ? 'zstd -T0' : 'zstdmt'), + ]) + case CompressionMethod.Gzip: + return Promise.resolve(utils.getCompressionArgs() + ? ['--use-compress-program', getUseCompressProgram('gzip')] + : ['-z']) default: return Promise.resolve(['-z']) } diff --git a/src/constants.ts b/src/constants.ts index 05958e4c..4cfb2ff9 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -4,7 +4,8 @@ export enum Inputs { RestoreKeys = 'restore-keys', // Input for cache, restore action FailOnCacheMiss = 'fail-on-cache-miss', // Input for cache, restore action LookupOnly = 'lookup-only', // Input for cache, restore action - CompressionMode = 'compression-mode' // Input for cache, restore, save action + CompressionMode = 'compression-mode', // Input for cache, restore, save action + CompressionArgs = 'compression-args' // Input for cache, restore, save action } export enum Outputs { diff --git a/src/utils/testUtils.ts b/src/utils/testUtils.ts index 8fbec33c..da21190a 100644 --- a/src/utils/testUtils.ts +++ b/src/utils/testUtils.ts @@ -17,6 +17,7 @@ interface CacheInput { failOnCacheMiss?: boolean, lookupOnly?: boolean, compressionMode?: string, + compressionArgs?: string, } export function setInputs(input: CacheInput): void { @@ -32,6 +33,9 @@ export function setInputs(input: CacheInput): void { if (input.compressionMode !== undefined) { setInput(Inputs.CompressionMode, input.compressionMode) } + if (input.compressionArgs !== undefined) { + setInput(Inputs.CompressionArgs, input.compressionArgs) + } } export function clearInputs(): void { @@ -41,4 +45,5 @@ export function clearInputs(): void { delete process.env[getInputName(Inputs.FailOnCacheMiss)] delete process.env[getInputName(Inputs.LookupOnly)] delete process.env[getInputName(Inputs.CompressionMode)] + delete process.env[getInputName(Inputs.CompressionArgs)] }