diff --git a/examples/cli/src/commands/pieces-removal.ts b/examples/cli/src/commands/pieces-removal.ts index 613d41321..cfbaea5b4 100644 --- a/examples/cli/src/commands/pieces-removal.ts +++ b/examples/cli/src/commands/pieces-removal.ts @@ -1,5 +1,5 @@ import * as p from '@clack/prompts' -import { schedulePieceDeletion } from '@filoz/synapse-core/sp' +import { schedulePieceDeletions } from '@filoz/synapse-core/sp' import { getPdpDataSet } from '@filoz/synapse-core/warm-storage' import { type Command, command } from 'cleye' import { waitForTransactionReceipt } from 'viem/actions' @@ -42,10 +42,10 @@ export const piecesRemoval: Command = command( : await selectPiece(client, dataSet, argv.flags) p.log.info(`Removing piece ${pieceId} from data set ${dataSetId}...`) - const result = await schedulePieceDeletion(client, { + const result = await schedulePieceDeletions(client, { dataSetId, clientDataSetId: dataSet.clientDataSetId, - pieceId, + pieceIds: [pieceId], serviceURL: dataSet.provider.pdp.serviceURL, }) diff --git a/packages/synapse-core/src/errors/pdp.ts b/packages/synapse-core/src/errors/pdp.ts index 01d3ed4d1..cf54ba2fb 100644 --- a/packages/synapse-core/src/errors/pdp.ts +++ b/packages/synapse-core/src/errors/pdp.ts @@ -179,6 +179,21 @@ export class DeletePieceError extends SynapseError { } } +export class TooManyPiecesQueuedError extends SynapseError { + override name: 'TooManyPiecesQueuedError' = 'TooManyPiecesQueuedError' + + constructor() { + super(`Too many pieces queued.`, { + details: + 'The data set already has 200 or more scheduled removals queued on-chain; retry after the next proving period flushes the queue.', + }) + } + + static override is(value: unknown): value is TooManyPiecesQueuedError { + return isSynapseError(value) && value.name === 'TooManyPiecesQueuedError' + } +} + export class TerminateServiceError extends SynapseError { override name: 'TerminateServiceError' = 'TerminateServiceError' diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index da3eda33f..57916cef2 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -1,14 +1,17 @@ -import { HttpError, type RequestJsonErrors, request } from 'iso-web/http' +import { HttpError, type RequestErrors, request } from 'iso-web/http' import type { Account, Chain, Client, Hex, Transport } from 'viem' -import { DeletePieceError } from '../errors/pdp.ts' +import { DeletePieceError, TooManyPiecesQueuedError } from '../errors/pdp.ts' +import { AtLeastOnePieceRequiredError, TooManyPiecesError } from '../errors/warm-storage.ts' import { signSchedulePieceRemovals } from '../typed-data/sign-schedule-piece-removals.ts' -import { RETRY_CONSTANTS } from '../utils/constants.ts' +import { RETRY_CONSTANTS, SIZE_CONSTANTS } from '../utils/constants.ts' -export namespace deletePiece { +const MAX_CURIO_PIECE_ID = (1n << 63n) - 1n + +export namespace deletePieces { export type OptionsType = { serviceURL: string dataSetId: bigint - pieceId: bigint + pieceIds: bigint[] extraData: Hex /** The number of retries. Defaults to 2. */ retryCount?: number @@ -18,47 +21,86 @@ export namespace deletePiece { export type OutputType = { hash: Hex } - export type ErrorType = DeletePieceError | RequestJsonErrors + export type ErrorType = + | AtLeastOnePieceRequiredError + | TooManyPiecesError + | RangeError + | DeletePieceError + | RequestErrors } /** - * Delete a piece from a data set on the PDP API. + * Delete pieces from a data set on the PDP API in one transaction. * * DELETE /pdp/data-sets/{dataSetId}/pieces/{pieceId} * - * @param options - {@link deletePiece.OptionsType} - * @returns Hash of the delete operation {@link deletePiece.OutputType} - * @throws Errors {@link deletePiece.ErrorType} + * Curio uses the first piece ID in the URL for backwards-compatible routing and + * the pieceIds request field as the authoritative list when it is non-empty. + * + * @param options - {@link deletePieces.OptionsType} + * @returns Hash of the delete operation {@link deletePieces.OutputType} + * @throws Errors {@link deletePieces.ErrorType} */ -export async function deletePiece(options: deletePiece.OptionsType): Promise { - const { serviceURL, dataSetId, pieceId, extraData } = options - const response = await request.json.delete<{ txHash: Hex }>( - new URL(`pdp/data-sets/${dataSetId}/pieces/${pieceId}`, serviceURL), - { - body: { extraData }, - timeout: RETRY_CONSTANTS.TIMEOUT, - retry: { - retries: options.retryCount, - minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, - shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, - }, - } - ) +export async function deletePieces(options: deletePieces.OptionsType): Promise { + const { serviceURL, dataSetId, extraData } = options + const pieceIds = normalizeDeletePieceIds(options.pieceIds) + + // Curio accepts uint64 JSON numbers. Construct the array from bigint decimal + // strings so IDs above Number.MAX_SAFE_INTEGER are not rounded by JSON.stringify. + const body = `{"extraData":${JSON.stringify(extraData)},"pieceIds":[${pieceIds.join(',')}]}` + const response = await request.delete(new URL(`pdp/data-sets/${dataSetId}/pieces/${pieceIds[0]}`, serviceURL), { + body, + headers: { 'content-type': 'application/json' }, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + }) if (response.error) { if (HttpError.is(response.error)) { + if (response.error.code === 429) { + throw new TooManyPiecesQueuedError() + } throw new DeletePieceError(await response.error.response.text()) } throw response.error } - return { hash: response.result.txHash } + const result = (await response.result.json()) as { txHash: Hex } + return { hash: result.txHash } } -export namespace schedulePieceDeletion { +/** + * Validate a delete-pieces batch before signing or sending it. + */ +export function validateDeletePiecesBatch(pieceCount: number): void { + if (!Number.isInteger(pieceCount) || pieceCount < 1) { + throw new AtLeastOnePieceRequiredError() + } + if (pieceCount > SIZE_CONSTANTS.MAX_DELETE_PIECES_BATCH_SIZE) { + throw new TooManyPiecesError(pieceCount, SIZE_CONSTANTS.MAX_DELETE_PIECES_BATCH_SIZE) + } +} + +function normalizeDeletePieceIds(pieceIds: bigint[]): bigint[] { + const normalized = [...new Set(pieceIds)] + validateDeletePiecesBatch(normalized.length) + + for (const pieceId of normalized) { + if (pieceId < 0n || pieceId > MAX_CURIO_PIECE_ID) { + throw new RangeError(`Piece ID ${pieceId} is outside Curio's supported range of 0 to ${MAX_CURIO_PIECE_ID}`) + } + } + + return normalized +} + +export namespace schedulePieceDeletions { export type OptionsType = { - /** The piece ID to delete. */ - pieceId: bigint + /** The piece IDs to delete. Duplicate IDs are removed before signing. */ + pieceIds: bigint[] /** The data set ID to delete the piece from. */ dataSetId: bigint /** The client data set id (nonce) to use for the signature. Must be unique for each data set. */ @@ -70,23 +112,23 @@ export namespace schedulePieceDeletion { /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } - export type OutputType = deletePiece.OutputType - export type ErrorType = deletePiece.ErrorType + export type OutputType = deletePieces.OutputType + export type ErrorType = deletePieces.ErrorType } /** - * Schedule a piece deletion + * Schedule piece deletions in one transaction. * * Call the Service Provider API to schedule the piece deletion. * * @param client - The client to use to schedule the piece deletion. - * @param options - {@link schedulePieceDeletion.OptionsType} - * @returns schedule piece deletion operation hash {@link schedulePieceDeletion.OutputType} - * @throws Errors {@link schedulePieceDeletion.ErrorType} + * @param options - {@link schedulePieceDeletions.OptionsType} + * @returns Schedule piece deletions operation hash {@link schedulePieceDeletions.OutputType} + * @throws Errors {@link schedulePieceDeletions.ErrorType} * * @example * ```ts - * import { schedulePieceDeletion } from '@filoz/synapse-core/sp' + * import { schedulePieceDeletions } from '@filoz/synapse-core/sp' * import { createWalletClient, http } from 'viem' * import { privateKeyToAccount } from 'viem/accounts' * import { calibration } from '@filoz/synapse-core/chains' @@ -98,8 +140,8 @@ export namespace schedulePieceDeletion { * transport: http(), * }) * - * const result = await schedulePieceDeletion(client, { - * pieceId: 1n, + * const result = await schedulePieceDeletions(client, { + * pieceIds: [1n, 2n], * dataSetId: 1n, * clientDataSetId: 1n, * serviceURL: 'https://pdp.example.com', @@ -108,19 +150,56 @@ export namespace schedulePieceDeletion { * console.log(result.hash) * ``` */ -export async function schedulePieceDeletion( +export async function schedulePieceDeletions( client: Client, - options: schedulePieceDeletion.OptionsType -): Promise { - return deletePiece({ + options: schedulePieceDeletions.OptionsType +): Promise { + const pieceIds = normalizeDeletePieceIds(options.pieceIds) + + return deletePieces({ serviceURL: options.serviceURL, dataSetId: options.dataSetId, - pieceId: options.pieceId, + pieceIds, extraData: await signSchedulePieceRemovals(client, { clientDataSetId: options.clientDataSetId, - pieceIds: [options.pieceId], + pieceIds, }), retryCount: options.retryCount, retryDelay: options.retryDelay, }) } + +export namespace deletePiece { + export type OptionsType = Omit & { pieceId: bigint } + export type OutputType = deletePieces.OutputType + export type ErrorType = deletePieces.ErrorType +} + +/** + * Delete one piece from a data set on the PDP API. + * + * @deprecated Use {@link deletePieces} instead. + */ +export function deletePiece(options: deletePiece.OptionsType): Promise { + const { pieceId, ...rest } = options + return deletePieces({ ...rest, pieceIds: [pieceId] }) +} + +export namespace schedulePieceDeletion { + export type OptionsType = Omit & { pieceId: bigint } + export type OutputType = schedulePieceDeletions.OutputType + export type ErrorType = schedulePieceDeletions.ErrorType +} + +/** + * Schedule one piece deletion. + * + * @deprecated Use {@link schedulePieceDeletions} instead. + */ +export function schedulePieceDeletion( + client: Client, + options: schedulePieceDeletion.OptionsType +): Promise { + const { pieceId, ...rest } = options + return schedulePieceDeletions(client, { ...rest, pieceIds: [pieceId] }) +} diff --git a/packages/synapse-core/src/utils/constants.ts b/packages/synapse-core/src/utils/constants.ts index 42f4a2887..8ce65fcca 100644 --- a/packages/synapse-core/src/utils/constants.ts +++ b/packages/synapse-core/src/utils/constants.ts @@ -99,6 +99,14 @@ export const SIZE_CONSTANTS = { */ MAX_ADD_PIECES_BATCH_SIZE: 40, + /** + * Maximum pieces per schedulePieceDeletions call accepted by the Curio PDP API. + * + * Curio also rejects requests (429) when the data set already has 200 or more + * removals queued on-chain; the queue only drains at the next proving period. + */ + MAX_DELETE_PIECES_BATCH_SIZE: 200, + /** * Bytes per leaf in the PDP merkle tree. * The FWSS contract converts leaf counts to bytes via `totalBytes = leafCount * BYTES_PER_LEAF`. diff --git a/packages/synapse-core/test/sp.test.ts b/packages/synapse-core/test/sp.test.ts index d156f3502..0ad4276e0 100644 --- a/packages/synapse-core/test/sp.test.ts +++ b/packages/synapse-core/test/sp.test.ts @@ -13,10 +13,12 @@ import { InvalidUploadSizeError, LocationHeaderError, PostPieceError, + TooManyPiecesQueuedError, UploadPieceError, WaitForAddPiecesError, WaitForCreateDataSetError, } from '../src/errors/pdp.ts' +import { AtLeastOnePieceRequiredError, TooManyPiecesError } from '../src/errors/warm-storage.ts' import { ADDRESSES, PRIVATE_KEYS } from '../src/mocks/index.ts' import { createAndAddPiecesHandler, @@ -33,10 +35,12 @@ import { createDataSetAndAddPiecesApiRequest, createDataSetApiRequest, deletePiece, + deletePieces, findPiece, getDataSet, NetworkError, ping, + schedulePieceDeletions, TimeoutError, uploadPiece, waitForAddPieces, @@ -935,9 +939,10 @@ InvalidSignature(address expected, address actual) server.use( http.delete('http://pdp.local/pdp/data-sets/1/pieces/2', async ({ request }) => { - const body = (await request.json()) as { extraData: string } - assert.hasAllKeys(body, ['extraData']) + const body = (await request.json()) as { extraData: string; pieceIds: number[] } + assert.hasAllKeys(body, ['extraData', 'pieceIds']) assert.isDefined(body.extraData) + assert.deepEqual(body.pieceIds, [2]) return HttpResponse.json(mockResponse, { status: 200, }) @@ -990,6 +995,104 @@ InvalidSignature(address expected, address actual) }) }) + describe('deletePieces', () => { + it('deletes multiple pieces with one request and matching authorization', async () => { + const mockTxHash = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef' + const submittedPieceIds = [2n, 3n, 2n, 9_007_199_254_740_993n] + const expectedPieceIds = [2n, 3n, 9_007_199_254_740_993n] + + server.use( + http.delete('http://pdp.local/pdp/data-sets/1/pieces/2', async ({ request }) => { + const rawBody = await request.text() + assert.include(rawBody, '"pieceIds":[2,3,9007199254740993]') + + const body = JSON.parse(rawBody) as { extraData: `0x${string}` } + const expectedExtraData = await TypedData.signSchedulePieceRemovals(client, { + clientDataSetId: 0n, + pieceIds: expectedPieceIds, + }) + assert.equal(body.extraData, expectedExtraData) + + return HttpResponse.json({ txHash: mockTxHash }) + }) + ) + + const result = await schedulePieceDeletions(client, { + serviceURL: 'http://pdp.local', + dataSetId: 1n, + clientDataSetId: 0n, + pieceIds: submittedPieceIds, + }) + + assert.equal(result.hash, mockTxHash) + }) + + it('deletes multiple pieces should fail when too many pieces are queued', async () => { + const submittedPieceIds = [2n, 3n, 2n, 9_007_199_254_740_993n] + + server.use( + http.delete('http://pdp.local/pdp/data-sets/1/pieces/2', async () => { + return new HttpResponse(null, { status: 429 }) + }) + ) + try { + await schedulePieceDeletions(client, { + serviceURL: 'http://pdp.local', + dataSetId: 1n, + clientDataSetId: 0n, + pieceIds: submittedPieceIds, + retryCount: 1, + retryDelay: 10, + }) + assert.fail('Should have thrown TooManyPiecesQueuedError') + } catch (error) { + assert.instanceOf(error, TooManyPiecesQueuedError) + } + }) + + it('rejects an empty batch', async () => { + try { + await deletePieces({ + serviceURL: 'http://pdp.local', + dataSetId: 1n, + pieceIds: [], + extraData: '0x', + }) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, AtLeastOnePieceRequiredError) + } + }) + + it('rejects batches above the Curio limit', async () => { + try { + await deletePieces({ + serviceURL: 'http://pdp.local', + dataSetId: 1n, + pieceIds: Array.from({ length: SIZE_CONSTANTS.MAX_DELETE_PIECES_BATCH_SIZE + 1 }, (_, i) => BigInt(i)), + extraData: '0x', + }) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, TooManyPiecesError) + } + }) + + it("rejects piece IDs outside Curio's signed 64-bit range", async () => { + try { + await deletePieces({ + serviceURL: 'http://pdp.local', + dataSetId: 1n, + pieceIds: [1n << 63n], + extraData: '0x', + }) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, RangeError) + } + }) + }) + describe('findPiece', () => { const mockPieceCidStr = 'bafkzcibcd4bdomn3tgwgrh3g532zopskstnbrd2n3sxfqbze7rxt7vqn7veigmy' diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 4e974b5e9..9723cd6c8 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -32,7 +32,7 @@ import { InvalidPieceCIDError } from '@filoz/synapse-core/errors' import * as PDPVerifier from '@filoz/synapse-core/pdp-verifier' import * as Piece from '@filoz/synapse-core/piece' import * as SP from '@filoz/synapse-core/sp' -import { schedulePieceDeletion, type UploadPieceStreamingData } from '@filoz/synapse-core/sp' +import { schedulePieceDeletions, type UploadPieceStreamingData } from '@filoz/synapse-core/sp' import { signAddPieces, signCreateDataSetAndAddPieces } from '@filoz/synapse-core/typed-data' import { calculateLastProofDate, @@ -1148,31 +1148,51 @@ export class StorageContext { } /** - * Delete a piece with given CID from this data set. + * Delete pieces with the given CIDs or piece IDs from this data set in one transaction. * * @param options - Options for the delete operation - * @param options.piece - The PieceCID identifier or a piece number to delete by pieceID + * @param options.pieces - PieceCID identifiers or piece numbers to delete by piece ID * @returns Transaction hash of the delete operation + * + * @remarks + * Curio accepts at most 200 pieces per request and rejects requests (429) when + * the data set already has 200 or more removals queued on-chain; the queue only + * drains at the next proving period. */ - async deletePiece(options: { piece: string | PieceCID | bigint }): Promise { - const { piece } = options + async deletePieces(options: { pieces: Array }): Promise { if (this.dataSetId == null) { - throw createError('StorageContext', 'deletePiece', 'Data set not found') + throw createError('StorageContext', 'deletePieces', 'Data set not found') } - const pieceId = typeof piece === 'bigint' ? piece : await this._getPieceIdByCID(piece) + + const resolvedPieceIds = await Promise.all( + options.pieces.map((piece) => (typeof piece === 'bigint' ? piece : this._getPieceIdByCID(piece))) + ) + const pieceIds = [...new Set(resolvedPieceIds)] + SP.validateDeletePiecesBatch(pieceIds.length) const clientDataSetId = await this.getClientDataSetId() - const { hash } = await schedulePieceDeletion(this._synapse.sessionClient ?? this._synapse.client, { + const { hash } = await schedulePieceDeletions(this._synapse.sessionClient ?? this._synapse.client, { serviceURL: this._pdpEndpoint, dataSetId: this.dataSetId, - pieceId: pieceId, + pieceIds, clientDataSetId: clientDataSetId, }) return hash } + /** + * Delete a piece with the given CID or piece ID from this data set. + * + * @param options - Options for the delete operation + * @param options.piece - The PieceCID identifier or a piece number to delete by piece ID + * @returns Transaction hash of the delete operation + */ + async deletePiece(options: { piece: string | PieceCID | bigint }): Promise { + return this.deletePieces({ pieces: [options.piece] }) + } + /** * Check if a piece exists on this service provider and get its proof status. * Also returns timing information about when the piece was last proven and when the next diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index c9005425b..929fb6855 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -13,6 +13,7 @@ import { bytesToHex, type Client, createWalletClient, + type Hex, numberToHex, type Transport, toFunctionSelector, @@ -1947,6 +1948,31 @@ describe('StorageService', () => { }) }) + describe('deletePieces', () => { + it('schedules multiple unique piece IDs in one request', async () => { + const txHash = `0x${'12'.repeat(32)}` as Hex + + server.use( + Mocks.JSONRPC({ ...Mocks.presets.basic }), + Mocks.PING(), + http.delete('https://pdp.example.com/pdp/data-sets/1/pieces/2', async ({ request }) => { + const body = (await request.json()) as { extraData: Hex; pieceIds: number[] } + assert.deepEqual(body.pieceIds, [2, 3]) + assert.isDefined(body.extraData) + return HttpResponse.json({ txHash }) + }) + ) + + const synapse = new Synapse({ client, source: null }) + const warmStorageService = new WarmStorageService({ client }) + const context = await StorageContext.create({ synapse, warmStorageService, dataSetId: 1n }) + + const hash = await context.deletePieces({ pieces: [2n, 3n, 2n] }) + + assert.equal(hash, txHash) + }) + }) + describe('getPieces', () => { it('should get all active pieces with pagination', async () => { // Use actual valid PieceCIDs from test data