diff --git a/.dev/tech-debt.md b/.dev/tech-debt.md
index 373f33e2..17458bf9 100644
--- a/.dev/tech-debt.md
+++ b/.dev/tech-debt.md
@@ -59,8 +59,51 @@ context: `dictionary_categories.id` is a plain auto-increment `serial`, used dir
---
+### No integration tests exist for the submission edit, delete, or commit endpoints — let alone mixed insert+update+delete scenarios
+standalone: yes
+context: `packages/data-provider/test/integration/routers/submission/` only covers `submissionRouter-submit*` (insert-only file/JSON uploads). There is no integration spec for `editSubmittedData`, `deleteSubmittedDataBySystemId`, or the commit flow (`performCommitSubmissionAsync`/`commitSubmissionWorker.ts`) at all, despite the test infra (`test/integration/dependencies/containers.ts`) already running a real Postgres container capable of exercising the real commit transaction. The new UPDATE/DELETE conflict resolution added 2026-08-12 has unit coverage only (`test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts` and friends) — still no HTTP+DB-level test proving a submission with a staged conflict actually ends up INVALID end-to-end. Fix: add integration specs that stage inserts+updates+deletes in the same active submission (including same-systemId collisions) and assert the committed `submitted_data` table end state and the active submission's status/record states.
+
+### `GET /submission/:submissionId/details` can't filter records by state
+standalone: yes
+context: `submissionController.ts::getSubmissionDetailsById` (294-321) and `submissionService.ts::getSubmissionDetailsById` (326-372) only accept `entityNames`/`actionTypes` filters (`submissionDetailsRequestSchema`, `schemas.ts:339-351`), then call `submissionRecordsRepository.getBySubmissionId` (365-369) without a `states` filter. The repository itself already supports it — `getBySubmissionId`/`getByFileIds` (`submissionRecordsRepository.ts`) both accept `filterOptions.states?: SubmissionRecordState[]` and `SUBMISSION_RECORD_STATE` (`types.ts`) already enumerates `RECEIVED`/`VALID`/`INVALID` — so this is a plumbing gap, not a missing capability. Fix: add a `states`/`state` query param to `submissionDetailsRequestSchema`, thread it through `getSubmissionDetailsById`'s `filterOptions` in both the controller and service, and pass it to `submissionRecordsRepository.getBySubmissionId`.
+
+### API to download the originally uploaded file
+standalone: yes
+context: No original file bytes are stored anywhere today. Uploads land in a multer temp path (`dest: '/tmp'`, `submissionRouter.ts:26`), get streamed and parsed by `collectRows` (`fileUtils.ts:62-80`), and the temp file is deleted immediately after parsing (`fileUtils.ts:78`, `fs.unlink`). `submission_files` (`packages/data-model/src/models/submission_files.ts:7-17`) only keeps metadata (`fileName`, `entityName`, `fileSize`); `submission_records` only keeps parsed row data (jsonb), not the raw file. Needs a scope decision before implementation: (a) persist the raw uploaded bytes somewhere (object store or DB blob) at upload time going forward — adds storage/retention cost and doesn't help for submissions already committed under the old behavior, or (b) reconstruct a file from the stored parsed rows — lossy, won't reproduce original column order, formatting, or any extra/ignored columns, and "recreate" may not satisfy whatever this is needed for (audit, re-upload, external sharing). No download/`Content-Disposition` endpoint exists for submission files today; the only precedent for that response pattern in the codebase is `dictionaryController.ts:74-87` (zips dictionary templates, unrelated data).
+
+### Download error report for a file
+standalone: no
+context: Depends on the storage/identity decision above and needs its own format decision. Per-record errors already exist as `SubmissionRecordError[]` in `submission_records.errors` (jsonb, `packages/data-model/src/models/submission_records.ts:59-73`), and a file already has a stable identity distinct from `entityName` (`submission_files.id`, `submission_records.fileId`) — so per-file error retrieval is possible today via `getBySubmissionId`/`getByFileIds` (`submissionRecordsRepository.ts:34-63,149-177`), just not exposed as a dedicated download endpoint. Open question flagged by the requirement itself: return the raw per-record `errors` JSON as-is, or design a purpose-built report format (e.g. row/field/message table)? Note that line numbers are computed at parse time (`fileUtils.ts:104`, `+1 for header row, +1 for 1-based line numbers`) but aren't currently persisted into the stored `errors` — worth carrying through if the report should reference original file line numbers.
+
+### Download error report as a zip for all files in a submission
+standalone: no
+context: Depends on the single-file error report above being defined first — this is just "download that report N times, zipped." `jszip` is already a dependency (`package.json:48`) and already used for exactly this response shape (`zip.generateAsync` + `Content-Disposition: attachment` + `application/zip`) in `dictionaryController.ts:74-87`, so no new library or pattern work is needed once the per-file report format exists.
+
+### List submissions endpoint needs sorting and filtering
+standalone: yes
+context: `GET /submission/category/:categoryId` (`submissionController.ts:233-276`) only accepts `onlyActive`, `organization`, `username` (`submissionsByCategoryRequestSchema`, `schemas.ts:318-329`), plus `page`/`pageSize`. Sort order is hardcoded to `desc(submissions.createdAt)` (`activeSubmissionRepository.ts:216`) with no sort param at all. The mandatory requirement (reverse creation-date sort) already matches today's fixed default — the gap is that it isn't documented as a stable, guaranteed default, and there's no way to choose anything else. Nice-to-have filters not yet supported: study/category beyond the path param, date range (created/updated), creator, contributing users, statuses. `auditRepository.ts` already has a directly reusable pattern for both pieces: date-range filtering (lines 76-81, `lt`/`gt` on `createdAt` against `startDate`/`endDate`) and configurable-direction `orderBy` (line 126), driven by `AuditFilterOptions` (`types.ts:89-98`) — worth modeling the new filter options after that rather than inventing a new shape. Swagger (`submission-api.yml:144-171`) will need the new params documented too.
+
+### `GET /submission/:submissionId` needs richer per-file details
+standalone: yes
+context: The response (`SubmissionSummaryResponse`, `types.ts:302-305`, built by `createSubmissionSummaryResponse`/`submissionResponseParser.ts:65-78`) groups `inserts`/`updates` by `entityName`, each entry only exposing `batchName` (the original filename), `recordsCount`, and `errors` (a count, not detail) — see `submissionResponseParser.ts:14-58` and the source rows shape in `submissionRecordsRepository.ts:20-26,188-208`. Missing relative to the ask: no `fileId` in the response (so a client can't correlate a listed file to a future per-file download/error-report endpoint, above); no file size, even though `submission_files.fileSize` is already stored in the DB (`submission_files.ts:16`) and just isn't selected by `getRecordsSummaryBySubmissionId` (`submissionRecordsRepository.ts:188-197`); no rolled-up per-file validation status (only an error count — the per-record `state` enum `RECEIVED`/`VALID`/`INVALID` exists at `submission_records.ts:12` but isn't aggregated to file level); `deletes` has no per-file breakdown at all in the current model (`DataDeletesSubmissionSummary` is not batched by file). Also found in passing: swagger's `SubmissionDetailsResult` schema (`schemas.yml:87-111`) documents a shape that doesn't match what `/submission/:submissionId/details` actually returns (`SubmissionRecordWithEntityName[]`, i.e. `{id, actionType, state, fileId, data, errors, entityName}[]`) — worth correcting alongside this work since both touch the same response family.
+
## Resolved
+### Flaky integration test: dictionary migration force-retry intermittently returned 409 instead of 200
+resolved: 2026-08-12, made the test wait for the background migration worker to reach a terminal status before mutating it directly, removing the race. `dictionaryMigration.spec.ts`'s "should retry migration..." test now calls a `waitForMigrationToFinish` helper (same polling pattern already used in `dictionaryMigrationData.spec.ts`) right before overwriting the migration's status to `FAILED`, instead of assuming `initiateMigration`'s fire-and-forget worker had already finished. Verified with 3 consecutive full integration-suite runs, 380/380 passing each time, no failures.
+
+### Staging a DELETE submission record never checked for or merged with an existing pending record for the same systemId
+resolved: 2026-08-12, added a staging-time check instead of letting one action override the other silently. `resolveDeleteStagingConflicts` (`packages/data-provider/src/utils/submissionUtils.ts`) is now called from `deleteSubmittedDataBySystemId` (`submmittedData.ts`) before any new DELETE record is inserted: it fetches the Active Submission's existing UPDATE/DELETE records and cross-checks them against the systemIds about to be deleted (the target record plus its dependents). A systemId with a pending UPDATE is now rejected outright (`INVALID_SUBMISSION` response, no record staged) — consistent with the "both sides invalid" policy used for the same conflict at validation time — instead of quietly reaching `performDataValidation` later. A systemId that already has a pending DELETE is treated as a duplicate and skipped rather than inserted a second time, addressing the original in-code TODO directly. Previously the function always blindly inserted new DELETE rows regardless of what was already staged.
+
+### Duplicate UPDATE submission records for the same systemId collapsed via undefined last-write-wins at commit time
+resolved: 2026-08-12, added a deterministic `ORDER BY` instead of relying on unspecified Postgres row order. `submissionRecordsRepository.ts::getByFileIds` now does `.orderBy(submissionRecords.id)` (ascending) — `id` is a `serial` primary key, monotonic with insertion order, so `commitSubmissionWorker.ts`'s `record[systemId] = record` reduction now deterministically keeps the most recently inserted UPDATE row for a given systemId instead of whichever row Postgres happened to scan last. Considered adding a `created_at` timestamp column instead, but `id` already gives the same ordering guarantee without a schema migration, so that was dropped in favour of the simpler fix.
+
+### UPDATE/DELETE conflict on the same systemId was silently dropped instead of surfaced as an error
+resolved: 2026-08-12, added explicit conflict detection ahead of dictionary validation. `findUpdateDeleteConflicts` (`packages/data-provider/src/utils/submissionUtils.ts`) scans staged submission records for entityName+systemId pairs with both an UPDATE and a DELETE, before `performDataValidation` (`submissionProcessor.ts`) runs `validateSchemas`. Conflicting records are excluded from validation, both sides are marked `INVALID` with a new `CONFLICTING_ACTION` error (`RecordErrorActionConflict`, added to `SubmissionRecordError` in `@overture-stack/lyric-data-model`), the active submission is marked `INVALID`, and the scenario is `logger.error`-logged. Previously this was only prevented by incidental JS array-filtering order with no error surfaced (see the still-open items above for what remains: the staging-time TODO, and integration coverage).
+
+### `filterDeletesFromUpdates`/`filterRecordsByConflicts` removed as dead code
+resolved: 2026-08-12, same session as the fix above. Once `findUpdateDeleteConflicts` became the real, wired-in conflict detector, the unused `filterRecordsByConflicts`/`filterDeletesFromUpdates` pair (`submissionUtils.ts`, previously lines 234-280) had no remaining reason to exist — deleted both functions and their orphaned spec (`test/unit/utils/submission/filterDeletesFromUpdates.spec.ts`). Confirmed via repo-wide grep that nothing else referenced them before removing.
+
### Kafka publish tracking: no unit tests for `createPublishTracker`
diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml
index 0e18592a..72046796 100644
--- a/apps/server/swagger/schemas.yml
+++ b/apps/server/swagger/schemas.yml
@@ -95,7 +95,7 @@ components:
type:
type: string
description: Type of action
- enum: ['INSERTS', 'UPDATES', 'DELETES']
+ enum: ['INSERT', 'UPDATE', 'DELETE']
entity:
type: string
description: Name of the entity
diff --git a/apps/server/swagger/submission-api.yml b/apps/server/swagger/submission-api.yml
index e7665673..8c0bb3fd 100644
--- a/apps/server/swagger/submission-api.yml
+++ b/apps/server/swagger/submission-api.yml
@@ -54,9 +54,9 @@
503:
$ref: '#/components/responses/ServiceUnavailableError'
-/submission/{submissionId}/details:
+/submission/{submissionId}/data:
get:
- summary: Fetch Submission Data records. Sorted in their original file order and grouped by `inserts`, `updates`, and `deletes`.
+ summary: Fetch Submission Data records. Sorted in their original file order and grouped by `insert`, `update`, and `delete`.
tags:
- Submission
parameters:
@@ -67,9 +67,9 @@
type: array
items:
type: string
- enum: [inserts, updates, deletes]
+ enum: [insert, update, delete]
uniqueItems: true
- description: Filters the Submission Data records by action type. Valid values are `inserts`, `updates`, and `deletes` (case insensitive). If not provided, all action types are returned.
+ description: Filters the Submission Data records by action type. Valid values are `insert`, `update`, and `delete` (case insensitive). If not provided, all action types are returned.
- name: entityNames
in: query
schema:
@@ -78,6 +78,12 @@
type: string
uniqueItems: true
description: Filters the Submission Data records by entity name. Must match names listed in the Submission Summary endpoint. If not provided, all entity names are returned.
+ - name: fileId
+ in: query
+ required: false
+ schema:
+ type: integer
+ description: An optional query parameter used to specify the file ID within the submission to be retrieved.
- $ref: '#/components/parameters/query/Page'
- $ref: '#/components/parameters/query/PageSize'
responses:
@@ -98,7 +104,6 @@
503:
$ref: '#/components/responses/ServiceUnavailableError'
-/submission/{submissionId}/{actionType}:
delete:
summary: Clear Active Submission by entity name
tags:
@@ -109,24 +114,18 @@
type: string
required: true
description: The ID of the Submission
- - name: actionType
- in: path
- required: true
- schema:
- type: string
- enum: [inserts, updates, deletes]
- description: Parameter to specify the type of record to remove from the Submission. Must be one of `inserts`, `updates`, or `deletes` (case insensitive)
- - name: entityName
+ - name: recordId
in: query
- type: string
- required: true
- description: The name of the entity
- - name: index
+ required: false
+ schema:
+ type: integer
+ description: An optional query parameter used to specify the record ID within the submission to be deleted.
Only one of `recordId` or `fileId` can be provided
+ - name: fileId
in: query
required: false
schema:
type: integer
- description: An optional query parameter used to specify the index of the item within the submission type to be deleted.
If not provided all the items within the submission type will be deleted.
+ description: An optional query parameter used to specify the file ID within the submission to be deleted.
Only one of `recordId` or `fileId` can be provided
responses:
200:
description: Submission cleared successfully. Returns the current Active Submission
diff --git a/packages/data-model/docs/schema.dbml b/packages/data-model/docs/schema.dbml
index 54b4e98b..0460ab44 100644
--- a/packages/data-model/docs/schema.dbml
+++ b/packages/data-model/docs/schema.dbml
@@ -128,10 +128,8 @@ table submission_records {
table submissions {
id serial [pk, not null, increment]
- data jsonb [not null]
dictionary_category_id integer [not null]
dictionary_id integer [not null]
- errors jsonb
organization varchar [not null]
status submission_status [not null]
created_at timestamp [default: `now()`]
@@ -187,7 +185,7 @@ ref: dictionary_migration.to_dictionary_id - dictionaries.id
ref: dictionary_migration.submission_id - submissions.id
-ref: submission_files.submission_id - submissions.id
+ref: submission_files.submission_id > submissions.id
ref: submission_records.file_id > submission_files.id
diff --git a/packages/data-model/migrations/0017_remove_submission_data.sql b/packages/data-model/migrations/0017_remove_submission_data.sql
new file mode 100644
index 00000000..a262dd9a
--- /dev/null
+++ b/packages/data-model/migrations/0017_remove_submission_data.sql
@@ -0,0 +1,2 @@
+ALTER TABLE "submissions" DROP COLUMN IF EXISTS "data";--> statement-breakpoint
+ALTER TABLE "submissions" DROP COLUMN IF EXISTS "errors";
\ No newline at end of file
diff --git a/packages/data-model/migrations/meta/0017_snapshot.json b/packages/data-model/migrations/meta/0017_snapshot.json
new file mode 100644
index 00000000..919f3f5a
--- /dev/null
+++ b/packages/data-model/migrations/meta/0017_snapshot.json
@@ -0,0 +1,959 @@
+{
+ "id": "02b8e9d6-f2dc-4b13-9795-78b274688e7a",
+ "prevId": "d21aa03b-6754-4978-9209-7113bb3d5890",
+ "version": "5",
+ "dialect": "pg",
+ "tables": {
+ "audit_submitted_data": {
+ "name": "audit_submitted_data",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "action": {
+ "name": "action",
+ "type": "audit_action",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "dictionary_category_id": {
+ "name": "dictionary_category_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "data_diff": {
+ "name": "data_diff",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "entity_name": {
+ "name": "entity_name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "errors": {
+ "name": "errors",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_valid_schema_id": {
+ "name": "last_valid_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "new_data_is_valid": {
+ "name": "new_data_is_valid",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "old_data_is_valid": {
+ "name": "old_data_is_valid",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "organization": {
+ "name": "organization",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "original_schema_id": {
+ "name": "original_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "submission_id": {
+ "name": "submission_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "system_id": {
+ "name": "system_id",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "audit_submitted_data_dictionary_index": {
+ "name": "audit_submitted_data_dictionary_index",
+ "columns": [
+ "dictionary_category_id"
+ ],
+ "isUnique": false
+ },
+ "audit_submitted_data_organization_index": {
+ "name": "audit_submitted_data_organization_index",
+ "columns": [
+ "organization"
+ ],
+ "isUnique": false
+ },
+ "audit_submitted_data_submission_index": {
+ "name": "audit_submitted_data_submission_index",
+ "columns": [
+ "submission_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk": {
+ "name": "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "dictionary_categories",
+ "columnsFrom": [
+ "dictionary_category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk": {
+ "name": "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "last_valid_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "audit_submitted_data_original_schema_id_dictionaries_id_fk": {
+ "name": "audit_submitted_data_original_schema_id_dictionaries_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "original_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "audit_submitted_data_submission_id_submissions_id_fk": {
+ "name": "audit_submitted_data_submission_id_submissions_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "submissions",
+ "columnsFrom": [
+ "submission_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "dictionaries": {
+ "name": "dictionaries",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "dictionary": {
+ "name": "dictionary",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "version": {
+ "name": "version",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "dictionary_categories": {
+ "name": "dictionary_categories",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "active_dictionary_id": {
+ "name": "active_dictionary_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "alias": {
+ "name": "alias",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "default_centric_entity": {
+ "name": "default_centric_entity",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "name": {
+ "name": "name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_by": {
+ "name": "updated_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "dictionary_categories_alias_unique": {
+ "name": "dictionary_categories_alias_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "alias"
+ ]
+ },
+ "dictionary_categories_name_unique": {
+ "name": "dictionary_categories_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ }
+ },
+ "dictionary_migration": {
+ "name": "dictionary_migration",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "category_id": {
+ "name": "category_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "from_dictionary_id": {
+ "name": "from_dictionary_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "to_dictionary_id": {
+ "name": "to_dictionary_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "submission_id": {
+ "name": "submission_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "migration_status",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "retries": {
+ "name": "retries",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_by": {
+ "name": "updated_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "dictionary_migration_category_id_index": {
+ "name": "dictionary_migration_category_id_index",
+ "columns": [
+ "category_id"
+ ],
+ "isUnique": false
+ },
+ "dictionary_migration_from_dictionary_id_index": {
+ "name": "dictionary_migration_from_dictionary_id_index",
+ "columns": [
+ "from_dictionary_id"
+ ],
+ "isUnique": false
+ },
+ "dictionary_migration_to_dictionary_id_index": {
+ "name": "dictionary_migration_to_dictionary_id_index",
+ "columns": [
+ "to_dictionary_id"
+ ],
+ "isUnique": false
+ },
+ "dictionary_migration_submission_id_index": {
+ "name": "dictionary_migration_submission_id_index",
+ "columns": [
+ "submission_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "dictionary_migration_category_id_dictionary_categories_id_fk": {
+ "name": "dictionary_migration_category_id_dictionary_categories_id_fk",
+ "tableFrom": "dictionary_migration",
+ "tableTo": "dictionary_categories",
+ "columnsFrom": [
+ "category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "dictionary_migration_from_dictionary_id_dictionaries_id_fk": {
+ "name": "dictionary_migration_from_dictionary_id_dictionaries_id_fk",
+ "tableFrom": "dictionary_migration",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "from_dictionary_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "dictionary_migration_to_dictionary_id_dictionaries_id_fk": {
+ "name": "dictionary_migration_to_dictionary_id_dictionaries_id_fk",
+ "tableFrom": "dictionary_migration",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "to_dictionary_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "dictionary_migration_submission_id_submissions_id_fk": {
+ "name": "dictionary_migration_submission_id_submissions_id_fk",
+ "tableFrom": "dictionary_migration",
+ "tableTo": "submissions",
+ "columnsFrom": [
+ "submission_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "submission_files": {
+ "name": "submission_files",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "submission_id": {
+ "name": "submission_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_name": {
+ "name": "file_name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "entity_name": {
+ "name": "entity_name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_size": {
+ "name": "file_size",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "submission_files_submission_id_index": {
+ "name": "submission_files_submission_id_index",
+ "columns": [
+ "submission_id"
+ ],
+ "isUnique": false
+ },
+ "submission_files_submission_entity_file_index": {
+ "name": "submission_files_submission_entity_file_index",
+ "columns": [
+ "submission_id",
+ "entity_name",
+ "file_name"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "submission_files_submission_id_submissions_id_fk": {
+ "name": "submission_files_submission_id_submissions_id_fk",
+ "tableFrom": "submission_files",
+ "tableTo": "submissions",
+ "columnsFrom": [
+ "submission_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "submission_records": {
+ "name": "submission_records",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "file_id": {
+ "name": "file_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "data": {
+ "name": "data",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action_type": {
+ "name": "action_type",
+ "type": "submission_record_type",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "errors": {
+ "name": "errors",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "state": {
+ "name": "state",
+ "type": "submission_record_state",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "submission_records_file_id_index": {
+ "name": "submission_records_file_id_index",
+ "columns": [
+ "file_id"
+ ],
+ "isUnique": false
+ },
+ "submission_records_file_id_action_type_index": {
+ "name": "submission_records_file_id_action_type_index",
+ "columns": [
+ "file_id",
+ "action_type"
+ ],
+ "isUnique": false
+ },
+ "submission_records_file_id_state_action_type_index": {
+ "name": "submission_records_file_id_state_action_type_index",
+ "columns": [
+ "file_id",
+ "state",
+ "action_type"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "submission_records_file_id_submission_files_id_fk": {
+ "name": "submission_records_file_id_submission_files_id_fk",
+ "tableFrom": "submission_records",
+ "tableTo": "submission_files",
+ "columnsFrom": [
+ "file_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "submissions": {
+ "name": "submissions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "dictionary_category_id": {
+ "name": "dictionary_category_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "dictionary_id": {
+ "name": "dictionary_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "organization": {
+ "name": "organization",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "submission_status",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "updated_by": {
+ "name": "updated_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "submission_organization_index": {
+ "name": "submission_organization_index",
+ "columns": [
+ "organization"
+ ],
+ "isUnique": false
+ },
+ "submission_category_index": {
+ "name": "submission_category_index",
+ "columns": [
+ "dictionary_category_id"
+ ],
+ "isUnique": false
+ },
+ "submission_created_by_index": {
+ "name": "submission_created_by_index",
+ "columns": [
+ "created_by"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "submissions_dictionary_category_id_dictionary_categories_id_fk": {
+ "name": "submissions_dictionary_category_id_dictionary_categories_id_fk",
+ "tableFrom": "submissions",
+ "tableTo": "dictionary_categories",
+ "columnsFrom": [
+ "dictionary_category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "submissions_dictionary_id_dictionaries_id_fk": {
+ "name": "submissions_dictionary_id_dictionaries_id_fk",
+ "tableFrom": "submissions",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "dictionary_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "submitted_data": {
+ "name": "submitted_data",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "data": {
+ "name": "data",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "dictionary_category_id": {
+ "name": "dictionary_category_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "entity_name": {
+ "name": "entity_name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_valid": {
+ "name": "is_valid",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "last_valid_schema_id": {
+ "name": "last_valid_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "organization": {
+ "name": "organization",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "original_schema_id": {
+ "name": "original_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "system_id": {
+ "name": "system_id",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "updated_by": {
+ "name": "updated_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "submitted_data_organization_index": {
+ "name": "submitted_data_organization_index",
+ "columns": [
+ "organization"
+ ],
+ "isUnique": false
+ },
+ "submitted_data_category_index": {
+ "name": "submitted_data_category_index",
+ "columns": [
+ "dictionary_category_id"
+ ],
+ "isUnique": false
+ },
+ "submitted_data_system_id_index": {
+ "name": "submitted_data_system_id_index",
+ "columns": [
+ "system_id"
+ ],
+ "isUnique": false
+ },
+ "submitted_data_entity_name_index": {
+ "name": "submitted_data_entity_name_index",
+ "columns": [
+ "entity_name"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "submitted_data_dictionary_category_id_dictionary_categories_id_fk": {
+ "name": "submitted_data_dictionary_category_id_dictionary_categories_id_fk",
+ "tableFrom": "submitted_data",
+ "tableTo": "dictionary_categories",
+ "columnsFrom": [
+ "dictionary_category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "submitted_data_last_valid_schema_id_dictionaries_id_fk": {
+ "name": "submitted_data_last_valid_schema_id_dictionaries_id_fk",
+ "tableFrom": "submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "last_valid_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "submitted_data_original_schema_id_dictionaries_id_fk": {
+ "name": "submitted_data_original_schema_id_dictionaries_id_fk",
+ "tableFrom": "submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "original_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "submitted_data_system_id_unique": {
+ "name": "submitted_data_system_id_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "system_id"
+ ]
+ }
+ }
+ }
+ },
+ "enums": {
+ "audit_action": {
+ "name": "audit_action",
+ "values": {
+ "UPDATE": "UPDATE",
+ "DELETE": "DELETE",
+ "MIGRATION": "MIGRATION"
+ }
+ },
+ "migration_status": {
+ "name": "migration_status",
+ "values": {
+ "IN_PROGRESS": "IN_PROGRESS",
+ "COMPLETED": "COMPLETED",
+ "FAILED": "FAILED"
+ }
+ },
+ "submission_record_state": {
+ "name": "submission_record_state",
+ "values": {
+ "RECEIVED": "RECEIVED",
+ "VALID": "VALID",
+ "INVALID": "INVALID"
+ }
+ },
+ "submission_record_type": {
+ "name": "submission_record_type",
+ "values": {
+ "INSERT": "INSERT",
+ "UPDATE": "UPDATE",
+ "DELETE": "DELETE"
+ }
+ },
+ "submission_status": {
+ "name": "submission_status",
+ "values": {
+ "OPEN": "OPEN",
+ "VALIDATING": "VALIDATING",
+ "VALID": "VALID",
+ "INVALID": "INVALID",
+ "CLOSED": "CLOSED",
+ "COMMITTING": "COMMITTING",
+ "COMMITTED": "COMMITTED"
+ }
+ }
+ },
+ "schemas": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/packages/data-model/migrations/meta/_journal.json b/packages/data-model/migrations/meta/_journal.json
index f24afaef..1c159718 100644
--- a/packages/data-model/migrations/meta/_journal.json
+++ b/packages/data-model/migrations/meta/_journal.json
@@ -120,6 +120,13 @@
"when": 1786388108952,
"tag": "0016_CUSTOM_migrate_submission_records",
"breakpoints": true
+ },
+ {
+ "idx": 17,
+ "version": "5",
+ "when": 1786414328157,
+ "tag": "0017_remove_submission_data",
+ "breakpoints": true
}
]
}
diff --git a/packages/data-model/src/models/submission_records.ts b/packages/data-model/src/models/submission_records.ts
index 01ab8a66..d6d91e95 100644
--- a/packages/data-model/src/models/submission_records.ts
+++ b/packages/data-model/src/models/submission_records.ts
@@ -13,43 +13,53 @@ export const submissionRecordState = pgEnum('submission_record_state', ['RECEIVE
export const submissionRecordType = pgEnum('submission_record_type', ['INSERT', 'UPDATE', 'DELETE']);
-// TODO: export this type
-type SubmissionInsertData = DataRecord;
+export type SubmissionInsertData = DataRecord;
-// TODO: export this type
-type SubmissionUpdateData = {
+export type SubmissionUpdateData = {
systemId: string;
old: DataRecord;
new: DataRecord;
};
-// TODO: export this type
-type SubmissionDeleteData = {
+export type SubmissionDeleteData = {
systemId: string;
data: DataRecord;
isValid: boolean;
organization: string;
};
-// TODO: export this type
-type SubmissionData = SubmissionInsertData | SubmissionUpdateData | SubmissionDeleteData;
+export type SubmissionData = SubmissionInsertData | SubmissionUpdateData | SubmissionDeleteData;
-// TODO: export this type
-type FieldDetails = {
+export type FieldDetails = {
fieldName: string;
fieldValue: DataRecordValue;
};
-// TODO: export this type
-type UnrecognizedValueReason = {
+export type UnrecognizedValueReason = {
reason: 'UNRECOGNIZED_VALUE';
};
-// TODO: export this type
-type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason;
+export type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason;
-// TODO: export this type
-type SubmissionRecordError = DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue;
+export type ConflictingActionReason = {
+ reason: 'CONFLICTING_ACTION';
+};
+
+/**
+ * Raised when a record's `systemId` has both an UPDATE and a DELETE staged in the same
+ * Active Submission. `conflictingActionType` names the *other* action type this record
+ * conflicts with, so both sides of the conflict can be reported independently.
+ */
+export type RecordErrorActionConflict = ConflictingActionReason & {
+ systemId: string;
+ conflictingActionType: 'UPDATE' | 'DELETE';
+ message: string;
+};
+
+export type SubmissionRecordError =
+ | DictionaryValidationRecordErrorDetails
+ | RecordErrorInvalidValue
+ | RecordErrorActionConflict;
export const submissionRecords = pgTable(
'submission_records',
diff --git a/packages/data-model/src/models/submissions.ts b/packages/data-model/src/models/submissions.ts
index 7020a81e..d264628a 100644
--- a/packages/data-model/src/models/submissions.ts
+++ b/packages/data-model/src/models/submissions.ts
@@ -1,14 +1,9 @@
import { relations } from 'drizzle-orm';
-import { index, integer, jsonb, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core';
-
-import {
- type DataRecord,
- type DataRecordValue,
- type DictionaryValidationRecordErrorDetails,
-} from '@overture-stack/lectern-client';
+import { index, integer, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core';
import { dictionaries } from './dictionaries.js';
import { dictionaryCategories } from './dictionary_categories.js';
+import { submissionFiles } from './submission_files.js';
export const submissionStatusEnum = pgEnum('submission_status', [
'OPEN',
@@ -20,64 +15,16 @@ export const submissionStatusEnum = pgEnum('submission_status', [
'COMMITTED',
]);
-export type SubmissionInsertData = {
- batchName: string;
- records: DataRecord[];
-};
-
-export type SubmissionUpdateData = {
- systemId: string;
- old: DataRecord;
- new: DataRecord;
-};
-
-export type SubmissionDeleteData = {
- systemId: string;
- data: DataRecord;
- entityName: string;
- isValid: boolean;
- organization: string;
-};
-
-export type SubmissionData = {
- inserts?: Record;
- updates?: Record;
- deletes?: Record;
-};
-
-export type FieldDetails = {
- fieldName: string;
- fieldValue: DataRecordValue;
-};
-
-export type UnrecognizedValueReason = {
- reason: 'UNRECOGNIZED_VALUE';
-};
-
-export type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason;
-
-export type SubmissionRecordErrorDetails = {
- index: number;
-} & (DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue);
-
-export type SubmissionErrors = {
- inserts?: Record;
- updates?: Record;
- deletes?: Record;
-};
-
export const submissions = pgTable(
'submissions',
{
id: serial('id').primaryKey(),
- data: jsonb('data').$type().notNull(),
dictionaryCategoryId: integer('dictionary_category_id')
.references(() => dictionaryCategories.id)
.notNull(),
dictionaryId: integer('dictionary_id')
.references(() => dictionaries.id)
.notNull(),
- errors: jsonb('errors').$type(),
organization: varchar('organization').notNull(),
status: submissionStatusEnum('status').notNull(),
createdAt: timestamp('created_at').defaultNow(),
@@ -94,7 +41,7 @@ export const submissions = pgTable(
},
);
-export const submissionRelations = relations(submissions, ({ one }) => ({
+export const submissionRelations = relations(submissions, ({ one, many }) => ({
dictionary: one(dictionaries, {
fields: [submissions.dictionaryId],
references: [dictionaries.id],
@@ -103,6 +50,7 @@ export const submissionRelations = relations(submissions, ({ one }) => ({
fields: [submissions.dictionaryCategoryId],
references: [dictionaryCategories.id],
}),
+ submissionFiles: many(submissionFiles),
}));
export type Submission = typeof submissions.$inferSelect; // return type when queried
diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts
index 29ab35a5..e2ea9bf0 100644
--- a/packages/data-provider/src/controllers/submissionController.ts
+++ b/packages/data-provider/src/controllers/submissionController.ts
@@ -18,9 +18,9 @@ import {
submissionActiveByOrganizationRequestSchema,
submissionByIdRequestSchema,
submissionCommitRequestSchema,
- submissionDeleteEntityNameRequestSchema,
submissionDeleteRequestSchema,
submissionDetailsRequestSchema,
+ submissionRecordDeleteRequestSchema,
submissionsByCategoryRequestSchema,
uploadSingleEntitySubmissionDataRequestSchema,
uploadSubmissionRequestSchema,
@@ -30,8 +30,8 @@ import {
BATCH_ERROR_TYPE,
BatchError,
type PaginatedResponse,
- SUBMISSION_ACTION_TYPE,
- type SubmissionSummary,
+ SUBMISSION_RECORD_ACTION_TYPE,
+ type SubmissionSummaryResponse,
} from '../utils/types.js';
const controller = ({
@@ -112,17 +112,22 @@ const controller = ({
next(error);
}
}),
- deleteEntityName: validateRequest(submissionDeleteEntityNameRequestSchema, async (req, res, next) => {
+ deleteByRecordIdOrFileId: validateRequest(submissionRecordDeleteRequestSchema, async (req, res, next) => {
try {
const submissionId = Number(req.params.submissionId);
- const actionType = SUBMISSION_ACTION_TYPE.parse(req.params.actionType.toUpperCase());
- const entityName = req.query.entityName;
- const index = req.query.index ? parseInt(req.query.index) : null;
+ const recordId = req.query.recordId ? parseInt(req.query.recordId) : null;
+ const fileId = req.query.fileId ? parseInt(req.query.fileId) : null;
const user = req.user;
+ if (!recordId && !fileId) {
+ throw new BadRequest('Either "recordId" or "fileId" query parameter must be provided for deletion.');
+ } else if (recordId && fileId) {
+ throw new BadRequest('Only one of "recordId" or "fileId" query parameter can be provided for deletion.');
+ }
+
logger.info(
LOG_MODULE,
- `Request Delete '${entityName ? entityName : 'all'}' records on '{${actionType}}' Active Submission '${submissionId}'`,
+ `Request Delete records with ${recordId ? `ID '${recordId}'` : ''}${fileId ? ` file ID '${fileId}'` : ''} on Submission '${submissionId}'`,
);
const submission = await submissionService.getSubmissionById(submissionId);
@@ -136,15 +141,10 @@ const controller = ({
const username = user?.username || '';
- const deleteSubmissionEntityResult = await submissionService.deleteActiveSubmissionEntity(
- submissionId,
- username,
- {
- actionType,
- entityName,
- index,
- },
- );
+ const deleteSubmissionEntityResult = await submissionService.deleteByRecordIdOrFileId(submissionId, username, {
+ recordId,
+ fileId,
+ });
if (isEmpty(deleteSubmissionEntityResult)) {
throw new NotFound('Active Submission not found');
@@ -232,7 +232,7 @@ const controller = ({
}),
getSubmissionsByCategory: validateRequest(
submissionsByCategoryRequestSchema,
- async (req, res: Response>, next) => {
+ async (req, res: Response>, next) => {
try {
const categoryId = await resolveCategoryId(baseDependencies, req.params.categoryId);
if (categoryId === undefined) {
@@ -258,7 +258,7 @@ const controller = ({
{ onlyActive, username, organization },
);
- const response: PaginatedResponse = {
+ const response: PaginatedResponse = {
pagination: {
currentPage: page,
pageSize: pageSize,
@@ -295,8 +295,9 @@ const controller = ({
try {
const submissionId = Number(req.params.submissionId);
const entityNames = asArray(req.query.entityNames || []);
+ const fileId = req.query.fileId ? parseInt(req.query.fileId) : undefined;
- const actionTypes = parseSubmissionActionTypes(req.query.actionTypes || SUBMISSION_ACTION_TYPE.options);
+ const actionTypes = parseSubmissionActionTypes(req.query.actionTypes || SUBMISSION_RECORD_ACTION_TYPE.options);
// query params
const page = parseInt(String(req.query.page)) || DEFAULT_PAGE;
@@ -307,13 +308,9 @@ const controller = ({
const submission = await submissionService.getSubmissionDetailsById({
submissionId,
paginationOptions: { page, pageSize },
- filterOptions: { entityNames, actionTypes },
+ filterOptions: { entityNames, actionTypes, fileId },
});
- if (isEmpty(submission)) {
- throw new NotFound('Submission not found');
- }
-
return res.status(200).json(submission);
} catch (error) {
next(error);
diff --git a/packages/data-provider/src/core/provider.ts b/packages/data-provider/src/core/provider.ts
index 37bd5c70..65588a1e 100644
--- a/packages/data-provider/src/core/provider.ts
+++ b/packages/data-provider/src/core/provider.ts
@@ -13,6 +13,8 @@ import auditRepository from '../repository/auditRepository.js';
import categoryRepository from '../repository/categoryRepository.js';
import migrationRepository from '../repository/dictionaryMigrationRepository.js';
import dictionaryRepository from '../repository/dictionaryRepository.js';
+import submissionFilesRepository from '../repository/submissionFilesRepository.js';
+import submissionRecordsRepository from '../repository/submissionRecordsRepository.js';
import submittedDataRepository from '../repository/submittedRepository.js';
import auditRouter from '../routers/auditRouter.js';
import categoryRouter from '../routers/categoryRouter.js';
@@ -104,6 +106,8 @@ const provider = (configData: AppConfig, options?: ProviderOptions) => {
category: categoryRepository(baseDeps),
dictionary: dictionaryRepository(baseDeps),
migration: migrationRepository(baseDeps),
+ submissionFiles: submissionFilesRepository(baseDeps),
+ submissionRecords: submissionRecordsRepository(baseDeps),
submission: submissionRepository(baseDeps),
submittedData: submittedDataRepository(baseDeps),
},
diff --git a/packages/data-provider/src/repository/activeSubmissionRepository.ts b/packages/data-provider/src/repository/activeSubmissionRepository.ts
index e9e1f6b5..a5434886 100644
--- a/packages/data-provider/src/repository/activeSubmissionRepository.ts
+++ b/packages/data-provider/src/repository/activeSubmissionRepository.ts
@@ -1,7 +1,7 @@
import type { ExtractTablesWithRelations, SQL } from 'drizzle-orm';
import type { PgTransaction } from 'drizzle-orm/pg-core';
import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js';
-import { and, count, eq, inArray, sql } from 'drizzle-orm/sql';
+import { and, count, eq, inArray } from 'drizzle-orm/sql';
import { type NewSubmission, type Submission, submissions } from '@overture-stack/lyric-data-model/models';
@@ -11,18 +11,16 @@ import { inProcessSubmissionStatus, openSubmissionStatus } from '../utils/submis
import type {
BooleanTrueObject,
PaginationOptions,
- SubmissionDataDetailsRepositoryRecord,
- SubmissionDataSummary,
- SubmissionDataSummaryRepositoryRecord,
- SubmissionErrorsSummary,
+ PartialColumns,
+ SubmissionWithDictionaryAndCategoryRepositoryRecord,
} from '../utils/types.js';
const activeSubmissionRepository = (dependencies: BaseDependencies) => {
const LOG_MODULE = 'ACTIVE_SUBMISSION_REPOSITORY';
const { db, logger } = dependencies;
- // Submission columns for lightweight queries to exclude `data` and `errors` columns to improve performance
- const submissionColumns: BooleanTrueObject = {
+ // Submission columns for lightweight queries to exclude foreign ID fields
+ const submissionColumns = {
id: true,
status: true,
organization: true,
@@ -30,14 +28,7 @@ const activeSubmissionRepository = (dependencies: BaseDependencies) => {
createdBy: true,
updatedAt: true,
updatedBy: true,
- };
-
- // Submission columns for full detail queries including `data` and `errors` columns
- const submissionColumnsWithData: BooleanTrueObject = {
- ...submissionColumns,
- data: true,
- errors: true,
- };
+ } as const satisfies PartialColumns>;
const submissionDictionaryRelationColumns = {
dictionary: {
@@ -67,116 +58,6 @@ const activeSubmissionRepository = (dependencies: BaseDependencies) => {
dictionaryCategory: Omit & { alias?: string };
};
- /**
- * A query to generate a summarized JSON object of the 'data' column
- * Returns a JSON object of type SubmissionDataSummary
- */
- const dataSummaryQuery = sql`
-jsonb_build_object(
- 'inserts',
- (
- SELECT jsonb_object_agg(
- i.key,
- jsonb_build_object(
- 'batchName', i.value->>'batchName',
- 'recordsCount',
- CASE
- WHEN jsonb_typeof(i.value->'records') = 'array'
- THEN jsonb_array_length(i.value->'records')
- ELSE 0
- END
- )
- )
- FROM jsonb_each(${submissions.data}->'inserts') AS i(key, value)
- ),
-
- 'updates',
- (
- SELECT jsonb_object_agg(
- u.key,
- jsonb_build_object(
- 'recordsCount',
- CASE
- WHEN jsonb_typeof(u.value) = 'array'
- THEN jsonb_array_length(u.value)
- ELSE 0
- END
- )
- )
- FROM jsonb_each(${submissions.data}->'updates') AS u(key, value)
- ),
-
- 'deletes',
- (
- SELECT jsonb_object_agg(
- d.key,
- jsonb_build_object(
- 'recordsCount',
- CASE
- WHEN jsonb_typeof(d.value) = 'array'
- THEN jsonb_array_length(d.value)
- ELSE 0
- END
- )
- )
- FROM jsonb_each(${submissions.data}->'deletes') AS d(key, value)
- )
-)`.as('data');
-
- /**
- * A query to generate a summarized JSON object of the 'errors' column
- * Returns a json object of type SubmissionErrorsSummary
- */
- const errorsSummaryQuery = sql`jsonb_build_object(
- 'inserts',
- (
- SELECT jsonb_object_agg(
- i.key,
- jsonb_build_object(
- 'recordsCount',
- CASE
- WHEN jsonb_typeof(i.value) = 'array'
- THEN jsonb_array_length(i.value)
- ELSE 0
- END
- )
- )
- FROM jsonb_each(${submissions.errors}->'inserts') AS i(key, value)
- ),
-
- 'updates',
- (
- SELECT jsonb_object_agg(
- u.key,
- jsonb_build_object(
- 'recordsCount',
- CASE
- WHEN jsonb_typeof(u.value) = 'array'
- THEN jsonb_array_length(u.value)
- ELSE 0
- END
- )
- )
- FROM jsonb_each(${submissions.errors}->'updates') AS u(key, value)
- ),
-
- 'deletes',
- (
- SELECT jsonb_object_agg(
- d.key,
- jsonb_build_object(
- 'recordsCount',
- CASE
- WHEN jsonb_typeof(d.value) = 'array'
- THEN jsonb_array_length(d.value)
- ELSE 0
- END
- )
- )
- FROM jsonb_each(${submissions.errors}->'deletes') AS d(key, value)
- )
-)`.as('errors');
-
/**
* SQL condition used to filter submissions that are in an active state.
* Example usage:
@@ -212,45 +93,17 @@ jsonb_build_object(
}
},
- /**
- * Returns the entire active submission, including all data.
- */
- getActiveSubmissionDetails: async ({
- categoryId,
- organization,
- username,
- }: {
- categoryId: number;
- username: string;
- organization: string;
- }): Promise | undefined> => {
- try {
- const dbResponse = await db.query.submissions.findFirst({
- where: and(
- eq(submissions.dictionaryCategoryId, categoryId),
- eq(submissions.createdBy, username),
- eq(submissions.organization, organization),
- activeStatusesCondition,
- ),
- columns: submissionColumnsWithData,
- with: submissionDictionaryRelationColumns,
- });
- return dbResponse;
- } catch (error) {
- logger.error(LOG_MODULE, `Failed getting active submission data`, error);
- throw new ServiceUnavailable();
- }
- },
-
/**
* Finds the current Active Submission by parameters
+ * Returns general information about the Submission, including its dictionary and category relations,
+ * omitting its submissionFiles or submissionRecords relations.
* @param {Object} params
* @param {number} params.categoryId Category ID
* @param {string} params.username Name of the user
* @param {string} params.organization Organization name
* @returns
*/
- getActiveSubmissionSummary: async ({
+ getActiveSubmission: async ({
categoryId,
username,
organization,
@@ -258,7 +111,7 @@ jsonb_build_object(
categoryId: number;
username: string;
organization: string;
- }): Promise => {
+ }): Promise => {
try {
const result = await db.query.submissions.findFirst({
where: and(
@@ -269,7 +122,6 @@ jsonb_build_object(
),
columns: submissionColumns,
with: submissionDictionaryRelationColumns,
- extras: { data: dataSummaryQuery, errors: errorsSummaryQuery },
});
return result ? withAliasNormalized(result) : undefined;
} catch (error) {
@@ -280,16 +132,19 @@ jsonb_build_object(
/**
* Finds a Submission by ID
+ * Returns general information about the Submission, including its dictionary and category relations,
+ * omitting its submissionFiles or submissionRecords relations.
* @param {number} submissionId Submission ID
* @returns The Submission found
*/
- getSubmissionById: async (submissionId: number): Promise => {
+ getSubmissionById: async (
+ submissionId: number,
+ ): Promise => {
try {
const result = await db.query.submissions.findFirst({
where: and(eq(submissions.id, submissionId)),
columns: submissionColumns,
with: submissionDictionaryRelationColumns,
- extras: { data: dataSummaryQuery, errors: errorsSummaryQuery },
});
return result ? withAliasNormalized(result) : undefined;
} catch (error) {
@@ -298,28 +153,6 @@ jsonb_build_object(
}
},
- /**
- * Retun the Submission with data details by ID
- * This includes the `data` and `errors` columns
- * @param {number} submissionId Submission ID
- * @returns The Submission found
- */
- getSubmissionDetailsById: async (
- submissionId: number,
- ): Promise => {
- try {
- const result = await db.query.submissions.findFirst({
- where: and(eq(submissions.id, submissionId)),
- columns: submissionColumnsWithData,
- with: submissionDictionaryRelationColumns,
- });
- return result ? withAliasNormalized(result) : undefined;
- } catch (error) {
- logger.error(LOG_MODULE, `Failed getting Submission details with id '${submissionId}'`, error);
- throw new ServiceUnavailable();
- }
- },
-
/**
* Update a Submission record in database
* @param {number} submissionId Submission ID to update
@@ -368,7 +201,7 @@ jsonb_build_object(
username?: string;
organization?: string;
},
- ): Promise => {
+ ): Promise => {
const { page, pageSize } = paginationOptions;
try {
const results = await db.query.submissions.findMany({
@@ -379,7 +212,6 @@ jsonb_build_object(
filterOptions.organization ? eq(submissions.organization, filterOptions.organization) : undefined,
),
columns: submissionColumns,
- extras: { data: dataSummaryQuery, errors: errorsSummaryQuery },
with: submissionDictionaryRelationColumns,
orderBy: (submissions, { desc }) => desc(submissions.createdAt),
limit: pageSize,
diff --git a/packages/data-provider/src/repository/submissionFilesRepository.ts b/packages/data-provider/src/repository/submissionFilesRepository.ts
new file mode 100644
index 00000000..bed127d2
--- /dev/null
+++ b/packages/data-provider/src/repository/submissionFilesRepository.ts
@@ -0,0 +1,102 @@
+import type { ExtractTablesWithRelations } from 'drizzle-orm';
+import type { PgTransaction } from 'drizzle-orm/pg-core';
+import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js';
+import { eq } from 'drizzle-orm/sql';
+
+import { type NewSubmissionFile, type SubmissionFile, submissionFiles } from '@overture-stack/lyric-data-model/models';
+
+import { BaseDependencies } from '../config/config.js';
+import { ServiceUnavailable } from '../utils/errors.js';
+
+const submissionFilesRepository = (dependencies: BaseDependencies) => {
+ const LOG_MODULE = 'SUBMISSION_FILES_REPOSITORY';
+ const { db, logger } = dependencies;
+
+ return {
+ save: async (
+ input: NewSubmissionFile,
+ tx?: PgTransaction>,
+ ): Promise => {
+ try {
+ const [savedSubmissionFile] = await (tx || db)
+ .insert(submissionFiles)
+ .values(input)
+ .returning({ id: submissionFiles.id });
+ if (!savedSubmissionFile) {
+ throw new Error('Failed to insert Submission File, no row returned');
+ }
+ logger.info(LOG_MODULE, `New Submission File saved successfully`);
+ return savedSubmissionFile.id;
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed saving Submission File`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ getById: async (fileId: number): Promise => {
+ try {
+ return await db.query.submissionFiles.findFirst({
+ where: eq(submissionFiles.id, fileId),
+ });
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed getting Submission File by id '${fileId}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ getBySubmissionId: async (submissionId: number): Promise => {
+ try {
+ return await db.query.submissionFiles.findMany({
+ where: eq(submissionFiles.submissionId, submissionId),
+ });
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed getting Submission Files by submissionId '${submissionId}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ deleteById: async (
+ fileId: number,
+ tx?: PgTransaction>,
+ ): Promise => {
+ try {
+ const deletedFiles = await (tx || db)
+ .delete(submissionFiles)
+ .where(eq(submissionFiles.id, fileId))
+ .returning({ id: submissionFiles.id });
+ logger.info(LOG_MODULE, `Deleted Submission Files with id '${fileId}'`);
+ return deletedFiles[0]?.id;
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed deleting Submission File by fileId '${fileId}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ /**
+ * Deletes the files associated with a specific submission ID.
+ * This function does not delete cascadeingly related submission records, it will throw an error if
+ * foreign key constraints are violated.
+ * @param submissionId
+ * @param tx
+ * @returns
+ */
+ deleteBySubmissionId: async (
+ submissionId: number,
+ tx?: PgTransaction>,
+ ): Promise => {
+ try {
+ const deletedFiles = await (tx || db)
+ .delete(submissionFiles)
+ .where(eq(submissionFiles.submissionId, submissionId))
+ .returning({ id: submissionFiles.id });
+ logger.info(LOG_MODULE, `Deleted '${deletedFiles.length}' Submission Files for submissionId '${submissionId}'`);
+ return deletedFiles.map((file) => file.id);
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed deleting Submission Files by submissionId '${submissionId}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+ };
+};
+
+export default submissionFilesRepository;
diff --git a/packages/data-provider/src/repository/submissionRecordsRepository.ts b/packages/data-provider/src/repository/submissionRecordsRepository.ts
new file mode 100644
index 00000000..82c106a4
--- /dev/null
+++ b/packages/data-provider/src/repository/submissionRecordsRepository.ts
@@ -0,0 +1,361 @@
+import { type ExtractTablesWithRelations } from 'drizzle-orm';
+import type { PgTransaction } from 'drizzle-orm/pg-core';
+import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js';
+import { and, count, eq, inArray } from 'drizzle-orm/sql';
+
+import {
+ type NewSubmissionRecord,
+ submissionFiles,
+ type SubmissionRecord,
+ type SubmissionRecordError,
+ submissionRecords,
+} from '@overture-stack/lyric-data-model/models';
+
+import { BaseDependencies } from '../config/config.js';
+import { ServiceUnavailable } from '../utils/errors.js';
+import type { PaginationOptions, SubmissionRecordActionType, SubmissionRecordState } from '../utils/types.js';
+
+// This is the information stored about each individual submission record in the database, including it's entity name.
+export type SubmissionRecordWithEntityName = SubmissionRecord & { entityName: string };
+
+// Raw data returned from the database
+export type RecordsSummaryRepository = {
+ actionType: SubmissionRecordActionType;
+ entityName: string;
+ totalRecords: number;
+ batchName?: string;
+ errors: number;
+};
+
+const submissionRecordsRepository = (dependencies: BaseDependencies) => {
+ const LOG_MODULE = 'SUBMISSION_RECORDS_REPOSITORY';
+ const { db, logger } = dependencies;
+
+ const getByFileIds = async (
+ fileIds: number[],
+ paginationOptions?: PaginationOptions,
+ filterOptions?: { actionTypes?: SubmissionRecordActionType[]; states?: SubmissionRecordState[] },
+ ): Promise => {
+ const query = db
+ .select({
+ id: submissionRecords.id,
+ actionType: submissionRecords.actionType,
+ state: submissionRecords.state,
+ fileId: submissionRecords.fileId,
+ data: submissionRecords.data,
+ errors: submissionRecords.errors,
+ entityName: submissionFiles.entityName,
+ })
+ .from(submissionRecords)
+ .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id))
+ .where(
+ and(
+ inArray(submissionRecords.fileId, fileIds),
+ filterOptions?.actionTypes ? inArray(submissionRecords.actionType, filterOptions.actionTypes) : undefined,
+ filterOptions?.states?.length ? inArray(submissionRecords.state, filterOptions.states) : undefined,
+ ),
+ )
+ .orderBy(submissionRecords.id);
+
+ if (paginationOptions) {
+ query.limit(paginationOptions.pageSize).offset((paginationOptions.page - 1) * paginationOptions.pageSize);
+ }
+
+ return await query;
+ };
+
+ const saveMany = async (
+ inputs: NewSubmissionRecord[],
+ tx?: PgTransaction>,
+ ): Promise => {
+ if (!inputs.length) {
+ return [];
+ }
+ try {
+ // TODO: Insert in batches if inputs.length > 1000 to avoid exceeding the maximum number of parameters in a single query
+ const savedSubmissionRecords = await (tx || db)
+ .insert(submissionRecords)
+ .values(inputs)
+ .returning({ id: submissionRecords.id });
+ logger.info(LOG_MODULE, `Saved '${savedSubmissionRecords.length}' Submission Record records successfully`);
+ return savedSubmissionRecords.map((record) => record.id);
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed saving '${inputs.length}' Submission Record records`, error);
+ throw new ServiceUnavailable();
+ }
+ };
+
+ const deleteByFileIds = async (
+ fileIds: number[],
+ tx?: PgTransaction>,
+ ): Promise => {
+ if (!fileIds.length) {
+ return 0;
+ }
+ try {
+ const deletedRecords = await (tx || db)
+ .delete(submissionRecords)
+ .where(inArray(submissionRecords.fileId, fileIds))
+ .returning({ id: submissionRecords.id });
+ logger.info(LOG_MODULE, `Deleted '${deletedRecords.length}' Submission Record records by fileIds`);
+ return deletedRecords.length;
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed deleting Submission Records by fileIds`, error);
+ throw new ServiceUnavailable();
+ }
+ };
+
+ return {
+ saveMany,
+
+ saveManyForFile: async (
+ fileId: number,
+ records: Omit[],
+ tx?: PgTransaction>,
+ ): Promise => {
+ // TODO: Batch insert records
+ const inputs: NewSubmissionRecord[] = records.map((record) => ({ ...record, fileId }));
+ return await saveMany(inputs, tx);
+ },
+
+ getById: async (id: number): Promise => {
+ try {
+ const query = await db
+ .select({
+ id: submissionRecords.id,
+ actionType: submissionRecords.actionType,
+ state: submissionRecords.state,
+ fileId: submissionRecords.fileId,
+ data: submissionRecords.data,
+ errors: submissionRecords.errors,
+ entityName: submissionFiles.entityName,
+ })
+ .from(submissionRecords)
+ .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id))
+ .where(eq(submissionRecords.id, id))
+ .limit(1);
+
+ if (query.length === 0) {
+ return undefined;
+ }
+ return query[0];
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed getting Submission Record by id '${id}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ getByFileIds,
+
+ getBySubmissionId: async (
+ submissionId: number,
+ paginationOptions?: PaginationOptions,
+ filterOptions?: {
+ actionTypes?: SubmissionRecordActionType[];
+ states?: SubmissionRecordState[];
+ entityNames?: string[];
+ fileId?: number;
+ },
+ ): Promise => {
+ try {
+ const submissionFileIds = await db
+ .select({ id: submissionFiles.id, entityName: submissionFiles.entityName })
+ .from(submissionFiles)
+ .where(
+ and(
+ eq(submissionFiles.submissionId, submissionId),
+ filterOptions?.entityNames?.length
+ ? inArray(submissionFiles.entityName, filterOptions.entityNames)
+ : undefined,
+ filterOptions?.fileId ? eq(submissionFiles.id, filterOptions.fileId) : undefined,
+ ),
+ );
+
+ if (submissionFileIds.length === 0) {
+ logger.info(
+ LOG_MODULE,
+ `No submission files found for submissionId '${submissionId}' with the provided filter options.`,
+ );
+ return [];
+ }
+
+ return await getByFileIds(
+ submissionFileIds.map((file) => file.id),
+ paginationOptions,
+ {
+ actionTypes: filterOptions?.actionTypes,
+ states: filterOptions?.states,
+ },
+ );
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed getting Submission Records by submissionId '${submissionId}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ getRecordsSummaryBySubmissionId: async (submissionId: number): Promise => {
+ try {
+ const submissionFileRecords = await db
+ .select({
+ actionType: submissionRecords.actionType,
+ batchName: submissionFiles.fileName,
+ entityName: submissionFiles.entityName,
+ totalRecords: count(),
+ errors: count(submissionRecords.errors),
+ })
+ .from(submissionRecords)
+ .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id))
+ .where(eq(submissionFiles.submissionId, submissionId))
+ .groupBy(submissionRecords.actionType, submissionFiles.fileName, submissionFiles.entityName);
+
+ return submissionFileRecords;
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed getting Submission Records summary by submissionId '${submissionId}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ /**
+ * This function updates the validation state of submission records based on the provided parameters.
+ * It can update records to 'VALID', 'RECEIVED', or 'INVALID' states, and also set errors for invalid records.
+ * @param params
+ * @param tx
+ * @returns
+ */
+ updateValidationState: async (
+ params: {
+ validRecordIds?: number[];
+ receivedRecordIds?: number[];
+ invalidRecords?: Array<{ id: number; errors?: SubmissionRecordError[] }>;
+ },
+ tx?: PgTransaction>,
+ ): Promise => {
+ const executor = tx || db;
+
+ const validRecordIds = params.validRecordIds ?? [];
+ const receivedRecordIds = params.receivedRecordIds ?? [];
+ const invalidRecords = params.invalidRecords ?? [];
+ if (!validRecordIds.length && !receivedRecordIds.length && !invalidRecords.length) {
+ return [];
+ }
+
+ try {
+ const updatedIds: number[] = [];
+
+ if (validRecordIds.length) {
+ const validUpdates = await executor
+ .update(submissionRecords)
+ .set({ state: 'VALID', errors: null })
+ .where(inArray(submissionRecords.id, validRecordIds))
+ .returning({ id: submissionRecords.id });
+ updatedIds.push(...validUpdates.map((record) => record.id));
+ }
+
+ if (receivedRecordIds.length) {
+ const receivedUpdates = await executor
+ .update(submissionRecords)
+ .set({ state: 'RECEIVED', errors: null })
+ .where(inArray(submissionRecords.id, receivedRecordIds))
+ .returning({ id: submissionRecords.id });
+ updatedIds.push(...receivedUpdates.map((record) => record.id));
+ }
+
+ if (invalidRecords.length) {
+ const invalidUpdates = await Promise.all(
+ invalidRecords.map(async ({ id, errors }) => {
+ const [updatedRecord] = await executor
+ .update(submissionRecords)
+ .set({ state: 'INVALID', errors: errors ?? null })
+ .where(eq(submissionRecords.id, id))
+ .returning({ id: submissionRecords.id });
+ return updatedRecord?.id;
+ }),
+ );
+ updatedIds.push(...invalidUpdates.filter((id): id is number => id !== undefined));
+ }
+
+ logger.info(
+ LOG_MODULE,
+ `Updated Submission Record states: VALID='${validRecordIds.length}', RECEIVED='${receivedRecordIds.length}', INVALID='${invalidRecords.length}'`,
+ );
+ return [...new Set(updatedIds)];
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed updating Submission Record validation state`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ countBySubmissionId: async (
+ submissionId: number,
+ ): Promise> => {
+ try {
+ return await db
+ .select({ actionType: submissionRecords.actionType, total: count() })
+ .from(submissionRecords)
+ .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id))
+ .where(eq(submissionFiles.submissionId, submissionId))
+ .groupBy(submissionRecords.actionType);
+ } catch (error) {
+ logger.error(
+ LOG_MODULE,
+ `Failed counting Submission Records by action for submissionId '${submissionId}'`,
+ error,
+ );
+ throw new ServiceUnavailable();
+ }
+ },
+
+ countInvalidBySubmissionId: async (
+ submissionId: number,
+ ): Promise> => {
+ try {
+ return await db
+ .select({ actionType: submissionRecords.actionType, total: count() })
+ .from(submissionRecords)
+ .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id))
+ .where(and(eq(submissionFiles.submissionId, submissionId), eq(submissionRecords.state, 'INVALID')))
+ .groupBy(submissionRecords.actionType);
+ } catch (error) {
+ logger.error(
+ LOG_MODULE,
+ `Failed counting invalid Submission Records by action for submissionId '${submissionId}'`,
+ error,
+ );
+ throw new ServiceUnavailable();
+ }
+ },
+
+ deleteByIds: async (
+ ids: number[],
+ tx?: PgTransaction>,
+ ): Promise => {
+ try {
+ return await (tx || db).delete(submissionRecords).where(inArray(submissionRecords.id, ids));
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed deleting Submission Record by ids '${ids}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+
+ deleteByFileIds,
+
+ deleteBySubmissionId: async (
+ submissionId: number,
+ tx?: PgTransaction>,
+ ): Promise => {
+ try {
+ const submissionFileIds = await (tx || db)
+ .select({ id: submissionFiles.id })
+ .from(submissionFiles)
+ .where(eq(submissionFiles.submissionId, submissionId));
+ const fileIds = submissionFileIds.map((file) => file.id);
+ return await deleteByFileIds(fileIds, tx);
+ } catch (error) {
+ logger.error(LOG_MODULE, `Failed deleting Submission Records by submissionId '${submissionId}'`, error);
+ throw new ServiceUnavailable();
+ }
+ },
+ };
+};
+
+export default submissionRecordsRepository;
diff --git a/packages/data-provider/src/routers/submissionRouter.ts b/packages/data-provider/src/routers/submissionRouter.ts
index 02ddcb86..a522d142 100644
--- a/packages/data-provider/src/routers/submissionRouter.ts
+++ b/packages/data-provider/src/routers/submissionRouter.ts
@@ -71,11 +71,11 @@ const router = ({
router.get('/:submissionId', submissionController.getSubmissionById);
- router.get('/:submissionId/details', submissionController.getSubmissionDetailsById);
-
router.delete('/:submissionId', submissionController.delete);
- router.delete('/:submissionId/:actionType', submissionController.deleteEntityName);
+ router.get('/:submissionId/data', submissionController.getSubmissionDetailsById);
+
+ router.delete('/:submissionId/data', submissionController.deleteByRecordIdOrFileId);
router.get('/category/:categoryId', submissionController.getSubmissionsByCategory);
diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts
index e4ec10ba..b0ef49c0 100644
--- a/packages/data-provider/src/services/submission/submissionProcessor.ts
+++ b/packages/data-provider/src/services/submission/submissionProcessor.ts
@@ -1,4 +1,3 @@
-import bytes from 'bytes';
import * as _ from 'lodash-es';
import type { DataRecord, DictionaryValidationRecordErrorDetails, Schema } from '@overture-stack/lectern-client';
@@ -6,9 +5,7 @@ import type {
DataDiff,
NewSubmittedData,
SubmissionDeleteData,
- SubmissionErrors,
SubmissionInsertData,
- SubmissionRecordErrorDetails,
SubmissionUpdateData,
SubmittedData,
} from '@overture-stack/lyric-data-model/models';
@@ -17,27 +14,30 @@ import { BaseDependencies } from '../../config/config.js';
import createSubmissionRepository from '../../repository/activeSubmissionRepository.js';
import createCategoryRepository from '../../repository/categoryRepository.js';
import createDictionaryRepository from '../../repository/dictionaryRepository.js';
+import createSubmissionFilesRepository from '../../repository/submissionFilesRepository.js';
+import createSubmissionRecordsRepository from '../../repository/submissionRecordsRepository.js';
import createSubmittedDataRepository from '../../repository/submittedRepository.js';
import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/dictionarySchemaRelations.js';
import { BadRequest } from '../../utils/errors.js';
+import { formatByteSize, genericSubmissionFileName, getSizeInBytes } from '../../utils/fileUtils.js';
import { convertRecordToString } from '../../utils/formatUtils.js';
import { parseRecordsToInsert } from '../../utils/recordsParser.js';
import {
+ extractRecordIdsFromSubmissionErrors,
extractSchemaDataFromMergedDataRecords,
type FileParseResult,
- filterDeletesFromUpdates,
filterRelationsForPrimaryIdUpdate,
- findEditSubmittedData,
findInvalidRecordErrorsBySchemaName,
+ findUpdateDeleteConflicts,
groupSchemaErrorsByEntity,
isSubmissionActive,
mapGroupedUpdateSubmissionData,
mergeAndReferenceEntityData,
- mergeDeleteRecords,
- mergeInsertsRecords,
+ mergeSubmissionErrors,
mergeUpdatesBySystemId,
parseToSchema,
segregateFieldChangeRecords,
+ type SubmissionErrors,
submissionInsertDataFromFiles,
validateSchemas,
} from '../../utils/submissionUtils.js';
@@ -57,7 +57,6 @@ import {
type ResultOnCommit,
type SchemasDictionary,
SUBMISSION_STATUS,
- type ValidateFilesParams,
} from '../../utils/types.js';
import createSubmittedDataRelationsSearch from '../submittedData/searchDataRelations.js';
@@ -68,6 +67,8 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
const submissionRepository = createSubmissionRepository(dependencies);
const submittedDataRepository = createSubmittedDataRepository(dependencies);
const submittedDataRelationsSearch = createSubmittedDataRelationsSearch(dependencies);
+ const submissionRecordsRepository = createSubmissionRecordsRepository(dependencies);
+ const submissionFilesRepository = createSubmissionFilesRepository(dependencies);
const { logger } = dependencies;
/**
@@ -223,7 +224,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
return Object.entries(idFieldChangeRecord).reduce<
Promise<{
- inserts: Record;
+ inserts: Record;
deletes: Record;
}>
>(
@@ -233,7 +234,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
// iterate each record on this entity
const result = await updRecord.reduce<
Promise<{
- inserts: DataRecord[];
+ inserts: SubmissionInsertData[];
deletes: SubmissionDeleteData[];
}>
>(
@@ -248,12 +249,11 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
const deleteRecord: SubmissionDeleteData = {
systemId: foundSubmittedData.systemId,
data: foundSubmittedData.data,
- entityName: foundSubmittedData.entityName,
isValid: foundSubmittedData.isValid,
organization: foundSubmittedData.organization,
};
- const insertDataRecord: DataRecord = { ...foundSubmittedData.data, ...u.new };
+ const insertDataRecord: SubmissionInsertData = { ...foundSubmittedData.data, ...u.new };
acc2.inserts.push(insertDataRecord);
acc2.deletes.push(deleteRecord);
@@ -263,7 +263,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
);
acc.deletes[entityName] = result.deletes;
- acc.inserts[entityName] = { batchName: entityName, records: result.inserts };
+ acc.inserts[entityName] = result.inserts;
return acc;
},
@@ -430,7 +430,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
// iterate if there are any record to be deleted
dataToValidate?.deletes?.forEach((item) => {
- const { data, entityName, isValid, organization, systemId } = item;
+ const { data, isValid, organization, systemId } = item;
deletesToProcess.push({
submissionId: submission.id,
@@ -441,7 +441,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
resultCommit.deletes.push({
data,
- entityName,
+ entityName: '', // TODO: need to fetch the entityName
isValid,
organization,
systemId,
@@ -503,10 +503,10 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
const performDataValidation = async (submissionId: number): Promise => {
const { getActiveDictionaryByCategory } = categoryRepository;
const { getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository;
- const { getSubmissionDetailsById } = submissionRepository;
+ const { getSubmissionById } = submissionRepository;
// Get Active Submission from database
- const activeSubmission = await getSubmissionDetailsById(submissionId);
+ const activeSubmission = await getSubmissionById(submissionId);
if (!activeSubmission) {
throw new Error(`Submission '${submissionId}' not found`);
@@ -523,10 +523,31 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
throw new BadRequest(`Dictionary in category '${activeSubmission.dictionaryCategory.id}' not found`);
}
+ const submissionRecords = await submissionRecordsRepository.getBySubmissionId(submissionId);
+
+ // Detect records where the same systemId has both an UPDATE and a DELETE staged, before
+ // running dictionary validation. Both sides of a conflict are rejected explicitly instead
+ // of letting one action silently win.
+ const conflictErrors = findUpdateDeleteConflicts(submissionRecords);
+ const conflictingRecordIds = extractRecordIdsFromSubmissionErrors(conflictErrors);
+
+ if (conflictingRecordIds.size > 0) {
+ logger.error(
+ LOG_MODULE,
+ `Detected '${conflictingRecordIds.size}' Submission Record(s) with conflicting UPDATE/DELETE actions on the same systemId in Submission '${submissionId}'`,
+ JSON.stringify(conflictErrors),
+ );
+ }
+
+ // Exclude conflicting records from validation; neither side of a conflict should be applied
+ const nonConflictingSubmissionRecords = conflictingRecordIds.size
+ ? submissionRecords.filter((record) => !conflictingRecordIds.has(record.id))
+ : submissionRecords;
+
// Merge Submitted Data with Active Submission keepping reference of each record ID
const dataMergedByEntityName = mergeAndReferenceEntityData({
submissionId,
- submissionData: activeSubmission.data,
+ submissionData: nonConflictingSubmissionRecords,
submittedData,
});
@@ -537,44 +558,12 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
const resultValidation = validateSchemas(currentDictionary, crossSchemasDataToValidate);
// Collect errors of the Active Submission
- const submissionSchemaErrors = groupSchemaErrorsByEntity({
+ const schemaValidationErrors = groupSchemaErrorsByEntity({
resultValidation,
dataValidated: dataMergedByEntityName,
});
- // Check for records to be updated that its systemId was not found in the Submitted Data collection.
- // Any error found will cause the submission to be marked as 'invalid'
- Object.entries(activeSubmission.data.updates ?? {}).forEach(([entityName, recordsToUpdate]) => {
- recordsToUpdate.forEach((submissionEditData, index) => {
- const found = findEditSubmittedData(entityName, submissionEditData.systemId, dataMergedByEntityName);
-
- if (found) {
- return;
- }
-
- logger.error(
- LOG_MODULE,
- `Record with systemId '${submissionEditData.systemId}' not found in entity '${entityName}'`,
- );
-
- if (!submissionSchemaErrors.updates) {
- submissionSchemaErrors.updates = {};
- }
-
- if (!submissionSchemaErrors.updates[entityName]) {
- submissionSchemaErrors.updates[entityName] = [];
- }
-
- const unrecodgnizedValueError: SubmissionRecordErrorDetails = {
- fieldName: 'systemId',
- fieldValue: submissionEditData.systemId,
- index,
- reason: 'UNRECOGNIZED_VALUE',
- };
-
- submissionSchemaErrors.updates[entityName].push(unrecodgnizedValueError);
- });
- });
+ const submissionSchemaErrors = mergeSubmissionErrors(conflictErrors, schemaValidationErrors);
if (_.isEmpty(submissionSchemaErrors)) {
logger.info(LOG_MODULE, `No error found on data submission`);
@@ -618,7 +607,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
},
): Promise => {
const { getDictionary } = dictionaryRepository;
- const { getSubmissionDetailsById, update } = submissionRepository;
+ const { getSubmissionById, update } = submissionRepository;
try {
// Parse file data
@@ -626,7 +615,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
const filesDataProcessed = await compareUpdatedData(recordsParsed, schema.name);
- const submission = await getSubmissionDetailsById(submissionId);
+ const submission = await getSubmissionById(submissionId);
if (!submission) {
throw new Error(`Submission '${submissionId}' not found`);
}
@@ -677,7 +666,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
// Aggegates all Update changes on Submission
// Note: We do not include records involving primary ID fields changes in here. We would rather do a DELETE and an INSERT
const updatedActiveSubmissionData: Record = mergeUpdatesBySystemId(
- submission.data.updates ?? {},
+ // formattedSubmissionRecordsToUpdate,
totalDependants,
nonIdFieldChangeRecord,
);
@@ -685,24 +674,69 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
// Creates insert and delete records based on primary ID field change records.
const additions = await handleIdFieldChanges(idFieldChangeRecord);
- // Merge Active Submission Inserts with Edit generated new Inserts
- const mergedInserts = mergeInsertsRecords(submission.data.inserts ?? {}, additions.inserts);
+ // Updating the Submission with the new data and 'VALIDATING' status before validation starts
+ await dependencies.db.transaction(async (tx) => {
+ await update(
+ submission.id,
+ {
+ updatedBy: username,
+ status: 'VALIDATING',
+ },
+ tx,
+ );
- // Merge Active Submission Deletes with Edit generated new Deletes
- const mergedDeletes = mergeDeleteRecords(submission.data.deletes ?? {}, additions.deletes);
+ const entityNames: Set = new Set([
+ ...Object.keys(additions.inserts),
+ ...Object.keys(additions.deletes),
+ ...Object.keys(updatedActiveSubmissionData),
+ ]);
+
+ for (const entityName of entityNames) {
+ const savedFileId = await submissionFilesRepository.save(
+ {
+ entityName: entityName,
+ fileName: genericSubmissionFileName(),
+ fileSize: getSizeInBytes(JSON.stringify(recordsParsed)),
+ submissionId: submission.id,
+ },
+ tx,
+ );
- // filter out delete records found on update records
- const filteredDeletes = filterDeletesFromUpdates(mergedDeletes, updatedActiveSubmissionData);
+ if (updatedActiveSubmissionData[entityName]) {
+ await submissionRecordsRepository.saveManyForFile(
+ savedFileId,
+ updatedActiveSubmissionData[entityName].map((record) => ({
+ actionType: 'UPDATE',
+ data: record,
+ state: 'RECEIVED',
+ })),
+ tx,
+ );
+ }
- // Updating the Submission with the new data and 'VALIDATING' status before validation starts
- await update(submission.id, {
- data: {
- inserts: mergedInserts,
- deletes: filteredDeletes,
- updates: updatedActiveSubmissionData,
- },
- updatedBy: username,
- status: 'VALIDATING',
+ if (additions.inserts[entityName]) {
+ await submissionRecordsRepository.saveManyForFile(
+ savedFileId,
+ additions.inserts[entityName].map((record) => ({
+ actionType: 'INSERT',
+ data: record,
+ state: 'RECEIVED',
+ })),
+ tx,
+ );
+ }
+ if (additions.deletes[entityName]) {
+ await submissionRecordsRepository.saveManyForFile(
+ savedFileId,
+ additions.deletes[entityName]?.map((record) => ({
+ actionType: 'DELETE',
+ data: record,
+ state: 'RECEIVED',
+ })),
+ tx,
+ );
+ }
+ }
});
// Perform Schema Data validation in a worker thread
@@ -739,35 +773,52 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
submissionId: number;
username: string;
}) => {
- const { getSubmissionDetailsById, update } = submissionRepository;
+ const { getSubmissionById, update } = submissionRepository;
try {
// Get Active Submission from database
- const activeSubmission = await getSubmissionDetailsById(submissionId);
+ const activeSubmission = await getSubmissionById(submissionId);
if (!activeSubmission) {
- throw new Error(`Submission '${activeSubmission}' not found`);
+ throw new Error(`Submission '${submissionId}' not found`);
}
if (!isSubmissionActive(activeSubmission.status)) {
throw new Error(`Submission '${activeSubmission.id}' is not active`);
}
- const insertRecords = parseRecordsToInsert(records, schemasDictionary);
-
- // Merge Active Submission insert records with incoming TSV file data processed
- const insertActiveSubmissionData = mergeInsertsRecords(activeSubmission.data.inserts ?? {}, insertRecords);
-
// Updating the Submission with the new data and 'VALIDATING' status before validation starts
await update(activeSubmission.id, {
- data: {
- inserts: insertActiveSubmissionData,
- deletes: activeSubmission.data.deletes,
- updates: activeSubmission.data.updates,
- },
updatedBy: username,
status: 'VALIDATING',
});
+ const insertRecords = parseRecordsToInsert(records, schemasDictionary);
+
+ await Promise.all(
+ Object.entries(insertRecords).map(([entityName, entityRecords]) =>
+ dependencies.db.transaction(async (tx) => {
+ const savedFileId = await submissionFilesRepository.save(
+ {
+ entityName,
+ fileName: genericSubmissionFileName(),
+ fileSize: getSizeInBytes(JSON.stringify(entityRecords)),
+ submissionId,
+ },
+ tx,
+ );
+ await submissionRecordsRepository.saveManyForFile(
+ savedFileId,
+ entityRecords.map((record) => ({
+ actionType: 'INSERT',
+ data: record,
+ state: 'RECEIVED',
+ })),
+ tx,
+ );
+ }),
+ ),
+ );
+
// Perform Schema Data validation in a worker thread
dependencies.workerPool.dataValidation({ submissionId: activeSubmission.id });
} catch (error) {
@@ -782,7 +833,9 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
/**
* Update Active Submission in database
- * Updates the status of the Submission to 'VALID' if there is no errors, otherwise updates it to 'INVALID'
+ * Updates the Submission status to 'VALID' if there is no errors, otherwise updates it to 'INVALID'
+ * Updates all the records of the submission with the validation state, marking records with errors as 'INVALID'
+ * and records without errors as 'VALID'
* @param {Object} input
* @param {number} input.dictionaryId The Dictionary ID of the Submission
* @param {number} input.idActiveSubmission ID of the Submission
@@ -795,43 +848,72 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
schemaErrors: SubmissionErrors;
}): Promise => {
const { dictionaryId, idActiveSubmission, schemaErrors } = input;
- const { update } = submissionRepository;
const newStatusSubmission =
Object.keys(schemaErrors).length > 0 ? SUBMISSION_STATUS.INVALID : SUBMISSION_STATUS.VALID;
- // Update with new data
- const updatedActiveSubmissionId = await update(idActiveSubmission, {
- status: newStatusSubmission,
- dictionaryId: dictionaryId,
- errors: schemaErrors,
+
+ await dependencies.db.transaction(async (tx) => {
+ // Update with new data
+ const updatedActiveSubmissionId = await submissionRepository.update(
+ idActiveSubmission,
+ {
+ status: newStatusSubmission,
+ dictionaryId: dictionaryId,
+ },
+ tx,
+ );
+
+ const invalidRecords = Object.values(schemaErrors).flatMap((entityErrors) =>
+ Object.values(entityErrors).flatMap((recordErrors) =>
+ recordErrors.map(({ recordId, errors }) => ({
+ id: recordId,
+ errors,
+ })),
+ ),
+ );
+
+ const submissionRecords = await submissionRecordsRepository.getBySubmissionId(idActiveSubmission);
+ const recordsWithoutError = submissionRecords
+ .filter((record) => !invalidRecords.some((invalidRecord) => invalidRecord.id === record.id))
+ .map((record) => record.id);
+
+ // Update records with validation state, marking records with errors as 'INVALID' and records without errors as 'VALID'
+ await submissionRecordsRepository.updateValidationState(
+ {
+ invalidRecords,
+ validRecordIds: recordsWithoutError,
+ },
+ tx,
+ );
+ logger.info(
+ LOG_MODULE,
+ `Updated Active submission '${updatedActiveSubmissionId}' with status '${newStatusSubmission}'`,
+ );
+ return updatedActiveSubmissionId;
});
- logger.info(
- LOG_MODULE,
- `Updated Active submission '${updatedActiveSubmissionId}' with status '${newStatusSubmission}'`,
- );
- return updatedActiveSubmissionId;
+ return 0;
};
- const logFileResults = (fileResults: FileParseResult[]) => {
- for (const result of fileResults) {
- if (result.status === 'error') {
- logger.error(LOG_MODULE, `Failed to parse file`, {
- fileName: result.fileName,
- entityName: result.entityName,
- error: result.streamError,
- });
- } else if (result.status === 'invalid') {
- // Log field names and line numbers only — not field values (OWASP A03).
- logger.warn(LOG_MODULE, `File parsed with schema validation issues`, {
- fileName: result.fileName,
- entityName: result.entityName,
- errorCount: result.parseErrors.length,
- issues: result.parseErrors.slice(0, 10).map((e) => ({
- line: e.recordIndex,
- fields: e.recordErrors.map((re) => re.fieldName),
- })),
- });
- }
+ const logFileResult = (result: FileParseResult) => {
+ if (result.status === 'error') {
+ logger.error(LOG_MODULE, `Failed to parse file`, {
+ fileName: result.fileName,
+ fileSize: formatByteSize(result.fileSize, 'MB', 2),
+ entityName: result.entityName,
+ error: result.streamError,
+ });
+ } else if (result.status === 'invalid') {
+ // Log field names and line numbers only — not field values (OWASP A03).
+ logger.warn(LOG_MODULE, `File parsed with schema validation issues`, {
+ fileName: result.fileName,
+ fileSize: formatByteSize(result.fileSize, 'MB', 2),
+ entityName: result.entityName,
+ errorCount: result.parseErrors.length,
+ issues: result.parseErrors.slice(0, 10).map((e) => ({
+ line: e.recordIndex,
+ fields: e.recordErrors.map((re) => re.fieldName),
+ })),
+ });
}
};
@@ -844,65 +926,72 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => {
*/
const addFilesToSubmissionAsync = async (
fileSchemaMap: FileSchemaMap,
- params: ValidateFilesParams,
+ submissionId: number,
+ username: string,
): Promise => {
- const fileSummaries = Object.entries(fileSchemaMap)
- .flatMap(([_, { files, schema }]) =>
- files.map(
- (file) => `'${file.originalname}' (${bytes.format(file.size, { decimalPlaces: 2 })}, entity: ${schema.name})`,
- ),
- )
- .join(', ');
- logger.info(LOG_MODULE, `Processing files: ${fileSummaries}`);
-
- const { categoryId, organization, username } = params;
- let fileResults: FileParseResult[] = [];
+ const fileResult: FileParseResult[] = [];
try {
- // Verify an active submission exists before doing any parsing work.
- const activeSubmission = await submissionRepository.getActiveSubmissionDetails({
- categoryId,
- username,
- organization,
- });
- if (!activeSubmission) {
- throw new BadRequest(`No active submission found for category '${categoryId}' organization '${organization}'`);
- }
+ await dependencies.db.transaction(async (tx) => {
+ // Updating the Submission with the new data and 'VALIDATING' status before validation starts
+ await submissionRepository.update(
+ submissionId,
+ {
+ updatedBy: username,
+ status: 'VALIDATING',
+ },
+ tx,
+ );
- // Parse file data — each file is isolated; a failure on one does not block others.
- const { data: filesDataProcessed, fileResults: parsed } = await submissionInsertDataFromFiles(fileSchemaMap);
- fileResults = parsed;
- logFileResults(fileResults);
+ // Parse file data — each file is isolated; a failure on one does not block others.
+ const parsingFileDataResult = await submissionInsertDataFromFiles(fileSchemaMap);
- // Merge Active Submission data with incoming TSV file data processed
- const insertActiveSubmissionData = mergeInsertsRecords(activeSubmission.data.inserts ?? {}, filesDataProcessed);
+ for (const fileProcessed of parsingFileDataResult) {
+ logFileResult(fileProcessed.fileResult);
+ const {
+ data,
+ fileResult: { entityName, fileName, fileSize, status },
+ } = fileProcessed;
+ fileResult.push(fileProcessed.fileResult);
- // Updating the Submission with the new data and 'VALIDATING' status before validation starts
- await submissionRepository.update(activeSubmission.id, {
- data: {
- inserts: insertActiveSubmissionData,
- deletes: activeSubmission.data.deletes,
- updates: activeSubmission.data.updates,
- },
- updatedBy: username,
- status: 'VALIDATING',
+ if (status === 'ok') {
+ const fileId = await submissionFilesRepository.save(
+ {
+ entityName,
+ fileName,
+ fileSize,
+ submissionId,
+ },
+ tx,
+ );
+
+ await submissionRecordsRepository.saveManyForFile(
+ fileId,
+ data.map((record) => ({
+ actionType: 'INSERT',
+ data: record,
+ state: 'RECEIVED',
+ })),
+ tx,
+ );
+ }
+ }
});
// Perform Schema Data validation in a worker thread
- dependencies.workerPool.dataValidation({ submissionId: activeSubmission.id });
+ dependencies.workerPool.dataValidation({ submissionId });
} catch (error) {
logger.error(LOG_MODULE, `Error processing submitted files`, {
- files: fileSummaries,
error: error instanceof Error ? error.message : String(error),
errorType: error instanceof Error ? error.name : 'unknown',
});
}
logger.info(
LOG_MODULE,
- `Finished addFilesToSubmissionAsync for active submission in category "${params.categoryId}" for organization "${params.organization}" submitted by user "${params.username}"`,
+ `Finished addFilesToSubmissionAsync for active submission with ID "${submissionId}" submitted by user "${username}"`,
);
- return fileResults;
+ return fileResult;
};
return {
diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts
index d453b0d1..8986e987 100644
--- a/packages/data-provider/src/services/submission/submissionService.ts
+++ b/packages/data-provider/src/services/submission/submissionService.ts
@@ -1,22 +1,25 @@
import * as _ from 'lodash-es';
import { Dictionary as SchemasDictionary } from '@overture-stack/lectern-client';
-import { type NewSubmission, type SubmissionRecordErrorDetails } from '@overture-stack/lyric-data-model/models';
+import { type NewSubmission } from '@overture-stack/lyric-data-model/models';
import { BaseDependencies } from '../../config/config.js';
import createSubmissionRepository from '../../repository/activeSubmissionRepository.js';
import createCategoryRepository from '../../repository/categoryRepository.js';
+import createDictionaryRepository from '../../repository/dictionaryRepository.js';
+import createSubmissionFilesRepository from '../../repository/submissionFilesRepository.js';
+import createSubmissionRecordsRepository, {
+ type SubmissionRecordWithEntityName,
+} from '../../repository/submissionRecordsRepository.js';
import { getSchemaByName } from '../../utils/dictionaryUtils.js';
import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js';
import type { PaginatedResult } from '../../utils/result.js';
import type { FilenameEntityPair } from '../../utils/schemas.js';
-import { filterAndPaginateSubmissionData, type FlattenedSubmissionData } from '../../utils/submissionResponseParser.js';
+import { buildDataSummary, createSubmissionSummaryResponse } from '../../utils/submissionResponseParser.js';
import {
checkEntityFieldNames,
- createSubmissionSummaryResponse,
type FileParseResult,
isSubmissionActive,
- removeItemsFromSubmission,
resolveFileEntities,
} from '../../utils/submissionUtils.js';
import {
@@ -25,10 +28,9 @@ import {
type DeleteSubmissionResult,
type EntityData,
type PaginationOptions,
- SUBMISSION_ACTION_TYPE,
SUBMISSION_STATUS,
- type SubmissionActionType,
- SubmissionSummary,
+ type SubmissionRecordActionType,
+ type SubmissionSummaryResponse,
type SubmitDataResult,
type SubmitFileResult,
} from '../../utils/types.js';
@@ -43,6 +45,9 @@ const submissionService = (dependencies: BaseDependencies) => {
const categoryRepository = createCategoryRepository(dependencies);
const submissionProcessor = submissionProcessorFactory.create(dependencies);
const submissionRepository = createSubmissionRepository(dependencies);
+ const submissionRecordsRepository = createSubmissionRecordsRepository(dependencies);
+ const dictionaryRepository = createDictionaryRepository(dependencies);
+ const submissionFilesRepository = createSubmissionFilesRepository(dependencies);
/**
* Runs Schema validation asynchronously in a worker thread and moves the Active Submission to Submitted Data
@@ -84,11 +89,8 @@ const submissionService = (dependencies: BaseDependencies) => {
await submissionRepository.update(submissionId, { status: SUBMISSION_STATUS.COMMITTING, updatedBy: username });
// Get entities to process
- const entitiesToProcess = new Set([
- ...Object.keys(submission.data?.inserts ?? {}),
- ...Object.keys(submission.data?.updates ?? {}),
- ...Object.keys(submission.data?.deletes ?? {}),
- ]);
+ const filesOnSubmission = await submissionFilesRepository.getBySubmissionId(submissionId);
+ const entitiesToProcess = new Set(filesOnSubmission.map((file) => file.entityName));
// Execute commit submission in worker pool
const commitData: CommitWorkerInput = {
@@ -147,24 +149,28 @@ const submissionService = (dependencies: BaseDependencies) => {
};
/**
- * Function to remove an entity from an Active Submission by given Submission ID
+ * Function to remove specific records from an Active Submission
+ * If fileID is provided, all records associated with that file will be removed from the Submission
+ * If recordID is provided, only that specific record will be removed from the Submission
+ * If both fileID and recordID are provided, only the recordID will be removed from the Submission
+ * If neither fileID nor recordID are provided, an error will be thrown
+ * The function will check if the Submission is Active and if the record or file belongs to the Submission
* It validates resulting Active Submission running cross schema validation along with the existing Submitted Data
* Returns the resulting ID of the Active Submission
- * @param {number} submissionId
- * @param {string} entityName
- * @param {string} username
+ * @param {number} submissionId - Submission ID
+ * @param {string} username - User name performing the action
+ * @param {object} filter - Filter to identify the entity to be removed
* @returns { Promise}
*/
- const deleteActiveSubmissionEntity = async (
+ const deleteByRecordIdOrFileId = async (
submissionId: number,
username: string,
filter: {
- actionType: SubmissionActionType;
- entityName: string;
- index: number | null;
+ recordId: number | null;
+ fileId: number | null;
},
): Promise => {
- const submission = await submissionRepository.getSubmissionDetailsById(submissionId);
+ const submission = await submissionRepository.getSubmissionById(submissionId);
if (!submission) {
throw new BadRequest(`Submission '${submissionId}' not found`);
}
@@ -173,35 +179,43 @@ const submissionService = (dependencies: BaseDependencies) => {
throw new StatusConflict('Submission is not active. Only Active Submission can be modified');
}
- if (
- SUBMISSION_ACTION_TYPE.Values.INSERTS.includes(filter.actionType) &&
- !_.has(submission.data.inserts, filter.entityName)
- ) {
- throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
- }
+ const filesOnSubmission = await submissionFilesRepository.getBySubmissionId(submissionId);
- if (
- SUBMISSION_ACTION_TYPE.Values.UPDATES.includes(filter.actionType) &&
- !_.has(submission.data.updates, filter.entityName)
- ) {
- throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
+ if (filesOnSubmission.length === 0) {
+ throw new BadRequest(`Submission '${submissionId}' has no records or files to delete`);
}
- if (
- SUBMISSION_ACTION_TYPE.Values.DELETES.includes(filter.actionType) &&
- !_.has(submission.data.deletes, filter.entityName)
- ) {
- throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
- }
+ // Remove record by ID from the Submission
+ if (filter.recordId) {
+ const recordFoundInDB = await submissionRecordsRepository.getById(filter.recordId);
+ if (!recordFoundInDB) {
+ throw new BadRequest(`Record with ID '${filter.recordId}' not found in Submission '${submissionId}'`);
+ }
- // Remove entity from the Submission
- const updatedActiveSubmissionData = removeItemsFromSubmission(submission.data, {
- ...filter,
- });
+ const fileReference = filesOnSubmission.find((file) => file.id === recordFoundInDB.fileId);
+ if (fileReference?.submissionId !== submissionId) {
+ throw new BadRequest(`Record with ID '${filter.recordId}' does not belong to Submission '${submissionId}'`);
+ }
+
+ await submissionRecordsRepository.deleteByIds([filter.recordId]);
+ } else if (filter.fileId != null) {
+ const fileId = filter.fileId;
+ // Verify the requested FileId belongs to the Submission before deleting
+ const fileReference = filesOnSubmission.find((f) => f.id === fileId);
+ if (!fileReference) {
+ throw new BadRequest(`File with ID '${fileId}' not found in Submission '${submissionId}'`);
+ }
+
+ await dependencies.db.transaction(async (tx) => {
+ await submissionRecordsRepository.deleteByFileIds([fileId], tx);
+ await submissionFilesRepository.deleteById(fileId, tx);
+ });
+ } else {
+ throw new BadRequest('Either recordId or fileId must be provided to delete a record or file from the Submission');
+ }
// Updating the Submission with the new data and 'VALIDATING' status before validation starts
await submissionRepository.update(submission.id, {
- data: updatedActiveSubmissionData,
updatedBy: username,
status: 'VALIDATING',
});
@@ -209,7 +223,10 @@ const submissionService = (dependencies: BaseDependencies) => {
// Perform Schema Data validation in a worker thread
dependencies.workerPool.dataValidation({ submissionId: submission.id });
- logger.info(LOG_MODULE, `Submission '${submission.id}' updated after removing entity '${filter.entityName}'`);
+ logger.info(
+ LOG_MODULE,
+ `Submission '${submission.id}' updated after removing entity with recordId '${filter.recordId}' and fileId '${filter.fileId}'`,
+ );
return {
status: ACTIVE_SUBMISSION_STATUS.PROCESSING,
@@ -238,7 +255,7 @@ const submissionService = (dependencies: BaseDependencies) => {
username?: string;
organization?: string;
},
- ): Promise> => {
+ ): Promise> => {
const recordsPaginated = await submissionRepository.getSubmissionsByCategory(
categoryId,
paginationOptions,
@@ -253,17 +270,30 @@ const submissionService = (dependencies: BaseDependencies) => {
};
}
- const totalRecords = await submissionRepository.getTotalSubmissionsByCategory(categoryId, filterOptions);
+ const totalSubmissions = await submissionRepository.getTotalSubmissionsByCategory(categoryId, filterOptions);
+ const result = await Promise.all(
+ recordsPaginated.map(async (response) => {
+ const submissionRecordsSummary = await submissionRecordsRepository.getRecordsSummaryBySubmissionId(response.id);
+ const formattedDataSummary = buildDataSummary(submissionRecordsSummary);
+
+ return createSubmissionSummaryResponse({
+ ...response,
+ data: formattedDataSummary,
+ });
+ }),
+ );
+
return {
metadata: {
- totalRecords,
+ totalRecords: totalSubmissions,
},
- result: recordsPaginated.map((response) => createSubmissionSummaryResponse(response)),
+ result,
};
};
/**
* Get Submission by Submission ID
+ * Returns the submission general information and includes the summary of the data and errors
* @param {number} submissionId A Submission ID
* @returns One Submission
*/
@@ -273,7 +303,13 @@ const submissionService = (dependencies: BaseDependencies) => {
return;
}
- return createSubmissionSummaryResponse(submission);
+ const submissionDataSummary = await submissionRecordsRepository.getRecordsSummaryBySubmissionId(submissionId);
+ const formattedDataSummary = buildDataSummary(submissionDataSummary);
+
+ return createSubmissionSummaryResponse({
+ ...submission,
+ data: formattedDataSummary,
+ });
};
/**
@@ -294,20 +330,31 @@ const submissionService = (dependencies: BaseDependencies) => {
}: {
submissionId: number;
paginationOptions: PaginationOptions;
- filterOptions: { entityNames: string[]; actionTypes: SubmissionActionType[] };
- }): Promise<{ data: FlattenedSubmissionData[]; errors?: SubmissionRecordErrorDetails[] }> => {
- const submission = await submissionRepository.getSubmissionDetailsById(submissionId);
+ filterOptions: { entityNames: string[]; actionTypes: SubmissionRecordActionType[]; fileId?: number };
+ }): Promise => {
+ const submission = await submissionRepository.getSubmissionById(submissionId);
if (!submission) {
throw new BadRequest(`Submission '${submissionId}' not found`);
}
- const submissionEntityNames = [
- ...Object.keys(submission.data.inserts ?? {}),
- ...Object.keys(submission.data.updates ?? {}),
- ...Object.keys(submission.data.deletes ?? {}),
- ];
+ const dictionary = await dictionaryRepository.getDictionary(
+ submission.dictionary.name,
+ submission.dictionary.version,
+ );
+
+ if (!dictionary) {
+ throw new InternalServerError(
+ `Dictionary '${submission.dictionary.name}' version '${submission.dictionary.version}' not found`,
+ );
+ }
+
+ const schemasDictionary: SchemasDictionary = {
+ name: dictionary.name,
+ version: dictionary.version,
+ schemas: dictionary.dictionary,
+ };
- const missingEntityNames = filterOptions.entityNames.filter((name) => !submissionEntityNames.includes(name));
+ const missingEntityNames = filterOptions.entityNames.filter((name) => !getSchemaByName(name, schemasDictionary));
if (filterOptions.entityNames.length > 0 && missingEntityNames.length > 0) {
throw new BadRequest(
@@ -315,12 +362,13 @@ const submissionService = (dependencies: BaseDependencies) => {
);
}
- return filterAndPaginateSubmissionData({
- data: submission.data,
- errors: submission.errors || {},
- filterOptions,
+ const submissionRecords = await submissionRecordsRepository.getBySubmissionId(
+ submissionId,
paginationOptions,
- });
+ filterOptions,
+ );
+
+ return submissionRecords;
};
/**
@@ -339,8 +387,8 @@ const submissionService = (dependencies: BaseDependencies) => {
categoryId: number;
username: string;
organization: string;
- }): Promise => {
- const submission = await submissionRepository.getActiveSubmissionSummary({
+ }): Promise => {
+ const submission = await submissionRepository.getActiveSubmission({
organization,
username,
categoryId,
@@ -349,7 +397,13 @@ const submissionService = (dependencies: BaseDependencies) => {
return;
}
- return createSubmissionSummaryResponse(submission);
+ const submissionDataSummary = await submissionRecordsRepository.getRecordsSummaryBySubmissionId(submission.id);
+ const formattedDataSummary = buildDataSummary(submissionDataSummary);
+
+ return createSubmissionSummaryResponse({
+ ...submission,
+ data: formattedDataSummary,
+ });
};
/**
@@ -369,7 +423,7 @@ const submissionService = (dependencies: BaseDependencies) => {
const { categoryId, username, organization } = params;
const { getActiveDictionaryByCategory } = categoryRepository;
- const activeSubmission = await submissionRepository.getActiveSubmissionSummary({
+ const activeSubmission = await submissionRepository.getActiveSubmission({
categoryId,
username,
organization,
@@ -390,10 +444,8 @@ const submissionService = (dependencies: BaseDependencies) => {
const newSubmissionInput: NewSubmission = {
createdBy: username,
- data: {},
dictionaryCategoryId: categoryId,
dictionaryId: currentDictionary.id,
- errors: {},
organization: organization,
status: SUBMISSION_STATUS.OPEN,
};
@@ -587,11 +639,7 @@ const submissionService = (dependencies: BaseDependencies) => {
// Parsing always starts immediately. When sync=true (default) the response waits for results;
// when sync=false it runs in the background and fileResults will be empty in the response.
// Schema validation always runs in a background worker thread regardless of this flag.
- const parsePromise = submissionProcessor.addFilesToSubmissionAsync(checkedEntities, {
- categoryId,
- organization,
- username,
- });
+ const parsePromise = submissionProcessor.addFilesToSubmissionAsync(checkedEntities, activeSubmissionId, username);
const fileResults: FileParseResult[] = sync ? await parsePromise : [];
if (batchErrors.length === 0) {
@@ -618,7 +666,7 @@ const submissionService = (dependencies: BaseDependencies) => {
return {
commitSubmission,
deleteActiveSubmissionById,
- deleteActiveSubmissionEntity,
+ deleteByRecordIdOrFileId,
getSubmissionsByCategory,
getSubmissionById,
getSubmissionDetailsById,
diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts
index fa95240a..c2bcadd8 100644
--- a/packages/data-provider/src/services/submittedData/submmittedData.ts
+++ b/packages/data-provider/src/services/submittedData/submmittedData.ts
@@ -6,12 +6,15 @@ import { SQON } from '@overture-stack/sqon-builder';
import { BaseDependencies } from '../../config/config.js';
import submissionRepository from '../../repository/activeSubmissionRepository.js';
import categoryRepository from '../../repository/categoryRepository.js';
+import createSubmissionFilesRepository from '../../repository/submissionFilesRepository.js';
+import createSubmissionRecordsRepository from '../../repository/submissionRecordsRepository.js';
import submittedRepository from '../../repository/submittedRepository.js';
import { convertSqonToQuery } from '../../utils/convertSqonToQuery.js';
import { getDictionarySchemaRelations } from '../../utils/dictionarySchemaRelations.js';
import { InternalServerError, StatusConflict } from '../../utils/errors.js';
+import { genericSubmissionFileName, getSizeInBytes } from '../../utils/fileUtils.js';
import type { PaginatedResult } from '../../utils/result.js';
-import { filterUpdatesFromDeletes, mergeDeleteRecords } from '../../utils/submissionUtils.js';
+import { resolveDeleteStagingConflicts } from '../../utils/submissionUtils.js';
import {
fetchDataErrorResponse,
getEntityNamesFromFilterOptions,
@@ -39,6 +42,8 @@ const submittedData = (dependencies: BaseDependencies) => {
const LOG_MODULE = 'SUBMITTED_DATA_SERVICE';
const submittedDataRepo = submittedRepository(dependencies);
const submissionProcessor = submissionProcessorFactory.create(dependencies);
+ const submissionRecordsRepository = createSubmissionRecordsRepository(dependencies);
+ const submissionFilesRepository = createSubmissionFilesRepository(dependencies);
const { logger } = dependencies;
const { convertRecordsToCompoundDocuments } = viewMode(dependencies);
@@ -56,7 +61,7 @@ const submittedData = (dependencies: BaseDependencies) => {
}> => {
const { getSubmittedDataBySystemId } = submittedDataRepo;
const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
- const { getSubmissionDetailsById, update } = submissionRepository(dependencies);
+ const { update: udpateSubmission } = submissionRepository(dependencies);
const { getOrCreateActiveSubmission } = submissionService(dependencies);
// get SubmittedData by SystemId
@@ -125,44 +130,96 @@ const submittedData = (dependencies: BaseDependencies) => {
throw error;
}
- const activeSubmission = await getSubmissionDetailsById(activeSubmissionId);
+ // Check what the Active Submission already has pending for these systemIds before staging
+ // anything new: a pending UPDATE is a conflict (reject, consistent with how the same
+ // conflict is handled at validation time), a pending DELETE is a duplicate (skip it).
+ const existingSubmissionRecords = await submissionRecordsRepository.getBySubmissionId(
+ activeSubmissionId,
+ undefined,
+ {
+ actionTypes: ['UPDATE', 'DELETE'],
+ },
+ );
+
+ const { filteredRecordsToDeleteMap, conflictingSystemIds, duplicateSystemIds } = resolveDeleteStagingConflicts(
+ recordsToDeleteMap,
+ existingSubmissionRecords,
+ );
- if (!activeSubmission) {
+ if (conflictingSystemIds.length > 0) {
+ logger.error(
+ LOG_MODULE,
+ `Cannot delete system ID(s) '${conflictingSystemIds.join(', ')}' on Submission '${activeSubmissionId}': a pending update already exists for the same system ID`,
+ );
return {
status: ACTIVE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: 'Active Submission not found',
+ description: `System ID(s) '${conflictingSystemIds.join(', ')}' already have a pending update staged on Submission '${activeSubmissionId}'. Resolve the conflicting update before deleting.`,
inProcessEntities: [],
};
}
- // Merge current Active Submission delete entities with unique records to delete based on systemId
- const mergedSubmissionDeletes = mergeDeleteRecords(activeSubmission.data.deletes || {}, recordsToDeleteMap);
+ if (duplicateSystemIds.length > 0) {
+ logger.info(
+ LOG_MODULE,
+ `System ID(s) '${duplicateSystemIds.join(', ')}' are already staged for deletion on Submission '${activeSubmissionId}', skipping duplicate`,
+ );
+ }
+
+ const entitiesToProcess = Object.keys(filteredRecordsToDeleteMap);
- const entitiesToProcess = Object.keys(mergedSubmissionDeletes);
+ if (entitiesToProcess.length === 0) {
+ return {
+ status: ACTIVE_SUBMISSION_STATUS.PROCESSING,
+ description: 'All requested records are already staged for deletion on the Active Submission',
+ submissionId: activeSubmissionId.toString(),
+ inProcessEntities: [],
+ };
+ }
- // filter out update records found matching systemID on delete records
- const filteredUpdates = filterUpdatesFromDeletes(activeSubmission.data.updates ?? {}, mergedSubmissionDeletes);
+ await dependencies.db.transaction(async (tx) => {
+ // Updating the Submission with the new data and 'VALIDATING' status before validation starts
+ await udpateSubmission(
+ activeSubmissionId,
+ {
+ updatedBy: username,
+ status: 'VALIDATING',
+ },
+ tx,
+ );
- // Updating the Submission with the new data and 'VALIDATING' status before validation starts
- await update(activeSubmission.id, {
- data: {
- inserts: activeSubmission.data.inserts,
- updates: filteredUpdates,
- deletes: mergedSubmissionDeletes,
- },
- updatedBy: username,
- status: 'VALIDATING',
+ await Promise.all(
+ Object.entries(filteredRecordsToDeleteMap).map(async ([entityName, entityRecords]) => {
+ const savedFileId = await submissionFilesRepository.save(
+ {
+ entityName,
+ fileName: genericSubmissionFileName(),
+ fileSize: getSizeInBytes(JSON.stringify(entityRecords)),
+ submissionId: activeSubmissionId,
+ },
+ tx,
+ );
+ await submissionRecordsRepository.saveManyForFile(
+ savedFileId,
+ entityRecords.map((record) => ({
+ actionType: 'DELETE',
+ data: record,
+ state: 'RECEIVED',
+ })),
+ tx,
+ );
+ }),
+ );
});
// Perform Schema Data validation in a worker thread
- dependencies.workerPool.dataValidation({ submissionId: activeSubmission.id });
+ dependencies.workerPool.dataValidation({ submissionId: activeSubmissionId });
logger.info(LOG_MODULE, `Added '${entitiesToProcess.length}' records to be deleted on the Active Submission`);
return {
status: ACTIVE_SUBMISSION_STATUS.PROCESSING,
description: 'Submission data is being processed',
- submissionId: activeSubmission.id.toString(),
+ submissionId: activeSubmissionId.toString(),
inProcessEntities: entitiesToProcess,
};
};
diff --git a/packages/data-provider/src/utils/auditUtils.ts b/packages/data-provider/src/utils/auditUtils.ts
index aaad3324..c69f50d4 100644
--- a/packages/data-provider/src/utils/auditUtils.ts
+++ b/packages/data-provider/src/utils/auditUtils.ts
@@ -5,7 +5,7 @@ import {
AuditAction,
AuditDataResponse,
AuditRepositoryRecord,
- SUBMISSION_ACTION_TYPE,
+ SUBMISSION_RECORD_ACTION_TYPE,
} from './types.js';
/**
@@ -18,7 +18,7 @@ export const isAuditEventValid = (value: unknown): boolean =>
typeof value === 'string' && AUDIT_ACTION.safeParse(value.toUpperCase()).success;
export const isSubmissionActionTypeValid = (value: unknown): boolean =>
- typeof value === 'string' && SUBMISSION_ACTION_TYPE.safeParse(value.toUpperCase()).success;
+ typeof value === 'string' && SUBMISSION_RECORD_ACTION_TYPE.safeParse(value.toUpperCase()).success;
/**
* Convert a value string into it's Audit event type if it matches.
diff --git a/packages/data-provider/src/utils/fileUtils.ts b/packages/data-provider/src/utils/fileUtils.ts
index 7998d170..9e46826f 100644
--- a/packages/data-provider/src/utils/fileUtils.ts
+++ b/packages/data-provider/src/utils/fileUtils.ts
@@ -13,7 +13,7 @@ import {
} from '@overture-stack/lectern-client';
import { getSubmittedFileType } from '../services/submission/submissionFile.js';
-import { failure, success, type Result } from './result.js';
+import { failure, type Result, success } from './result.js';
import { BATCH_ERROR_TYPE, type BatchError } from './types.js';
export const SUPPORTED_FILE_EXTENSIONS = z.enum(['tsv', 'csv']);
@@ -125,6 +125,13 @@ function formatForExcelCompatibility(data: string) {
.trim();
}
+/**
+ * Generates a generic file name for submission files with `.json` extension.
+ * Based on the current date and time in ISO format.
+ * @returns
+ */
+export const genericSubmissionFileName = () => `submission-${new Date().toISOString()}.json`;
+
export function getSizeInBytes(size: string | number): number {
// Parse the string value into an integer in bytes.
// If value is a number it is assumed is in bytes.
diff --git a/packages/data-provider/src/utils/recordsParser.ts b/packages/data-provider/src/utils/recordsParser.ts
index 295fed97..5317cb2d 100644
--- a/packages/data-provider/src/utils/recordsParser.ts
+++ b/packages/data-provider/src/utils/recordsParser.ts
@@ -1,9 +1,7 @@
import { type DataRecord, parse, type Schema } from '@overture-stack/lectern-client';
-import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models';
import { getSchemaByName } from './dictionaryUtils.js';
import { convertRecordToString, notEmpty } from './formatUtils.js';
-import { createBatchResponse } from './submissionResponseParser.js';
import type { EntityData, SchemasDictionary } from './types.js';
/**
@@ -37,7 +35,7 @@ export const convertToTypedRecords = (dataRecords: Record[], sc
export const parseRecordsToInsert = (
records: EntityData,
schemasDictionary: SchemasDictionary,
-): Record => {
+): Record => {
return Object.fromEntries(
Object.entries(records)
.map(([schemaName, dataRecords]) => {
@@ -53,7 +51,7 @@ export const parseRecordsToInsert = (
return null;
}
- return [schemaName, createBatchResponse(schemaName, parsedRecords)];
+ return [schemaName, parsedRecords];
})
.filter(notEmpty),
);
diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts
index 3a7c2b02..66edfb49 100644
--- a/packages/data-provider/src/utils/schemas.ts
+++ b/packages/data-provider/src/utils/schemas.ts
@@ -35,7 +35,10 @@ const categoryIdSchema = zod
const categoryAliasSchema = zod
.string()
.trim()
- .refine((value) => value === '' || isValidCategoryAlias(value), 'alias must contain only letters, numbers, hyphens, and underscores');
+ .refine(
+ (value) => value === '' || isValidCategoryAlias(value),
+ 'alias must contain only letters, numbers, hyphens, and underscores',
+ );
const endDateSchema = zod
.string()
@@ -67,17 +70,6 @@ const pageSizeSchema = zod.string().superRefine((value, ctx) => {
}
});
-const indexIntegerSchema = zod.string().superRefine((value, ctx) => {
- const parsed = parseInt(value);
- if (isNaN(parsed)) {
- ctx.addIssue({
- code: zod.ZodIssueCode.invalid_type,
- expected: 'number',
- received: 'nan',
- });
- }
-});
-
const positiveInteger = zod.string().superRefine((value, ctx) => {
const parsed = parseInt(value);
if (isNaN(parsed)) {
@@ -342,6 +334,7 @@ export const submissionByIdRequestSchema: RequestValidation = {
query: zod.object({
- entityName: entityNameSchema,
- index: indexIntegerSchema.optional(),
+ recordId: positiveInteger.optional(),
+ fileId: positiveInteger.optional(),
}),
pathParams: zod.object({
- actionType: submissionActionTypeSchema,
submissionId: submissionIdSchema,
}),
};
diff --git a/packages/data-provider/src/utils/submissionResponseParser.ts b/packages/data-provider/src/utils/submissionResponseParser.ts
index dcc54958..0129958c 100644
--- a/packages/data-provider/src/utils/submissionResponseParser.ts
+++ b/packages/data-provider/src/utils/submissionResponseParser.ts
@@ -1,163 +1,78 @@
-import type { DataRecord } from '@overture-stack/lectern-client';
-import type {
- SubmissionData,
- SubmissionDeleteData,
- SubmissionErrors,
- SubmissionInsertData,
- SubmissionRecordErrorDetails,
- SubmissionUpdateData,
-} from '@overture-stack/lyric-data-model/models';
-
-import { type PaginationOptions, type SubmissionActionType } from './types.js';
-
-export const createBatchResponse = (schemaName: string, records: DataRecord[]): SubmissionInsertData => {
- return { batchName: schemaName, records };
-};
-
-export type FlattenedSubmissionData =
- | { type: 'INSERTS'; entity: string; value: DataRecord; index: number }
- | { type: 'UPDATES'; entity: string; value: SubmissionUpdateData; index: number }
- | { type: 'DELETES'; entity: string; value: SubmissionDeleteData; index: number };
-
-/**
- * Filters and paginates submission `data` and `errors` based on specified action types and entity names.
- * Returns the paginated data along with the corresponding filtered errors.
- */
-export const filterAndPaginateSubmissionData = ({
- data,
- errors,
- filterOptions,
- paginationOptions,
-}: {
- data: SubmissionData;
- errors: SubmissionErrors;
- filterOptions: {
- actionTypes: SubmissionActionType[];
- entityNames: string[];
- };
- paginationOptions: PaginationOptions;
-}): { data: FlattenedSubmissionData[]; errors: SubmissionRecordErrorDetails[] } => {
- const { page, pageSize } = paginationOptions;
- const { actionTypes, entityNames } = filterOptions;
-
- const flattenedRecords = flattenData(data, actionTypes, entityNames);
-
- const startIndex = (page - 1) * pageSize;
- const paginatedRecords = flattenedRecords.slice(startIndex, startIndex + pageSize);
-
- // Extract indexes that belongs to paginated records
- const paginatedRecordIndexes = paginatedRecords.map((record) => record.index);
- const relevantErrors = getFilteredErrors({
- errors: errors || {},
- actionTypes,
- entityNames,
- indices: paginatedRecordIndexes,
- });
-
- return { data: paginatedRecords, errors: relevantErrors };
-};
-
-/**
- * Flattens submission data into a list of records based on specified action types and entity names.
- */
-const flattenData = (
- data: SubmissionData,
- actionTypes: SubmissionActionType[],
- entityNames: string[],
-): FlattenedSubmissionData[] => {
- const list: FlattenedSubmissionData[] = [];
-
- for (const action of actionTypes) {
- const bucket = getActionData(data, action);
-
- if (!bucket) {
- continue;
- }
-
- for (const [entity, value] of Object.entries(bucket)) {
- if ((entityNames.length > 0 && !entityNames.includes(entity)) || !value) {
- continue;
+import * as _ from 'lodash-es';
+
+import type { RecordsSummaryRepository } from '../repository/submissionRecordsRepository.js';
+import {
+ type DataDeletesSubmissionSummary,
+ type DataInsertsSubmissionSummary,
+ type DataUpdatesSubmissionSummary,
+ type SubmissionDataSummaryWithTotal,
+ type SubmissionSummary,
+ type SubmissionSummaryResponse,
+} from './types.js';
+
+// This function accepts a raw array of submission records from the database and builds a summary response.
+export const buildDataSummary = (rows: RecordsSummaryRepository[]): SubmissionDataSummaryWithTotal => {
+ const inserts: Record = {};
+ const updates: Record = {};
+ const deletes: Record = {};
+ let totalRecords = 0;
+ let errors = 0;
+
+ for (const row of rows) {
+ const { actionType, entityName, totalRecords: rowTotalRecords, batchName, errors: rowErrors } = row;
+ const summaryItem = { batchName: batchName ?? '', recordsCount: rowTotalRecords, errors: rowErrors };
+
+ totalRecords += rowTotalRecords;
+ errors += rowErrors;
+
+ switch (actionType) {
+ case 'INSERT': {
+ const entitySummaries = inserts[entityName] ?? [];
+ entitySummaries.push(summaryItem);
+ inserts[entityName] = entitySummaries;
+ break;
}
-
- if (action === 'INSERTS') {
- for (const [index, record] of value.records.entries()) {
- list.push({ type: 'INSERTS', entity, value: record, index });
- }
- continue;
+ case 'UPDATE': {
+ const entitySummaries = updates[entityName] ?? [];
+ entitySummaries.push(summaryItem);
+ updates[entityName] = entitySummaries;
+ break;
}
-
- for (const [index, record] of value.entries()) {
- list.push({ type: action, entity, value: record, index });
+ case 'DELETE': {
+ const entitySummary = deletes[entityName] ?? { recordsCount: 0, errors: 0 };
+ entitySummary.recordsCount += rowTotalRecords;
+ entitySummary.errors += rowErrors;
+ deletes[entityName] = entitySummary;
+ break;
}
}
}
- return list;
-};
-
-/**
- * Retrieves the set of submission data corresponding to a specific action type.
- */
-export const getActionData = (data: SubmissionData, actionType: SubmissionActionType) => {
- switch (actionType) {
- case 'INSERTS':
- return data.inserts ?? {};
-
- case 'UPDATES':
- return data.updates ?? {};
-
- case 'DELETES':
- return data.deletes ?? {};
- }
-};
-
-/**
- * Retrieves the set of submission errors corresponding to a specific action type.
- */
-export const getActionErrors = (errors: SubmissionErrors, actionType: SubmissionActionType) => {
- switch (actionType) {
- case 'INSERTS':
- return errors.inserts ?? {};
-
- case 'UPDATES':
- return errors.updates ?? {};
-
- case 'DELETES':
- return errors.deletes ?? {};
- }
+ return {
+ inserts,
+ updates,
+ deletes,
+ totalRecords,
+ errors,
+ };
};
/**
- * Filters submission errors based on specified action types, entity names, and record indices.
+ * Utility to convert a raw Submission record to a Response type
+ * @param {SubmissionSummary} submission
+ * @returns {SubmissionSummaryResponse}
*/
-export const getFilteredErrors = ({
- errors,
- actionTypes,
- entityNames,
- indices,
-}: {
- errors: SubmissionErrors;
- actionTypes: SubmissionActionType[];
- entityNames: string[];
- indices: number[];
-}): SubmissionRecordErrorDetails[] => {
- const allErrors: SubmissionRecordErrorDetails[] = [];
-
- for (const actionType of actionTypes) {
- const bucket = getActionErrors(errors, actionType);
-
- if (bucket) {
- for (const [entityName, records] of Object.entries(bucket)) {
- if (entityNames.length > 0 && !entityNames.includes(entityName)) {
- continue;
- }
-
- for (const record of records) {
- allErrors.push(record);
- }
- }
- }
- }
-
- return allErrors.filter((err) => indices.includes(err.index));
+export const createSubmissionSummaryResponse = (submission: SubmissionSummary): SubmissionSummaryResponse => {
+ return {
+ id: submission.id,
+ data: submission.data,
+ dictionary: submission.dictionary,
+ dictionaryCategory: submission.dictionaryCategory,
+ organization: submission.organization,
+ status: submission.status,
+ createdAt: _.toString(submission.createdAt?.toISOString()),
+ createdBy: _.toString(submission.createdBy),
+ updatedAt: _.toString(submission.updatedAt?.toISOString()),
+ updatedBy: _.toString(submission.updatedBy),
+ };
};
diff --git a/packages/data-provider/src/utils/submissionUtils.spec.ts b/packages/data-provider/src/utils/submissionUtils.spec.ts
index 72c70ade..127decb1 100644
--- a/packages/data-provider/src/utils/submissionUtils.spec.ts
+++ b/packages/data-provider/src/utils/submissionUtils.spec.ts
@@ -1,9 +1,9 @@
import { expect } from 'chai';
import { writeFileSync } from 'fs';
import { describe, it } from 'mocha';
+import { tmpdir } from 'os';
import { join } from 'path';
import { Readable } from 'stream';
-import { tmpdir } from 'os';
import { type Schema } from '@overture-stack/lectern-client';
@@ -65,11 +65,11 @@ describe('submissionInsertDataFromFiles', () => {
items: { files: [makeFile(path, 'items.tsv')], schema: minimalSchema('items') },
};
- const { fileResults } = await submissionInsertDataFromFiles(fileSchemaMap);
+ const fileResults = await submissionInsertDataFromFiles(fileSchemaMap);
expect(fileResults).to.have.length(1);
- expect(fileResults[0]?.status).to.equal('ok');
- expect(fileResults[0]?.fileName).to.equal('items.tsv');
+ expect(fileResults[0]?.fileResult.status).to.equal('ok');
+ expect(fileResults[0]?.fileResult.fileName).to.equal('items.tsv');
});
it('returns status invalid for a file with schema validation failures', async () => {
@@ -78,13 +78,13 @@ describe('submissionInsertDataFromFiles', () => {
items: { files: [makeFile(path, 'items.tsv')], schema: integerSchema('items') },
};
- const { fileResults } = await submissionInsertDataFromFiles(fileSchemaMap);
+ const fileResults = await submissionInsertDataFromFiles(fileSchemaMap);
expect(fileResults).to.have.length(1);
const result = fileResults[0];
- expect(result?.status).to.equal('invalid');
- if (result?.status === 'invalid') {
- expect(result.parseErrors).to.have.length.greaterThan(0);
+ expect(result?.fileResult.status).to.equal('invalid');
+ if (result?.fileResult.status === 'invalid') {
+ expect(result.fileResult.parseErrors).to.have.length.greaterThan(0);
}
});
@@ -93,13 +93,13 @@ describe('submissionInsertDataFromFiles', () => {
items: { files: [makeFile('/nonexistent/path/items.tsv', 'items.tsv')], schema: minimalSchema('items') },
};
- const { fileResults } = await submissionInsertDataFromFiles(fileSchemaMap);
+ const fileResults = await submissionInsertDataFromFiles(fileSchemaMap);
expect(fileResults).to.have.length(1);
const result = fileResults[0];
- expect(result?.status).to.equal('error');
- if (result?.status === 'error') {
- expect(result.streamError).to.be.a('string').and.not.be.empty;
+ expect(result?.fileResult.status).to.equal('error');
+ if (result?.fileResult.status === 'error') {
+ expect(result.fileResult.streamError).to.be.a('string').and.not.be.empty;
}
});
@@ -107,19 +107,16 @@ describe('submissionInsertDataFromFiles', () => {
const validPath = writeTsv([['item_id'], ['A']]);
const fileSchemaMap: FileSchemaMap = {
items: {
- files: [
- makeFile('/nonexistent/path/missing.tsv', 'missing.tsv'),
- makeFile(validPath, 'valid.tsv'),
- ],
+ files: [makeFile('/nonexistent/path/missing.tsv', 'missing.tsv'), makeFile(validPath, 'valid.tsv')],
schema: minimalSchema('items'),
},
};
- const { fileResults } = await submissionInsertDataFromFiles(fileSchemaMap);
+ const fileResults = await submissionInsertDataFromFiles(fileSchemaMap);
expect(fileResults).to.have.length(2);
- expect(fileResults.find((r) => r.fileName === 'missing.tsv')?.status).to.equal('error');
- expect(fileResults.find((r) => r.fileName === 'valid.tsv')?.status).to.equal('ok');
+ expect(fileResults.find((r) => r.fileResult.fileName === 'missing.tsv')?.fileResult.status).to.equal('error');
+ expect(fileResults.find((r) => r.fileResult.fileName === 'valid.tsv')?.fileResult.status).to.equal('ok');
});
it('accumulates records from multiple successful files for the same entity', async () => {
@@ -130,10 +127,10 @@ describe('submissionInsertDataFromFiles', () => {
items: { files: [makeFile(pathA, 'a.tsv'), makeFile(pathB, 'b.tsv')], schema },
};
- const { data, fileResults } = await submissionInsertDataFromFiles(fileSchemaMap);
+ const fileResults = await submissionInsertDataFromFiles(fileSchemaMap);
expect(fileResults).to.have.length(2);
- expect(fileResults.every((r) => r.status === 'ok')).to.be.true;
- expect(data['items']?.records).to.have.length(2);
+ expect(fileResults.every((r) => r.fileResult.status === 'ok')).to.be.true;
+ expect(fileResults.flatMap((r) => r.data)).to.have.length(2);
});
});
diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts
index 56d565ce..9c4ac9d5 100644
--- a/packages/data-provider/src/utils/submissionUtils.ts
+++ b/packages/data-provider/src/utils/submissionUtils.ts
@@ -12,20 +12,21 @@ import {
validate,
} from '@overture-stack/lectern-client';
import {
- SubmissionData,
+ type RecordErrorActionConflict,
type SubmissionDeleteData,
- type SubmissionErrors,
type SubmissionInsertData,
+ type SubmissionRecordError,
type SubmissionUpdateData,
type SubmittedData,
} from '@overture-stack/lyric-data-model/models';
+import type { SubmissionRecordWithEntityName } from '../repository/submissionRecordsRepository.js';
import { getSubmittedFileEntity } from '../services/submission/submissionFile.js';
import { isSubmissionActionTypeValid } from './auditUtils.js';
import type { SchemaChildNode } from './dictionarySchemaRelations.js';
import { getSchemaFieldNames } from './dictionaryUtils.js';
import { readHeaders, readTextFile } from './fileUtils.js';
-import { asArray, deepCompare } from './formatUtils.js';
+import { asArray } from './formatUtils.js';
import type { FilenameEntityPair } from './schemas.js';
import { groupErrorsByIndex, mapAndMergeSubmittedDataToRecordReferences } from './submittedDataUtils.js';
import {
@@ -36,16 +37,12 @@ import {
type FileSchemaMap as FileSchemaMap,
MERGE_REFERENCE_TYPE,
type NewSubmittedDataReference,
- SUBMISSION_ACTION_TYPE,
+ SUBMISSION_RECORD_ACTION_TYPE,
SUBMISSION_STATUS,
- type SubmissionActionType,
- type SubmissionDataDetailsRepositoryRecord,
- type SubmissionDataSummary,
- type SubmissionDataSummaryRepositoryRecord,
- type SubmissionDetailsResponse,
- type SubmissionErrorsSummary,
+ type SubmissionInsertRecordWithEntityName,
+ type SubmissionRecordActionType,
type SubmissionStatus,
- type SubmissionSummary,
+ type SubmissionUpdateRecordWithEntityName,
SubmittedDataReference,
} from './types.js';
@@ -223,86 +220,6 @@ export const findInvalidRecordErrorsBySchemaName = (
: [];
};
-/**
- * Generalized function to filter out conflicting records between two data sets based on `systemId`.
- *
- * This function can be used to either filter updates from deletes or deletes from updates, depending on the provided parameters.
- * It removes records from the `sourceData` that have a matching `systemId` in the `conflictData`.
- *
- * @param sourceData - A record of the primary data (e.g., updates or deletes) to be filtered, grouped by entity name.
- * @param conflictData - A record of data that might conflict (e.g., deletes or updates), grouped by entity name.
- * @param entitySelector - A function to select the `systemId` from the source records.
- * @param conflictSelector - A function to select the `systemId` from the conflict records.
- * @returns A record of filtered source data, excluding records that conflict based on `systemId`.
- */
-export const filterRecordsByConflicts = (
- sourceData: Record,
- conflictData: Record,
- entitySelector: (item: SourceData) => string,
- conflictSelector: (item: ConflictData) => string,
-): Record => {
- return Object.entries(sourceData).reduce>((acc, [entityName, sourceItems]) => {
- const conflicts = conflictData[entityName];
-
- if (conflicts) {
- // Create a Set of systemIds from conflict records for faster lookup
- const conflictIdsSet = new Set(conflicts.map(conflictSelector));
-
- // Filter source data that does not have a matching systemId in the conflict set
- const filteredValues = sourceItems.filter((item) => !conflictIdsSet.has(entitySelector(item)));
-
- if (filteredValues.length > 0) {
- acc[entityName] = filteredValues;
- }
- } else {
- // If no conflicts, keep the source data as is
- acc[entityName] = sourceItems;
- }
-
- return acc;
- }, {});
-};
-
-/**
- * Filters updates from the provided `submissionUpdateData` based on conflicts found in the `submissionDeleteData`.
- * Conflicts are determined by matching the `systemId` of the items in both records.
- *
- * @param submissionUpdateData - A record containing arrays of `SubmissionUpdateData` to be filtered.
- * @param submissionDeleteData - A record containing arrays of `SubmissionDeleteData` that defines the conflicts.
- * @returns A filtered record of `SubmissionUpdateData[]` where no items conflict with those in `submissionDeleteData`.
- */
-export const filterUpdatesFromDeletes = (
- submissionUpdateData: Record,
- submissionDeleteData: Record,
-): Record => {
- return filterRecordsByConflicts(
- submissionUpdateData,
- submissionDeleteData,
- (itemToUpdate) => itemToUpdate.systemId,
- (itemToDelete) => itemToDelete.systemId,
- );
-};
-
-/**
- * Filters deletes from the provided `submissionDeleteData` based on conflicts found in the `submissionUpdateData`.
- * Conflicts are determined by matching the `systemId` of the items in both records.
- *
- * @param submissionDeleteData - A record containing arrays of `SubmissionDeleteData` to be filtered.
- * @param submissionUpdateData - A record containing arrays of `SubmissionUpdateData` that defines the conflicts.
- * @returns A filtered record of `SubmissionDeleteData[]` where no items conflict with those in `submissionUpdateData`.
- */
-export const filterDeletesFromUpdates = (
- submissionDeleteData: Record,
- submissionUpdateData: Record,
-): Record => {
- return filterRecordsByConflicts(
- submissionDeleteData,
- submissionUpdateData,
- (itemToDelete) => itemToDelete.systemId,
- (itemToUpdate) => itemToUpdate.systemId,
- );
-};
-
/**
* Returns a filter to query the database used to find dependents records when the update record involves changes of an primary ID field
*
@@ -332,6 +249,16 @@ export const filterRelationsForPrimaryIdUpdate = (
})
);
};
+type SubmissionRecordErrorDetails = {
+ recordId: number;
+ errors: SubmissionRecordError[];
+};
+
+export type SubmissionErrors = {
+ inserts?: Record;
+ updates?: Record;
+ deletes?: Record;
+};
/**
* Returns only the schema errors corresponding to the Active Submission.
@@ -370,14 +297,9 @@ export const groupSchemaErrorsByEntity = (input: {
return;
}
- const submissionIndex = mapping.reference.index;
+ const submissionRecordId = mapping.reference.recordId;
const actionType = mapping.reference.type === MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA ? 'inserts' : 'updates';
- const mutableSchemaValidationErrors = schemaValidationErrors.map((errors) => ({
- ...errors,
- index: submissionIndex,
- }));
-
if (!submissionSchemaErrors[actionType]) {
submissionSchemaErrors[actionType] = {};
}
@@ -386,35 +308,246 @@ export const groupSchemaErrorsByEntity = (input: {
submissionSchemaErrors[actionType][entityName] = [];
}
- submissionSchemaErrors[actionType][entityName].push(...mutableSchemaValidationErrors);
+ submissionSchemaErrors[actionType][entityName].push({
+ recordId: submissionRecordId,
+ errors: schemaValidationErrors,
+ });
});
});
return submissionSchemaErrors;
};
+/**
+ * Scans the Active Submission for `systemId`s (scoped per entity) that have both an UPDATE and a
+ * DELETE record staged at the same time. This is meant to run *before* dictionary validation:
+ * detecting the conflict up front lets both conflicting records be rejected explicitly, instead of
+ * one action silently winning based on array-filtering order later in the validation/merge pipeline.
+ * @param {SubmissionRecordWithEntityName[]} submissionData The Active Submission data
+ * @returns {SubmissionErrors} Conflict errors under the 'updates' and 'deletes' buckets, grouped by entity name
+ */
+export const findUpdateDeleteConflicts = (submissionData: SubmissionRecordWithEntityName[]): SubmissionErrors => {
+ const updatesByEntity = new Map>();
+ const deletesByEntity = new Map>();
+
+ const trackRecordId = (
+ bucket: Map>,
+ entityName: string,
+ systemId: string,
+ recordId: number,
+ ) => {
+ const bySystemId = bucket.get(entityName) ?? new Map();
+ bySystemId.set(systemId, [...(bySystemId.get(systemId) ?? []), recordId]);
+ bucket.set(entityName, bySystemId);
+ };
+
+ submissionData.forEach((record) => {
+ if (isUpdateSubmissionRecord(record)) {
+ trackRecordId(updatesByEntity, record.entityName, record.data.systemId, record.id);
+ } else if (isDeleteSubmissionRecord(record)) {
+ trackRecordId(deletesByEntity, record.entityName, record.data.systemId, record.id);
+ }
+ });
+
+ const conflictErrorFor = (
+ systemId: string,
+ conflictingActionType: RecordErrorActionConflict['conflictingActionType'],
+ ): RecordErrorActionConflict => ({
+ reason: 'CONFLICTING_ACTION',
+ systemId,
+ conflictingActionType,
+ message: `Record with systemId '${systemId}' has both an UPDATE and a DELETE staged in the same Active Submission`,
+ });
+
+ const conflictErrors: SubmissionErrors = {};
+
+ updatesByEntity.forEach((updateSystemIds, entityName) => {
+ const deleteSystemIds = deletesByEntity.get(entityName);
+ if (!deleteSystemIds) {
+ return;
+ }
+
+ updateSystemIds.forEach((updateRecordIds, systemId) => {
+ const deleteRecordIds = deleteSystemIds.get(systemId);
+ if (!deleteRecordIds) {
+ return;
+ }
+
+ conflictErrors.updates ??= {};
+ conflictErrors.updates[entityName] = [
+ ...(conflictErrors.updates[entityName] ?? []),
+ ...updateRecordIds.map((recordId) => ({ recordId, errors: [conflictErrorFor(systemId, 'DELETE')] })),
+ ];
+
+ conflictErrors.deletes ??= {};
+ conflictErrors.deletes[entityName] = [
+ ...(conflictErrors.deletes[entityName] ?? []),
+ ...deleteRecordIds.map((recordId) => ({ recordId, errors: [conflictErrorFor(systemId, 'UPDATE')] })),
+ ];
+ });
+ });
+
+ return conflictErrors;
+};
+
+export type DeleteStagingConflicts = {
+ /** `recordsToDeleteMap` with systemIds that already have a pending DELETE removed, so they aren't staged twice */
+ filteredRecordsToDeleteMap: Record;
+ /** systemIds that already have a pending UPDATE staged for the same entity in the Active Submission */
+ conflictingSystemIds: string[];
+ /** systemIds that already have a pending DELETE staged for the same entity — skipped instead of duplicated */
+ duplicateSystemIds: string[];
+};
+
+/**
+ * Checks systemIds about to be staged for deletion against what the Active Submission already has
+ * pending for the same entity, before a new DELETE record is ever inserted. A systemId with a
+ * pending UPDATE is reported as a conflict — the caller should reject the delete rather than
+ * silently letting one action override the other, consistent with how `findUpdateDeleteConflicts`
+ * treats the same conflict at validation time. A systemId that already has a pending DELETE is
+ * treated as a duplicate and dropped from the result, instead of inserting a second DELETE record.
+ * @param {Record} recordsToDeleteMap New deletes, grouped by entity name
+ * @param {SubmissionRecordWithEntityName[]} existingSubmissionRecords The Active Submission's current UPDATE/DELETE records
+ * @returns {DeleteStagingConflicts}
+ */
+export const resolveDeleteStagingConflicts = (
+ recordsToDeleteMap: Record,
+ existingSubmissionRecords: SubmissionRecordWithEntityName[],
+): DeleteStagingConflicts => {
+ const existingUpdateSystemIds = new Map>();
+ const existingDeleteSystemIds = new Map>();
+
+ const trackSystemId = (bucket: Map>, entityName: string, systemId: string) => {
+ const systemIds = bucket.get(entityName) ?? new Set();
+ systemIds.add(systemId);
+ bucket.set(entityName, systemIds);
+ };
+
+ existingSubmissionRecords.forEach((record) => {
+ if (isUpdateSubmissionRecord(record)) {
+ trackSystemId(existingUpdateSystemIds, record.entityName, record.data.systemId);
+ } else if (isDeleteSubmissionRecord(record)) {
+ trackSystemId(existingDeleteSystemIds, record.entityName, record.data.systemId);
+ }
+ });
+
+ const conflictingSystemIds: string[] = [];
+ const duplicateSystemIds: string[] = [];
+ const filteredRecordsToDeleteMap: Record = {};
+
+ Object.entries(recordsToDeleteMap).forEach(([entityName, records]) => {
+ const conflictingUpdateIds = existingUpdateSystemIds.get(entityName);
+ const duplicateDeleteIds = existingDeleteSystemIds.get(entityName);
+
+ const recordsToKeep = records.filter((record) => {
+ if (conflictingUpdateIds?.has(record.systemId)) {
+ conflictingSystemIds.push(record.systemId);
+ return false;
+ }
+ if (duplicateDeleteIds?.has(record.systemId)) {
+ duplicateSystemIds.push(record.systemId);
+ return false;
+ }
+ return true;
+ });
+
+ if (recordsToKeep.length > 0) {
+ filteredRecordsToDeleteMap[entityName] = recordsToKeep;
+ }
+ });
+
+ return { filteredRecordsToDeleteMap, conflictingSystemIds, duplicateSystemIds };
+};
+
+/**
+ * Collects every `recordId` referenced across all buckets of a `SubmissionErrors` object.
+ * @param {SubmissionErrors} errors
+ * @returns {Set}
+ */
+export const extractRecordIdsFromSubmissionErrors = (errors: SubmissionErrors): Set => {
+ const recordIds = new Set();
+ for (const entities of Object.values(errors)) {
+ if (!entities) {
+ continue;
+ }
+ for (const records of Object.values(entities)) {
+ records.forEach(({ recordId }) => recordIds.add(recordId));
+ }
+ }
+ return recordIds;
+};
+
+/**
+ * Merges two `SubmissionErrors` objects together, concatenating each entity's error array
+ * bucket-by-bucket instead of overwriting it.
+ * @param {SubmissionErrors} a
+ * @param {SubmissionErrors} b
+ * @returns {SubmissionErrors}
+ */
+export const mergeSubmissionErrors = (a: SubmissionErrors, b: SubmissionErrors): SubmissionErrors => {
+ const mergeBucket = (
+ bucketA?: Record,
+ bucketB?: Record,
+ ): Record | undefined => {
+ if (!bucketA && !bucketB) {
+ return undefined;
+ }
+ const merged: Record = { ...bucketA };
+ for (const [entityName, records] of Object.entries(bucketB ?? {})) {
+ merged[entityName] = [...(merged[entityName] ?? []), ...records];
+ }
+ return merged;
+ };
+
+ // Only set a bucket key when it actually has content — callers rely on `Object.keys(...).length`
+ // (and `_.isEmpty`) to detect the "no errors" case, so an always-present `undefined` value would
+ // make every submission look like it has errors.
+ const merged: SubmissionErrors = {};
+ const inserts = mergeBucket(a.inserts, b.inserts);
+ if (inserts) {
+ merged.inserts = inserts;
+ }
+ const updates = mergeBucket(a.updates, b.updates);
+ if (updates) {
+ merged.updates = updates;
+ }
+ const deletes = mergeBucket(a.deletes, b.deletes);
+ if (deletes) {
+ merged.deletes = deletes;
+ }
+ return merged;
+};
+
/**
* This function extracts the Schema Data from the Active Submission
* and maps it to it's original reference Id
* The result mapping is used to perform the cross schema validation
* @param {number} activeSubmissionId
- * @param {Record} activeSubmissionInsertDataEntities
+ * @param {SubmissionInsertRecordWithEntityName[]} activeSubmissionInsertDataEntities
* @returns {Record}
*/
export const mapInsertDataToRecordReferences = (
activeSubmissionId: number,
- activeSubmissionInsertDataEntities: Record,
+ activeSubmissionInsertDataEntities: SubmissionInsertRecordWithEntityName[],
): Record => {
- return _.mapValues(activeSubmissionInsertDataEntities, (submissionInsertData) =>
- submissionInsertData.records.map((record, index) => {
- return {
- dataRecord: record,
+ return activeSubmissionInsertDataEntities.reduce>(
+ (acc, submissionInsertData) => {
+ const entityName = submissionInsertData.entityName;
+ let entityRecords = acc[entityName];
+ if (!entityRecords) {
+ entityRecords = [];
+ acc[entityName] = entityRecords;
+ }
+ entityRecords.push({
+ dataRecord: submissionInsertData.data,
reference: {
submissionId: activeSubmissionId,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
- index: index,
+ recordId: submissionInsertData.recordId,
},
- };
- }),
+ });
+ return acc;
+ },
+ {},
);
};
@@ -455,15 +588,63 @@ export const mapGroupedUpdateSubmissionData = ({
);
};
+export const isUpdateSubmissionRecord = (
+ item: SubmissionRecordWithEntityName,
+): item is SubmissionRecordWithEntityName & {
+ actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE;
+ data: SubmissionUpdateData;
+} => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE;
+
+export const isInsertSubmissionRecord = (
+ item: SubmissionRecordWithEntityName,
+): item is SubmissionRecordWithEntityName & {
+ actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT;
+ data: SubmissionInsertData;
+} => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT;
+
+export const isDeleteSubmissionRecord = (
+ item: SubmissionRecordWithEntityName,
+): item is SubmissionRecordWithEntityName & {
+ actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.DELETE;
+ data: SubmissionDeleteData;
+} => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.DELETE;
+
+export const createSubmissionUpdateRecords = (
+ submissionData: SubmissionRecordWithEntityName[],
+): SubmissionUpdateRecordWithEntityName[] => {
+ return submissionData.reduce((acc, item) => {
+ if (isUpdateSubmissionRecord(item)) {
+ acc.push({
+ recordId: item.id,
+ entityName: item.entityName,
+ data: item.data,
+ });
+ }
+ return acc;
+ }, []);
+};
+
+export const createSubmissionInsertRecords = (
+ submissionData: SubmissionRecordWithEntityName[],
+): SubmissionInsertRecordWithEntityName[] => {
+ return submissionData.reduce((acc, item) => {
+ if (isInsertSubmissionRecord(item)) {
+ acc.push({
+ recordId: item.id,
+ entityName: item.entityName,
+ data: item.data,
+ });
+ }
+ return acc;
+ }, []);
+};
+
/**
* Combines **Active Submission** and the **Submitted Data** recevied as arguments.
* Then, the Schema Data is extracted and mapped with its internal reference ID.
* The returned Object is a collection of the raw Schema Data with it's reference ID grouped by entity name.
* @param {number} submissionId ID of the Active Submission
- * @param {Object} submissionData
- * @param {Record} submissionData.insertData Collection of Data records of the Active Submission
- * @param {Record} submissionData.updateData Collection of Data records of the Active Submission
- * @param {Record} submissionData.deleteData Collection of Data records of the Active Submission
+ * @param {SubmissionRecordWithEntityName[]} submissionData The Active Submission data
* @param {SubmittedData[]} submittedData An array of Submitted Data
* @returns {Record}
*/
@@ -473,12 +654,10 @@ export const mergeAndReferenceEntityData = ({
submittedData,
}: {
submissionId: number;
- submissionData: SubmissionData;
+ submissionData: SubmissionRecordWithEntityName[];
submittedData: SubmittedData[];
}): Record => {
- const systemsIdsToRemove = submissionData.deletes
- ? Object.values(submissionData.deletes).flatMap((entityData) => entityData.map(({ systemId }) => systemId))
- : [];
+ const systemsIdsToRemove = submissionData.filter(isDeleteSubmissionRecord).map((item) => item.data.systemId);
// Exclude items that are marked for deletion
const submittedDataFiltered =
@@ -486,15 +665,17 @@ export const mergeAndReferenceEntityData = ({
? submittedData.filter(({ systemId }) => !systemsIdsToRemove.includes(systemId))
: submittedData;
+ const dataToUpdate = createSubmissionUpdateRecords(submissionData);
+
const submittedDataWithRef = mapAndMergeSubmittedDataToRecordReferences({
submittedData: submittedDataFiltered,
- editSubmittedData: submissionData.updates,
+ editSubmittedData: dataToUpdate,
submissionId,
});
- const insertDataWithRef = submissionData.inserts
- ? mapInsertDataToRecordReferences(submissionId, submissionData.inserts)
- : {};
+ const dataToInsert = createSubmissionInsertRecords(submissionData);
+
+ const insertDataWithRef = dataToInsert.length > 0 ? mapInsertDataToRecordReferences(submissionId, dataToInsert) : {};
// This object will merge existing data + new data for validation (Submitted data + active Submission)
return _.mergeWith(submittedDataWithRef, insertDataWithRef, (objValue, srcValue) => {
@@ -505,90 +686,6 @@ export const mergeAndReferenceEntityData = ({
});
};
-/**
- * Merges multiple `Record` objects into a single object.
- * If there are duplicate keys between the objects, the `records` arrays of `SubmissionInsertData`
- * are concatenated for the matching keys, ensuring no duplicates.
- *
- * @param objects An array of objects where each object is a `Record`.
- * Each key represents the entityName, and the value is an object of type `SubmissionInsertData`.
- *
- * @returns A new `Record` where:
- * - If a key is unique across all objects, its value is directly included.
- * - If a key appears in multiple objects, the `records` arrays are concatenated for that key, avoiding duplicates.
- */
-export const mergeInsertsRecords = (
- ...objects: Record[]
-): Record => {
- const result: Record = {};
-
- let seen: DataRecord[] = [];
- // Iterate over all objects
- objects.forEach((obj) => {
- // Iterate over each key in the current object
- Object.entries(obj).forEach(([key, value]) => {
- if (result[key]) {
- // The key already exists in the result, concatenate the `records` arrays, avoiding duplicates
- let uniqueData: DataRecord[] = [];
-
- result[key].records.concat(value.records).forEach((item) => {
- if (!seen.some((existingItem) => deepCompare(existingItem, item))) {
- uniqueData = uniqueData.concat(item);
- seen = seen.concat(item);
- }
- });
-
- result[key].records = uniqueData;
- return;
- } else {
- // The key doesn't exists in the result, create as it comes
- result[key] = value;
- return;
- }
- });
- });
-
- return result;
-};
-
-/**
- * Merges multiple `Record` objects into a single object.
- * For each key, the `SubmissionDeleteData[]` arrays are concatenated, ensuring no duplicate
- * `SubmissionDeleteData` objects based on the `systemId` field.
- *
- * @param objects Multiple `Record` objects to be merged.
- * Each key represents an identifier, and the value is an array of `SubmissionDeleteData`.
- *
- * @returns
- */
-export const mergeDeleteRecords = (
- ...objects: Record[]
-): Record => {
- const result: Record = {};
-
- // Iterate over all objects
- objects.forEach((obj) => {
- // Iterate over each key in the current object
- Object.entries(obj).forEach(([key, value]) => {
- if (!result[key]) {
- result[key] = [];
- }
- const uniqueRecords = new Map();
-
- // Add existing records to the map
- result[key].forEach((record) => uniqueRecords.set(record.systemId, record));
-
- // Add new records, overriding duplicates based on systemId
- value.forEach((record) => uniqueRecords.set(record.systemId, record));
-
- // Convert the map back to an array
- result[key] = Array.from(uniqueRecords.values());
- });
- });
-
- return result;
-};
-
/**
* Merge Active Submission data with incoming TSV file data processed
*
@@ -624,170 +721,10 @@ export const mergeUpdatesBySystemId = (
return result;
};
-/**
- * Utility to convert a raw Submission record to a Response type
- * @param {SubmissionDataDetailsRepositoryRecord} submission
- * @returns {SubmissionDetailsResponse}
- */
-export const createSubmissionDetailsResponse = (
- submission: SubmissionDataDetailsRepositoryRecord,
-): SubmissionDetailsResponse => {
- return {
- id: submission.id,
- data: submission.data,
- dictionary: submission.dictionary,
- dictionaryCategory: submission.dictionaryCategory,
- errors: submission.errors || {},
- organization: submission.organization,
- status: submission.status,
- createdAt: _.toString(submission.createdAt?.toISOString()),
- createdBy: _.toString(submission.createdBy),
- updatedAt: _.toString(submission.updatedAt?.toISOString()),
- updatedBy: _.toString(submission.updatedBy),
- };
-};
-
-/**
- * Utility to sum the recordsCount from a SubmissionDataSummary or SubmissionErrorsSummary
- */
-const sumRecordsCount = (buckets: SubmissionDataSummary | SubmissionErrorsSummary): number => {
- return Object.values(buckets)
- .flatMap((bucket) => (bucket ? Object.values(bucket) : []))
- .reduce((total, { recordsCount }) => total + recordsCount, 0);
-};
-
-/**
- * Utility to convert the raw SubmissionDataSummaryRepositoryRecord into a SubmissionSummaryResponse.
- * It includes a `total` value representing the sum of changes of each `data` and `errors`
- * @param {SubmissionDataSummaryRepositoryRecord} submission
- * @returns {SubmissionSummary}
- */
-export const createSubmissionSummaryResponse = (
- submission: SubmissionDataSummaryRepositoryRecord,
-): SubmissionSummary => {
- return {
- id: submission.id,
- data: {
- ...submission.data,
- total: sumRecordsCount(submission.data),
- },
- dictionary: submission.dictionary,
- dictionaryCategory: submission.dictionaryCategory,
- errors: {
- ...submission.errors,
- total: sumRecordsCount(submission.errors ?? {}),
- },
- organization: submission.organization,
- status: submission.status,
- createdAt: _.toString(submission.createdAt?.toISOString()),
- createdBy: _.toString(submission.createdBy),
- updatedAt: _.toString(submission.updatedAt?.toISOString()),
- updatedBy: _.toString(submission.updatedBy),
- };
-};
-
export const pluralizeSchemaName = (schemaName: string) => {
return pluralize(schemaName);
};
-export const removeItemsFromSubmission = (
- submissionData: SubmissionData,
- filter: { actionType: SubmissionActionType; entityName: string; index: number | null },
-): SubmissionData => {
- const filteredSubmissionData = _.cloneDeep(submissionData);
- switch (filter.actionType) {
- case SUBMISSION_ACTION_TYPE.Values.INSERTS:
- if (submissionData.inserts) {
- const filteredInserts = Object.entries(submissionData.inserts).reduce>(
- (acc, [insertsEntityName, insertsSubmissionData]) => {
- if (insertsEntityName === filter.entityName && filter.index == null) {
- // remove this whole entity
- return acc;
- } else if (insertsEntityName === filter.entityName && filter.index != null) {
- // remove an item on records based on it's index
- const filteredRecords = insertsSubmissionData.records.filter(
- (_, recordIndex) => recordIndex !== filter.index,
- );
- if (filteredRecords.length > 0) {
- acc[insertsEntityName] = {
- batchName: insertsSubmissionData.batchName,
- records: filteredRecords,
- };
- }
- } else {
- acc[insertsEntityName] = insertsSubmissionData;
- }
-
- return acc;
- },
- {},
- );
- if (Object.keys(filteredInserts).length === 0) {
- delete filteredSubmissionData.inserts;
- } else {
- filteredSubmissionData.inserts = filteredInserts;
- }
- }
- break;
- case SUBMISSION_ACTION_TYPE.Values.UPDATES:
- if (submissionData.updates) {
- const filteredUpdates = Object.entries(submissionData.updates).reduce>(
- (acc, [updatesEntityName, updatesSubmissionData]) => {
- if (updatesEntityName === filter.entityName && filter.index == null) {
- // remove this whole entity
- return acc;
- } else if (updatesEntityName === filter.entityName && filter.index != null) {
- // remove an item on records based on it's index
- const filteredRecords = updatesSubmissionData.filter((_, recordIndex) => recordIndex !== filter.index);
- if (filteredRecords.length > 0) {
- acc[updatesEntityName] = filteredRecords;
- }
- } else {
- acc[updatesEntityName] = updatesSubmissionData;
- }
-
- return acc;
- },
- {},
- );
- if (Object.keys(filteredUpdates).length === 0) {
- delete filteredSubmissionData.updates;
- } else {
- filteredSubmissionData.updates = filteredUpdates;
- }
- }
- break;
- case SUBMISSION_ACTION_TYPE.Values.DELETES:
- if (submissionData.deletes) {
- const filteredDeletes = Object.entries(submissionData.deletes).reduce>(
- (acc, [deletesEntityName, deletesSubmissionData]) => {
- if (deletesEntityName === filter.entityName && filter.index == null) {
- // remove this whole entity
- return acc;
- } else if (deletesEntityName === filter.entityName && filter.index != null) {
- // remove an item on records based on it's index
- const filteredRecords = deletesSubmissionData.filter((_, recordIndex) => recordIndex !== filter.index);
- if (filteredRecords.length > 0) {
- acc[deletesEntityName] = filteredRecords;
- }
- } else {
- acc[deletesEntityName] = deletesSubmissionData;
- }
- return acc;
- },
- {},
- );
- if (Object.keys(filteredDeletes).length === 0) {
- delete filteredSubmissionData.deletes;
- } else {
- filteredSubmissionData.deletes = filteredDeletes;
- }
- }
- break;
- }
- return filteredSubmissionData;
-};
-
/**
* Processes the `foundDependentUpdates` array and segregates the updates based on
* whether they involve ID fields (dependent fields) or non-ID fields.
@@ -833,7 +770,7 @@ export const segregateFieldChangeRecords = (
};
/** Per-file outcome from `submissionInsertDataFromFiles`. */
-export type FileParseResult = { fileName: string; entityName: string } & (
+export type FileParseResult = { fileName: string; entityName: string; fileSize: number } & (
| { status: 'ok' }
| { status: 'invalid'; parseErrors: ParseSchemaError[] }
| { status: 'error'; streamError: string }
@@ -841,8 +778,8 @@ export type FileParseResult = { fileName: string; entityName: string } & (
/** Return type of `submissionInsertDataFromFiles`. */
export type FileInsertResult = {
- data: Record;
- fileResults: FileParseResult[];
+ data: DataRecord[];
+ fileResult: FileParseResult;
};
/**
@@ -850,33 +787,40 @@ export type FileInsertResult = {
* Each file is processed independently: a stream or parse failure on one file is captured and
* reported without interrupting processing of the remaining files.
*/
-export const submissionInsertDataFromFiles = async (fileSchemaMap: FileSchemaMap): Promise => {
- const data: Record = {};
- const fileResults: FileParseResult[] = [];
+export const submissionInsertDataFromFiles = async (fileSchemaMap: FileSchemaMap): Promise => {
+ const result: FileInsertResult[] = [];
for (const [entityName, { files, schema }] of Object.entries(fileSchemaMap)) {
for (const file of files) {
try {
const parsedFileData = await readTextFile(file, schema);
- const existing = data[schema.name] ?? { batchName: entityName, records: [] };
- data[schema.name] = { ...existing, records: [...existing.records, ...parsedFileData.records] };
- fileResults.push(
- parsedFileData.errors.length > 0
- ? { status: 'invalid', fileName: file.originalname, entityName, parseErrors: parsedFileData.errors }
- : { status: 'ok', fileName: file.originalname, entityName },
- );
+ result.push({
+ data: parsedFileData.records,
+ fileResult: {
+ entityName,
+ fileName: file.originalname,
+ fileSize: file.size,
+ ...(parsedFileData.errors.length > 0
+ ? { status: 'invalid', parseErrors: parsedFileData.errors }
+ : { status: 'ok' }),
+ },
+ });
} catch (err) {
- fileResults.push({
- status: 'error',
- fileName: file.originalname,
- entityName,
- streamError: err instanceof Error ? err.message : String(err),
+ result.push({
+ data: [],
+ fileResult: {
+ status: 'error',
+ fileName: file.originalname,
+ fileSize: file.size,
+ entityName,
+ streamError: err instanceof Error ? err.message : String(err),
+ },
});
}
}
}
- return { data, fileResults };
+ return result;
};
/**
@@ -907,9 +851,9 @@ export const parseToSchema = (schema: Schema) => (record: Record
return parsedRecord.data.record;
};
-export const parseSubmissionActionTypes = (values: unknown): SubmissionActionType[] => {
+export const parseSubmissionActionTypes = (values: unknown): SubmissionRecordActionType[] => {
return asArray(values || [])
.map((value) => value.toString().toUpperCase())
.filter(isSubmissionActionTypeValid)
- .map((value) => SUBMISSION_ACTION_TYPE.parse(value));
+ .map((value) => SUBMISSION_RECORD_ACTION_TYPE.parse(value));
};
diff --git a/packages/data-provider/src/utils/submittedDataUtils.ts b/packages/data-provider/src/utils/submittedDataUtils.ts
index bcd42478..c460ff99 100644
--- a/packages/data-provider/src/utils/submittedDataUtils.ts
+++ b/packages/data-provider/src/utils/submittedDataUtils.ts
@@ -21,6 +21,7 @@ import {
MERGE_REFERENCE_TYPE,
type MutableDataDiff,
type MutableDataRecord,
+ type SubmissionUpdateRecordWithEntityName,
VIEW_TYPE,
type ViewType,
} from './types.js';
@@ -237,8 +238,8 @@ export const groupSchemaDataByEntityName = (data: {
* Edits each record that is marked to be edited on the Submission
* @param {object} params
* @param {SubmittedData[] | undefined} params.submittedData An array of `SubmittedData` objects to be transformed.
- * @param {Record} params.editSubmittedData An Array of `SubmittedData` objects to be updated
- * @param {Rnumber} params.submissionId The ID of the Active Submission
+ * @param {SubmissionRecordWithEntityName[]} params.editSubmittedData An Array of `SubmittedData` objects to be updated
+ * @param {number} params.submissionId The ID of the Active Submission
* @returns {Record}
*/
export const mapAndMergeSubmittedDataToRecordReferences = ({
@@ -247,26 +248,20 @@ export const mapAndMergeSubmittedDataToRecordReferences = ({
submissionId,
}: {
submittedData?: SubmittedData[];
- editSubmittedData?: Record;
+ editSubmittedData?: SubmissionUpdateRecordWithEntityName[];
submissionId: number;
}): Record => {
if (!submittedData) {
return {};
}
return submittedData.reduce>((acc, entityData) => {
- const entityEditData = editSubmittedData?.[entityData.entityName];
- const foundRecordToUpdateIndex = entityEditData
- ? entityEditData.findIndex((item) => item.systemId === entityData.systemId)
- : -1;
- let record: DataRecordReference;
- if (entityEditData && foundRecordToUpdateIndex >= 0) {
- const recordToUpdate = entityEditData[foundRecordToUpdateIndex];
-
- if (!recordToUpdate) {
- return acc;
- }
+ const recordToUpdate = editSubmittedData?.find(
+ (item) => item.entityName === entityData.entityName && item.data.systemId === entityData.systemId,
+ );
- const newDataToUpdate = updateEntityData(entityData.data, recordToUpdate);
+ let record: DataRecordReference;
+ if (recordToUpdate) {
+ const newDataToUpdate = updateEntityData(entityData.data, recordToUpdate.data);
record = {
dataRecord: newDataToUpdate,
@@ -274,7 +269,7 @@ export const mapAndMergeSubmittedDataToRecordReferences = ({
type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA,
systemId: entityData.systemId,
submissionId,
- index: foundRecordToUpdateIndex,
+ recordId: recordToUpdate.recordId,
},
};
} else {
diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts
index 64b4b949..8e02fb56 100644
--- a/packages/data-provider/src/utils/types.ts
+++ b/packages/data-provider/src/utils/types.ts
@@ -12,9 +12,8 @@ import {
type DataDiff,
type Dictionary,
NewSubmittedData,
- SubmissionData,
type SubmissionDeleteData,
- type SubmissionErrors,
+ type SubmissionInsertData,
type SubmissionUpdateData,
type SubmittedData,
} from '@overture-stack/lyric-data-model/models';
@@ -170,16 +169,19 @@ export type MigrationAuditRecord = Omit;
-
/** Action field included in each Kafka message emitted after a successful commit. */
export const KAFKA_ACTION = zod.enum(['delete', 'insert', 'update']);
export type KafkaAction = zod.infer;
+/**
+ * Enum matching Submission Record state in database
+ */
+export const SUBMISSION_RECORD_STATE = zod.enum(['RECEIVED', 'VALID', 'INVALID']);
+export type SubmissionRecordState = zod.infer;
+
+export const SUBMISSION_RECORD_ACTION_TYPE = zod.enum(['INSERT', 'UPDATE', 'DELETE']);
+export type SubmissionRecordActionType = zod.infer;
+
/**
* File upload validation error types
*/
@@ -264,85 +266,52 @@ export type PaginationOptions = {
export type DataInsertsSubmissionSummary = {
batchName: string;
recordsCount: number;
+ errors: number;
};
export type DataUpdatesSubmissionSummary = {
+ batchName: string;
recordsCount: number;
+ errors: number;
};
export type DataDeletesSubmissionSummary = {
recordsCount: number;
-};
-
-export type DataErrorsSubmissionSummary = {
- recordsCount: number;
-};
-
-/**
- * Response type for Get Submission by Submission ID endpoint
- */
-export type SubmissionDetailsResponse = {
- id: number;
- data: SubmissionData;
- dictionary: DictionarySummary;
- dictionaryCategory: CategorySummary;
- errors: SubmissionErrors;
- organization: string;
- status: SubmissionStatus;
- createdAt: string;
- createdBy: string;
- updatedAt: string;
- updatedBy: string;
+ errors: number;
};
export type SubmissionDataSummary = {
- inserts?: Record;
- updates?: Record;
+ inserts?: Record;
+ updates?: Record;
deletes?: Record;
};
-export type SubmissionErrorsSummary = {
- inserts?: Record;
- updates?: Record;
- deletes?: Record;
+export type SubmissionDataSummaryWithTotal = SubmissionDataSummary & {
+ totalRecords: number;
+ errors: number;
};
/**
* Shortened version of the Submission record that omits the data changes and error details
* in favour of the count of records changed and errors for each entity type.
*/
-export type SubmissionSummary = Omit & {
- data: SubmissionDataSummary & { total: number };
-} & {
- errors: SubmissionErrorsSummary & { total: number };
+export type SubmissionSummary = SubmissionWithDictionaryAndCategoryRepositoryRecord & {
+ data: SubmissionDataSummaryWithTotal;
};
-/**
- * Retrieve Submission object with data summary from repository
- */
-export type SubmissionDataSummaryRepositoryRecord = {
- id: number;
- data: SubmissionDataSummary;
- dictionary: DictionarySummary;
- dictionaryCategory: CategorySummary;
- errors: SubmissionErrorsSummary | null;
- organization: string;
- status: SubmissionStatus;
- createdAt: Date | null;
- createdBy: string | null;
- updatedAt: Date | null;
- updatedBy: string | null;
+export type SubmissionSummaryResponse = Omit & {
+ createdAt: string;
+ updatedAt: string;
};
/**
- * Retrieve Submission object with data details from repository
+ * Retrieve Submission object with Dictionary and Category from repository
*/
-export type SubmissionDataDetailsRepositoryRecord = {
+
+export type SubmissionWithDictionaryAndCategoryRepositoryRecord = {
id: number;
- data: SubmissionData;
dictionary: DictionarySummary;
dictionaryCategory: CategorySummary;
- errors: SubmissionErrors | null;
organization: string;
status: SubmissionStatus;
createdAt: Date | null;
@@ -464,13 +433,13 @@ export interface SubmittedDataReference {
}
export interface NewSubmittedDataReference {
- index: number;
+ recordId: number;
submissionId: number;
type: typeof MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA;
}
export interface EditSubmittedDataReference {
- index: number;
+ recordId: number;
systemId?: string;
submissionId: number;
type: typeof MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA;
@@ -485,6 +454,17 @@ export interface DataRecordNested {
[key: string]: DataRecordValue | DataRecordNested | DataRecordNested[];
}
+export type SubmissionInsertRecordWithEntityName = {
+ recordId: number;
+ entityName: string;
+ data: SubmissionInsertData;
+};
+export type SubmissionUpdateRecordWithEntityName = {
+ recordId: number;
+ entityName: string;
+ data: SubmissionUpdateData;
+};
+
/**
* Keys of an object type as a union
*
diff --git a/packages/data-provider/src/workers/commitSubmissionWorker.ts b/packages/data-provider/src/workers/commitSubmissionWorker.ts
index 982e486f..a6249471 100644
--- a/packages/data-provider/src/workers/commitSubmissionWorker.ts
+++ b/packages/data-provider/src/workers/commitSubmissionWorker.ts
@@ -3,10 +3,15 @@ import type { SubmissionUpdateData } from '@overture-stack/lyric-data-model/mode
import systemIdGenerator from '../external/systemIdGenerator.js';
import createSubmissionRepository from '../repository/activeSubmissionRepository.js';
import createCategoryRepository from '../repository/categoryRepository.js';
-import submittedRepository from '../repository/submittedRepository.js';
+import createSubmissionRecordsRepository from '../repository/submissionRecordsRepository.js';
+import createSubmittedRepository from '../repository/submittedRepository.js';
import submissionProcessorFactory from '../services/submission/submissionProcessor.js';
-import type { ResultOnCommit } from '../utils/types.js';
-import { SUBMISSION_STATUS } from '../utils/types.js';
+import {
+ isDeleteSubmissionRecord,
+ isInsertSubmissionRecord,
+ isUpdateSubmissionRecord,
+} from '../utils/submissionUtils.js';
+import { type ResultOnCommit, SUBMISSION_STATUS } from '../utils/types.js';
import type { CommitWorkerInput } from './types.js';
import { getWorkerDependencies } from './workerContext.js';
@@ -23,12 +28,13 @@ export const processCommitSubmission = async (message: CommitWorkerInput): Promi
const submissionRepo = createSubmissionRepository(dependencies);
const categoryRepo = createCategoryRepository(dependencies);
- const submittedDataRepo = submittedRepository(dependencies);
+ const submittedDataRepo = createSubmittedRepository(dependencies);
+ const submissionRecordsRepo = createSubmissionRecordsRepository(dependencies);
const submissionProcessor = submissionProcessorFactory.create(dependencies);
// Fetch submission
- const submission = await submissionRepo.getSubmissionDetailsById(submissionId);
+ const submission = await submissionRepo.getSubmissionById(submissionId);
if (!submission) {
throw new Error(`Submission '${submissionId}' not found`);
}
@@ -54,39 +60,40 @@ export const processCommitSubmission = async (message: CommitWorkerInput): Promi
const { generateIdentifier } = systemIdGenerator(dependencies);
+ const recordsToInsert = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, {
+ actionTypes: ['INSERT'],
+ });
+
// Build inserts for validation
- const insertsToValidate = submission.data?.inserts
- ? Object.entries(submission.data.inserts).flatMap(([entityName, submissionData]) => {
- return submissionData.records.map((record) => ({
- data: record,
- dictionaryCategoryId: categoryId,
- entityName,
- isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise
- organization: submission.organization,
- originalSchemaId: currentDictionary.id,
- systemId: generateIdentifier(entityName, record),
- createdBy: username,
- }));
- })
- : [];
-
- const deleteDataArray = submission.data?.deletes
- ? Object.entries(submission.data.deletes).flatMap(([_entityName, submissionDeleteData]) => {
- return submissionDeleteData;
- })
- : [];
-
- const updateDataArray =
- submission.data?.updates &&
- Object.entries(submission.data.updates).reduce>(
- (acc, [_entityName, submissionUpdateData]) => {
- submissionUpdateData.forEach((record) => {
- acc[record.systemId] = record;
- });
- return acc;
- },
- {},
- );
+ const insertsToValidate = recordsToInsert.filter(isInsertSubmissionRecord).map(({ entityName, data }) => {
+ return {
+ data,
+ dictionaryCategoryId: categoryId,
+ entityName,
+ isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise
+ organization: submission.organization,
+ originalSchemaId: currentDictionary.id,
+ systemId: generateIdentifier(entityName, data),
+ createdBy: username,
+ };
+ });
+
+ const recordsToDelete = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, {
+ actionTypes: ['DELETE'],
+ });
+
+ const deleteDataArray = recordsToDelete.filter(isDeleteSubmissionRecord).map(({ data }) => data);
+
+ const recordsToUpdate = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, {
+ actionTypes: ['UPDATE'],
+ });
+
+ const updatesBySystemId = recordsToUpdate
+ .filter(isUpdateSubmissionRecord)
+ .reduce>((acc, { data }) => {
+ acc[data.systemId] = data;
+ return acc;
+ }, {});
try {
return await submissionProcessor.performCommitSubmissionAsync({
@@ -94,7 +101,7 @@ export const processCommitSubmission = async (message: CommitWorkerInput): Promi
inserts: insertsToValidate,
submittedData: submittedDataToValidate,
deletes: deleteDataArray,
- updates: updateDataArray,
+ updates: updatesBySystemId,
},
submissionId: submission.id,
dictionary: currentDictionary,
diff --git a/packages/data-provider/src/workers/workerPoolManager.ts b/packages/data-provider/src/workers/workerPoolManager.ts
index d02e55bf..9809a725 100644
--- a/packages/data-provider/src/workers/workerPoolManager.ts
+++ b/packages/data-provider/src/workers/workerPoolManager.ts
@@ -133,6 +133,10 @@ export const createWorkerPool = (configData: AppConfig, options?: CreateWorkerPo
}
},
terminate: async (): Promise => {
+ // Wait for the worker's startup handshake to settle before terminating the pool.
+ // Terminating while the child process is still registering closes the IPC channel
+ // out from under its own `process.send()`, which crashes the process with EPIPE.
+ await readyProxy.catch(() => undefined);
await pool.terminate();
},
};
diff --git a/packages/data-provider/test/assertions.ts b/packages/data-provider/test/assertions.ts
new file mode 100644
index 00000000..4b493b03
--- /dev/null
+++ b/packages/data-provider/test/assertions.ts
@@ -0,0 +1,10 @@
+import { expect } from 'chai';
+
+/**
+ * Asserts that the value is not null or undefined.
+ * This lets TypeScript know the value is of type T after this check.
+ * @param value The value to check for existence.
+ */
+export function assertExists(value: T | null | undefined): asserts value is T {
+ expect(value).to.exist;
+}
diff --git a/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts b/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts
index d01ded51..6f783e5c 100644
--- a/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts
+++ b/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts
@@ -14,6 +14,8 @@ import {
VALID_DICTIONARY_VERSION,
} from './fixtures.js';
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
describe('Integration - Dictionary Migration', () => {
let appDictionary: supertest.Agent;
let appMigration: supertest.Agent;
@@ -43,6 +45,23 @@ describe('Integration - Dictionary Migration', () => {
const registerDictionary = async (payload: RegisterPayload, force = false) =>
appDictionary.post(`/register${force ? '?force=true' : ''}`).send(payload);
+ // The migration itself runs in a background worker (fired without being awaited by
+ // `registerDictionary`), so a test that mutates a migration's status directly must first wait
+ // for that worker to reach a terminal status — otherwise the manual override races with the
+ // worker's own status transition.
+ const waitForMigrationToFinish = async (migrationId: number, maxRetries = 20, delayMs = 300) => {
+ let response = await appMigration.get(`/${migrationId}`);
+ let attempts = 0;
+
+ while (response.body.status === 'IN_PROGRESS' && attempts < maxRetries) {
+ await sleep(delayMs);
+ response = await appMigration.get(`/${migrationId}`);
+ attempts += 1;
+ }
+
+ return response;
+ };
+
before(async () => {
schemaServiceUrl = getContainers().providerConfig.schemaService.url;
lyricProvider = await createLyricProvider(getContainers().providerConfig);
@@ -160,6 +179,10 @@ describe('Integration - Dictionary Migration', () => {
const migrationId = migrationResponse.body.migrationId;
+ // Wait for the background migration worker to finish before overriding its status below,
+ // otherwise the worker's own status write can race with this test's manual override.
+ await waitForMigrationToFinish(migrationId);
+
// Making the migration fail by force registering the same new version again
await lyricProvider.repositories.migration.update(migrationId, {
status: 'FAILED',
diff --git a/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts b/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts
index 721f4127..59d74953 100644
--- a/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts
+++ b/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts
@@ -3,8 +3,10 @@ import { after, afterEach, before, beforeEach, describe, it } from 'mocha';
import supertest from 'supertest';
import submissionProcessorFactory from '../../../../src/services/submission/submissionProcessor.js';
+import type { FileParseResult } from '../../../../src/utils/submissionUtils.js';
import { createTsvFileContent } from '../../../fixtures/createTsvContent.js';
import { dictionarySportsData } from '../../../fixtures/dictionarySchemasTestData.js';
+import { assertExists } from '../../assertions.js';
import { createLyricProvider, type LyricProvider } from '../../dependencies/lyricProvider.js';
import { createTestApp } from '../../dependencies/testServer.js';
import { getContainers } from '../../globalSetup.js';
@@ -28,7 +30,7 @@ const waitForSubmissionToStopValidating = async ({
let submission;
do {
await sleep(delayMs);
- submission = await lyricProvider.repositories.submission.getActiveSubmissionSummary({
+ submission = await lyricProvider.repositories.submission.getActiveSubmission({
categoryId,
username: '',
organization,
@@ -48,7 +50,7 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D
let lyricProvider: LyricProvider;
let categoryId: number;
let originalCreate: typeof submissionProcessorFactory.create;
- let pendingAsyncWork: Promise | undefined;
+ let pendingAsyncWork: Promise | undefined;
before(async () => {
originalCreate = submissionProcessorFactory.create;
@@ -100,46 +102,46 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D
it('should save submitted file records to the active submission', async () => {
const sportTsv = createTsvFileContent(['sport_id', 'name'], [['1', 'Soccer']]);
- await app.post(`/category/${categoryId}/files?organization=testOrg`).attach('files', sportTsv, 'sport.tsv');
+ const submitResponse = await app
+ .post(`/category/${categoryId}/files?organization=testOrg`)
+ .attach('files', sportTsv, 'sport.tsv');
await pendingAsyncWork;
- const submission = await lyricProvider.repositories.submission.getActiveSubmissionDetails({
- categoryId,
- username: '',
- organization: 'testOrg',
- });
+ const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId(
+ submitResponse.body.submissionId,
+ );
- expect(submission).to.exist;
- expect(submission!.data.inserts).to.have.property('sport');
- expect(submission!.data.inserts!['sport'].records).to.have.length(1);
- expect(submission!.data.inserts!['sport'].records[0]).to.include({ sport_id: '1', name: 'Soccer' });
+ expect(submissionRecords.length).to.eq(1);
+ assertExists(submissionRecords[0]);
+ expect(submissionRecords[0].entityName).to.eql('sport');
+ expect(submissionRecords[0].actionType).to.eql('INSERT');
+ expect(submissionRecords[0].data).to.eql({ sport_id: '1', name: 'Soccer' });
});
it('should save records for each entity when multiple files are submitted', async () => {
const sportTsv = createTsvFileContent(['sport_id', 'name'], [['1', 'Soccer']]);
const teamTsv = createTsvFileContent(['team_id', 'sport_id', 'name'], [['1', '1', 'Team A']]);
- await app
+ const submitResponse = await app
.post(`/category/${categoryId}/files?organization=testOrg`)
.attach('files', sportTsv, 'sport.tsv')
.attach('files', teamTsv, 'team.tsv');
await pendingAsyncWork;
- const submission = await lyricProvider.repositories.submission.getActiveSubmissionDetails({
- categoryId,
- username: '',
- organization: 'testOrg',
- });
-
- expect(submission).to.exist;
- expect(submission!.data.inserts).to.have.property('sport');
- expect(submission!.data.inserts!['sport'].records).to.have.length(1);
- expect(submission!.data.inserts!['sport'].records[0]).to.include({ sport_id: '1', name: 'Soccer' });
- expect(submission!.data.inserts).to.have.property('team');
- expect(submission!.data.inserts!['team'].records).to.have.length(1);
- expect(submission!.data.inserts!['team'].records[0]).to.include({ team_id: '1', sport_id: '1', name: 'Team A' });
+ const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId(
+ submitResponse.body.submissionId,
+ );
+
+ expect(submissionRecords).to.exist;
+ expect(submissionRecords.length).to.eq(2);
+ expect(submissionRecords.map((record) => record.entityName)).to.eql(['sport', 'team']);
+ expect(submissionRecords.map((record) => record.actionType)).to.eql(['INSERT', 'INSERT']);
+ expect(submissionRecords.map((record) => record.data)).to.eql([
+ { sport_id: '1', name: 'Soccer' },
+ { team_id: '1', sport_id: '1', name: 'Team A' },
+ ]);
});
it('should merge records from multiple files for the same entity into a single batch', async () => {
@@ -150,7 +152,7 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D
{ filename: 'sports_batch2.tsv', entity: 'sport' },
]);
- await app
+ const submitResponse = await app
.post(`/category/${categoryId}/files?organization=testOrg`)
.attach('files', batch1, 'sports_batch1.tsv')
.attach('files', batch2, 'sports_batch2.tsv')
@@ -158,15 +160,18 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D
await pendingAsyncWork;
- const submission = await lyricProvider.repositories.submission.getActiveSubmissionDetails({
- categoryId,
- username: '',
- organization: 'testOrg',
- });
-
- expect(submission).to.exist;
- expect(submission!.data.inserts).to.have.property('sport');
- expect(submission!.data.inserts!['sport'].records).to.have.length(2);
+ const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId(
+ submitResponse.body.submissionId,
+ );
+
+ expect(submissionRecords).to.exist;
+ expect(submissionRecords.length).to.eq(2);
+ expect(submissionRecords.map((record) => record.entityName)).to.eql(['sport', 'sport']);
+ expect(submissionRecords.map((record) => record.actionType)).to.eql(['INSERT', 'INSERT']);
+ expect(submissionRecords.map((record) => record.data)).to.eql([
+ { sport_id: '1', name: 'Soccer' },
+ { sport_id: '2', name: 'Basketball' },
+ ]);
});
it('should accumulate records across sequential submissions to the same active submission', async () => {
@@ -189,7 +194,9 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D
expect(resultFirstSubmission).to.exist;
expect(resultFirstSubmission!.status).to.equal('VALID');
- await app.post(`/category/${categoryId}/files?organization=${organization}`).attach('files', teamTsv, 'team.tsv');
+ const submitResponse = await app
+ .post(`/category/${categoryId}/files?organization=${organization}`)
+ .attach('files', teamTsv, 'team.tsv');
await pendingAsyncWork;
const resultFinalSubmission = await waitForSubmissionToStopValidating({
@@ -200,19 +207,20 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D
delayMs: 500,
});
- expect(resultFinalSubmission).to.exist;
- expect(resultFinalSubmission!.status).to.equal('VALID');
+ assertExists(resultFinalSubmission);
+ expect(resultFinalSubmission.status).to.equal('VALID');
- const submissionDetails = await lyricProvider.repositories.submission.getActiveSubmissionDetails({
- categoryId,
- username: '',
- organization,
- });
+ const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId(
+ submitResponse.body.submissionId,
+ );
- expect(submissionDetails).to.exist;
- expect(submissionDetails!.data.inserts).to.have.property('sport');
- expect(submissionDetails!.data.inserts!['sport'].records).to.have.length(1);
- expect(submissionDetails!.data.inserts).to.have.property('team');
- expect(submissionDetails!.data.inserts!['team'].records).to.have.length(1);
+ expect(submissionRecords).to.exist;
+ expect(submissionRecords.length).to.eq(2);
+ expect(submissionRecords.map((record) => record.entityName)).to.eql(['sport', 'team']);
+ expect(submissionRecords.map((record) => record.actionType)).to.eql(['INSERT', 'INSERT']);
+ expect(submissionRecords.map((record) => record.data)).to.eql([
+ { sport_id: '1', name: 'Soccer' },
+ { team_id: '1', sport_id: '1', name: 'Team A' },
+ ]);
});
});
diff --git a/packages/data-provider/test/unit/utils/result.spec.ts b/packages/data-provider/test/unit/utils/result.spec.ts
new file mode 100644
index 00000000..261551ee
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/result.spec.ts
@@ -0,0 +1,25 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import { failure, success } from '../../../src/utils/result.js';
+
+describe('Result', () => {
+ describe('success', () => {
+ it('should wrap data in a success result', () => {
+ const result = success({ id: 1, name: 'sport' });
+ expect(result).to.eql({ success: true, data: { id: 1, name: 'sport' } });
+ });
+ });
+
+ describe('failure', () => {
+ it('should wrap data in a failure result', () => {
+ const result = failure('something went wrong');
+ expect(result).to.eql({ success: false, data: 'something went wrong' });
+ });
+
+ it('should support a non-string failure data type', () => {
+ const result = failure({ code: 'NOT_FOUND' });
+ expect(result).to.eql({ success: false, data: { code: 'NOT_FOUND' } });
+ });
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/createSubmissionInsertRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/createSubmissionInsertRecords.spec.ts
new file mode 100644
index 00000000..ea6e310d
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/createSubmissionInsertRecords.spec.ts
@@ -0,0 +1,71 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
+import { createSubmissionInsertRecords } from '../../../../src/utils/submissionUtils.js';
+
+describe('createSubmissionInsertRecords', () => {
+ it('should map insert records to SubmissionInsertRecordWithEntityName, dropping non-insert records', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 8,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'elephant', color: 'gray' },
+ },
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' },
+ },
+ ];
+
+ const result = createSubmissionInsertRecords(submissionData);
+
+ expect(result).to.eql([
+ {
+ recordId: 8,
+ entityName: 'animals',
+ data: { name: 'elephant', color: 'gray' },
+ },
+ ]);
+ });
+
+ it('should return an empty array when there are no insert records', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ ];
+
+ const result = createSubmissionInsertRecords(submissionData);
+
+ expect(result).to.eql([]);
+ });
+
+ it('should return an empty array when given no records', () => {
+ expect(createSubmissionInsertRecords([])).to.eql([]);
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/createSubmissionUpdateRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/createSubmissionUpdateRecords.spec.ts
new file mode 100644
index 00000000..49b3f703
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/createSubmissionUpdateRecords.spec.ts
@@ -0,0 +1,71 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
+import { createSubmissionUpdateRecords } from '../../../../src/utils/submissionUtils.js';
+
+describe('createSubmissionUpdateRecords', () => {
+ it('should map update records to SubmissionUpdateRecordWithEntityName, dropping non-update records', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 8,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'elephant', color: 'gray' },
+ },
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' },
+ },
+ ];
+
+ const result = createSubmissionUpdateRecords(submissionData);
+
+ expect(result).to.eql([
+ {
+ recordId: 10,
+ entityName: 'animals',
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ ]);
+ });
+
+ it('should return an empty array when there are no update records', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 8,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'elephant', color: 'gray' },
+ },
+ ];
+
+ const result = createSubmissionUpdateRecords(submissionData);
+
+ expect(result).to.eql([]);
+ });
+
+ it('should return an empty array when given no records', () => {
+ expect(createSubmissionUpdateRecords([])).to.eql([]);
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/extractRecordIdsFromSubmissionErrors.spec.ts b/packages/data-provider/test/unit/utils/submission/extractRecordIdsFromSubmissionErrors.spec.ts
new file mode 100644
index 00000000..2c6bf626
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/extractRecordIdsFromSubmissionErrors.spec.ts
@@ -0,0 +1,30 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import { extractRecordIdsFromSubmissionErrors, type SubmissionErrors } from '../../../../src/utils/submissionUtils.js';
+
+describe('Submission Utils - Extract Record Ids From Submission Errors', () => {
+ it('returns an empty Set when there are no errors', () => {
+ const response = extractRecordIdsFromSubmissionErrors({});
+ expect(response.size).to.eq(0);
+ });
+
+ it('collects recordIds across every bucket and entity', () => {
+ const errors: SubmissionErrors = {
+ inserts: { plants: [{ recordId: 1, errors: [] }] },
+ updates: { animals: [{ recordId: 2, errors: [] }, { recordId: 3, errors: [] }] },
+ deletes: { animals: [{ recordId: 4, errors: [] }] },
+ };
+ const response = extractRecordIdsFromSubmissionErrors(errors);
+ expect([...response]).to.have.members([1, 2, 3, 4]);
+ });
+
+ it('deduplicates a recordId that appears more than once', () => {
+ const errors: SubmissionErrors = {
+ updates: { animals: [{ recordId: 5, errors: [] }] },
+ deletes: { animals: [{ recordId: 5, errors: [] }] },
+ };
+ const response = extractRecordIdsFromSubmissionErrors(errors);
+ expect([...response]).to.eql([5]);
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/filterDeletesFromUpdates.spec.ts b/packages/data-provider/test/unit/utils/submission/filterDeletesFromUpdates.spec.ts
deleted file mode 100644
index 7c4194c4..00000000
--- a/packages/data-provider/test/unit/utils/submission/filterDeletesFromUpdates.spec.ts
+++ /dev/null
@@ -1,136 +0,0 @@
-import { expect } from 'chai';
-import { describe, it } from 'mocha';
-
-import type { SubmissionDeleteData, SubmissionUpdateData } from '@overture-stack/lyric-data-model/models';
-
-import { filterDeletesFromUpdates } from '../../../../src/utils/submissionUtils.js';
-
-describe('Submission Utils - Remove conflicts on Submission with records to be delete', () => {
- it('should remove 1 matching record from the Delete records', () => {
- const submissionDeleteData: Record = {
- cars: [
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ],
- };
-
- const submissionUpdateData: Record = {
- cars: [{ systemId: 'AAA111', new: { name: 'lamborghini' }, old: { name: 'Lambo' } }],
- };
- const result = filterDeletesFromUpdates(submissionDeleteData, submissionUpdateData);
- expect(Object.keys(result)).to.eql(['cars']);
- expect(result['cars'].length).to.eq(1);
- expect(result['cars'][0]).to.eql({
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- });
- });
- it('should not remove any record if there is no matching Id', () => {
- const submissionDeleteData: Record = {
- cars: [
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ],
- };
-
- const submissionUpdateData: Record = {
- cars: [{ systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } }],
- };
- const result = filterDeletesFromUpdates(submissionDeleteData, submissionUpdateData);
- expect(Object.keys(result)).to.eql(['cars']);
- expect(result['cars'].length).to.eq(2);
- expect(result['cars']).to.eql([
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ]);
- });
- it('should not remove any record if there is no matching entity name', () => {
- const submissionDeleteData: Record = {
- cars: [
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ],
- };
-
- const submissionUpdateData: Record = {
- food: [{ systemId: 'PTT123', new: { name: 'Potato' }, old: { name: 'Tomato' } }],
- };
- const result = filterDeletesFromUpdates(submissionDeleteData, submissionUpdateData);
- expect(Object.keys(result)).to.eql(['cars']);
- expect(result['cars'].length).to.eq(2);
- expect(result['cars']).to.eql([
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ]);
- });
- it('should return empty object when passing an empty object', () => {
- const submissionUpdateData: Record = {
- cars: [{ systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } }],
- };
- const result = filterDeletesFromUpdates({}, submissionUpdateData);
- expect(Object.keys(result).length).to.eq(0);
- });
-});
diff --git a/packages/data-provider/test/unit/utils/submission/filterUpdatesFromDeletes.spec.ts b/packages/data-provider/test/unit/utils/submission/filterUpdatesFromDeletes.spec.ts
deleted file mode 100644
index 883c2807..00000000
--- a/packages/data-provider/test/unit/utils/submission/filterUpdatesFromDeletes.spec.ts
+++ /dev/null
@@ -1,128 +0,0 @@
-import { expect } from 'chai';
-import { describe, it } from 'mocha';
-
-import type { SubmissionDeleteData, SubmissionUpdateData } from '@overture-stack/lyric-data-model/models';
-
-import { filterUpdatesFromDeletes } from '../../../../src/utils/submissionUtils.js';
-
-describe('Submission Utils - Remove conflicts on Submission with records to be updated', () => {
- it('should remove 1 matching record from the updates records', () => {
- const submissionUpdateData: Record = {
- cars: [
- { systemId: 'AAA111', new: { name: 'lamborghini Huracan' }, old: { name: 'Lambo' } },
- { systemId: 'BBB222', new: { name: 'Volkswagen Beettle' }, old: { name: 'Beettle' } },
- ],
- };
-
- const submissionDeleteData: Record = {
- cars: [
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ],
- };
- const result = filterUpdatesFromDeletes(submissionUpdateData, submissionDeleteData);
- expect(Object.keys(result)).to.eql(['cars']);
- expect(result['cars'].length).to.eq(1);
- expect(result['cars'][0]).to.eql({
- systemId: 'BBB222',
- new: { name: 'Volkswagen Beettle' },
- old: { name: 'Beettle' },
- });
- });
- it('should not remove any record if there is no matching Id', () => {
- const submissionUpdateData: Record = {
- cars: [
- { systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } },
- { systemId: 'DDDD444', new: { name: 'Audi' }, old: { name: 'Q5' } },
- ],
- };
-
- const submissionDeleteData: Record = {
- cars: [
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ],
- };
-
- const result = filterUpdatesFromDeletes(submissionUpdateData, submissionDeleteData);
- expect(Object.keys(result)).to.eql(['cars']);
- expect(result['cars'].length).to.eq(2);
- expect(result['cars']).to.eql([
- { systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } },
- { systemId: 'DDDD444', new: { name: 'Audi' }, old: { name: 'Q5' } },
- ]);
- });
- it('should not remove any record if there is no matching entity name', () => {
- const submissionUpdateData: Record = {
- food: [
- { systemId: 'PTT123', new: { name: 'Potato' }, old: { name: 'Tomato' } },
- { systemId: 'SPNCH889', new: { name: 'Spinach' }, old: { name: 'Lettuce' } },
- ],
- };
-
- const submissionDeleteData: Record = {
- cars: [
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ],
- };
- const result = filterUpdatesFromDeletes(submissionUpdateData, submissionDeleteData);
- expect(Object.keys(result)).to.eql(['food']);
- expect(result['food'].length).to.eq(2);
- expect(result['food']).to.eql([
- { systemId: 'PTT123', new: { name: 'Potato' }, old: { name: 'Tomato' } },
- { systemId: 'SPNCH889', new: { name: 'Spinach' }, old: { name: 'Lettuce' } },
- ]);
- });
- it('should return empty object when passing an empty object', () => {
- const submissionDeleteData: Record = {
- cars: [
- {
- systemId: 'AAA111',
- data: { name: 'Lambo' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- {
- systemId: 'BBB222',
- data: { name: 'Beettle' },
- entityName: 'cars',
- isValid: true,
- organization: 'myCollection',
- },
- ],
- };
- const result = filterUpdatesFromDeletes({}, submissionDeleteData);
- expect(Object.keys(result).length).to.eq(0);
- });
-});
diff --git a/packages/data-provider/test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts b/packages/data-provider/test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts
new file mode 100644
index 00000000..a3a521b0
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts
@@ -0,0 +1,216 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
+import { findUpdateDeleteConflicts } from '../../../../src/utils/submissionUtils.js';
+
+describe('Submission Utils - Find Update/Delete Conflicts', () => {
+ it('returns no conflicts when there are no Submission records', () => {
+ const response = findUpdateDeleteConflicts([]);
+ expect(response).eql({});
+ });
+
+ it('returns no conflicts when UPDATE and DELETE records target different systemIds', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 11,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' },
+ },
+ ];
+ const response = findUpdateDeleteConflicts(submissionData);
+ expect(response).eql({});
+ });
+
+ it('returns no conflicts when the matching systemId belongs to a different entity', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'zookeepers',
+ id: 11,
+ fileId: 2,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'someone' }, isValid: true, organization: 'zoo' },
+ },
+ ];
+ const response = findUpdateDeleteConflicts(submissionData);
+ expect(response).eql({});
+ });
+
+ it('ignores INSERT records and does not treat them as part of a conflict', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 9,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'beaver', color: 'brown' },
+ },
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ ];
+ const response = findUpdateDeleteConflicts(submissionData);
+ expect(response).eql({});
+ });
+
+ it('flags both the UPDATE and DELETE record when they share a systemId in the same entity', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' },
+ },
+ ];
+ const response = findUpdateDeleteConflicts(submissionData);
+ expect(response).eql({
+ updates: {
+ animals: [
+ {
+ recordId: 10,
+ errors: [
+ {
+ reason: 'CONFLICTING_ACTION',
+ systemId: 'TGR1425',
+ conflictingActionType: 'DELETE',
+ message:
+ "Record with systemId 'TGR1425' has both an UPDATE and a DELETE staged in the same Active Submission",
+ },
+ ],
+ },
+ ],
+ },
+ deletes: {
+ animals: [
+ {
+ recordId: 12,
+ errors: [
+ {
+ reason: 'CONFLICTING_ACTION',
+ systemId: 'TGR1425',
+ conflictingActionType: 'UPDATE',
+ message:
+ "Record with systemId 'TGR1425' has both an UPDATE and a DELETE staged in the same Active Submission",
+ },
+ ],
+ },
+ ],
+ },
+ });
+ });
+
+ it('flags every UPDATE and DELETE row when more than one row exists for the same systemId', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 20,
+ fileId: 2,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { weight: '200kg' }, old: { weight: '190kg' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' },
+ },
+ ];
+ const response = findUpdateDeleteConflicts(submissionData);
+ expect(response.updates?.['animals']?.map((record) => record.recordId)).to.have.members([10, 20]);
+ expect(response.deletes?.['animals']?.map((record) => record.recordId)).to.eql([12]);
+ });
+
+ it('only flags the entities/systemIds that actually conflict, leaving others untouched', () => {
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' },
+ },
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 11,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'BR8912', new: { color: 'brown' }, old: { color: 'black' } },
+ },
+ ];
+ const response = findUpdateDeleteConflicts(submissionData);
+ expect(Object.keys(response.updates ?? {})).to.eql(['animals']);
+ expect(response.updates?.['animals']?.map((record) => record.recordId)).to.eql([10]);
+ expect(response.deletes?.['animals']?.map((record) => record.recordId)).to.eql([12]);
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts b/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts
index 28097c56..700846dd 100644
--- a/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts
+++ b/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts
@@ -5,6 +5,7 @@ import type { DictionaryValidationError, TestResult } from '@overture-stack/lect
import { groupSchemaErrorsByEntity } from '../../../../src/utils/submissionUtils.js';
import { type DataRecordReference, MERGE_REFERENCE_TYPE } from '../../../../src/utils/types.js';
+import { assertExists } from '../../../assertions.js';
describe('Submission Utils - Group validation errors by entity', () => {
it('retuns empty object when there is no data being processed', () => {
@@ -71,7 +72,7 @@ describe('Submission Utils - Group validation errors by entity', () => {
{
dataRecord: { title: 'XYZ123' },
reference: {
- index: 12,
+ recordId: 12,
submissionId: 23,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
},
@@ -79,7 +80,7 @@ describe('Submission Utils - Group validation errors by entity', () => {
{
dataRecord: { sex_at_birth: 'Homme' },
reference: {
- index: 12,
+ recordId: 13,
submissionId: 23,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
},
@@ -89,11 +90,13 @@ describe('Submission Utils - Group validation errors by entity', () => {
const response = groupSchemaErrorsByEntity({ resultValidation, dataValidated });
expect(Object.keys(response)).to.eql(['inserts']);
+ assertExists(response['inserts']);
expect(Object.keys(response['inserts'])).to.eql(['sports']);
+ assertExists(response['inserts']['sports']);
expect(response['inserts']['sports'].length).to.eq(2);
expect(response['inserts']['sports']).to.eql([
- { fieldName: 'systemId', reason: 'UNRECOGNIZED_FIELD', fieldValue: '', index: 12 },
- { fieldName: 'sex_at_birth', reason: 'UNRECOGNIZED_FIELD', fieldValue: 'Homme', index: 12 },
+ { errors: [{ fieldName: 'systemId', reason: 'UNRECOGNIZED_FIELD', fieldValue: '' }], recordId: 12 },
+ { errors: [{ fieldName: 'sex_at_birth', reason: 'UNRECOGNIZED_FIELD', fieldValue: 'Homme' }], recordId: 13 },
]);
});
it('retuns errors found on the Submission updates', () => {
@@ -123,7 +126,7 @@ describe('Submission Utils - Group validation errors by entity', () => {
{
dataRecord: { title: 'XYZ123' },
reference: {
- index: 12,
+ recordId: 12,
submissionId: 23,
type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA,
},
@@ -131,7 +134,7 @@ describe('Submission Utils - Group validation errors by entity', () => {
{
dataRecord: { sex_at_birth: 'Homme' },
reference: {
- index: 12,
+ recordId: 13,
submissionId: 23,
type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA,
},
@@ -141,22 +144,32 @@ describe('Submission Utils - Group validation errors by entity', () => {
const response = groupSchemaErrorsByEntity({ resultValidation, dataValidated });
expect(Object.keys(response)).to.eql(['updates']);
+ assertExists(response['updates']);
expect(Object.keys(response['updates'])).to.eql(['sports']);
+ assertExists(response['updates']['sports']);
expect(response['updates']['sports'].length).to.eq(2);
expect(response['updates']['sports']).to.eql([
{
- errors: [],
- index: 12,
- reason: 'INVALID_BY_RESTRICTION',
- fieldName: 'systemId',
- fieldValue: '',
+ errors: [
+ {
+ reason: 'INVALID_BY_RESTRICTION',
+ fieldName: 'systemId',
+ fieldValue: '',
+ errors: [],
+ },
+ ],
+ recordId: 12,
},
{
- errors: [],
- index: 12,
- reason: 'INVALID_BY_RESTRICTION',
- fieldName: 'sex_at_birth',
- fieldValue: '',
+ errors: [
+ {
+ reason: 'INVALID_BY_RESTRICTION',
+ fieldName: 'sex_at_birth',
+ fieldValue: '',
+ errors: [],
+ },
+ ],
+ recordId: 13,
},
]);
});
diff --git a/packages/data-provider/test/unit/utils/submission/isDeleteSubmissionRecord.spec.ts b/packages/data-provider/test/unit/utils/submission/isDeleteSubmissionRecord.spec.ts
new file mode 100644
index 00000000..dd960bb6
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/isDeleteSubmissionRecord.spec.ts
@@ -0,0 +1,49 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
+import { isDeleteSubmissionRecord } from '../../../../src/utils/submissionUtils.js';
+
+describe('isDeleteSubmissionRecord', () => {
+ it('should return true when actionType is DELETE', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 1,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' },
+ };
+
+ expect(isDeleteSubmissionRecord(record)).to.be.true;
+ });
+
+ it('should return false when actionType is INSERT', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 2,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'elephant', color: 'gray' },
+ };
+
+ expect(isDeleteSubmissionRecord(record)).to.be.false;
+ });
+
+ it('should return false when actionType is UPDATE', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 3,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ };
+
+ expect(isDeleteSubmissionRecord(record)).to.be.false;
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/isInsertSubmissionRecord.spec.ts b/packages/data-provider/test/unit/utils/submission/isInsertSubmissionRecord.spec.ts
new file mode 100644
index 00000000..effe8dc4
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/isInsertSubmissionRecord.spec.ts
@@ -0,0 +1,49 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
+import { isInsertSubmissionRecord } from '../../../../src/utils/submissionUtils.js';
+
+describe('isInsertSubmissionRecord', () => {
+ it('should return true when actionType is INSERT', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 1,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'elephant', color: 'gray' },
+ };
+
+ expect(isInsertSubmissionRecord(record)).to.be.true;
+ });
+
+ it('should return false when actionType is UPDATE', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 2,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ };
+
+ expect(isInsertSubmissionRecord(record)).to.be.false;
+ });
+
+ it('should return false when actionType is DELETE', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 3,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' },
+ };
+
+ expect(isInsertSubmissionRecord(record)).to.be.false;
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/isUpdateSubmissionRecord.spec.ts b/packages/data-provider/test/unit/utils/submission/isUpdateSubmissionRecord.spec.ts
new file mode 100644
index 00000000..45a69342
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/isUpdateSubmissionRecord.spec.ts
@@ -0,0 +1,49 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
+import { isUpdateSubmissionRecord } from '../../../../src/utils/submissionUtils.js';
+
+describe('isUpdateSubmissionRecord', () => {
+ it('should return true when actionType is UPDATE', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 1,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ };
+
+ expect(isUpdateSubmissionRecord(record)).to.be.true;
+ });
+
+ it('should return false when actionType is INSERT', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 2,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'elephant', color: 'gray' },
+ };
+
+ expect(isUpdateSubmissionRecord(record)).to.be.false;
+ });
+
+ it('should return false when actionType is DELETE', () => {
+ const record: SubmissionRecordWithEntityName = {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 3,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' },
+ };
+
+ expect(isUpdateSubmissionRecord(record)).to.be.false;
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts b/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts
index 94773ff4..dfe2c30b 100644
--- a/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts
+++ b/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts
@@ -1,28 +1,32 @@
import { expect } from 'chai';
import { describe, it } from 'mocha';
-import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models';
-
+import type { SubmissionInsertRecordWithEntityName } from '../../../../index.js';
import { mapInsertDataToRecordReferences } from '../../../../src/utils/submissionUtils.js';
import { MERGE_REFERENCE_TYPE } from '../../../../src/utils/types.js';
+import { assertExists } from '../../../assertions.js';
describe('Submission Utils - Transforms inserts from the Submission object into a Record grouped by entityName', () => {
it('should return an object grouped by entity name with 2 records', () => {
- const submissionInsertData: SubmissionInsertData = {
- batchName: 'cars.tsv',
- records: [
- {
+ const insertDataEntity: SubmissionInsertRecordWithEntityName[] = [
+ {
+ data: {
name: 'Lamborghini Murcielago',
},
- {
+ entityName: 'cars',
+ recordId: 100,
+ },
+ {
+ data: {
name: 'Lamborghini Gallardo',
},
- ],
- };
+ entityName: 'cars',
+ recordId: 101,
+ },
+ ];
- const response = mapInsertDataToRecordReferences(100, { cars: submissionInsertData });
- expect(Object.keys(response)).to.eql(['cars']);
- expect(response['cars'].length).to.eq(2);
+ const response = mapInsertDataToRecordReferences(100, insertDataEntity);
+ expect(Object.keys(response).length).to.eq(1);
expect(response['cars']).to.eql([
{
dataRecord: {
@@ -31,7 +35,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into
reference: {
submissionId: 100,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
- index: 0,
+ recordId: 100,
},
},
{
@@ -41,41 +45,39 @@ describe('Submission Utils - Transforms inserts from the Submission object into
reference: {
submissionId: 100,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
- index: 1,
+ recordId: 101,
},
},
]);
});
- it('should return 2 objects grouped by entity names with 2 records each one', () => {
- const submissionInsertDataCars: SubmissionInsertData = {
- batchName: 'cars.tsv',
- records: [
- {
- name: 'Lamborghini Murcielago',
- },
- {
- name: 'Lamborghini Gallardo',
- },
- ],
- };
-
- const submissionInsertDataAnimals: SubmissionInsertData = {
- batchName: 'animals.tsv',
- records: [
- {
- name: 'Cat',
- },
- {
- name: 'Dog',
- },
- ],
- };
+ it('should return an array of 4 record references', () => {
+ const submissionInsertRecords: SubmissionInsertRecordWithEntityName[] = [
+ {
+ data: { name: 'Lamborghini Murcielago' },
+ entityName: 'cars',
+ recordId: 100,
+ },
+ {
+ data: { name: 'Lamborghini Gallardo' },
+ entityName: 'cars',
+ recordId: 101,
+ },
+ {
+ data: { name: 'Cat' },
+ entityName: 'animals',
+ recordId: 102,
+ },
+ {
+ data: { name: 'Dog' },
+ entityName: 'animals',
+ recordId: 103,
+ },
+ ];
- const response = mapInsertDataToRecordReferences(100, {
- cars: submissionInsertDataCars,
- animals: submissionInsertDataAnimals,
- });
- expect(Object.keys(response)).to.eql(['cars', 'animals']);
+ const response = mapInsertDataToRecordReferences(100, submissionInsertRecords);
+ expect(Object.keys(response).length).to.eq(2);
+ assertExists(response['cars']);
+ assertExists(response['animals']);
expect(response['cars'].length).to.eq(2);
expect(response['animals'].length).to.eq(2);
expect(response['cars']).to.eql([
@@ -86,7 +88,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into
reference: {
submissionId: 100,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
- index: 0,
+ recordId: 100,
},
},
{
@@ -96,7 +98,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into
reference: {
submissionId: 100,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
- index: 1,
+ recordId: 101,
},
},
]);
@@ -108,7 +110,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into
reference: {
submissionId: 100,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
- index: 0,
+ recordId: 102,
},
},
{
@@ -118,35 +120,15 @@ describe('Submission Utils - Transforms inserts from the Submission object into
reference: {
submissionId: 100,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
- index: 1,
+ recordId: 103,
},
},
]);
});
- it('should return an objects grouped by entity names with zero records', () => {
- const submissionInsertDataFruits: SubmissionInsertData = {
- batchName: 'fruit.tsv',
- records: [],
- };
-
- const response = mapInsertDataToRecordReferences(101, {
- fruit: submissionInsertDataFruits,
- });
- expect(Object.keys(response)).to.eql(['fruit']);
- expect(response['fruit'].length).to.eq(0);
- expect(response['fruit']).to.eql([]);
- });
- it('should return an empty object', () => {
- const submissionInsertDataFruits: SubmissionInsertData = {
- batchName: '',
- records: [],
- };
+ it('should return an empty array', () => {
+ const submissionInsertRecords: SubmissionInsertRecordWithEntityName[] = [];
- const response = mapInsertDataToRecordReferences(103, {
- '': submissionInsertDataFruits,
- });
- expect(Object.keys(response)).to.eql(['']);
- expect(response[''].length).to.eq(0);
- expect(response['']).to.eql([]);
+ const response = mapInsertDataToRecordReferences(103, submissionInsertRecords);
+ expect(Object.keys(response).length).to.eql(0);
});
});
diff --git a/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts
index 6ef32ab2..681318f6 100644
--- a/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts
+++ b/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts
@@ -1,20 +1,20 @@
import { expect } from 'chai';
import { describe, it } from 'mocha';
-import type { Submission, SubmissionData, SubmittedData } from '@overture-stack/lyric-data-model/models';
+import type { Submission, SubmittedData } from '@overture-stack/lyric-data-model/models';
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
import { mergeAndReferenceEntityData } from '../../../../src/utils/submissionUtils.js';
import { MERGE_REFERENCE_TYPE, SUBMISSION_STATUS } from '../../../../src/utils/types.js';
+import { assertExists } from '../../../assertions.js';
describe('Submission Utils - Combine Active Submission and the Submitted Data with reference', () => {
const todaysDate = new Date();
it('returns only SubmittedData data when Submission doesnt contain data', () => {
const originalSubmission: Submission = {
id: 2,
- data: {},
dictionaryId: 14,
dictionaryCategoryId: 20,
- errors: {},
organization: 'zoo',
status: SUBMISSION_STATUS.OPEN,
createdAt: todaysDate,
@@ -22,7 +22,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
updatedAt: null,
updatedBy: null,
};
- const submissionData: SubmissionData = {};
+ const submissionData: SubmissionRecordWithEntityName[] = [];
const submittedData: SubmittedData[] = [
{
id: 5,
@@ -47,6 +47,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
});
expect(Object.keys(response).length).to.eq(1);
expect(Object.keys(response)).to.eql(['animals']);
+ assertExists(response['animals']);
expect(response['animals'].length).eq(1);
expect(response['animals']).eql([
{
@@ -62,10 +63,8 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
it('returns combination of SubmittedData and Submission insert data', () => {
const originalSubmission: Submission = {
id: 2,
- data: {},
dictionaryId: 14,
dictionaryCategoryId: 20,
- errors: {},
organization: 'zoo',
status: SUBMISSION_STATUS.OPEN,
createdAt: todaysDate,
@@ -73,17 +72,26 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
updatedAt: null,
updatedBy: null,
};
- const submissionData: SubmissionData = {
- inserts: {
- animals: {
- batchName: 'animals.tsv',
- records: [
- { name: 'elephant', color: 'gray' },
- { name: 'beaver', color: 'brown' },
- ],
- },
+ const submissionData: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 8,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'elephant', color: 'gray' },
},
- };
+ {
+ actionType: 'INSERT',
+ entityName: 'animals',
+ id: 9,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { name: 'beaver', color: 'brown' },
+ },
+ ];
const submittedData: SubmittedData[] = [
{
id: 5,
@@ -106,8 +114,10 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
submissionData,
submittedData,
});
+
expect(Object.keys(response).length).to.eq(1);
expect(Object.keys(response)).to.eql(['animals']);
+ assertExists(response['animals']);
expect(response['animals'].length).eq(3);
expect(response['animals']).eql([
{
@@ -121,16 +131,16 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
{
dataRecord: { name: 'elephant', color: 'gray' },
reference: {
- index: 0,
- submissionId: 2,
+ recordId: 8,
+ submissionId: originalSubmission.id,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
},
},
{
dataRecord: { name: 'beaver', color: 'brown' },
reference: {
- index: 1,
- submissionId: 2,
+ recordId: 9,
+ submissionId: originalSubmission.id,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
},
},
@@ -139,10 +149,8 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
it('returns combination of SubmittedData and Submission update data', () => {
const originalSubmission: Submission = {
id: 2,
- data: {},
dictionaryId: 14,
dictionaryCategoryId: 20,
- errors: {},
organization: 'zoo',
status: SUBMISSION_STATUS.OPEN,
createdAt: todaysDate,
@@ -150,14 +158,26 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
updatedAt: null,
updatedBy: null,
};
- const submissionData: SubmissionData = {
- updates: {
- animals: [
- { systemId: 'TGR1425', old: { color: 'yellow' }, new: { color: 'orange' } },
- { systemId: 'BR8912', old: { color: 'black' }, new: { color: 'brown' } },
- ],
+ const submissionRecords: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
},
- };
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 11,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'BR8912', new: { color: 'brown' }, old: { color: 'black' } },
+ },
+ ];
const submittedData: SubmittedData[] = [
{
id: 5,
@@ -192,11 +212,12 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
];
const response = mergeAndReferenceEntityData({
submissionId: originalSubmission.id,
- submissionData,
+ submissionData: submissionRecords,
submittedData,
});
expect(Object.keys(response).length).to.eq(1);
expect(Object.keys(response)).to.eql(['animals']);
+ assertExists(response['animals']);
expect(response['animals'].length).eq(2);
expect(response['animals']).eql([
{
@@ -204,7 +225,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
reference: {
systemId: 'TGR1425',
submissionId: 2,
- index: 0,
+ recordId: 10,
type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA,
},
},
@@ -213,7 +234,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
reference: {
systemId: 'BR8912',
submissionId: 2,
- index: 1,
+ recordId: 11,
type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA,
},
},
@@ -222,10 +243,8 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
it('returns combination of SubmittedData and Submission delete data', () => {
const originalSubmission: Submission = {
id: 2,
- data: {},
dictionaryId: 14,
dictionaryCategoryId: 20,
- errors: {},
organization: 'zoo',
status: SUBMISSION_STATUS.OPEN,
createdAt: todaysDate,
@@ -233,26 +252,26 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
updatedAt: null,
updatedBy: null,
};
- const submissionData: SubmissionData = {
- deletes: {
- animals: [
- {
- systemId: 'TGR1425',
- data: { name: 'tiger', color: 'yellow' },
- entityName: 'animals',
- isValid: true,
- organization: 'zoo',
- },
- {
- systemId: 'BR8912',
- data: { name: 'bear', color: 'black' },
- entityName: 'animals',
- isValid: true,
- organization: 'zoo',
- },
- ],
+ const submissionRecords: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' },
},
- };
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 13,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' },
+ },
+ ];
const submittedData: SubmittedData[] = [
{
id: 5,
@@ -287,7 +306,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi
];
const response = mergeAndReferenceEntityData({
submissionId: originalSubmission.id,
- submissionData,
+ submissionData: submissionRecords,
submittedData,
});
expect(Object.keys(response).length).to.eq(0);
diff --git a/packages/data-provider/test/unit/utils/submission/mergeDeleteRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeDeleteRecords.spec.ts
deleted file mode 100644
index 3ea06c0d..00000000
--- a/packages/data-provider/test/unit/utils/submission/mergeDeleteRecords.spec.ts
+++ /dev/null
@@ -1,106 +0,0 @@
-import { expect } from 'chai';
-import { describe, it } from 'mocha';
-
-import type { SubmissionDeleteData } from '@overture-stack/lyric-data-model/models';
-
-import { mergeDeleteRecords } from '../../../../src/utils/submissionUtils.js';
-
-describe('Submission Utils - Merge multiple Submission delete records', () => {
- it('should return an object with 2 records within the same key', () => {
- const deletes1: Record = {
- food: [
- { data: { name: 'pizza' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PZ8900' },
- ],
- };
- const deletes2: Record = {
- food: [
- {
- data: { name: 'pizza' },
- entityName: 'hamburger',
- isValid: true,
- organization: 'kitchen',
- systemId: 'HG1234',
- },
- ],
- };
- const response = mergeDeleteRecords(deletes1, deletes2);
- expect(Object.keys(response).length).eq(1);
- expect(Object.keys(response)[0]).eql('food');
- expect(response['food'].length).eq(2);
- expect(response['food']).eql([
- { data: { name: 'pizza' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PZ8900' },
- {
- data: { name: 'pizza' },
- entityName: 'hamburger',
- isValid: true,
- organization: 'kitchen',
- systemId: 'HG1234',
- },
- ]);
- });
- it('should return an object with 2 records with different key', () => {
- const deletes1: Record = {
- food: [
- { data: { name: 'pizza' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PZ8900' },
- ],
- };
- const deletes2: Record = {
- animal: [
- {
- data: { name: 'lion' },
- entityName: 'animal',
- isValid: false,
- organization: 'zoo',
- systemId: 'LN5566',
- },
- ],
- };
- const response = mergeDeleteRecords(deletes1, deletes2);
- expect(Object.keys(response).length).eq(2);
- expect(Object.keys(response)).eql(['food', 'animal']);
- expect(response['food'].length).eq(1);
- expect(response['animal'].length).eq(1);
- expect(response['food'][0]).eql({
- data: { name: 'pizza' },
- entityName: 'food',
- isValid: true,
- organization: 'kitchen',
- systemId: 'PZ8900',
- });
- expect(response['animal'][0]).eql({
- data: { name: 'lion' },
- entityName: 'animal',
- isValid: false,
- organization: 'zoo',
- systemId: 'LN5566',
- });
- });
- it('should avoid duplication and return an object with 1 record', () => {
- const deletes1: Record = {
- food: [
- { data: { name: 'Paella' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PAE344' },
- ],
- };
- const deletes2: Record = {
- food: [
- { data: { name: 'Paella' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PAE344' },
- ],
- };
- const deletes3: Record = {
- food: [
- { data: { name: 'Paella' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PAE344' },
- ],
- };
- const response = mergeDeleteRecords(deletes1, deletes2, deletes3);
- expect(Object.keys(response).length).eq(1);
- expect(Object.keys(response)).eql(['food']);
- expect(response['food'].length).eq(1);
- expect(response['food'][0]).eql({
- data: { name: 'Paella' },
- entityName: 'food',
- isValid: true,
- organization: 'kitchen',
- systemId: 'PAE344',
- });
- });
-});
diff --git a/packages/data-provider/test/unit/utils/submission/mergeInsertsRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeInsertsRecords.spec.ts
deleted file mode 100644
index 9f1c6665..00000000
--- a/packages/data-provider/test/unit/utils/submission/mergeInsertsRecords.spec.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import { expect } from 'chai';
-import { describe, it } from 'mocha';
-
-import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models';
-
-import { mergeInsertsRecords } from '../../../../src/utils/submissionUtils.js';
-
-describe('Submission Utils - Merge multiple Submission insert records', () => {
- it('should return a record object with one key and merged array items', () => {
- const obj1: Record = {
- sports: { batchName: 'sports.tsv', records: [{ title: 'footbal' }] },
- };
- const obj2: Record = {
- sports: { batchName: 'sports', records: [{ title: 'basketball' }] },
- };
- const result = mergeInsertsRecords(obj1, obj2);
- expect(Object.keys(result).length).to.eq(1);
- expect(result['sports'].records.length).eql(2);
- });
-
- it('should return a record object with two different keys', () => {
- const obj1: Record = {
- food: { batchName: 'food.tsv', records: [{ title: 'apple' }] },
- };
- const obj2: Record = {
- sports: { batchName: 'sports', records: [{ title: 'basketball' }] },
- };
- const result = mergeInsertsRecords(obj1, obj2);
- expect(Object.keys(result).length).to.eq(2);
- expect(result['sports'].records.length).eql(1);
- expect(result['food'].records.length).eql(1);
- });
-
- it('should return a record object with one key and merged array items without duplication', () => {
- const obj1: Record = {
- sports: { batchName: 'sports.tsv', records: [{ title: 'snowboarding' }] },
- };
- const obj2: Record = {
- sports: { batchName: 'sports.csv', records: [{ title: 'snowboarding' }] },
- };
- const result = mergeInsertsRecords(obj1, obj2);
- expect(Object.keys(result).length).to.eq(1);
- expect(result['sports'].records.length).to.eq(1);
- expect(result['sports'].records[0]).eql({ title: 'snowboarding' });
- });
-});
diff --git a/packages/data-provider/test/unit/utils/submission/mergeSubmissionErrors.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeSubmissionErrors.spec.ts
new file mode 100644
index 00000000..3b500300
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/mergeSubmissionErrors.spec.ts
@@ -0,0 +1,49 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import { mergeSubmissionErrors, type SubmissionErrors } from '../../../../src/utils/submissionUtils.js';
+
+describe('Submission Utils - Merge Submission Errors', () => {
+ it('returns an empty object when both inputs are empty', () => {
+ const response = mergeSubmissionErrors({}, {});
+ expect(response).eql({});
+ });
+
+ it('returns the empty object without setting keys for buckets neither input has', () => {
+ const response = mergeSubmissionErrors({}, {});
+ expect(Object.keys(response)).to.eql([]);
+ });
+
+ it('returns the other input unchanged when one side is empty', () => {
+ const a: SubmissionErrors = { updates: { animals: [{ recordId: 1, errors: [] }] } };
+ const response = mergeSubmissionErrors(a, {});
+ expect(response).eql(a);
+ });
+
+ it('concatenates entity error arrays instead of overwriting them', () => {
+ const a: SubmissionErrors = {
+ updates: { animals: [{ recordId: 1, errors: [] }] },
+ };
+ const b: SubmissionErrors = {
+ updates: { animals: [{ recordId: 2, errors: [] }] },
+ };
+ const response = mergeSubmissionErrors(a, b);
+ expect(response.updates?.['animals']?.map((record) => record.recordId)).to.eql([1, 2]);
+ });
+
+ it('merges different buckets and different entities independently', () => {
+ const a: SubmissionErrors = {
+ updates: { animals: [{ recordId: 1, errors: [] }] },
+ deletes: { animals: [{ recordId: 2, errors: [] }] },
+ };
+ const b: SubmissionErrors = {
+ inserts: { plants: [{ recordId: 3, errors: [] }] },
+ updates: { plants: [{ recordId: 4, errors: [] }] },
+ };
+ const response = mergeSubmissionErrors(a, b);
+ expect(response.updates?.['animals']?.map((record) => record.recordId)).to.eql([1]);
+ expect(response.updates?.['plants']?.map((record) => record.recordId)).to.eql([4]);
+ expect(response.deletes?.['animals']?.map((record) => record.recordId)).to.eql([2]);
+ expect(response.inserts?.['plants']?.map((record) => record.recordId)).to.eql([3]);
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts b/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts
index 90cfbadc..417e33fe 100644
--- a/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts
+++ b/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts
@@ -44,21 +44,15 @@ describe('parseRecordsToInsert', () => {
const result = parseRecordsToInsert(records, inventoryDictionary);
// Fields properly formatted with the corresponding data type based on Dictionary
- const expectedResult: Record = {
- user: {
- batchName: 'user',
- records: [
- { id: 1, name: 'Alice', birthYear: 2000, hasAllergies: false },
- { id: 2, name: 'Pedro', birthYear: 1990, hasAllergies: true },
- ],
- },
- product: {
- batchName: 'product',
- records: [
- { id: 101, name: 'Laptop' },
- { id: 102, name: 'Monitor' },
- ],
- },
+ const expectedResult: Record = {
+ user: [
+ { id: 1, name: 'Alice', birthYear: 2000, hasAllergies: false },
+ { id: 2, name: 'Pedro', birthYear: 1990, hasAllergies: true },
+ ],
+ product: [
+ { id: 101, name: 'Laptop' },
+ { id: 102, name: 'Monitor' },
+ ],
};
expect(Object.keys(result).length).to.eql(2);
@@ -83,14 +77,11 @@ describe('parseRecordsToInsert', () => {
],
};
- const expectedResult: Record = {
- product: {
- batchName: 'product',
- records: [
- { id: 101, name: 'Laptop' },
- { id: 102, name: 'Monitor' },
- ],
- },
+ const expectedResult: Record = {
+ product: [
+ { id: 101, name: 'Laptop' },
+ { id: 102, name: 'Monitor' },
+ ],
};
const result = parseRecordsToInsert(records, inventoryDictionary);
diff --git a/packages/data-provider/test/unit/utils/submission/parseSubmissionResponse.spec.ts b/packages/data-provider/test/unit/utils/submission/parseSubmissionResponse.spec.ts
deleted file mode 100644
index dc1f96e8..00000000
--- a/packages/data-provider/test/unit/utils/submission/parseSubmissionResponse.spec.ts
+++ /dev/null
@@ -1,152 +0,0 @@
-import { expect } from 'chai';
-import { describe, it } from 'mocha';
-
-import type { DataRecord } from '@overture-stack/lectern-client';
-
-import { createBatchResponse } from '../../../../src/utils/submissionResponseParser.js';
-import { createSubmissionDetailsResponse } from '../../../../src/utils/submissionUtils.js';
-import { SUBMISSION_STATUS, type SubmissionDataDetailsRepositoryRecord } from '../../../../src/utils/types.js';
-
-describe('Submission Utils - Parse a Submisison object to a response format', () => {
- const todaysDate = new Date();
- it('return a Submission response with no data', () => {
- const submissionRepositoryRecord: SubmissionDataDetailsRepositoryRecord = {
- id: 2,
- data: {},
- dictionary: { name: 'books', version: '1' },
- dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: {},
- organization: 'oicr',
- status: SUBMISSION_STATUS.OPEN,
- createdAt: todaysDate,
- createdBy: 'me',
- updatedAt: null,
- updatedBy: null,
- };
- const response = createSubmissionDetailsResponse(submissionRepositoryRecord);
- expect(response).to.eql({
- id: 2,
- data: {},
- dictionary: { name: 'books', version: '1' },
- dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: {},
- organization: 'oicr',
- status: SUBMISSION_STATUS.OPEN,
- createdAt: todaysDate.toISOString(),
- createdBy: 'me',
- updatedAt: '',
- updatedBy: '',
- });
- });
- it('return a Submission response format with insert, update and delete data', () => {
- const submissionRepositoryRecord: SubmissionDataDetailsRepositoryRecord = {
- id: 2,
- data: {
- inserts: {
- books: {
- batchName: 'books.tsv',
- records: [
- {
- title: 'abc',
- },
- ],
- },
- },
- updates: {
- books: [
- {
- systemId: 'QWE987',
- new: { title: 'The Little Prince' },
- old: { title: 'the little prince' },
- },
- ],
- },
- deletes: {
- books: [
- {
- systemId: 'ZXC678',
- entityName: 'books',
- organization: 'oicr',
- isValid: true,
- data: { title: 'batman' },
- },
- ],
- },
- },
- dictionary: { name: 'books', version: '1.1' },
- dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: {},
- organization: 'oicr',
- status: SUBMISSION_STATUS.OPEN,
- createdAt: todaysDate,
- createdBy: 'me',
- updatedAt: null,
- updatedBy: null,
- };
- const response = createSubmissionDetailsResponse(submissionRepositoryRecord);
- expect(response).to.eql({
- id: 2,
- data: {
- inserts: {
- books: {
- batchName: 'books.tsv',
- records: [
- {
- title: 'abc',
- },
- ],
- },
- },
- updates: {
- books: [
- {
- systemId: 'QWE987',
- new: { title: 'The Little Prince' },
- old: { title: 'the little prince' },
- },
- ],
- },
- deletes: {
- books: [
- {
- systemId: 'ZXC678',
- entityName: 'books',
- organization: 'oicr',
- isValid: true,
- data: { title: 'batman' },
- },
- ],
- },
- },
- dictionary: { name: 'books', version: '1.1' },
- dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: {},
- organization: 'oicr',
- status: SUBMISSION_STATUS.OPEN,
- createdAt: todaysDate.toISOString(),
- createdBy: 'me',
- updatedAt: '',
- updatedBy: '',
- });
- });
-
- describe('Create Submission Insert Response', () => {
- it('should return a Submission Insert Response with records', () => {
- const records: DataRecord[] = [
- { id: 1, name: 'ABC' },
- { id: 2, name: 'XYZ' },
- ];
- const result = createBatchResponse('sample', records);
- expect(result.batchName).eql('sample');
- expect(result.records.length).eql(2);
- expect(result.records).eql([
- { id: 1, name: 'ABC' },
- { id: 2, name: 'XYZ' },
- ]);
- });
- it('should return a Submission Insert Response with no records', () => {
- const result = createBatchResponse('sample', []);
- expect(result).eql({ batchName: 'sample', records: [] });
- });
- });
-});
diff --git a/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts b/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts
index fc18301a..2adb27ea 100644
--- a/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts
+++ b/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts
@@ -1,18 +1,20 @@
import { expect } from 'chai';
import { describe, it } from 'mocha';
-import { createSubmissionSummaryResponse } from '../../../../src/utils/submissionUtils.js';
-import { SUBMISSION_STATUS, type SubmissionDataSummaryRepositoryRecord } from '../../../../src/utils/types.js';
+import { createSubmissionSummaryResponse } from '../../../../src/utils/submissionResponseParser.js';
+import { SUBMISSION_STATUS, type SubmissionSummary } from '../../../../src/utils/types.js';
describe('Submission Utils - Parse a Submission object to a Summary of the Active Submission', () => {
const todaysDate = new Date();
it('should return a Summary without any data ', () => {
- const submissionDataSummaryRepositoryRecord: SubmissionDataSummaryRepositoryRecord = {
+ const submissionDataSummaryRepositoryRecord: SubmissionSummary = {
id: 4,
- data: {},
+ data: {
+ totalRecords: 0,
+ errors: 0,
+ },
dictionary: { name: 'books', version: '1' },
dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: null,
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
createdAt: todaysDate,
@@ -24,11 +26,11 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ
expect(response).to.eql({
id: 4,
data: {
- total: 0,
+ totalRecords: 0,
+ errors: 0,
},
dictionary: { name: 'books', version: '1' },
dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: { total: 0 },
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
createdAt: todaysDate.toISOString(),
@@ -38,29 +40,38 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ
});
});
it('should return a Summary with insert, update and delete data ', () => {
- const submissionDataSummaryRepositoryRecord: SubmissionDataSummaryRepositoryRecord = {
+ const submissionDataSummaryRepositoryRecord: SubmissionSummary = {
id: 3,
data: {
inserts: {
- books: {
- batchName: 'books.tsv',
- recordsCount: 1,
- },
+ books: [
+ {
+ batchName: 'books.tsv',
+ recordsCount: 1,
+ errors: 0,
+ },
+ ],
},
updates: {
- books: {
- recordsCount: 1,
- },
+ books: [
+ {
+ batchName: 'books.tsv',
+ recordsCount: 1,
+ errors: 0,
+ },
+ ],
},
deletes: {
books: {
recordsCount: 1,
+ errors: 0,
},
},
+ totalRecords: 3,
+ errors: 0,
},
dictionary: { name: 'books', version: '1' },
dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
createdAt: todaysDate,
@@ -73,26 +84,34 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ
id: 3,
data: {
inserts: {
- books: {
- batchName: 'books.tsv',
- recordsCount: 1,
- },
+ books: [
+ {
+ batchName: 'books.tsv',
+ recordsCount: 1,
+ errors: 0,
+ },
+ ],
},
updates: {
- books: {
- recordsCount: 1,
- },
+ books: [
+ {
+ batchName: 'books.tsv',
+ recordsCount: 1,
+ errors: 0,
+ },
+ ],
},
deletes: {
books: {
recordsCount: 1,
+ errors: 0,
},
},
- total: 3,
+ totalRecords: 3,
+ errors: 0,
},
dictionary: { name: 'books', version: '1' },
dictionaryCategory: { name: 'favorite books', id: 1 },
- errors: { total: 0 },
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
createdAt: todaysDate.toISOString(),
diff --git a/packages/data-provider/test/unit/utils/submission/removeItemsFromSubmission.spec.ts b/packages/data-provider/test/unit/utils/submission/removeItemsFromSubmission.spec.ts
deleted file mode 100644
index b3eb0cb8..00000000
--- a/packages/data-provider/test/unit/utils/submission/removeItemsFromSubmission.spec.ts
+++ /dev/null
@@ -1,392 +0,0 @@
-import { expect } from 'chai';
-import { describe, it } from 'mocha';
-
-import type { SubmissionData } from '@overture-stack/lyric-data-model/models';
-
-import { removeItemsFromSubmission } from '../../../../src/utils/submissionUtils.js';
-import { SUBMISSION_ACTION_TYPE, type SubmissionActionType } from '../../../../src/utils/types.js';
-
-describe('Submission Utils - Removes items from submission based on filter parameters', () => {
- const insertDataSubmission: SubmissionData = {
- inserts: {
- sports: {
- batchName: 'sports.tsv',
- records: [
- {
- name: 'Breakdance',
- },
- {
- name: 'Skateboarding',
- },
- ],
- },
- food: {
- batchName: 'food.xml',
- records: [
- {
- name: 'Poutine',
- },
- ],
- },
- },
- };
-
- const updateDataSubmission: SubmissionData = {
- updates: {
- sports: [
- {
- systemId: 'QWE987',
- new: { name: 'Basketball 3X3' },
- old: { name: 'Basketball' },
- },
- {
- systemId: 'SWI321',
- new: { name: 'Swimming' },
- old: { name: 'swiming' },
- },
- ],
- food: [
- {
- systemId: 'PTO456',
- new: { name: 'Potato' },
- old: { name: 'potahto' },
- },
- ],
- },
- };
-
- const deleteDataSubmission: SubmissionData = {
- deletes: {
- sports: [
- {
- systemId: 'ZXC678',
- entityName: 'sports',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Baseball' },
- },
- {
- systemId: 'SFT098',
- entityName: 'sports',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Softball' },
- },
- ],
- food: [
- {
- systemId: 'EGG789',
- entityName: 'food',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Eggpplant' },
- },
- ],
- },
- };
- it('should return an empty response when Submission is empty', () => {
- const submissionData: SubmissionData = {};
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS,
- entityName: 'sports',
- index: 0,
- };
- const response = removeItemsFromSubmission(submissionData, filter);
- expect(response).to.eql({});
- });
- it('should returm intact SubmissionData when filter doesnt find anything', () => {
- const fullSubmissionData: SubmissionData = {
- ...insertDataSubmission,
- ...updateDataSubmission,
- ...deleteDataSubmission,
- };
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS,
- entityName: 'animals', // item doesn't belong in the submission
- index: 0,
- };
- const response = removeItemsFromSubmission(fullSubmissionData, filter);
- expect(response).to.eql({
- inserts: {
- sports: {
- batchName: 'sports.tsv',
- records: [
- {
- name: 'Breakdance',
- },
- {
- name: 'Skateboarding',
- },
- ],
- },
- food: {
- batchName: 'food.xml',
- records: [
- {
- name: 'Poutine',
- },
- ],
- },
- },
- updates: {
- sports: [
- {
- systemId: 'QWE987',
- new: { name: 'Basketball 3X3' },
- old: { name: 'Basketball' },
- },
- {
- systemId: 'SWI321',
- new: { name: 'Swimming' },
- old: { name: 'swiming' },
- },
- ],
- food: [
- {
- systemId: 'PTO456',
- new: { name: 'Potato' },
- old: { name: 'potahto' },
- },
- ],
- },
- deletes: {
- sports: [
- {
- systemId: 'ZXC678',
- entityName: 'sports',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Baseball' },
- },
- {
- systemId: 'SFT098',
- entityName: 'sports',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Softball' },
- },
- ],
- food: [
- {
- systemId: 'EGG789',
- entityName: 'food',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Eggpplant' },
- },
- ],
- },
- });
- });
- it('should remove whole inserts object from SubmissionData', () => {
- const insertOnseSubmission: SubmissionData = {
- inserts: {
- sports: {
- batchName: 'sports.tsv',
- records: [
- {
- name: 'Snowboarding',
- },
- ],
- },
- },
- };
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS,
- entityName: 'sports',
- index: null,
- };
- const response = removeItemsFromSubmission(insertOnseSubmission, filter);
- expect(response).to.eql({});
- });
- it('should remove whole updates object from SubmissionData', () => {
- const updateOneSubmission: SubmissionData = {
- updates: {
- sports: [
- {
- systemId: 'QWE987',
- new: { name: 'Basketball 3X3' },
- old: { name: 'Basketball' },
- },
- ],
- },
- };
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.UPDATES,
- entityName: 'sports',
- index: null,
- };
- const response = removeItemsFromSubmission(updateOneSubmission, filter);
- expect(response).to.eql({});
- });
- it('should remove whole deletes object from SubmissionData', () => {
- const deleteOneSubmission: SubmissionData = {
- deletes: {
- sports: [
- {
- systemId: 'ZXC678',
- entityName: 'sports',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Baseball' },
- },
- ],
- },
- };
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.DELETES,
- entityName: 'sports',
- index: null,
- };
- const response = removeItemsFromSubmission(deleteOneSubmission, filter);
- expect(response).to.eql({});
- });
- it('should remove one item from inserts object on SubmissionData', () => {
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS,
- entityName: 'sports',
- index: 1,
- };
- const response = removeItemsFromSubmission(insertDataSubmission, filter);
- expect(response).to.eql({
- inserts: {
- sports: {
- batchName: 'sports.tsv',
- records: [
- {
- name: 'Breakdance',
- },
- ],
- },
- food: {
- batchName: 'food.xml',
- records: [
- {
- name: 'Poutine',
- },
- ],
- },
- },
- });
- });
- it('should remove one item from updates object on SubmissionData', () => {
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.UPDATES,
- entityName: 'sports',
- index: 1,
- };
- const response = removeItemsFromSubmission(updateDataSubmission, filter);
- expect(response).to.eql({
- updates: {
- sports: [
- {
- systemId: 'QWE987',
- new: { name: 'Basketball 3X3' },
- old: { name: 'Basketball' },
- },
- ],
- food: [
- {
- systemId: 'PTO456',
- new: { name: 'Potato' },
- old: { name: 'potahto' },
- },
- ],
- },
- });
- });
- it('should remove one item from deletes object on SubmissionData', () => {
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.DELETES,
- entityName: 'sports',
- index: 1,
- };
- const response = removeItemsFromSubmission(deleteDataSubmission, filter);
- expect(response).to.eql({
- deletes: {
- sports: [
- {
- systemId: 'ZXC678',
- entityName: 'sports',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Baseball' },
- },
- ],
- food: [
- {
- systemId: 'EGG789',
- entityName: 'food',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Eggpplant' },
- },
- ],
- },
- });
- });
- it('should remove inserts if no items are left on the records array', () => {
- const insertOnseSubmission: SubmissionData = {
- inserts: {
- sports: {
- batchName: 'sports.tsv',
- records: [
- {
- name: 'Snowboarding',
- },
- ],
- },
- },
- };
-
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS,
- entityName: 'sports',
- index: 0,
- };
- const response = removeItemsFromSubmission(insertOnseSubmission, filter);
- expect(response).to.eql({});
- });
- it('should remove updates if no items are left on the array', () => {
- const updateOneSubmission: SubmissionData = {
- updates: {
- sports: [
- {
- systemId: 'QWE987',
- new: { name: 'Basketball 3X3' },
- old: { name: 'Basketball' },
- },
- ],
- },
- };
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.UPDATES,
- entityName: 'sports',
- index: 0,
- };
- const response = removeItemsFromSubmission(updateOneSubmission, filter);
- expect(response).to.eql({});
- });
- it('should remove deletes if no items are left on the array', () => {
- const deleteOneSubmission: SubmissionData = {
- deletes: {
- sports: [
- {
- systemId: 'ZXC678',
- entityName: 'sports',
- organization: 'olimpics',
- isValid: true,
- data: { name: 'Baseball' },
- },
- ],
- },
- };
- const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = {
- actionType: SUBMISSION_ACTION_TYPE.Values.DELETES,
- entityName: 'sports',
- index: 0,
- };
- const response = removeItemsFromSubmission(deleteOneSubmission, filter);
- expect(response).to.eql({});
- });
-});
diff --git a/packages/data-provider/test/unit/utils/submission/resolveDeleteStagingConflicts.spec.ts b/packages/data-provider/test/unit/utils/submission/resolveDeleteStagingConflicts.spec.ts
new file mode 100644
index 00000000..215aba44
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submission/resolveDeleteStagingConflicts.spec.ts
@@ -0,0 +1,157 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { SubmissionDeleteData } from '@overture-stack/lyric-data-model/models';
+
+import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js';
+import { resolveDeleteStagingConflicts } from '../../../../src/utils/submissionUtils.js';
+
+const deleteRecord = (systemId: string): SubmissionDeleteData => ({
+ systemId,
+ data: { name: 'tiger', color: 'yellow' },
+ isValid: true,
+ organization: 'zoo',
+});
+
+describe('Submission Utils - Resolve Delete Staging Conflicts', () => {
+ it('keeps every record and reports no conflicts/duplicates when there are no existing records', () => {
+ const recordsToDeleteMap: Record = {
+ animals: [deleteRecord('TGR1425')],
+ };
+ const response = resolveDeleteStagingConflicts(recordsToDeleteMap, []);
+ expect(response).to.eql({
+ filteredRecordsToDeleteMap: recordsToDeleteMap,
+ conflictingSystemIds: [],
+ duplicateSystemIds: [],
+ });
+ });
+
+ it('reports a conflict and excludes the record when a pending UPDATE exists for the same entity+systemId', () => {
+ const recordsToDeleteMap: Record = {
+ animals: [deleteRecord('TGR1425')],
+ };
+ const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ ];
+ const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords);
+ expect(response).to.eql({
+ filteredRecordsToDeleteMap: {},
+ conflictingSystemIds: ['TGR1425'],
+ duplicateSystemIds: [],
+ });
+ });
+
+ it('reports a duplicate and excludes the record when a pending DELETE already exists for the same entity+systemId', () => {
+ const recordsToDeleteMap: Record = {
+ animals: [deleteRecord('TGR1425')],
+ };
+ const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: deleteRecord('TGR1425'),
+ },
+ ];
+ const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords);
+ expect(response).to.eql({
+ filteredRecordsToDeleteMap: {},
+ conflictingSystemIds: [],
+ duplicateSystemIds: ['TGR1425'],
+ });
+ });
+
+ it('does not treat a matching systemId in a different entity as a conflict or duplicate', () => {
+ const recordsToDeleteMap: Record = {
+ animals: [deleteRecord('TGR1425')],
+ };
+ const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'zookeepers',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { name: 'someone' }, old: { name: 'someone else' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'zookeepers',
+ id: 11,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: deleteRecord('TGR1425'),
+ },
+ ];
+ const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords);
+ expect(response).to.eql({
+ filteredRecordsToDeleteMap: recordsToDeleteMap,
+ conflictingSystemIds: [],
+ duplicateSystemIds: [],
+ });
+ });
+
+ it('keeps clean records while filtering out conflicting and duplicate ones within the same entity', () => {
+ const recordsToDeleteMap: Record = {
+ animals: [deleteRecord('TGR1425'), deleteRecord('BR8912'), deleteRecord('ZBR001')],
+ };
+ const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ {
+ actionType: 'DELETE',
+ entityName: 'animals',
+ id: 12,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: deleteRecord('BR8912'),
+ },
+ ];
+ const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords);
+ expect(response.conflictingSystemIds).to.eql(['TGR1425']);
+ expect(response.duplicateSystemIds).to.eql(['BR8912']);
+ expect(response.filteredRecordsToDeleteMap['animals']?.map((record) => record.systemId)).to.eql(['ZBR001']);
+ });
+
+ it('omits an entity entirely from the filtered map once every one of its records is filtered out', () => {
+ const recordsToDeleteMap: Record = {
+ animals: [deleteRecord('TGR1425')],
+ plants: [deleteRecord('OAK001')],
+ };
+ const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [
+ {
+ actionType: 'UPDATE',
+ entityName: 'animals',
+ id: 10,
+ fileId: 1,
+ state: 'RECEIVED',
+ errors: [],
+ data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } },
+ },
+ ];
+ const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords);
+ expect(Object.keys(response.filteredRecordsToDeleteMap)).to.eql(['plants']);
+ expect(response.filteredRecordsToDeleteMap['plants']?.map((record) => record.systemId)).to.eql(['OAK001']);
+ });
+});
diff --git a/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts b/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts
new file mode 100644
index 00000000..056e99bb
--- /dev/null
+++ b/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts
@@ -0,0 +1,71 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import type { RecordsSummaryRepository } from '../../../src/repository/submissionRecordsRepository.js';
+import { buildDataSummary } from '../../../src/utils/submissionResponseParser.js';
+
+describe('buildDataSummary', () => {
+ it('should return an empty summary when given no rows', () => {
+ const result = buildDataSummary([]);
+
+ expect(result).to.eql({
+ inserts: {},
+ updates: {},
+ deletes: {},
+ totalRecords: 0,
+ errors: 0,
+ });
+ });
+
+ it('should group insert and update rows by entity name and accumulate totals', () => {
+ const rows: RecordsSummaryRepository[] = [
+ { actionType: 'INSERT', entityName: 'sport', totalRecords: 3, batchName: 'batch1.tsv', errors: 0 },
+ { actionType: 'INSERT', entityName: 'sport', totalRecords: 2, batchName: 'batch2.tsv', errors: 1 },
+ { actionType: 'UPDATE', entityName: 'player', totalRecords: 5, batchName: 'batch3.tsv', errors: 0 },
+ ];
+
+ const result = buildDataSummary(rows);
+
+ expect(result).to.eql({
+ inserts: {
+ sport: [
+ { batchName: 'batch1.tsv', recordsCount: 3, errors: 0 },
+ { batchName: 'batch2.tsv', recordsCount: 2, errors: 1 },
+ ],
+ },
+ updates: {
+ player: [{ batchName: 'batch3.tsv', recordsCount: 5, errors: 0 }],
+ },
+ deletes: {},
+ totalRecords: 10,
+ errors: 1,
+ });
+ });
+
+ it('should aggregate delete rows for the same entity instead of listing them individually', () => {
+ const rows: RecordsSummaryRepository[] = [
+ { actionType: 'DELETE', entityName: 'sport', totalRecords: 2, batchName: 'batch1.tsv', errors: 0 },
+ { actionType: 'DELETE', entityName: 'sport', totalRecords: 1, batchName: 'batch2.tsv', errors: 1 },
+ ];
+
+ const result = buildDataSummary(rows);
+
+ expect(result.deletes).to.eql({
+ sport: { recordsCount: 3, errors: 1 },
+ });
+ expect(result.totalRecords).to.eq(3);
+ expect(result.errors).to.eq(1);
+ });
+
+ it('should default a missing batchName to an empty string', () => {
+ const rows: RecordsSummaryRepository[] = [
+ { actionType: 'INSERT', entityName: 'sport', totalRecords: 1, errors: 0 },
+ ];
+
+ const result = buildDataSummary(rows);
+
+ expect(result.inserts).to.eql({
+ sport: [{ batchName: '', recordsCount: 1, errors: 0 }],
+ });
+ });
+});