From 34acebc1f0e3fc17c2802c9d14f7954757cb4318 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 17 Jun 2026 15:21:11 -0400 Subject: [PATCH 01/20] Drizzle migration to remove data and errors --- packages/data-model/docs/schema.dbml | 6 +- .../0016_remove_submission_data.sql | 2 + .../migrations/meta/0016_snapshot.json | 920 ++++++++++++++++++ .../data-model/migrations/meta/_journal.json | 7 + 4 files changed, 931 insertions(+), 4 deletions(-) create mode 100644 packages/data-model/migrations/0016_remove_submission_data.sql create mode 100644 packages/data-model/migrations/meta/0016_snapshot.json diff --git a/packages/data-model/docs/schema.dbml b/packages/data-model/docs/schema.dbml index bb7ba9b9..f9f1f9b9 100644 --- a/packages/data-model/docs/schema.dbml +++ b/packages/data-model/docs/schema.dbml @@ -124,10 +124,8 @@ table submission_records { table submissions { id serial [pk, not null, increment] - data jsonb [not null] dictionary_category_id integer [not null] dictionary_id integer [not null] - errors jsonb organization varchar [not null] status submission_status [not null] created_at timestamp [default: `now()`] @@ -183,9 +181,9 @@ ref: dictionary_migration.to_dictionary_id - dictionaries.id ref: dictionary_migration.submission_id - submissions.id -ref: submission_files.submission_id - submissions.id +ref: submission_files.submission_id > submissions.id -ref: submission_records.file_id - submission_files.id +ref: submission_records.file_id > submission_files.id ref: submissions.dictionary_id - dictionaries.id diff --git a/packages/data-model/migrations/0016_remove_submission_data.sql b/packages/data-model/migrations/0016_remove_submission_data.sql new file mode 100644 index 00000000..a262dd9a --- /dev/null +++ b/packages/data-model/migrations/0016_remove_submission_data.sql @@ -0,0 +1,2 @@ +ALTER TABLE "submissions" DROP COLUMN IF EXISTS "data";--> statement-breakpoint +ALTER TABLE "submissions" DROP COLUMN IF EXISTS "errors"; \ No newline at end of file diff --git a/packages/data-model/migrations/meta/0016_snapshot.json b/packages/data-model/migrations/meta/0016_snapshot.json new file mode 100644 index 00000000..aa72d265 --- /dev/null +++ b/packages/data-model/migrations/meta/0016_snapshot.json @@ -0,0 +1,920 @@ +{ + "id": "1155508a-258f-4040-b54e-f8a1df1181a1", + "prevId": "ab9dad09-40e7-4df3-9b6d-57aa9ae809cf", + "version": "5", + "dialect": "pg", + "tables": { + "audit_submitted_data": { + "name": "audit_submitted_data", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action": { + "name": "action", + "type": "audit_action", + "primaryKey": false, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "data_diff": { + "name": "data_diff", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_valid_schema_id": { + "name": "last_valid_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "new_data_is_valid": { + "name": "new_data_is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "old_data_is_valid": { + "name": "old_data_is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "original_schema_id": { + "name": "original_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "system_id": { + "name": "system_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "audit_submitted_data_dictionary_index": { + "name": "audit_submitted_data_dictionary_index", + "columns": [ + "dictionary_category_id" + ], + "isUnique": false + }, + "audit_submitted_data_organization_index": { + "name": "audit_submitted_data_organization_index", + "columns": [ + "organization" + ], + "isUnique": false + }, + "audit_submitted_data_submission_index": { + "name": "audit_submitted_data_submission_index", + "columns": [ + "submission_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk": { + "name": "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk": { + "name": "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "last_valid_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_original_schema_id_dictionaries_id_fk": { + "name": "audit_submitted_data_original_schema_id_dictionaries_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "original_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_submission_id_submissions_id_fk": { + "name": "audit_submitted_data_submission_id_submissions_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dictionaries": { + "name": "dictionaries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dictionary": { + "name": "dictionary", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dictionary_categories": { + "name": "dictionary_categories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "active_dictionary_id": { + "name": "active_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "default_centric_entity": { + "name": "default_centric_entity", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "dictionary_categories_name_unique": { + "name": "dictionary_categories_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + } + }, + "dictionary_migration": { + "name": "dictionary_migration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "category_id": { + "name": "category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "from_dictionary_id": { + "name": "from_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "to_dictionary_id": { + "name": "to_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "migration_status", + "primaryKey": false, + "notNull": true + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "dictionary_migration_category_id_index": { + "name": "dictionary_migration_category_id_index", + "columns": [ + "category_id" + ], + "isUnique": false + }, + "dictionary_migration_from_dictionary_id_index": { + "name": "dictionary_migration_from_dictionary_id_index", + "columns": [ + "from_dictionary_id" + ], + "isUnique": false + }, + "dictionary_migration_to_dictionary_id_index": { + "name": "dictionary_migration_to_dictionary_id_index", + "columns": [ + "to_dictionary_id" + ], + "isUnique": false + }, + "dictionary_migration_submission_id_index": { + "name": "dictionary_migration_submission_id_index", + "columns": [ + "submission_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "dictionary_migration_category_id_dictionary_categories_id_fk": { + "name": "dictionary_migration_category_id_dictionary_categories_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_from_dictionary_id_dictionaries_id_fk": { + "name": "dictionary_migration_from_dictionary_id_dictionaries_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionaries", + "columnsFrom": [ + "from_dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_to_dictionary_id_dictionaries_id_fk": { + "name": "dictionary_migration_to_dictionary_id_dictionaries_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionaries", + "columnsFrom": [ + "to_dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_submission_id_submissions_id_fk": { + "name": "dictionary_migration_submission_id_submissions_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submission_files": { + "name": "submission_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "file_name": { + "name": "file_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "submission_files_submission_id_index": { + "name": "submission_files_submission_id_index", + "columns": [ + "submission_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submission_files_submission_id_submissions_id_fk": { + "name": "submission_files_submission_id_submissions_id_fk", + "tableFrom": "submission_files", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submission_records": { + "name": "submission_records", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "file_id": { + "name": "file_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "submission_record_type", + "primaryKey": false, + "notNull": true + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "submission_record_state", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "submission_records_file_id_index": { + "name": "submission_records_file_id_index", + "columns": [ + "file_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submission_records_file_id_submission_files_id_fk": { + "name": "submission_records_file_id_submission_files_id_fk", + "tableFrom": "submission_records", + "tableTo": "submission_files", + "columnsFrom": [ + "file_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submissions": { + "name": "submissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dictionary_id": { + "name": "dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "submission_status", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "submission_organization_index": { + "name": "submission_organization_index", + "columns": [ + "organization" + ], + "isUnique": false + }, + "submission_category_index": { + "name": "submission_category_index", + "columns": [ + "dictionary_category_id" + ], + "isUnique": false + }, + "submission_created_by_index": { + "name": "submission_created_by_index", + "columns": [ + "created_by" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submissions_dictionary_category_id_dictionary_categories_id_fk": { + "name": "submissions_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "submissions", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submissions_dictionary_id_dictionaries_id_fk": { + "name": "submissions_dictionary_id_dictionaries_id_fk", + "tableFrom": "submissions", + "tableTo": "dictionaries", + "columnsFrom": [ + "dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submitted_data": { + "name": "submitted_data", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_valid": { + "name": "is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "last_valid_schema_id": { + "name": "last_valid_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "original_schema_id": { + "name": "original_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "system_id": { + "name": "system_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "submitted_data_organization_index": { + "name": "submitted_data_organization_index", + "columns": [ + "organization" + ], + "isUnique": false + }, + "submitted_data_category_index": { + "name": "submitted_data_category_index", + "columns": [ + "dictionary_category_id" + ], + "isUnique": false + }, + "submitted_data_system_id_index": { + "name": "submitted_data_system_id_index", + "columns": [ + "system_id" + ], + "isUnique": false + }, + "submitted_data_entity_name_index": { + "name": "submitted_data_entity_name_index", + "columns": [ + "entity_name" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submitted_data_dictionary_category_id_dictionary_categories_id_fk": { + "name": "submitted_data_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submitted_data_last_valid_schema_id_dictionaries_id_fk": { + "name": "submitted_data_last_valid_schema_id_dictionaries_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "last_valid_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submitted_data_original_schema_id_dictionaries_id_fk": { + "name": "submitted_data_original_schema_id_dictionaries_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "original_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "submitted_data_system_id_unique": { + "name": "submitted_data_system_id_unique", + "nullsNotDistinct": false, + "columns": [ + "system_id" + ] + } + } + } + }, + "enums": { + "audit_action": { + "name": "audit_action", + "values": { + "UPDATE": "UPDATE", + "DELETE": "DELETE", + "MIGRATION": "MIGRATION" + } + }, + "migration_status": { + "name": "migration_status", + "values": { + "IN_PROGRESS": "IN_PROGRESS", + "COMPLETED": "COMPLETED", + "FAILED": "FAILED" + } + }, + "submission_record_state": { + "name": "submission_record_state", + "values": { + "RECEIVED": "RECEIVED", + "VALID": "VALID", + "INVALID": "INVALID" + } + }, + "submission_record_type": { + "name": "submission_record_type", + "values": { + "INSERT": "INSERT", + "UPDATE": "UPDATE", + "DELETE": "DELETE" + } + }, + "submission_status": { + "name": "submission_status", + "values": { + "OPEN": "OPEN", + "VALIDATING": "VALIDATING", + "VALID": "VALID", + "INVALID": "INVALID", + "CLOSED": "CLOSED", + "COMMITTING": "COMMITTING", + "COMMITTED": "COMMITTED" + } + } + }, + "schemas": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/data-model/migrations/meta/_journal.json b/packages/data-model/migrations/meta/_journal.json index 1e708e9e..a5cd0f59 100644 --- a/packages/data-model/migrations/meta/_journal.json +++ b/packages/data-model/migrations/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1781011323766, "tag": "0015_CUSTOM_migrate_submission_records", "breakpoints": true + }, + { + "idx": 16, + "version": "5", + "when": 1781210639932, + "tag": "0016_remove_submission_data", + "breakpoints": true } ] } From 54bd443d2c7750eb33d37471416337c48087c753 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 6 Aug 2026 15:08:07 -0400 Subject: [PATCH 02/20] refactoring submission files and records --- apps/server/swagger/submission-api.yml | 22 +- packages/data-model/docs/schema.dbml | 3 + .../migrations/0017_aspiring_leader.sql | 3 + .../migrations/meta/0017_snapshot.json | 946 ++++++++++++++++++ .../data-model/migrations/meta/_journal.json | 9 +- .../data-model/src/models/submission_files.ts | 9 +- .../src/models/submission_records.ts | 32 +- packages/data-model/src/models/submissions.ts | 60 +- .../src/controllers/submissionController.ts | 36 +- packages/data-provider/src/core/provider.ts | 4 + .../repository/activeSubmissionRepository.ts | 199 +--- .../repository/submissionFilesRepository.ts | 102 ++ .../repository/submissionRecordsRepository.ts | 347 +++++++ .../src/routers/submissionRouter.ts | 2 +- .../submission/submissionProcessor.ts | 297 +++--- .../services/submission/submissionService.ts | 197 ++-- .../services/submittedData/submmittedData.ts | 57 +- .../data-provider/src/utils/recordsParser.ts | 6 +- packages/data-provider/src/utils/schemas.ts | 31 +- .../src/utils/submissionResponseParser.ts | 191 +--- .../src/utils/submissionUtils.ts | 432 +++----- .../src/utils/submittedDataUtils.ts | 38 +- packages/data-provider/src/utils/types.ts | 82 +- .../src/workers/commitSubmissionWorker.ts | 106 +- 24 files changed, 2128 insertions(+), 1083 deletions(-) create mode 100644 packages/data-model/migrations/0017_aspiring_leader.sql create mode 100644 packages/data-model/migrations/meta/0017_snapshot.json create mode 100644 packages/data-provider/src/repository/submissionFilesRepository.ts create mode 100644 packages/data-provider/src/repository/submissionRecordsRepository.ts diff --git a/apps/server/swagger/submission-api.yml b/apps/server/swagger/submission-api.yml index ec520d3e..74941e1c 100644 --- a/apps/server/swagger/submission-api.yml +++ b/apps/server/swagger/submission-api.yml @@ -98,7 +98,7 @@ 503: $ref: '#/components/responses/ServiceUnavailableError' -/submission/{submissionId}/{actionType}: +/submission/{submissionId}/data: delete: summary: Clear Active Submission by entity name tags: @@ -109,24 +109,18 @@ type: string required: true description: The ID of the Submission - - name: actionType - in: path - required: true - schema: - type: string - enum: [inserts, updates, deletes] - description: Parameter to specify the type of record to remove from the Submission. Must be one of `inserts`, `updates`, or `deletes` (case insensitive) - - name: entityName + - name: recordId in: query - type: string - required: true - description: The name of the entity - - name: index + required: false + schema: + type: integer + description: An optional query parameter used to specify the record ID within the submission to be deleted.
Only one of `recordId` or `fileId` can be provided + - name: fileId in: query required: false schema: type: integer - description: An optional query parameter used to specify the index of the item within the submission type to be deleted.
If not provided all the items within the submission type will be deleted. + description: An optional query parameter used to specify the file ID within the submission to be deleted.
Only one of `recordId` or `fileId` can be provided responses: 200: description: Submission cleared successfully. Returns the current Active Submission diff --git a/packages/data-model/docs/schema.dbml b/packages/data-model/docs/schema.dbml index f9f1f9b9..5fe4cfeb 100644 --- a/packages/data-model/docs/schema.dbml +++ b/packages/data-model/docs/schema.dbml @@ -106,6 +106,7 @@ table submission_files { indexes { submission_id [name: 'submission_files_submission_id_index'] + (submission_id, entity_name, file_name) [name: 'submission_files_submission_entity_file_index'] } } @@ -119,6 +120,8 @@ table submission_records { indexes { file_id [name: 'submission_records_file_id_index'] + (file_id, action_type) [name: 'submission_records_file_id_action_type_index'] + (file_id, state, action_type) [name: 'submission_records_file_id_state_action_type_index'] } } diff --git a/packages/data-model/migrations/0017_aspiring_leader.sql b/packages/data-model/migrations/0017_aspiring_leader.sql new file mode 100644 index 00000000..724a02a2 --- /dev/null +++ b/packages/data-model/migrations/0017_aspiring_leader.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS "submission_files_submission_entity_file_index" ON "submission_files" ("submission_id","entity_name","file_name");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "submission_records_file_id_action_type_index" ON "submission_records" ("file_id","action_type");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "submission_records_file_id_state_action_type_index" ON "submission_records" ("file_id","state","action_type"); \ No newline at end of file diff --git a/packages/data-model/migrations/meta/0017_snapshot.json b/packages/data-model/migrations/meta/0017_snapshot.json new file mode 100644 index 00000000..bb06e629 --- /dev/null +++ b/packages/data-model/migrations/meta/0017_snapshot.json @@ -0,0 +1,946 @@ +{ + "id": "b363a43f-2207-44d0-9789-eaf16ea50064", + "prevId": "1155508a-258f-4040-b54e-f8a1df1181a1", + "version": "5", + "dialect": "pg", + "tables": { + "audit_submitted_data": { + "name": "audit_submitted_data", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action": { + "name": "action", + "type": "audit_action", + "primaryKey": false, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "data_diff": { + "name": "data_diff", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_valid_schema_id": { + "name": "last_valid_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "new_data_is_valid": { + "name": "new_data_is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "old_data_is_valid": { + "name": "old_data_is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "original_schema_id": { + "name": "original_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "system_id": { + "name": "system_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "audit_submitted_data_dictionary_index": { + "name": "audit_submitted_data_dictionary_index", + "columns": [ + "dictionary_category_id" + ], + "isUnique": false + }, + "audit_submitted_data_organization_index": { + "name": "audit_submitted_data_organization_index", + "columns": [ + "organization" + ], + "isUnique": false + }, + "audit_submitted_data_submission_index": { + "name": "audit_submitted_data_submission_index", + "columns": [ + "submission_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk": { + "name": "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk": { + "name": "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "last_valid_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_original_schema_id_dictionaries_id_fk": { + "name": "audit_submitted_data_original_schema_id_dictionaries_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "original_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "audit_submitted_data_submission_id_submissions_id_fk": { + "name": "audit_submitted_data_submission_id_submissions_id_fk", + "tableFrom": "audit_submitted_data", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dictionaries": { + "name": "dictionaries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dictionary": { + "name": "dictionary", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dictionary_categories": { + "name": "dictionary_categories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "active_dictionary_id": { + "name": "active_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "default_centric_entity": { + "name": "default_centric_entity", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "dictionary_categories_name_unique": { + "name": "dictionary_categories_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + } + }, + "dictionary_migration": { + "name": "dictionary_migration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "category_id": { + "name": "category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "from_dictionary_id": { + "name": "from_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "to_dictionary_id": { + "name": "to_dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "migration_status", + "primaryKey": false, + "notNull": true + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "dictionary_migration_category_id_index": { + "name": "dictionary_migration_category_id_index", + "columns": [ + "category_id" + ], + "isUnique": false + }, + "dictionary_migration_from_dictionary_id_index": { + "name": "dictionary_migration_from_dictionary_id_index", + "columns": [ + "from_dictionary_id" + ], + "isUnique": false + }, + "dictionary_migration_to_dictionary_id_index": { + "name": "dictionary_migration_to_dictionary_id_index", + "columns": [ + "to_dictionary_id" + ], + "isUnique": false + }, + "dictionary_migration_submission_id_index": { + "name": "dictionary_migration_submission_id_index", + "columns": [ + "submission_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "dictionary_migration_category_id_dictionary_categories_id_fk": { + "name": "dictionary_migration_category_id_dictionary_categories_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_from_dictionary_id_dictionaries_id_fk": { + "name": "dictionary_migration_from_dictionary_id_dictionaries_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionaries", + "columnsFrom": [ + "from_dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_to_dictionary_id_dictionaries_id_fk": { + "name": "dictionary_migration_to_dictionary_id_dictionaries_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "dictionaries", + "columnsFrom": [ + "to_dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dictionary_migration_submission_id_submissions_id_fk": { + "name": "dictionary_migration_submission_id_submissions_id_fk", + "tableFrom": "dictionary_migration", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submission_files": { + "name": "submission_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "submission_id": { + "name": "submission_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "file_name": { + "name": "file_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "submission_files_submission_id_index": { + "name": "submission_files_submission_id_index", + "columns": [ + "submission_id" + ], + "isUnique": false + }, + "submission_files_submission_entity_file_index": { + "name": "submission_files_submission_entity_file_index", + "columns": [ + "submission_id", + "entity_name", + "file_name" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submission_files_submission_id_submissions_id_fk": { + "name": "submission_files_submission_id_submissions_id_fk", + "tableFrom": "submission_files", + "tableTo": "submissions", + "columnsFrom": [ + "submission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submission_records": { + "name": "submission_records", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "file_id": { + "name": "file_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "submission_record_type", + "primaryKey": false, + "notNull": true + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "submission_record_state", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "submission_records_file_id_index": { + "name": "submission_records_file_id_index", + "columns": [ + "file_id" + ], + "isUnique": false + }, + "submission_records_file_id_action_type_index": { + "name": "submission_records_file_id_action_type_index", + "columns": [ + "file_id", + "action_type" + ], + "isUnique": false + }, + "submission_records_file_id_state_action_type_index": { + "name": "submission_records_file_id_state_action_type_index", + "columns": [ + "file_id", + "state", + "action_type" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submission_records_file_id_submission_files_id_fk": { + "name": "submission_records_file_id_submission_files_id_fk", + "tableFrom": "submission_records", + "tableTo": "submission_files", + "columnsFrom": [ + "file_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submissions": { + "name": "submissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dictionary_id": { + "name": "dictionary_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "submission_status", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "submission_organization_index": { + "name": "submission_organization_index", + "columns": [ + "organization" + ], + "isUnique": false + }, + "submission_category_index": { + "name": "submission_category_index", + "columns": [ + "dictionary_category_id" + ], + "isUnique": false + }, + "submission_created_by_index": { + "name": "submission_created_by_index", + "columns": [ + "created_by" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submissions_dictionary_category_id_dictionary_categories_id_fk": { + "name": "submissions_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "submissions", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submissions_dictionary_id_dictionaries_id_fk": { + "name": "submissions_dictionary_id_dictionaries_id_fk", + "tableFrom": "submissions", + "tableTo": "dictionaries", + "columnsFrom": [ + "dictionary_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "submitted_data": { + "name": "submitted_data", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "dictionary_category_id": { + "name": "dictionary_category_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "entity_name": { + "name": "entity_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_valid": { + "name": "is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "last_valid_schema_id": { + "name": "last_valid_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "organization": { + "name": "organization", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "original_schema_id": { + "name": "original_schema_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "system_id": { + "name": "system_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "varchar", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "submitted_data_organization_index": { + "name": "submitted_data_organization_index", + "columns": [ + "organization" + ], + "isUnique": false + }, + "submitted_data_category_index": { + "name": "submitted_data_category_index", + "columns": [ + "dictionary_category_id" + ], + "isUnique": false + }, + "submitted_data_system_id_index": { + "name": "submitted_data_system_id_index", + "columns": [ + "system_id" + ], + "isUnique": false + }, + "submitted_data_entity_name_index": { + "name": "submitted_data_entity_name_index", + "columns": [ + "entity_name" + ], + "isUnique": false + } + }, + "foreignKeys": { + "submitted_data_dictionary_category_id_dictionary_categories_id_fk": { + "name": "submitted_data_dictionary_category_id_dictionary_categories_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionary_categories", + "columnsFrom": [ + "dictionary_category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submitted_data_last_valid_schema_id_dictionaries_id_fk": { + "name": "submitted_data_last_valid_schema_id_dictionaries_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "last_valid_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "submitted_data_original_schema_id_dictionaries_id_fk": { + "name": "submitted_data_original_schema_id_dictionaries_id_fk", + "tableFrom": "submitted_data", + "tableTo": "dictionaries", + "columnsFrom": [ + "original_schema_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "submitted_data_system_id_unique": { + "name": "submitted_data_system_id_unique", + "nullsNotDistinct": false, + "columns": [ + "system_id" + ] + } + } + } + }, + "enums": { + "audit_action": { + "name": "audit_action", + "values": { + "UPDATE": "UPDATE", + "DELETE": "DELETE", + "MIGRATION": "MIGRATION" + } + }, + "migration_status": { + "name": "migration_status", + "values": { + "IN_PROGRESS": "IN_PROGRESS", + "COMPLETED": "COMPLETED", + "FAILED": "FAILED" + } + }, + "submission_record_state": { + "name": "submission_record_state", + "values": { + "RECEIVED": "RECEIVED", + "VALID": "VALID", + "INVALID": "INVALID" + } + }, + "submission_record_type": { + "name": "submission_record_type", + "values": { + "INSERT": "INSERT", + "UPDATE": "UPDATE", + "DELETE": "DELETE" + } + }, + "submission_status": { + "name": "submission_status", + "values": { + "OPEN": "OPEN", + "VALIDATING": "VALIDATING", + "VALID": "VALID", + "INVALID": "INVALID", + "CLOSED": "CLOSED", + "COMMITTING": "COMMITTING", + "COMMITTED": "COMMITTED" + } + } + }, + "schemas": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/data-model/migrations/meta/_journal.json b/packages/data-model/migrations/meta/_journal.json index a5cd0f59..d36e3ffb 100644 --- a/packages/data-model/migrations/meta/_journal.json +++ b/packages/data-model/migrations/meta/_journal.json @@ -120,6 +120,13 @@ "when": 1781210639932, "tag": "0016_remove_submission_data", "breakpoints": true + }, + { + "idx": 17, + "version": "5", + "when": 1783109751012, + "tag": "0017_aspiring_leader", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/packages/data-model/src/models/submission_files.ts b/packages/data-model/src/models/submission_files.ts index 9a1f2636..00a3b3fb 100644 --- a/packages/data-model/src/models/submission_files.ts +++ b/packages/data-model/src/models/submission_files.ts @@ -1,6 +1,7 @@ import { relations } from 'drizzle-orm'; import { index, integer, pgTable, serial, varchar } from 'drizzle-orm/pg-core'; +import { submissionRecords } from './submission_records.js'; import { submissions } from './submissions.js'; export const submissionFiles = pgTable( @@ -17,14 +18,20 @@ export const submissionFiles = pgTable( (table) => { return { submissionIndex: index('submission_files_submission_id_index').on(table.submissionId), + submissionEntityFileIndex: index('submission_files_submission_entity_file_index').on( + table.submissionId, + table.entityName, + table.fileName, + ), }; }, ); -export const submissionFileRelations = relations(submissionFiles, ({ one }) => ({ +export const submissionFileRelations = relations(submissionFiles, ({ one, many }) => ({ submission: one(submissions, { fields: [submissionFiles.submissionId], references: [submissions.id], }), + submissionRecords: many(submissionRecords), })); export type SubmissionFile = typeof submissionFiles.$inferSelect; diff --git a/packages/data-model/src/models/submission_records.ts b/packages/data-model/src/models/submission_records.ts index 9d826b20..3bc6695f 100644 --- a/packages/data-model/src/models/submission_records.ts +++ b/packages/data-model/src/models/submission_records.ts @@ -13,43 +13,35 @@ export const submissionRecordState = pgEnum('submission_record_state', ['RECEIVE export const submissionRecordType = pgEnum('submission_record_type', ['INSERT', 'UPDATE', 'DELETE']); -// TODO: export this type -type SubmissionInsertData = DataRecord; +export type SubmissionInsertData = DataRecord; -// TODO: export this type -type SubmissionUpdateData = { +export type SubmissionUpdateData = { systemId: string; old: DataRecord; new: DataRecord; }; -// TODO: export this type -type SubmissionDeleteData = { +export type SubmissionDeleteData = { systemId: string; data: DataRecord; isValid: boolean; organization: string; }; -// TODO: export this type -type SubmissionData = SubmissionInsertData | SubmissionUpdateData | SubmissionDeleteData; +export type SubmissionData = SubmissionInsertData | SubmissionUpdateData | SubmissionDeleteData; -// TODO: export this type -type FieldDetails = { +export type FieldDetails = { fieldName: string; fieldValue: DataRecordValue; }; -// TODO: export this type -type UnrecognizedValueReason = { +export type UnrecognizedValueReason = { reason: 'UNRECOGNIZED_VALUE'; }; -// TODO: export this type -type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason; +export type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason; -// TODO: export this type -type SubmissionRecordErrors = (DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue)[]; +export type SubmissionRecordErrors = DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue; export const submissionRecords = pgTable( 'submission_records', @@ -60,12 +52,18 @@ export const submissionRecords = pgTable( .notNull(), data: jsonb('data').$type().notNull(), actionType: submissionRecordType('action_type').notNull(), - errors: jsonb('errors').$type(), + errors: jsonb('errors').$type(), state: submissionRecordState('state').notNull(), }, (table) => { return { fileIndex: index('submission_records_file_id_index').on(table.fileId), + fileActionIndex: index('submission_records_file_id_action_type_index').on(table.fileId, table.actionType), + fileStateActionIndex: index('submission_records_file_id_state_action_type_index').on( + table.fileId, + table.state, + table.actionType, + ), }; }, ); diff --git a/packages/data-model/src/models/submissions.ts b/packages/data-model/src/models/submissions.ts index 7020a81e..d264628a 100644 --- a/packages/data-model/src/models/submissions.ts +++ b/packages/data-model/src/models/submissions.ts @@ -1,14 +1,9 @@ import { relations } from 'drizzle-orm'; -import { index, integer, jsonb, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; - -import { - type DataRecord, - type DataRecordValue, - type DictionaryValidationRecordErrorDetails, -} from '@overture-stack/lectern-client'; +import { index, integer, pgEnum, pgTable, serial, timestamp, varchar } from 'drizzle-orm/pg-core'; import { dictionaries } from './dictionaries.js'; import { dictionaryCategories } from './dictionary_categories.js'; +import { submissionFiles } from './submission_files.js'; export const submissionStatusEnum = pgEnum('submission_status', [ 'OPEN', @@ -20,64 +15,16 @@ export const submissionStatusEnum = pgEnum('submission_status', [ 'COMMITTED', ]); -export type SubmissionInsertData = { - batchName: string; - records: DataRecord[]; -}; - -export type SubmissionUpdateData = { - systemId: string; - old: DataRecord; - new: DataRecord; -}; - -export type SubmissionDeleteData = { - systemId: string; - data: DataRecord; - entityName: string; - isValid: boolean; - organization: string; -}; - -export type SubmissionData = { - inserts?: Record; - updates?: Record; - deletes?: Record; -}; - -export type FieldDetails = { - fieldName: string; - fieldValue: DataRecordValue; -}; - -export type UnrecognizedValueReason = { - reason: 'UNRECOGNIZED_VALUE'; -}; - -export type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason; - -export type SubmissionRecordErrorDetails = { - index: number; -} & (DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue); - -export type SubmissionErrors = { - inserts?: Record; - updates?: Record; - deletes?: Record; -}; - export const submissions = pgTable( 'submissions', { id: serial('id').primaryKey(), - data: jsonb('data').$type().notNull(), dictionaryCategoryId: integer('dictionary_category_id') .references(() => dictionaryCategories.id) .notNull(), dictionaryId: integer('dictionary_id') .references(() => dictionaries.id) .notNull(), - errors: jsonb('errors').$type(), organization: varchar('organization').notNull(), status: submissionStatusEnum('status').notNull(), createdAt: timestamp('created_at').defaultNow(), @@ -94,7 +41,7 @@ export const submissions = pgTable( }, ); -export const submissionRelations = relations(submissions, ({ one }) => ({ +export const submissionRelations = relations(submissions, ({ one, many }) => ({ dictionary: one(dictionaries, { fields: [submissions.dictionaryId], references: [dictionaries.id], @@ -103,6 +50,7 @@ export const submissionRelations = relations(submissions, ({ one }) => ({ fields: [submissions.dictionaryCategoryId], references: [dictionaryCategories.id], }), + submissionFiles: many(submissionFiles), })); export type Submission = typeof submissions.$inferSelect; // return type when queried diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts index f202dce5..5a1f26aa 100644 --- a/packages/data-provider/src/controllers/submissionController.ts +++ b/packages/data-provider/src/controllers/submissionController.ts @@ -17,9 +17,9 @@ import { submissionActiveByOrganizationRequestSchema, submissionByIdRequestSchema, submissionCommitRequestSchema, - submissionDeleteEntityNameRequestSchema, submissionDeleteRequestSchema, submissionDetailsRequestSchema, + submissionRecordDeleteRequestSchema, submissionsByCategoryRequestSchema, uploadSingleEntitySubmissionDataRequestSchema, uploadSubmissionRequestSchema, @@ -30,7 +30,7 @@ import { BatchError, type PaginatedResponse, SUBMISSION_ACTION_TYPE, - type SubmissionSummary, + type SubmissionSummaryResponse, } from '../utils/types.js'; const controller = ({ @@ -108,17 +108,22 @@ const controller = ({ next(error); } }), - deleteEntityName: validateRequest(submissionDeleteEntityNameRequestSchema, async (req, res, next) => { + deleteByRecordIdOrFileId: validateRequest(submissionRecordDeleteRequestSchema, async (req, res, next) => { try { const submissionId = Number(req.params.submissionId); - const actionType = SUBMISSION_ACTION_TYPE.parse(req.params.actionType.toUpperCase()); - const entityName = req.query.entityName; - const index = req.query.index ? parseInt(req.query.index) : null; + const recordId = req.query.recordId ? parseInt(req.query.recordId) : null; + const fileId = req.query.fileId ? parseInt(req.query.fileId) : null; const user = req.user; + if (!recordId && !fileId) { + throw new BadRequest('Either "recordId" or "fileId" query parameter must be provided for deletion.'); + } else if (recordId && fileId) { + throw new BadRequest('Only one of "recordId" or "fileId" query parameter can be provided for deletion.'); + } + logger.info( LOG_MODULE, - `Request Delete '${entityName ? entityName : 'all'}' records on '{${actionType}}' Active Submission '${submissionId}'`, + `Request Delete records with ${recordId ? `ID '${recordId}'` : ''}${fileId ? ` file ID '${fileId}'` : ''} on Submission '${submissionId}'`, ); const submission = await submissionService.getSubmissionById(submissionId); @@ -132,15 +137,10 @@ const controller = ({ const username = user?.username || ''; - const deleteSubmissionEntityResult = await submissionService.deleteActiveSubmissionEntity( - submissionId, - username, - { - actionType, - entityName, - index, - }, - ); + const deleteSubmissionEntityResult = await submissionService.deleteByRecordIdOrFileId(submissionId, username, { + recordId, + fileId, + }); if (isEmpty(deleteSubmissionEntityResult)) { throw new NotFound('Active Submission not found'); @@ -222,7 +222,7 @@ const controller = ({ }), getSubmissionsByCategory: validateRequest( submissionsByCategoryRequestSchema, - async (req, res: Response>, next) => { + async (req, res: Response>, next) => { try { const categoryId = Number(req.params.categoryId); const onlyActive = req.query.onlyActive?.toLowerCase() === 'true'; @@ -245,7 +245,7 @@ const controller = ({ { onlyActive, username, organization }, ); - const response: PaginatedResponse = { + const response: PaginatedResponse = { pagination: { currentPage: page, pageSize: pageSize, diff --git a/packages/data-provider/src/core/provider.ts b/packages/data-provider/src/core/provider.ts index 37bd5c70..65588a1e 100644 --- a/packages/data-provider/src/core/provider.ts +++ b/packages/data-provider/src/core/provider.ts @@ -13,6 +13,8 @@ import auditRepository from '../repository/auditRepository.js'; import categoryRepository from '../repository/categoryRepository.js'; import migrationRepository from '../repository/dictionaryMigrationRepository.js'; import dictionaryRepository from '../repository/dictionaryRepository.js'; +import submissionFilesRepository from '../repository/submissionFilesRepository.js'; +import submissionRecordsRepository from '../repository/submissionRecordsRepository.js'; import submittedDataRepository from '../repository/submittedRepository.js'; import auditRouter from '../routers/auditRouter.js'; import categoryRouter from '../routers/categoryRouter.js'; @@ -104,6 +106,8 @@ const provider = (configData: AppConfig, options?: ProviderOptions) => { category: categoryRepository(baseDeps), dictionary: dictionaryRepository(baseDeps), migration: migrationRepository(baseDeps), + submissionFiles: submissionFilesRepository(baseDeps), + submissionRecords: submissionRecordsRepository(baseDeps), submission: submissionRepository(baseDeps), submittedData: submittedDataRepository(baseDeps), }, diff --git a/packages/data-provider/src/repository/activeSubmissionRepository.ts b/packages/data-provider/src/repository/activeSubmissionRepository.ts index eb9a2bdb..92a57c19 100644 --- a/packages/data-provider/src/repository/activeSubmissionRepository.ts +++ b/packages/data-provider/src/repository/activeSubmissionRepository.ts @@ -1,7 +1,7 @@ import type { ExtractTablesWithRelations, SQL } from 'drizzle-orm'; import type { PgTransaction } from 'drizzle-orm/pg-core'; import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js'; -import { and, count, eq, inArray, sql } from 'drizzle-orm/sql'; +import { and, count, eq, inArray } from 'drizzle-orm/sql'; import { type NewSubmission, type Submission, submissions } from '@overture-stack/lyric-data-model/models'; @@ -11,18 +11,16 @@ import { inProcessSubmissionStatus, openSubmissionStatus } from '../utils/submis import type { BooleanTrueObject, PaginationOptions, - SubmissionDataDetailsRepositoryRecord, - SubmissionDataSummary, - SubmissionDataSummaryRepositoryRecord, - SubmissionErrorsSummary, + PartialColumns, + SubmissionWithDictionaryAndCategoryRepositoryRecord, } from '../utils/types.js'; const activeSubmissionRepository = (dependencies: BaseDependencies) => { const LOG_MODULE = 'ACTIVE_SUBMISSION_REPOSITORY'; const { db, logger } = dependencies; - // Submission columns for lightweight queries to exclude `data` and `errors` columns to improve performance - const submissionColumns: BooleanTrueObject = { + // Submission columns for lightweight queries to exclude foreign ID fields + const submissionColumns = { id: true, status: true, organization: true, @@ -30,14 +28,7 @@ const activeSubmissionRepository = (dependencies: BaseDependencies) => { createdBy: true, updatedAt: true, updatedBy: true, - }; - - // Submission columns for full detail queries including `data` and `errors` columns - const submissionColumnsWithData: BooleanTrueObject = { - ...submissionColumns, - data: true, - errors: true, - }; + } as const satisfies PartialColumns>; const submissionDictionaryRelationColumns = { dictionary: { @@ -54,116 +45,6 @@ const activeSubmissionRepository = (dependencies: BaseDependencies) => { }, } as const satisfies Record; - /** - * A query to generate a summarized JSON object of the 'data' column - * Returns a JSON object of type SubmissionDataSummary - */ - const dataSummaryQuery = sql` -jsonb_build_object( - 'inserts', - ( - SELECT jsonb_object_agg( - i.key, - jsonb_build_object( - 'batchName', i.value->>'batchName', - 'recordsCount', - CASE - WHEN jsonb_typeof(i.value->'records') = 'array' - THEN jsonb_array_length(i.value->'records') - ELSE 0 - END - ) - ) - FROM jsonb_each(${submissions.data}->'inserts') AS i(key, value) - ), - - 'updates', - ( - SELECT jsonb_object_agg( - u.key, - jsonb_build_object( - 'recordsCount', - CASE - WHEN jsonb_typeof(u.value) = 'array' - THEN jsonb_array_length(u.value) - ELSE 0 - END - ) - ) - FROM jsonb_each(${submissions.data}->'updates') AS u(key, value) - ), - - 'deletes', - ( - SELECT jsonb_object_agg( - d.key, - jsonb_build_object( - 'recordsCount', - CASE - WHEN jsonb_typeof(d.value) = 'array' - THEN jsonb_array_length(d.value) - ELSE 0 - END - ) - ) - FROM jsonb_each(${submissions.data}->'deletes') AS d(key, value) - ) -)`.as('data'); - - /** - * A query to generate a summarized JSON object of the 'errors' column - * Returns a json object of type SubmissionErrorsSummary - */ - const errorsSummaryQuery = sql`jsonb_build_object( - 'inserts', - ( - SELECT jsonb_object_agg( - i.key, - jsonb_build_object( - 'recordsCount', - CASE - WHEN jsonb_typeof(i.value) = 'array' - THEN jsonb_array_length(i.value) - ELSE 0 - END - ) - ) - FROM jsonb_each(${submissions.errors}->'inserts') AS i(key, value) - ), - - 'updates', - ( - SELECT jsonb_object_agg( - u.key, - jsonb_build_object( - 'recordsCount', - CASE - WHEN jsonb_typeof(u.value) = 'array' - THEN jsonb_array_length(u.value) - ELSE 0 - END - ) - ) - FROM jsonb_each(${submissions.errors}->'updates') AS u(key, value) - ), - - 'deletes', - ( - SELECT jsonb_object_agg( - d.key, - jsonb_build_object( - 'recordsCount', - CASE - WHEN jsonb_typeof(d.value) = 'array' - THEN jsonb_array_length(d.value) - ELSE 0 - END - ) - ) - FROM jsonb_each(${submissions.errors}->'deletes') AS d(key, value) - ) -)`.as('errors'); - /** * SQL condition used to filter submissions that are in an active state. * Example usage: @@ -199,45 +80,17 @@ jsonb_build_object( } }, - /** - * Returns the entire active submission, including all data. - */ - getActiveSubmissionDetails: async ({ - categoryId, - organization, - username, - }: { - categoryId: number; - username: string; - organization: string; - }): Promise | undefined> => { - try { - const dbResponse = await db.query.submissions.findFirst({ - where: and( - eq(submissions.dictionaryCategoryId, categoryId), - eq(submissions.createdBy, username), - eq(submissions.organization, organization), - activeStatusesCondition, - ), - columns: submissionColumnsWithData, - with: submissionDictionaryRelationColumns, - }); - return dbResponse; - } catch (error) { - logger.error(LOG_MODULE, `Failed getting active submission data`, error); - throw new ServiceUnavailable(); - } - }, - /** * Finds the current Active Submission by parameters + * Returns general information about the Submission, including its dictionary and category relations, + * omitting its submissionFiles or submissionRecords relations. * @param {Object} params * @param {number} params.categoryId Category ID * @param {string} params.username Name of the user * @param {string} params.organization Organization name * @returns */ - getActiveSubmissionSummary: async ({ + getActiveSubmission: async ({ categoryId, username, organization, @@ -245,7 +98,7 @@ jsonb_build_object( categoryId: number; username: string; organization: string; - }): Promise => { + }): Promise => { try { return await db.query.submissions.findFirst({ where: and( @@ -256,7 +109,6 @@ jsonb_build_object( ), columns: submissionColumns, with: submissionDictionaryRelationColumns, - extras: { data: dataSummaryQuery, errors: errorsSummaryQuery }, }); } catch (error) { logger.error(LOG_MODULE, `Failed getting active submission summary`, error); @@ -266,16 +118,19 @@ jsonb_build_object( /** * Finds a Submission by ID + * Returns general information about the Submission, including its dictionary and category relations, + * omitting its submissionFiles or submissionRecords relations. * @param {number} submissionId Submission ID * @returns The Submission found */ - getSubmissionById: async (submissionId: number): Promise => { + getSubmissionById: async ( + submissionId: number, + ): Promise => { try { return await db.query.submissions.findFirst({ where: and(eq(submissions.id, submissionId)), columns: submissionColumns, with: submissionDictionaryRelationColumns, - extras: { data: dataSummaryQuery, errors: errorsSummaryQuery }, }); } catch (error) { logger.error(LOG_MODULE, `Failed getting Submission with id '${submissionId}'`, error); @@ -283,27 +138,6 @@ jsonb_build_object( } }, - /** - * Retun the Submission with data details by ID - * This includes the `data` and `errors` columns - * @param {number} submissionId Submission ID - * @returns The Submission found - */ - getSubmissionDetailsById: async ( - submissionId: number, - ): Promise => { - try { - return await db.query.submissions.findFirst({ - where: and(eq(submissions.id, submissionId)), - columns: submissionColumnsWithData, - with: submissionDictionaryRelationColumns, - }); - } catch (error) { - logger.error(LOG_MODULE, `Failed getting Submission details with id '${submissionId}'`, error); - throw new ServiceUnavailable(); - } - }, - /** * Update a Submission record in database * @param {number} submissionId Submission ID to update @@ -352,7 +186,7 @@ jsonb_build_object( username?: string; organization?: string; }, - ): Promise => { + ): Promise => { const { page, pageSize } = paginationOptions; try { return await db.query.submissions.findMany({ @@ -363,7 +197,6 @@ jsonb_build_object( filterOptions.organization ? eq(submissions.organization, filterOptions.organization) : undefined, ), columns: submissionColumns, - extras: { data: dataSummaryQuery, errors: errorsSummaryQuery }, with: submissionDictionaryRelationColumns, orderBy: (submissions, { desc }) => desc(submissions.createdAt), limit: pageSize, diff --git a/packages/data-provider/src/repository/submissionFilesRepository.ts b/packages/data-provider/src/repository/submissionFilesRepository.ts new file mode 100644 index 00000000..bed127d2 --- /dev/null +++ b/packages/data-provider/src/repository/submissionFilesRepository.ts @@ -0,0 +1,102 @@ +import type { ExtractTablesWithRelations } from 'drizzle-orm'; +import type { PgTransaction } from 'drizzle-orm/pg-core'; +import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js'; +import { eq } from 'drizzle-orm/sql'; + +import { type NewSubmissionFile, type SubmissionFile, submissionFiles } from '@overture-stack/lyric-data-model/models'; + +import { BaseDependencies } from '../config/config.js'; +import { ServiceUnavailable } from '../utils/errors.js'; + +const submissionFilesRepository = (dependencies: BaseDependencies) => { + const LOG_MODULE = 'SUBMISSION_FILES_REPOSITORY'; + const { db, logger } = dependencies; + + return { + save: async ( + input: NewSubmissionFile, + tx?: PgTransaction>, + ): Promise => { + try { + const [savedSubmissionFile] = await (tx || db) + .insert(submissionFiles) + .values(input) + .returning({ id: submissionFiles.id }); + if (!savedSubmissionFile) { + throw new Error('Failed to insert Submission File, no row returned'); + } + logger.info(LOG_MODULE, `New Submission File saved successfully`); + return savedSubmissionFile.id; + } catch (error) { + logger.error(LOG_MODULE, `Failed saving Submission File`, error); + throw new ServiceUnavailable(); + } + }, + + getById: async (fileId: number): Promise => { + try { + return await db.query.submissionFiles.findFirst({ + where: eq(submissionFiles.id, fileId), + }); + } catch (error) { + logger.error(LOG_MODULE, `Failed getting Submission File by id '${fileId}'`, error); + throw new ServiceUnavailable(); + } + }, + + getBySubmissionId: async (submissionId: number): Promise => { + try { + return await db.query.submissionFiles.findMany({ + where: eq(submissionFiles.submissionId, submissionId), + }); + } catch (error) { + logger.error(LOG_MODULE, `Failed getting Submission Files by submissionId '${submissionId}'`, error); + throw new ServiceUnavailable(); + } + }, + + deleteById: async ( + fileId: number, + tx?: PgTransaction>, + ): Promise => { + try { + const deletedFiles = await (tx || db) + .delete(submissionFiles) + .where(eq(submissionFiles.id, fileId)) + .returning({ id: submissionFiles.id }); + logger.info(LOG_MODULE, `Deleted Submission Files with id '${fileId}'`); + return deletedFiles[0]?.id; + } catch (error) { + logger.error(LOG_MODULE, `Failed deleting Submission File by fileId '${fileId}'`, error); + throw new ServiceUnavailable(); + } + }, + + /** + * Deletes the files associated with a specific submission ID. + * This function does not delete cascadeingly related submission records, it will throw an error if + * foreign key constraints are violated. + * @param submissionId + * @param tx + * @returns + */ + deleteBySubmissionId: async ( + submissionId: number, + tx?: PgTransaction>, + ): Promise => { + try { + const deletedFiles = await (tx || db) + .delete(submissionFiles) + .where(eq(submissionFiles.submissionId, submissionId)) + .returning({ id: submissionFiles.id }); + logger.info(LOG_MODULE, `Deleted '${deletedFiles.length}' Submission Files for submissionId '${submissionId}'`); + return deletedFiles.map((file) => file.id); + } catch (error) { + logger.error(LOG_MODULE, `Failed deleting Submission Files by submissionId '${submissionId}'`, error); + throw new ServiceUnavailable(); + } + }, + }; +}; + +export default submissionFilesRepository; diff --git a/packages/data-provider/src/repository/submissionRecordsRepository.ts b/packages/data-provider/src/repository/submissionRecordsRepository.ts new file mode 100644 index 00000000..65911053 --- /dev/null +++ b/packages/data-provider/src/repository/submissionRecordsRepository.ts @@ -0,0 +1,347 @@ +import { type ExtractTablesWithRelations } from 'drizzle-orm'; +import type { PgTransaction } from 'drizzle-orm/pg-core'; +import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js'; +import { and, count, eq, inArray } from 'drizzle-orm/sql'; + +import { + type NewSubmissionRecord, + submissionFiles, + type SubmissionRecord, + type SubmissionRecordErrors, + submissionRecords, +} from '@overture-stack/lyric-data-model/models'; + +import { BaseDependencies } from '../config/config.js'; +import { ServiceUnavailable } from '../utils/errors.js'; +import type { PaginationOptions, SubmissionRecordActionType, SubmissionRecordState } from '../utils/types.js'; + +// This is the information stored about each individual submission record in the database, including it's entity name. +export type SubmissionRecordWithEntityName = SubmissionRecord & { entityName: string }; + +// Raw data returned from the database +export type RecordsSummaryRepository = { + actionType: SubmissionRecordActionType; + entityName: string; + totalRecords: number; + batchName?: string; + errors: number; +}; + +const submissionRecordsRepository = (dependencies: BaseDependencies) => { + const LOG_MODULE = 'SUBMISSION_RECORDS_REPOSITORY'; + const { db, logger } = dependencies; + + const getByFileIds = async ( + fileIds: number[], + paginationOptions?: PaginationOptions, + filterOptions?: { actionTypes?: SubmissionRecordActionType[]; states?: SubmissionRecordState[] }, + ): Promise => { + const query = db + .select({ + id: submissionRecords.id, + actionType: submissionRecords.actionType, + state: submissionRecords.state, + fileId: submissionRecords.fileId, + data: submissionRecords.data, + errors: submissionRecords.errors, + entityName: submissionFiles.entityName, + }) + .from(submissionRecords) + .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id)) + .where( + and( + inArray(submissionRecords.fileId, fileIds), + filterOptions?.actionTypes ? inArray(submissionRecords.actionType, filterOptions.actionTypes) : undefined, + filterOptions?.states ? inArray(submissionRecords.state, filterOptions.states) : undefined, + ), + ); + + if (paginationOptions) { + query.limit(paginationOptions.pageSize).offset((paginationOptions.page - 1) * paginationOptions.pageSize); + } + + return await query; + }; + + const saveMany = async ( + inputs: NewSubmissionRecord[], + tx?: PgTransaction>, + ): Promise => { + if (!inputs.length) { + return []; + } + try { + // TODO: Insert in batches if inputs.length > 1000 to avoid exceeding the maximum number of parameters in a single query + const savedSubmissionRecords = await (tx || db) + .insert(submissionRecords) + .values(inputs) + .returning({ id: submissionRecords.id }); + logger.info(LOG_MODULE, `Saved '${savedSubmissionRecords.length}' Submission Record records successfully`); + return savedSubmissionRecords.map((record) => record.id); + } catch (error) { + logger.error(LOG_MODULE, `Failed saving '${inputs.length}' Submission Record records`, error); + throw new ServiceUnavailable(); + } + }; + + const deleteByFileIds = async ( + fileIds: number[], + tx?: PgTransaction>, + ): Promise => { + if (!fileIds.length) { + return 0; + } + try { + const deletedRecords = await (tx || db) + .delete(submissionRecords) + .where(inArray(submissionRecords.fileId, fileIds)) + .returning({ id: submissionRecords.id }); + logger.info(LOG_MODULE, `Deleted '${deletedRecords.length}' Submission Record records by fileIds`); + return deletedRecords.length; + } catch (error) { + logger.error(LOG_MODULE, `Failed deleting Submission Records by fileIds`, error); + throw new ServiceUnavailable(); + } + }; + + return { + saveMany, + + saveManyForFile: async ( + fileId: number, + records: Omit[], + tx?: PgTransaction>, + ): Promise => { + // TODO: Batch insert records + const inputs: NewSubmissionRecord[] = records.map((record) => ({ ...record, fileId })); + return await saveMany(inputs, tx); + }, + + getById: async (id: number): Promise => { + try { + const query = await db + .select({ + id: submissionRecords.id, + actionType: submissionRecords.actionType, + state: submissionRecords.state, + fileId: submissionRecords.fileId, + data: submissionRecords.data, + errors: submissionRecords.errors, + entityName: submissionFiles.entityName, + }) + .from(submissionRecords) + .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id)) + .where(eq(submissionRecords.id, id)) + .limit(1); + + if (query.length === 0) { + return undefined; + } + return query[0]; + } catch (error) { + logger.error(LOG_MODULE, `Failed getting Submission Record by id '${id}'`, error); + throw new ServiceUnavailable(); + } + }, + + getByFileIds, + + getBySubmissionId: async ( + submissionId: number, + paginationOptions?: PaginationOptions, + filterOptions?: { + actionTypes?: SubmissionRecordActionType[]; + states?: SubmissionRecordState[]; + entityNames?: string[]; + }, + ): Promise => { + try { + const submissionFileIds = await db + .select({ id: submissionFiles.id, entityName: submissionFiles.entityName }) + .from(submissionFiles) + .where( + and( + eq(submissionFiles.submissionId, submissionId), + filterOptions?.entityNames ? inArray(submissionFiles.entityName, filterOptions.entityNames) : undefined, + ), + ); + return await getByFileIds( + submissionFileIds.map((file) => file.id), + paginationOptions, + { + actionTypes: filterOptions?.actionTypes, + states: filterOptions?.states, + }, + ); + } catch (error) { + logger.error(LOG_MODULE, `Failed getting Submission Records by submissionId '${submissionId}'`, error); + throw new ServiceUnavailable(); + } + }, + + getRecordsSummaryBySubmissionId: async (submissionId: number): Promise => { + try { + const submissionFileRecords = await db + .select({ + actionType: submissionRecords.actionType, + batchName: submissionFiles.fileName, + entityName: submissionFiles.entityName, + totalRecords: count(), + errors: count(submissionRecords.errors), + }) + .from(submissionRecords) + .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id)) + .where(eq(submissionFiles.submissionId, submissionId)) + .groupBy(submissionRecords.actionType, submissionFiles.fileName, submissionFiles.entityName); + + return submissionFileRecords; + } catch (error) { + logger.error(LOG_MODULE, `Failed getting Submission Records summary by submissionId '${submissionId}'`, error); + throw new ServiceUnavailable(); + } + }, + + /** + * This function updates the validation state of submission records based on the provided parameters. + * It can update records to 'VALID', 'RECEIVED', or 'INVALID' states, and also set errors for invalid records. + * @param params + * @param tx + * @returns + */ + updateValidationState: async ( + params: { + validRecordIds?: number[]; + receivedRecordIds?: number[]; + invalidRecords?: Array<{ id: number; errors?: SubmissionRecordErrors[] }>; + }, + tx?: PgTransaction>, + ): Promise => { + const executor = tx || db; + + const validRecordIds = params.validRecordIds ?? []; + const receivedRecordIds = params.receivedRecordIds ?? []; + const invalidRecords = params.invalidRecords ?? []; + if (!validRecordIds.length && !receivedRecordIds.length && !invalidRecords.length) { + return []; + } + + try { + const updatedIds: number[] = []; + + if (validRecordIds.length) { + const validUpdates = await executor + .update(submissionRecords) + .set({ state: 'VALID', errors: null }) + .where(inArray(submissionRecords.id, validRecordIds)) + .returning({ id: submissionRecords.id }); + updatedIds.push(...validUpdates.map((record) => record.id)); + } + + if (receivedRecordIds.length) { + const receivedUpdates = await executor + .update(submissionRecords) + .set({ state: 'RECEIVED', errors: null }) + .where(inArray(submissionRecords.id, receivedRecordIds)) + .returning({ id: submissionRecords.id }); + updatedIds.push(...receivedUpdates.map((record) => record.id)); + } + + if (invalidRecords.length) { + const invalidUpdates = await Promise.all( + invalidRecords.map(async ({ id, errors }) => { + const [updatedRecord] = await executor + .update(submissionRecords) + .set({ state: 'INVALID', errors: errors ?? null }) + .where(eq(submissionRecords.id, id)) + .returning({ id: submissionRecords.id }); + return updatedRecord?.id; + }), + ); + updatedIds.push(...invalidUpdates.filter((id): id is number => id !== undefined)); + } + + logger.info( + LOG_MODULE, + `Updated Submission Record states: VALID='${validRecordIds.length}', RECEIVED='${receivedRecordIds.length}', INVALID='${invalidRecords.length}'`, + ); + return [...new Set(updatedIds)]; + } catch (error) { + logger.error(LOG_MODULE, `Failed updating Submission Record validation state`, error); + throw new ServiceUnavailable(); + } + }, + + countBySubmissionId: async ( + submissionId: number, + ): Promise> => { + try { + return await db + .select({ actionType: submissionRecords.actionType, total: count() }) + .from(submissionRecords) + .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id)) + .where(eq(submissionFiles.submissionId, submissionId)) + .groupBy(submissionRecords.actionType); + } catch (error) { + logger.error( + LOG_MODULE, + `Failed counting Submission Records by action for submissionId '${submissionId}'`, + error, + ); + throw new ServiceUnavailable(); + } + }, + + countInvalidBySubmissionId: async ( + submissionId: number, + ): Promise> => { + try { + return await db + .select({ actionType: submissionRecords.actionType, total: count() }) + .from(submissionRecords) + .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id)) + .where(and(eq(submissionFiles.submissionId, submissionId), eq(submissionRecords.state, 'INVALID'))) + .groupBy(submissionRecords.actionType); + } catch (error) { + logger.error( + LOG_MODULE, + `Failed counting invalid Submission Records by action for submissionId '${submissionId}'`, + error, + ); + throw new ServiceUnavailable(); + } + }, + + deleteByIds: async ( + ids: number[], + tx?: PgTransaction>, + ): Promise => { + try { + return await (tx || db).delete(submissionRecords).where(inArray(submissionRecords.id, ids)); + } catch (error) { + logger.error(LOG_MODULE, `Failed deleting Submission Record by ids '${ids}'`, error); + throw new ServiceUnavailable(); + } + }, + + deleteByFileIds, + + deleteBySubmissionId: async ( + submissionId: number, + tx?: PgTransaction>, + ): Promise => { + try { + const submissionFileIds = await (tx || db) + .select({ id: submissionFiles.id }) + .from(submissionFiles) + .where(eq(submissionFiles.submissionId, submissionId)); + const fileIds = submissionFileIds.map((file) => file.id); + return await deleteByFileIds(fileIds, tx); + } catch (error) { + logger.error(LOG_MODULE, `Failed deleting Submission Records by submissionId '${submissionId}'`, error); + throw new ServiceUnavailable(); + } + }, + }; +}; + +export default submissionRecordsRepository; diff --git a/packages/data-provider/src/routers/submissionRouter.ts b/packages/data-provider/src/routers/submissionRouter.ts index 02ddcb86..2f8c99fd 100644 --- a/packages/data-provider/src/routers/submissionRouter.ts +++ b/packages/data-provider/src/routers/submissionRouter.ts @@ -75,7 +75,7 @@ const router = ({ router.delete('/:submissionId', submissionController.delete); - router.delete('/:submissionId/:actionType', submissionController.deleteEntityName); + router.delete('/:submissionId/data', submissionController.deleteByRecordIdOrFileId); router.get('/category/:categoryId', submissionController.getSubmissionsByCategory); diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index 7f4a3a49..9f07a7f8 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -6,9 +6,7 @@ import type { DataDiff, NewSubmittedData, SubmissionDeleteData, - SubmissionErrors, SubmissionInsertData, - SubmissionRecordErrorDetails, SubmissionUpdateData, SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -17,27 +15,26 @@ import { BaseDependencies } from '../../config/config.js'; import createSubmissionRepository from '../../repository/activeSubmissionRepository.js'; import createCategoryRepository from '../../repository/categoryRepository.js'; import createDictionaryRepository from '../../repository/dictionaryRepository.js'; +import createSubmissionFilesRepository from '../../repository/submissionFilesRepository.js'; +import createSubmissionRecordsRepository from '../../repository/submissionRecordsRepository.js'; import createSubmittedDataRepository from '../../repository/submittedRepository.js'; import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/dictionarySchemaRelations.js'; import { BadRequest } from '../../utils/errors.js'; +import { readTextFile } from '../../utils/fileUtils.js'; import { convertRecordToString } from '../../utils/formatUtils.js'; import { parseRecordsToInsert } from '../../utils/recordsParser.js'; import { extractSchemaDataFromMergedDataRecords, - filterDeletesFromUpdates, filterRelationsForPrimaryIdUpdate, - findEditSubmittedData, findInvalidRecordErrorsBySchemaName, groupSchemaErrorsByEntity, isSubmissionActive, mapGroupedUpdateSubmissionData, mergeAndReferenceEntityData, - mergeDeleteRecords, - mergeInsertsRecords, mergeUpdatesBySystemId, parseToSchema, segregateFieldChangeRecords, - submissionInsertDataFromFiles, + type SubmissionErrors, validateSchemas, } from '../../utils/submissionUtils.js'; import { @@ -56,7 +53,6 @@ import { type ResultOnCommit, type SchemasDictionary, SUBMISSION_STATUS, - type ValidateFilesParams, } from '../../utils/types.js'; import createSubmittedDataRelationsSearch from '../submittedData/searchDataRelations.js'; @@ -67,6 +63,8 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const submissionRepository = createSubmissionRepository(dependencies); const submittedDataRepository = createSubmittedDataRepository(dependencies); const submittedDataRelationsSearch = createSubmittedDataRelationsSearch(dependencies); + const submissionRecordsRepository = createSubmissionRecordsRepository(dependencies); + const submissionFilesRepository = createSubmissionFilesRepository(dependencies); const { logger } = dependencies; /** @@ -222,7 +220,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { return Object.entries(idFieldChangeRecord).reduce< Promise<{ - inserts: Record; + inserts: Record; deletes: Record; }> >( @@ -232,7 +230,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { // iterate each record on this entity const result = await updRecord.reduce< Promise<{ - inserts: DataRecord[]; + inserts: SubmissionInsertData[]; deletes: SubmissionDeleteData[]; }> >( @@ -247,12 +245,11 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const deleteRecord: SubmissionDeleteData = { systemId: foundSubmittedData.systemId, data: foundSubmittedData.data, - entityName: foundSubmittedData.entityName, isValid: foundSubmittedData.isValid, organization: foundSubmittedData.organization, }; - const insertDataRecord: DataRecord = { ...foundSubmittedData.data, ...u.new }; + const insertDataRecord: SubmissionInsertData = { ...foundSubmittedData.data, ...u.new }; acc2.inserts.push(insertDataRecord); acc2.deletes.push(deleteRecord); @@ -262,7 +259,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { ); acc.deletes[entityName] = result.deletes; - acc.inserts[entityName] = { batchName: entityName, records: result.inserts }; + acc.inserts[entityName] = result.inserts; return acc; }, @@ -429,7 +426,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { // iterate if there are any record to be deleted dataToValidate?.deletes?.forEach((item) => { - const { data, entityName, isValid, organization, systemId } = item; + const { data, isValid, organization, systemId } = item; deletesToProcess.push({ submissionId: submission.id, @@ -440,7 +437,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { resultCommit.deletes.push({ data, - entityName, + entityName: '', // TODO: need to fetch the entityName isValid, organization, systemId, @@ -501,10 +498,10 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const performDataValidation = async (submissionId: number): Promise => { const { getActiveDictionaryByCategory } = categoryRepository; const { getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository; - const { getSubmissionDetailsById } = submissionRepository; + const { getSubmissionById } = submissionRepository; // Get Active Submission from database - const activeSubmission = await getSubmissionDetailsById(submissionId); + const activeSubmission = await getSubmissionById(submissionId); if (!activeSubmission) { throw new Error(`Submission '${submissionId}' not found`); @@ -521,10 +518,12 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { throw new BadRequest(`Dictionary in category '${activeSubmission.dictionaryCategory.id}' not found`); } + const submissionRecords = await submissionRecordsRepository.getBySubmissionId(submissionId); + // Merge Submitted Data with Active Submission keepping reference of each record ID const dataMergedByEntityName = mergeAndReferenceEntityData({ submissionId, - submissionData: activeSubmission.data, + submissionData: submissionRecords, submittedData, }); @@ -540,40 +539,6 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { dataValidated: dataMergedByEntityName, }); - // Check for records to be updated that its systemId was not found in the Submitted Data collection. - // Any error found will cause the submission to be marked as 'invalid' - Object.entries(activeSubmission.data.updates ?? {}).forEach(([entityName, recordsToUpdate]) => { - recordsToUpdate.forEach((submissionEditData, index) => { - const found = findEditSubmittedData(entityName, submissionEditData.systemId, dataMergedByEntityName); - - if (found) { - return; - } - - logger.error( - LOG_MODULE, - `Record with systemId '${submissionEditData.systemId}' not found in entity '${entityName}'`, - ); - - if (!submissionSchemaErrors.updates) { - submissionSchemaErrors.updates = {}; - } - - if (!submissionSchemaErrors.updates[entityName]) { - submissionSchemaErrors.updates[entityName] = []; - } - - const unrecodgnizedValueError: SubmissionRecordErrorDetails = { - fieldName: 'systemId', - fieldValue: submissionEditData.systemId, - index, - reason: 'UNRECOGNIZED_VALUE', - }; - - submissionSchemaErrors.updates[entityName].push(unrecodgnizedValueError); - }); - }); - if (_.isEmpty(submissionSchemaErrors)) { logger.info(LOG_MODULE, `No error found on data submission`); } else { @@ -616,7 +581,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { }, ): Promise => { const { getDictionary } = dictionaryRepository; - const { getSubmissionDetailsById, update } = submissionRepository; + const { getSubmissionById, update } = submissionRepository; try { // Parse file data @@ -624,7 +589,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const filesDataProcessed = await compareUpdatedData(recordsParsed, schema.name); - const submission = await getSubmissionDetailsById(submissionId); + const submission = await getSubmissionById(submissionId); if (!submission) { throw new Error(`Submission '${submissionId}' not found`); } @@ -672,10 +637,36 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { dictionaryRelations, ); + // const submissionRecordsToUpdate = await submissionRecordsRepository.getBySubmissionId(submissionId, undefined, { + // actionTypes: ['UPDATE'], + // }); + + // const formattedSubmissionRecordsToUpdate = submissionRecordsToUpdate + // .filter( + // ( + // record, + // ): record is (typeof submissionRecordsToUpdate)[number] & { + // actionType: 'UPDATE'; + // data: SubmissionUpdateData; + // } => record.actionType === 'UPDATE', + // ) + // .reduce>((acc, record) => { + // if (!acc[record.entityName]) { + // acc[record.entityName] = []; + // } + + // (acc[record.entityName] ?? []).push({ + // systemId: record.data.systemId.toString(), + // old: record.data.old ?? {}, + // new: record.data.new ?? {}, + // }); + // return acc; + // }, {}); + // Aggegates all Update changes on Submission // Note: We do not include records involving primary ID fields changes in here. We would rather do a DELETE and an INSERT const updatedActiveSubmissionData: Record = mergeUpdatesBySystemId( - submission.data.updates ?? {}, + // formattedSubmissionRecordsToUpdate, totalDependants, nonIdFieldChangeRecord, ); @@ -683,26 +674,73 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { // Creates insert and delete records based on primary ID field change records. const additions = await handleIdFieldChanges(idFieldChangeRecord); - // Merge Active Submission Inserts with Edit generated new Inserts - const mergedInserts = mergeInsertsRecords(submission.data.inserts ?? {}, additions.inserts); + // // Merge Active Submission Inserts with Edit generated new Inserts + // const mergedInserts = mergeInsertsRecords(submission.data.inserts ?? {}, additions.inserts); - // Merge Active Submission Deletes with Edit generated new Deletes - const mergedDeletes = mergeDeleteRecords(submission.data.deletes ?? {}, additions.deletes); + // // Merge Active Submission Deletes with Edit generated new Deletes + // const mergedDeletes = mergeDeleteRecords(submission.data.deletes ?? {}, additions.deletes); - // filter out delete records found on update records - const filteredDeletes = filterDeletesFromUpdates(mergedDeletes, updatedActiveSubmissionData); + // // filter out delete records found on update records + // const filteredDeletes = filterDeletesFromUpdates(mergedDeletes, updatedActiveSubmissionData); // Updating the Submission with the new data and 'VALIDATING' status before validation starts await update(submission.id, { - data: { - inserts: mergedInserts, - deletes: filteredDeletes, - updates: updatedActiveSubmissionData, - }, updatedBy: username, status: 'VALIDATING', }); + // TODO insert new data into submissionRecordsRepository, remove the code above to merge + // insert new file for submission + + const entityNames: Set = new Set([ + ...Object.keys(additions.inserts), + ...Object.keys(additions.deletes), + ...Object.keys(updatedActiveSubmissionData), + ]); + + for (const entityName of entityNames) { + const savedFileId = await submissionFilesRepository.save({ + entityName: entityName, + fileName: `${Date.now()}.json`, + fileSize: 0, + submissionId: submission.id, + }); + + if (updatedActiveSubmissionData[entityName]) { + await submissionRecordsRepository.saveManyForFile( + savedFileId, + updatedActiveSubmissionData[entityName].map((record) => ({ + actionType: 'UPDATE', + data: record, + state: 'RECEIVED', + fileId: savedFileId, + })), + ); + } + + if (additions.inserts[entityName]) { + await submissionRecordsRepository.saveManyForFile( + savedFileId, + additions.inserts[entityName].map((record) => ({ + actionType: 'INSERT', + data: record, + state: 'RECEIVED', + fileId: savedFileId, + })), + ); + } + if (additions.deletes[entityName]) { + await submissionRecordsRepository.saveManyForFile( + savedFileId, + additions.deletes[entityName]?.map((record) => ({ + actionType: 'DELETE', + data: record, + state: 'RECEIVED', + })), + ); + } + } + // Perform Schema Data validation in a worker thread dependencies.workerPool.dataValidation({ submissionId: submission.id }); } catch (error) { @@ -737,35 +775,46 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { submissionId: number; username: string; }) => { - const { getSubmissionDetailsById, update } = submissionRepository; + const { getSubmissionById, update } = submissionRepository; try { // Get Active Submission from database - const activeSubmission = await getSubmissionDetailsById(submissionId); + const activeSubmission = await getSubmissionById(submissionId); if (!activeSubmission) { - throw new Error(`Submission '${activeSubmission}' not found`); + throw new Error(`Submission '${submissionId}' not found`); } if (!isSubmissionActive(activeSubmission.status)) { throw new Error(`Submission '${activeSubmission.id}' is not active`); } - const insertRecords = parseRecordsToInsert(records, schemasDictionary); - - // Merge Active Submission insert records with incoming TSV file data processed - const insertActiveSubmissionData = mergeInsertsRecords(activeSubmission.data.inserts ?? {}, insertRecords); - // Updating the Submission with the new data and 'VALIDATING' status before validation starts await update(activeSubmission.id, { - data: { - inserts: insertActiveSubmissionData, - deletes: activeSubmission.data.deletes, - updates: activeSubmission.data.updates, - }, updatedBy: username, status: 'VALIDATING', }); + const insertRecords = parseRecordsToInsert(records, schemasDictionary); + + await Promise.all( + Object.entries(insertRecords).map(async ([entityName, entityRecords]) => { + const savedFileId = await submissionFilesRepository.save({ + entityName, + fileName: `insert-${entityName}-${Date.now()}.json`, // default file name for adding JSON records + fileSize: JSON.stringify(entityRecords).length, + submissionId, + }); + await submissionRecordsRepository.saveManyForFile( + savedFileId, + entityRecords.map((record) => ({ + actionType: 'INSERT', + data: record, + state: 'RECEIVED', + })), + ); + }), + ); + // Perform Schema Data validation in a worker thread dependencies.workerPool.dataValidation({ submissionId: activeSubmission.id }); } catch (error) { @@ -780,7 +829,9 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { /** * Update Active Submission in database - * Updates the status of the Submission to 'VALID' if there is no errors, otherwise updates it to 'INVALID' + * Updates the Submission status to 'VALID' if there is no errors, otherwise updates it to 'INVALID' + * Updates all the records of the submission with the validation state, marking records with errors as 'INVALID' + * and records without errors as 'VALID' * @param {Object} input * @param {number} input.dictionaryId The Dictionary ID of the Submission * @param {number} input.idActiveSubmission ID of the Submission @@ -800,7 +851,25 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const updatedActiveSubmissionId = await update(idActiveSubmission, { status: newStatusSubmission, dictionaryId: dictionaryId, - errors: schemaErrors, + }); + + const invalidRecords = Object.values(schemaErrors).flatMap((entityErrors) => + Object.values(entityErrors).flatMap((recordErrors) => + recordErrors.map(({ recordId, errors }) => ({ + id: recordId, + errors, + })), + ), + ); + + const submissionRecords = await submissionRecordsRepository.getBySubmissionId(idActiveSubmission); + const recordsWithoutError = submissionRecords + .filter((record) => !invalidRecords.some((invalidRecord) => invalidRecord.id === record.id)) + .map((record) => record.id); + + submissionRecordsRepository.updateValidationState({ + invalidRecords, + validRecordIds: recordsWithoutError, }); logger.info( @@ -813,14 +882,12 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { /** * Void function to process and validate uploaded files on an Active Submission. * Performs the schema data validation combined with all Submitted Data. - * @param {Record} fileSchemaMap Mapping the files with a schema - * @param {Object} params - * @param {number} params.categoryId Category Identifier - * @param {string} params.organization Organization name - * @param {string} params.username User who performs the action + * @param {FileSchemaMap} fileSchemaMap Mapping the files with a schema + * @param {number} submissionId Submission ID + * @param {string} username User who performs the action * @returns {void} */ - const addFilesToSubmissionAsync = async (fileSchemaMap: FileSchemaMap, params: ValidateFilesParams) => { + const addFilesToSubmissionAsync = async (fileSchemaMap: FileSchemaMap, submissionId: number, username: string) => { const fileSummaries = Object.entries(fileSchemaMap) .flatMap(([_, { files, schema }]) => files.map( @@ -830,47 +897,41 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { .join(', '); logger.info(LOG_MODULE, `Processing files: ${fileSummaries}`); - // TODO: This only gets a summary, we need to insert data into an active submission so we need all the insert statements. - - const { categoryId, organization, username } = params; - try { - // Parse file data - const filesDataProcessed = await submissionInsertDataFromFiles(fileSchemaMap); - - // Get Active Submission from database - const activeSubmission = await submissionRepository.getActiveSubmissionDetails({ - categoryId, - username, - organization, - }); - if (!activeSubmission) { - throw new BadRequest(`Submission '${activeSubmission}' not found`); - } - - // Merge Active Submission data with incoming TSV file data processed - const insertActiveSubmissionData = mergeInsertsRecords(activeSubmission.data.inserts ?? {}, filesDataProcessed); - // Updating the Submission with the new data and 'VALIDATING' status before validation starts - await submissionRepository.update(activeSubmission.id, { - data: { - inserts: insertActiveSubmissionData, - deletes: activeSubmission.data.deletes, - updates: activeSubmission.data.updates, - }, + await submissionRepository.update(submissionId, { updatedBy: username, status: 'VALIDATING', }); + for (const [entityName, { files, schema }] of Object.entries(fileSchemaMap)) { + for (const file of files) { + const parsedFileData = await readTextFile(file, schema); + + const fileId = await submissionFilesRepository.save({ + entityName, + fileName: file.filename, + fileSize: file.size, + submissionId, + }); + + await submissionRecordsRepository.saveManyForFile( + fileId, + parsedFileData.records.map((record) => ({ + actionType: 'INSERT', + data: record, + state: 'RECEIVED', + })), + ); + } + } + // Perform Schema Data validation in a worker thread - dependencies.workerPool.dataValidation({ submissionId: activeSubmission.id }); + dependencies.workerPool.dataValidation({ submissionId: submissionId }); } catch (error) { logger.error(`There was an error processing submitted files: ${fileSummaries}`, JSON.stringify(error)); } - logger.info( - LOG_MODULE, - `Finished addFilesToSubmissionAsync for active submission in category "${params.categoryId}" for organization "${params.organization}" submitted by user "${params.username}"`, - ); + logger.info(LOG_MODULE, `Finished addFilesToSubmissionAsync on submission "${submissionId}"`); }; return { diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts index 1ca243dc..7b0637d6 100644 --- a/packages/data-provider/src/services/submission/submissionService.ts +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -1,21 +1,25 @@ import * as _ from 'lodash-es'; import { Dictionary as SchemasDictionary } from '@overture-stack/lectern-client'; -import { type NewSubmission, type SubmissionRecordErrorDetails } from '@overture-stack/lyric-data-model/models'; +import { type NewSubmission } from '@overture-stack/lyric-data-model/models'; import { BaseDependencies } from '../../config/config.js'; import createSubmissionRepository from '../../repository/activeSubmissionRepository.js'; import createCategoryRepository from '../../repository/categoryRepository.js'; +import createDictionaryRepository from '../../repository/dictionaryRepository.js'; +import createSubmissionFilesRepository from '../../repository/submissionFilesRepository.js'; +import createSubmissionRecordsRepository, { + type SubmissionRecordWithEntityName, +} from '../../repository/submissionRecordsRepository.js'; import { getSchemaByName } from '../../utils/dictionaryUtils.js'; import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; import type { PaginatedResult } from '../../utils/result.js'; import type { FilenameEntityPair } from '../../utils/schemas.js'; -import { filterAndPaginateSubmissionData, type FlattenedSubmissionData } from '../../utils/submissionResponseParser.js'; +import { buildDataSummary } from '../../utils/submissionResponseParser.js'; import { checkEntityFieldNames, createSubmissionSummaryResponse, isSubmissionActive, - removeItemsFromSubmission, resolveFileEntities, } from '../../utils/submissionUtils.js'; import { @@ -24,10 +28,9 @@ import { type DeleteSubmissionResult, type EntityData, type PaginationOptions, - SUBMISSION_ACTION_TYPE, SUBMISSION_STATUS, - type SubmissionActionType, - SubmissionSummary, + type SubmissionRecordActionType, + type SubmissionSummaryResponse, type SubmitDataResult, type SubmitFileResult, } from '../../utils/types.js'; @@ -42,6 +45,9 @@ const submissionService = (dependencies: BaseDependencies) => { const categoryRepository = createCategoryRepository(dependencies); const submissionProcessor = submissionProcessorFactory.create(dependencies); const submissionRepository = createSubmissionRepository(dependencies); + const submissionRecordsRepository = createSubmissionRecordsRepository(dependencies); + const dictionaryRepository = createDictionaryRepository(dependencies); + const submissionFilesRepository = createSubmissionFilesRepository(dependencies); /** * Runs Schema validation asynchronously in a worker thread and moves the Active Submission to Submitted Data @@ -83,11 +89,8 @@ const submissionService = (dependencies: BaseDependencies) => { await submissionRepository.update(submissionId, { status: SUBMISSION_STATUS.COMMITTING, updatedBy: username }); // Get entities to process - const entitiesToProcess = new Set([ - ...Object.keys(submission.data?.inserts ?? {}), - ...Object.keys(submission.data?.updates ?? {}), - ...Object.keys(submission.data?.deletes ?? {}), - ]); + const filesOnSubmission = await submissionFilesRepository.getBySubmissionId(submissionId); + const entitiesToProcess = new Set(filesOnSubmission.map((file) => file.entityName)); // Execute commit submission in worker pool const commitData: CommitWorkerInput = { @@ -146,24 +149,28 @@ const submissionService = (dependencies: BaseDependencies) => { }; /** - * Function to remove an entity from an Active Submission by given Submission ID + * Function to remove specific records from an Active Submission + * If fileID is provided, all records associated with that file will be removed from the Submission + * If recordID is provided, only that specific record will be removed from the Submission + * If both fileID and recordID are provided, only the recordID will be removed from the Submission + * If neither fileID nor recordID are provided, an error will be thrown + * The function will check if the Submission is Active and if the record or file belongs to the Submission * It validates resulting Active Submission running cross schema validation along with the existing Submitted Data * Returns the resulting ID of the Active Submission - * @param {number} submissionId - * @param {string} entityName - * @param {string} username + * @param {number} submissionId - Submission ID + * @param {string} username - User name performing the action + * @param {object} filter - Filter to identify the entity to be removed * @returns { Promise} */ - const deleteActiveSubmissionEntity = async ( + const deleteByRecordIdOrFileId = async ( submissionId: number, username: string, filter: { - actionType: SubmissionActionType; - entityName: string; - index: number | null; + recordId: number | null; + fileId: number | null; }, ): Promise => { - const submission = await submissionRepository.getSubmissionDetailsById(submissionId); + const submission = await submissionRepository.getSubmissionById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); } @@ -172,35 +179,43 @@ const submissionService = (dependencies: BaseDependencies) => { throw new StatusConflict('Submission is not active. Only Active Submission can be modified'); } - if ( - SUBMISSION_ACTION_TYPE.Values.INSERTS.includes(filter.actionType) && - !_.has(submission.data.inserts, filter.entityName) - ) { - throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); - } + const filesOnSubmission = await submissionFilesRepository.getBySubmissionId(submissionId); - if ( - SUBMISSION_ACTION_TYPE.Values.UPDATES.includes(filter.actionType) && - !_.has(submission.data.updates, filter.entityName) - ) { - throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); + if (filesOnSubmission.length === 0) { + throw new BadRequest(`Submission '${submissionId}' has no records or files to delete`); } - if ( - SUBMISSION_ACTION_TYPE.Values.DELETES.includes(filter.actionType) && - !_.has(submission.data.deletes, filter.entityName) - ) { - throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); - } + // Remove record by ID from the Submission + if (filter.recordId) { + const recordFoundInDB = await submissionRecordsRepository.getById(filter.recordId); + if (!recordFoundInDB) { + throw new BadRequest(`Record with ID '${filter.recordId}' not found in Submission '${submissionId}'`); + } - // Remove entity from the Submission - const updatedActiveSubmissionData = removeItemsFromSubmission(submission.data, { - ...filter, - }); + const fileReference = filesOnSubmission.find((file) => file.id === recordFoundInDB.fileId); + if (fileReference?.submissionId !== submissionId) { + throw new BadRequest(`Record with ID '${filter.recordId}' does not belong to Submission '${submissionId}'`); + } + + await submissionRecordsRepository.deleteByIds([filter.recordId]); + } else if (filter.fileId != null) { + const fileId = filter.fileId; + // Verify the requested FileId belongs to the Submission before deleting + const fileReference = filesOnSubmission.find((f) => f.id === fileId); + if (!fileReference) { + throw new BadRequest(`File with ID '${fileId}' not found in Submission '${submissionId}'`); + } + + await dependencies.db.transaction(async (tx) => { + await submissionRecordsRepository.deleteByFileIds([fileId], tx); + await submissionFilesRepository.deleteById(fileId, tx); + }); + } else { + throw new BadRequest('Either recordId or fileId must be provided to delete a record or file from the Submission'); + } // Updating the Submission with the new data and 'VALIDATING' status before validation starts await submissionRepository.update(submission.id, { - data: updatedActiveSubmissionData, updatedBy: username, status: 'VALIDATING', }); @@ -208,7 +223,10 @@ const submissionService = (dependencies: BaseDependencies) => { // Perform Schema Data validation in a worker thread dependencies.workerPool.dataValidation({ submissionId: submission.id }); - logger.info(LOG_MODULE, `Submission '${submission.id}' updated after removing entity '${filter.entityName}'`); + logger.info( + LOG_MODULE, + `Submission '${submission.id}' updated after removing entity with recordId '${filter.recordId}' and fileId '${filter.fileId}'`, + ); return { status: ACTIVE_SUBMISSION_STATUS.PROCESSING, @@ -237,7 +255,7 @@ const submissionService = (dependencies: BaseDependencies) => { username?: string; organization?: string; }, - ): Promise> => { + ): Promise> => { const recordsPaginated = await submissionRepository.getSubmissionsByCategory( categoryId, paginationOptions, @@ -252,17 +270,30 @@ const submissionService = (dependencies: BaseDependencies) => { }; } - const totalRecords = await submissionRepository.getTotalSubmissionsByCategory(categoryId, filterOptions); + const totalSubmissions = await submissionRepository.getTotalSubmissionsByCategory(categoryId, filterOptions); + const result = await Promise.all( + recordsPaginated.map(async (response) => { + const submissionRecordsSummary = await submissionRecordsRepository.getRecordsSummaryBySubmissionId(response.id); + const formattedDataSummary = buildDataSummary(submissionRecordsSummary); + + return createSubmissionSummaryResponse({ + ...response, + data: formattedDataSummary, + }); + }), + ); + return { metadata: { - totalRecords, + totalRecords: totalSubmissions, }, - result: recordsPaginated.map((response) => createSubmissionSummaryResponse(response)), + result, }; }; /** * Get Submission by Submission ID + * Returns the submission general information and includes the summary of the data and errors * @param {number} submissionId A Submission ID * @returns One Submission */ @@ -272,7 +303,13 @@ const submissionService = (dependencies: BaseDependencies) => { return; } - return createSubmissionSummaryResponse(submission); + const submissionDataSummary = await submissionRecordsRepository.getRecordsSummaryBySubmissionId(submissionId); + const formattedDataSummary = buildDataSummary(submissionDataSummary); + + return createSubmissionSummaryResponse({ + ...submission, + data: formattedDataSummary, + }); }; /** @@ -293,20 +330,31 @@ const submissionService = (dependencies: BaseDependencies) => { }: { submissionId: number; paginationOptions: PaginationOptions; - filterOptions: { entityNames: string[]; actionTypes: SubmissionActionType[] }; - }): Promise<{ data: FlattenedSubmissionData[]; errors?: SubmissionRecordErrorDetails[] }> => { - const submission = await submissionRepository.getSubmissionDetailsById(submissionId); + filterOptions: { entityNames: string[]; actionTypes: SubmissionRecordActionType[] }; + }): Promise => { + const submission = await submissionRepository.getSubmissionById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); } - const submissionEntityNames = [ - ...Object.keys(submission.data.inserts ?? {}), - ...Object.keys(submission.data.updates ?? {}), - ...Object.keys(submission.data.deletes ?? {}), - ]; + const dictionary = await dictionaryRepository.getDictionary( + submission.dictionary.name, + submission.dictionary.version, + ); + + if (!dictionary) { + throw new InternalServerError( + `Dictionary '${submission.dictionary.name}' version '${submission.dictionary.version}' not found`, + ); + } + + const schemasDictionary: SchemasDictionary = { + name: dictionary.name, + version: dictionary.version, + schemas: dictionary.dictionary, + }; - const missingEntityNames = filterOptions.entityNames.filter((name) => !submissionEntityNames.includes(name)); + const missingEntityNames = filterOptions.entityNames.filter((name) => !getSchemaByName(name, schemasDictionary)); if (filterOptions.entityNames.length > 0 && missingEntityNames.length > 0) { throw new BadRequest( @@ -314,12 +362,13 @@ const submissionService = (dependencies: BaseDependencies) => { ); } - return filterAndPaginateSubmissionData({ - data: submission.data, - errors: submission.errors || {}, - filterOptions, + const submissionRecords = await submissionRecordsRepository.getBySubmissionId( + submissionId, paginationOptions, - }); + filterOptions, + ); + + return submissionRecords; }; /** @@ -338,8 +387,8 @@ const submissionService = (dependencies: BaseDependencies) => { categoryId: number; username: string; organization: string; - }): Promise => { - const submission = await submissionRepository.getActiveSubmissionSummary({ + }): Promise => { + const submission = await submissionRepository.getActiveSubmission({ organization, username, categoryId, @@ -348,7 +397,13 @@ const submissionService = (dependencies: BaseDependencies) => { return; } - return createSubmissionSummaryResponse(submission); + const submissionDataSummary = await submissionRecordsRepository.getRecordsSummaryBySubmissionId(submission.id); + const formattedDataSummary = buildDataSummary(submissionDataSummary); + + return createSubmissionSummaryResponse({ + ...submission, + data: formattedDataSummary, + }); }; /** @@ -368,7 +423,7 @@ const submissionService = (dependencies: BaseDependencies) => { const { categoryId, username, organization } = params; const { getActiveDictionaryByCategory } = categoryRepository; - const activeSubmission = await submissionRepository.getActiveSubmissionSummary({ + const activeSubmission = await submissionRepository.getActiveSubmission({ categoryId, username, organization, @@ -389,10 +444,8 @@ const submissionService = (dependencies: BaseDependencies) => { const newSubmissionInput: NewSubmission = { createdBy: username, - data: {}, dictionaryCategoryId: categoryId, dictionaryId: currentDictionary.id, - errors: {}, organization: organization, status: SUBMISSION_STATUS.OPEN, }; @@ -585,11 +638,7 @@ const submissionService = (dependencies: BaseDependencies) => { // Start background process of adding files to submission // Running Schema validation in the background do not need to wait // Result of validations will be stored in database - submissionProcessor.addFilesToSubmissionAsync(checkedEntities, { - categoryId, - organization, - username, - }); + submissionProcessor.addFilesToSubmissionAsync(checkedEntities, activeSubmissionId, username); if (batchErrors.length === 0) { return { @@ -613,7 +662,7 @@ const submissionService = (dependencies: BaseDependencies) => { return { commitSubmission, deleteActiveSubmissionById, - deleteActiveSubmissionEntity, + deleteByRecordIdOrFileId, getSubmissionsByCategory, getSubmissionById, getSubmissionDetailsById, diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index fa95240a..3102ec58 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -6,12 +6,13 @@ import { SQON } from '@overture-stack/sqon-builder'; import { BaseDependencies } from '../../config/config.js'; import submissionRepository from '../../repository/activeSubmissionRepository.js'; import categoryRepository from '../../repository/categoryRepository.js'; +import createSubmissionFilesRepository from '../../repository/submissionFilesRepository.js'; +import createSubmissionRecordsRepository from '../../repository/submissionRecordsRepository.js'; import submittedRepository from '../../repository/submittedRepository.js'; import { convertSqonToQuery } from '../../utils/convertSqonToQuery.js'; import { getDictionarySchemaRelations } from '../../utils/dictionarySchemaRelations.js'; import { InternalServerError, StatusConflict } from '../../utils/errors.js'; import type { PaginatedResult } from '../../utils/result.js'; -import { filterUpdatesFromDeletes, mergeDeleteRecords } from '../../utils/submissionUtils.js'; import { fetchDataErrorResponse, getEntityNamesFromFilterOptions, @@ -39,6 +40,8 @@ const submittedData = (dependencies: BaseDependencies) => { const LOG_MODULE = 'SUBMITTED_DATA_SERVICE'; const submittedDataRepo = submittedRepository(dependencies); const submissionProcessor = submissionProcessorFactory.create(dependencies); + const submissionRecordsRepository = createSubmissionRecordsRepository(dependencies); + const submissionFilesRepository = createSubmissionFilesRepository(dependencies); const { logger } = dependencies; const { convertRecordsToCompoundDocuments } = viewMode(dependencies); @@ -56,7 +59,7 @@ const submittedData = (dependencies: BaseDependencies) => { }> => { const { getSubmittedDataBySystemId } = submittedDataRepo; const { getActiveDictionaryByCategory } = categoryRepository(dependencies); - const { getSubmissionDetailsById, update } = submissionRepository(dependencies); + const { update: udpateSubmission } = submissionRepository(dependencies); const { getOrCreateActiveSubmission } = submissionService(dependencies); // get SubmittedData by SystemId @@ -125,44 +128,44 @@ const submittedData = (dependencies: BaseDependencies) => { throw error; } - const activeSubmission = await getSubmissionDetailsById(activeSubmissionId); - - if (!activeSubmission) { - return { - status: ACTIVE_SUBMISSION_STATUS.INVALID_SUBMISSION, - description: 'Active Submission not found', - inProcessEntities: [], - }; - } - - // Merge current Active Submission delete entities with unique records to delete based on systemId - const mergedSubmissionDeletes = mergeDeleteRecords(activeSubmission.data.deletes || {}, recordsToDeleteMap); - - const entitiesToProcess = Object.keys(mergedSubmissionDeletes); - - // filter out update records found matching systemID on delete records - const filteredUpdates = filterUpdatesFromDeletes(activeSubmission.data.updates ?? {}, mergedSubmissionDeletes); + // TODO: get the existing entity names for a submission and merge with the new ones to avoid duplicates, + // but for now just insert the new records + const entitiesToProcess = Object.keys(recordsToDeleteMap); // Updating the Submission with the new data and 'VALIDATING' status before validation starts - await update(activeSubmission.id, { - data: { - inserts: activeSubmission.data.inserts, - updates: filteredUpdates, - deletes: mergedSubmissionDeletes, - }, + await udpateSubmission(activeSubmissionId, { updatedBy: username, status: 'VALIDATING', }); + await Promise.all( + Object.entries(recordsToDeleteMap).map(async ([entityName, entityRecords]) => { + const savedFileId = await submissionFilesRepository.save({ + entityName, + fileName: `update-${entityName}-${Date.now()}.json`, // default file name for adding JSON records + fileSize: JSON.stringify(entityRecords).length, + submissionId: activeSubmissionId, + }); + await submissionRecordsRepository.saveManyForFile( + savedFileId, + entityRecords.map((record) => ({ + actionType: 'DELETE', + data: record, + state: 'RECEIVED', + })), + ); + }), + ); + // Perform Schema Data validation in a worker thread - dependencies.workerPool.dataValidation({ submissionId: activeSubmission.id }); + dependencies.workerPool.dataValidation({ submissionId: activeSubmissionId }); logger.info(LOG_MODULE, `Added '${entitiesToProcess.length}' records to be deleted on the Active Submission`); return { status: ACTIVE_SUBMISSION_STATUS.PROCESSING, description: 'Submission data is being processed', - submissionId: activeSubmission.id.toString(), + submissionId: activeSubmissionId.toString(), inProcessEntities: entitiesToProcess, }; }; diff --git a/packages/data-provider/src/utils/recordsParser.ts b/packages/data-provider/src/utils/recordsParser.ts index 295fed97..5317cb2d 100644 --- a/packages/data-provider/src/utils/recordsParser.ts +++ b/packages/data-provider/src/utils/recordsParser.ts @@ -1,9 +1,7 @@ import { type DataRecord, parse, type Schema } from '@overture-stack/lectern-client'; -import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models'; import { getSchemaByName } from './dictionaryUtils.js'; import { convertRecordToString, notEmpty } from './formatUtils.js'; -import { createBatchResponse } from './submissionResponseParser.js'; import type { EntityData, SchemasDictionary } from './types.js'; /** @@ -37,7 +35,7 @@ export const convertToTypedRecords = (dataRecords: Record[], sc export const parseRecordsToInsert = ( records: EntityData, schemasDictionary: SchemasDictionary, -): Record => { +): Record => { return Object.fromEntries( Object.entries(records) .map(([schemaName, dataRecords]) => { @@ -53,7 +51,7 @@ export const parseRecordsToInsert = ( return null; } - return [schemaName, createBatchResponse(schemaName, parsedRecords)]; + return [schemaName, parsedRecords]; }) .filter(notEmpty), ); diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts index 18985295..7bd6ecc3 100644 --- a/packages/data-provider/src/utils/schemas.ts +++ b/packages/data-provider/src/utils/schemas.ts @@ -63,17 +63,6 @@ const pageSizeSchema = z.string().superRefine((value, ctx) => { } }); -const indexIntegerSchema = z.string().superRefine((value, ctx) => { - const parsed = parseInt(value); - if (isNaN(parsed)) { - ctx.addIssue({ - code: z.ZodIssueCode.invalid_type, - expected: 'number', - received: 'nan', - }); - } -}); - const positiveInteger = z.string().superRefine((value, ctx) => { const parsed = parseInt(value); if (isNaN(parsed)) { @@ -360,27 +349,25 @@ export const submissionDeleteRequestSchema: RequestValidation< }), }; -export interface SubmissionDeleteEntityNameParams extends ParamsDictionary { - actionType: string; +export interface SubmissionRecordDeleteParams extends ParamsDictionary { submissionId: string; } -export interface SubmissionDeleteEntityNameQueryParams extends ParsedQs { - entityName: string; - index?: string; +export interface SubmissionRecordDeleteQueryParams extends ParsedQs { + recordId?: string; + fileId?: string; } -export const submissionDeleteEntityNameRequestSchema: RequestValidation< +export const submissionRecordDeleteRequestSchema: RequestValidation< object, - SubmissionDeleteEntityNameQueryParams, - SubmissionDeleteEntityNameParams + SubmissionRecordDeleteQueryParams, + SubmissionRecordDeleteParams > = { query: z.object({ - entityName: entityNameSchema, - index: indexIntegerSchema.optional(), + recordId: positiveInteger.optional(), + fileId: positiveInteger.optional(), }), pathParams: z.object({ - actionType: submissionActionTypeSchema, submissionId: submissionIdSchema, }), }; diff --git a/packages/data-provider/src/utils/submissionResponseParser.ts b/packages/data-provider/src/utils/submissionResponseParser.ts index dcc54958..087dbb33 100644 --- a/packages/data-provider/src/utils/submissionResponseParser.ts +++ b/packages/data-provider/src/utils/submissionResponseParser.ts @@ -1,163 +1,48 @@ -import type { DataRecord } from '@overture-stack/lectern-client'; -import type { - SubmissionData, - SubmissionDeleteData, - SubmissionErrors, - SubmissionInsertData, - SubmissionRecordErrorDetails, - SubmissionUpdateData, -} from '@overture-stack/lyric-data-model/models'; - -import { type PaginationOptions, type SubmissionActionType } from './types.js'; - -export const createBatchResponse = (schemaName: string, records: DataRecord[]): SubmissionInsertData => { - return { batchName: schemaName, records }; -}; - -export type FlattenedSubmissionData = - | { type: 'INSERTS'; entity: string; value: DataRecord; index: number } - | { type: 'UPDATES'; entity: string; value: SubmissionUpdateData; index: number } - | { type: 'DELETES'; entity: string; value: SubmissionDeleteData; index: number }; - -/** - * Filters and paginates submission `data` and `errors` based on specified action types and entity names. - * Returns the paginated data along with the corresponding filtered errors. - */ -export const filterAndPaginateSubmissionData = ({ - data, - errors, - filterOptions, - paginationOptions, -}: { - data: SubmissionData; - errors: SubmissionErrors; - filterOptions: { - actionTypes: SubmissionActionType[]; - entityNames: string[]; +import type { RecordsSummaryRepository } from '../repository/submissionRecordsRepository.js'; +import { type SubmissionDataSummaryWithTotal } from './types.js'; + +// This function accepts a raw array of submission records from the database and builds a summary response. +export const buildDataSummary = (rows: RecordsSummaryRepository[]): SubmissionDataSummaryWithTotal => { + const initialSummary: SubmissionDataSummaryWithTotal = { + inserts: {}, + updates: {}, + deletes: {}, + totalRecords: 0, + errors: 0, }; - paginationOptions: PaginationOptions; -}): { data: FlattenedSubmissionData[]; errors: SubmissionRecordErrorDetails[] } => { - const { page, pageSize } = paginationOptions; - const { actionTypes, entityNames } = filterOptions; - - const flattenedRecords = flattenData(data, actionTypes, entityNames); - const startIndex = (page - 1) * pageSize; - const paginatedRecords = flattenedRecords.slice(startIndex, startIndex + pageSize); + return rows.reduce((acc, row) => { + const { actionType, entityName, totalRecords, batchName, errors } = row; + const summaryItem = { batchName: batchName ?? '', recordsCount: totalRecords, errors }; - // Extract indexes that belongs to paginated records - const paginatedRecordIndexes = paginatedRecords.map((record) => record.index); - const relevantErrors = getFilteredErrors({ - errors: errors || {}, - actionTypes, - entityNames, - indices: paginatedRecordIndexes, - }); - - return { data: paginatedRecords, errors: relevantErrors }; -}; + acc.totalRecords += totalRecords; + acc.errors += errors; -/** - * Flattens submission data into a list of records based on specified action types and entity names. - */ -const flattenData = ( - data: SubmissionData, - actionTypes: SubmissionActionType[], - entityNames: string[], -): FlattenedSubmissionData[] => { - const list: FlattenedSubmissionData[] = []; - - for (const action of actionTypes) { - const bucket = getActionData(data, action); - - if (!bucket) { - continue; - } - - for (const [entity, value] of Object.entries(bucket)) { - if ((entityNames.length > 0 && !entityNames.includes(entity)) || !value) { - continue; + switch (actionType) { + case 'INSERT': { + const entitySummaries = acc.inserts ?? {}; + entitySummaries[entityName] = entitySummaries[entityName] ?? []; + entitySummaries[entityName].push(summaryItem); + acc.inserts = entitySummaries; + break; } - - if (action === 'INSERTS') { - for (const [index, record] of value.records.entries()) { - list.push({ type: 'INSERTS', entity, value: record, index }); - } - continue; - } - - for (const [index, record] of value.entries()) { - list.push({ type: action, entity, value: record, index }); + case 'UPDATE': { + const entitySummaries = acc.updates ?? {}; + entitySummaries[entityName] = entitySummaries[entityName] ?? []; + entitySummaries[entityName].push(summaryItem); + acc.updates = entitySummaries; + break; } - } - } - - return list; -}; - -/** - * Retrieves the set of submission data corresponding to a specific action type. - */ -export const getActionData = (data: SubmissionData, actionType: SubmissionActionType) => { - switch (actionType) { - case 'INSERTS': - return data.inserts ?? {}; - - case 'UPDATES': - return data.updates ?? {}; - - case 'DELETES': - return data.deletes ?? {}; - } -}; - -/** - * Retrieves the set of submission errors corresponding to a specific action type. - */ -export const getActionErrors = (errors: SubmissionErrors, actionType: SubmissionActionType) => { - switch (actionType) { - case 'INSERTS': - return errors.inserts ?? {}; - - case 'UPDATES': - return errors.updates ?? {}; - - case 'DELETES': - return errors.deletes ?? {}; - } -}; - -/** - * Filters submission errors based on specified action types, entity names, and record indices. - */ -export const getFilteredErrors = ({ - errors, - actionTypes, - entityNames, - indices, -}: { - errors: SubmissionErrors; - actionTypes: SubmissionActionType[]; - entityNames: string[]; - indices: number[]; -}): SubmissionRecordErrorDetails[] => { - const allErrors: SubmissionRecordErrorDetails[] = []; - - for (const actionType of actionTypes) { - const bucket = getActionErrors(errors, actionType); - - if (bucket) { - for (const [entityName, records] of Object.entries(bucket)) { - if (entityNames.length > 0 && !entityNames.includes(entityName)) { - continue; - } - - for (const record of records) { - allErrors.push(record); - } + case 'DELETE': { + const entitySummaries = acc.deletes ?? {}; + entitySummaries[entityName] = entitySummaries[entityName] ?? { recordsCount: 0, errors: 0 }; + entitySummaries[entityName].recordsCount += totalRecords; + entitySummaries[entityName].errors += errors; + acc.deletes = entitySummaries; + break; } } - } - return allErrors.filter((err) => indices.includes(err.index)); + return acc; + }, initialSummary); }; diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index b960bbbd..5d04e4dc 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -11,22 +11,27 @@ import { validate, } from '@overture-stack/lectern-client'; import { - SubmissionData, type SubmissionDeleteData, - type SubmissionErrors, type SubmissionInsertData, + type SubmissionRecordErrors, type SubmissionUpdateData, type SubmittedData, } from '@overture-stack/lyric-data-model/models'; +import type { SubmissionRecordWithEntityName } from '../repository/submissionRecordsRepository.js'; import { getSubmittedFileEntity } from '../services/submission/submissionFile.js'; import { isSubmissionActionTypeValid } from './auditUtils.js'; import type { SchemaChildNode } from './dictionarySchemaRelations.js'; import { getSchemaFieldNames } from './dictionaryUtils.js'; import { readHeaders, readTextFile } from './fileUtils.js'; -import { asArray, deepCompare } from './formatUtils.js'; +import { asArray } from './formatUtils.js'; import type { FilenameEntityPair } from './schemas.js'; -import { groupErrorsByIndex, mapAndMergeSubmittedDataToRecordReferences } from './submittedDataUtils.js'; +import { + groupErrorsByIndex, + mapAndMergeSubmittedDataToRecordReferences, + type SubmissionInsertRecordWithEntityName, + type SubmissionUpdateRecordWithEntityName, +} from './submittedDataUtils.js'; import { BATCH_ERROR_TYPE, type BatchError, @@ -35,22 +40,63 @@ import { type FileSchemaMap as FileSchemaMap, MERGE_REFERENCE_TYPE, type NewSubmittedDataReference, - SUBMISSION_ACTION_TYPE, + SUBMISSION_RECORD_ACTION_TYPE, SUBMISSION_STATUS, - type SubmissionActionType, - type SubmissionDataDetailsRepositoryRecord, - type SubmissionDataSummary, - type SubmissionDataSummaryRepositoryRecord, - type SubmissionDetailsResponse, - type SubmissionErrorsSummary, + type SubmissionRecordActionType, type SubmissionStatus, type SubmissionSummary, + type SubmissionSummaryResponse, SubmittedDataReference, } from './types.js'; export const inProcessSubmissionStatus = [SUBMISSION_STATUS.VALIDATING, SUBMISSION_STATUS.COMMITTING] as const; export type InProcessSubmissionStatus = typeof inProcessSubmissionStatus; +type SubmissionInsertBatch = { + batchName: string; + records: DataRecord[]; +}; + +const FILE_PARSE_CONCURRENCY = 4; + +const parseEntityFiles = async (files: Express.Multer.File[], schema: Schema): Promise => { + const parsedRecordsByIndex: DataRecord[][] = new Array(files.length); + let nextFileIndex = 0; + + const worker = async () => { + while (true) { + const fileIndex = nextFileIndex; + nextFileIndex++; + + if (fileIndex >= files.length) { + return; + } + + const file = files[fileIndex]; + if (!file) { + continue; + } + const parsedFileData = await readTextFile(file, schema); + + if (parsedFileData.errors && parsedFileData.errors.length > 0) { + throw new Error( + `File '${file.originalname}' has ${parsedFileData.errors.length} parsing ${plur( + 'error', + parsedFileData.errors.length, + )}`, + ); + } + + parsedRecordsByIndex[fileIndex] = parsedFileData.records; + } + }; + + const workers = Array.from({ length: Math.min(FILE_PARSE_CONCURRENCY, files.length) }, () => worker()); + await Promise.all(workers); + + return parsedRecordsByIndex; +}; + // Only "open", "valid", and "invalid" statuses are considered Active Submission export const openSubmissionStatus = [ SUBMISSION_STATUS.OPEN, @@ -269,6 +315,7 @@ export const filterRecordsByConflicts = ( * @param submissionUpdateData - A record containing arrays of `SubmissionUpdateData` to be filtered. * @param submissionDeleteData - A record containing arrays of `SubmissionDeleteData` that defines the conflicts. * @returns A filtered record of `SubmissionUpdateData[]` where no items conflict with those in `submissionDeleteData`. + * TODO: do not delete this function, this MIGHT be usefull in the future to avoid duplicates between submission records */ export const filterUpdatesFromDeletes = ( submissionUpdateData: Record, @@ -331,6 +378,16 @@ export const filterRelationsForPrimaryIdUpdate = ( }) ); }; +type SubmissionRecordErrorDetails = { + recordId: number; + errors: SubmissionRecordErrors[]; +}; + +export type SubmissionErrors = { + inserts?: Record; + updates?: Record; + deletes?: Record; +}; /** * Returns only the schema errors corresponding to the Active Submission. @@ -369,14 +426,9 @@ export const groupSchemaErrorsByEntity = (input: { return; } - const submissionIndex = mapping.reference.index; + const submissionRecordId = mapping.reference.recordId; const actionType = mapping.reference.type === MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA ? 'inserts' : 'updates'; - const mutableSchemaValidationErrors = schemaValidationErrors.map((errors) => ({ - ...errors, - index: submissionIndex, - })); - if (!submissionSchemaErrors[actionType]) { submissionSchemaErrors[actionType] = {}; } @@ -385,7 +437,10 @@ export const groupSchemaErrorsByEntity = (input: { submissionSchemaErrors[actionType][entityName] = []; } - submissionSchemaErrors[actionType][entityName].push(...mutableSchemaValidationErrors); + submissionSchemaErrors[actionType][entityName].push({ + recordId: submissionRecordId, + errors: schemaValidationErrors, + }); }); }); return submissionSchemaErrors; @@ -396,25 +451,21 @@ export const groupSchemaErrorsByEntity = (input: { * and maps it to it's original reference Id * The result mapping is used to perform the cross schema validation * @param {number} activeSubmissionId - * @param {Record} activeSubmissionInsertDataEntities - * @returns {Record} + * @param {SubmissionInsertRecordWithEntityName[]} activeSubmissionInsertDataEntities + * @returns {DataRecordReference[]} */ export const mapInsertDataToRecordReferences = ( activeSubmissionId: number, - activeSubmissionInsertDataEntities: Record, -): Record => { - return _.mapValues(activeSubmissionInsertDataEntities, (submissionInsertData) => - submissionInsertData.records.map((record, index) => { - return { - dataRecord: record, - reference: { - submissionId: activeSubmissionId, - type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - index: index, - }, - }; - }), - ); + activeSubmissionInsertDataEntities: SubmissionInsertRecordWithEntityName[], +): DataRecordReference[] => { + return activeSubmissionInsertDataEntities.map((submissionInsertData) => ({ + dataRecord: submissionInsertData.data, + reference: { + submissionId: activeSubmissionId, + type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, + recordId: submissionInsertData.recordId, + }, + })); }; /** @@ -460,9 +511,7 @@ export const mapGroupedUpdateSubmissionData = ({ * The returned Object is a collection of the raw Schema Data with it's reference ID grouped by entity name. * @param {number} submissionId ID of the Active Submission * @param {Object} submissionData - * @param {Record} submissionData.insertData Collection of Data records of the Active Submission - * @param {Record} submissionData.updateData Collection of Data records of the Active Submission - * @param {Record} submissionData.deleteData Collection of Data records of the Active Submission + * @param {SubmissionRecordWithEntityName[]} submissionData The Active Submission data * @param {SubmittedData[]} submittedData An array of Submitted Data * @returns {Record} */ @@ -472,12 +521,12 @@ export const mergeAndReferenceEntityData = ({ submittedData, }: { submissionId: number; - submissionData: SubmissionData; + submissionData: SubmissionRecordWithEntityName[]; submittedData: SubmittedData[]; }): Record => { - const systemsIdsToRemove = submissionData.deletes - ? Object.values(submissionData.deletes).flatMap((entityData) => entityData.map(({ systemId }) => systemId)) - : []; + const systemsIdsToRemove = submissionData + .filter((item) => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.DELETE) + .map((item) => item.data.systemId); // Exclude items that are marked for deletion const submittedDataFiltered = @@ -485,15 +534,43 @@ export const mergeAndReferenceEntityData = ({ ? submittedData.filter(({ systemId }) => !systemsIdsToRemove.includes(systemId)) : submittedData; + const dataToUpdate: SubmissionUpdateRecordWithEntityName[] = submissionData + .filter( + ( + item, + ): item is SubmissionRecordWithEntityName & { + actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE; + data: SubmissionUpdateData; + } => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE, + ) + .map((item) => ({ + recordId: item.id, + entityName: item.entityName, + data: item.data, + })); + const submittedDataWithRef = mapAndMergeSubmittedDataToRecordReferences({ submittedData: submittedDataFiltered, - editSubmittedData: submissionData.updates, + editSubmittedData: dataToUpdate, submissionId, }); - const insertDataWithRef = submissionData.inserts - ? mapInsertDataToRecordReferences(submissionId, submissionData.inserts) - : {}; + const dataToInsert: SubmissionInsertRecordWithEntityName[] = submissionData + .filter( + ( + item, + ): item is SubmissionRecordWithEntityName & { + actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT; + data: SubmissionInsertData; + } => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT, + ) + .map((item) => ({ + recordId: item.id, + entityName: item.entityName, + data: item.data, + })); + + const insertDataWithRef = dataToInsert.length > 0 ? mapInsertDataToRecordReferences(submissionId, dataToInsert) : {}; // This object will merge existing data + new data for validation (Submitted data + active Submission) return _.mergeWith(submittedDataWithRef, insertDataWithRef, (objValue, srcValue) => { @@ -504,90 +581,6 @@ export const mergeAndReferenceEntityData = ({ }); }; -/** - * Merges multiple `Record` objects into a single object. - * If there are duplicate keys between the objects, the `records` arrays of `SubmissionInsertData` - * are concatenated for the matching keys, ensuring no duplicates. - * - * @param objects An array of objects where each object is a `Record`. - * Each key represents the entityName, and the value is an object of type `SubmissionInsertData`. - * - * @returns A new `Record` where: - * - If a key is unique across all objects, its value is directly included. - * - If a key appears in multiple objects, the `records` arrays are concatenated for that key, avoiding duplicates. - */ -export const mergeInsertsRecords = ( - ...objects: Record[] -): Record => { - const result: Record = {}; - - let seen: DataRecord[] = []; - // Iterate over all objects - objects.forEach((obj) => { - // Iterate over each key in the current object - Object.entries(obj).forEach(([key, value]) => { - if (result[key]) { - // The key already exists in the result, concatenate the `records` arrays, avoiding duplicates - let uniqueData: DataRecord[] = []; - - result[key].records.concat(value.records).forEach((item) => { - if (!seen.some((existingItem) => deepCompare(existingItem, item))) { - uniqueData = uniqueData.concat(item); - seen = seen.concat(item); - } - }); - - result[key].records = uniqueData; - return; - } else { - // The key doesn't exists in the result, create as it comes - result[key] = value; - return; - } - }); - }); - - return result; -}; - -/** - * Merges multiple `Record` objects into a single object. - * For each key, the `SubmissionDeleteData[]` arrays are concatenated, ensuring no duplicate - * `SubmissionDeleteData` objects based on the `systemId` field. - * - * @param objects Multiple `Record` objects to be merged. - * Each key represents an identifier, and the value is an array of `SubmissionDeleteData`. - * - * @returns - */ -export const mergeDeleteRecords = ( - ...objects: Record[] -): Record => { - const result: Record = {}; - - // Iterate over all objects - objects.forEach((obj) => { - // Iterate over each key in the current object - Object.entries(obj).forEach(([key, value]) => { - if (!result[key]) { - result[key] = []; - } - const uniqueRecords = new Map(); - - // Add existing records to the map - result[key].forEach((record) => uniqueRecords.set(record.systemId, record)); - - // Add new records, overriding duplicates based on systemId - value.forEach((record) => uniqueRecords.set(record.systemId, record)); - - // Convert the map back to an array - result[key] = Array.from(uniqueRecords.values()); - }); - }); - - return result; -}; - /** * Merge Active Submission data with incoming TSV file data processed * @@ -624,58 +617,16 @@ export const mergeUpdatesBySystemId = ( }; /** - * Utility to convert a raw Submission record to a Response type - * @param {SubmissionDataDetailsRepositoryRecord} submission - * @returns {SubmissionDetailsResponse} + * Utility to convert Dates to ISO string for a SubmissionSummaryResponse + * @param {SubmissionSummary} submission + * @returns {SubmissionSummaryResponse} */ -export const createSubmissionDetailsResponse = ( - submission: SubmissionDataDetailsRepositoryRecord, -): SubmissionDetailsResponse => { +export const createSubmissionSummaryResponse = (submission: SubmissionSummary): SubmissionSummaryResponse => { return { id: submission.id, data: submission.data, dictionary: submission.dictionary, dictionaryCategory: submission.dictionaryCategory, - errors: submission.errors || {}, - organization: submission.organization, - status: submission.status, - createdAt: _.toString(submission.createdAt?.toISOString()), - createdBy: _.toString(submission.createdBy), - updatedAt: _.toString(submission.updatedAt?.toISOString()), - updatedBy: _.toString(submission.updatedBy), - }; -}; - -/** - * Utility to sum the recordsCount from a SubmissionDataSummary or SubmissionErrorsSummary - */ -const sumRecordsCount = (buckets: SubmissionDataSummary | SubmissionErrorsSummary): number => { - return Object.values(buckets) - .flatMap((bucket) => (bucket ? Object.values(bucket) : [])) - .reduce((total, { recordsCount }) => total + recordsCount, 0); -}; - -/** - * Utility to convert the raw SubmissionDataSummaryRepositoryRecord into a SubmissionSummaryResponse. - * It includes a `total` value representing the sum of changes of each `data` and `errors` - * @param {SubmissionDataSummaryRepositoryRecord} submission - * @returns {SubmissionSummary} - */ -export const createSubmissionSummaryResponse = ( - submission: SubmissionDataSummaryRepositoryRecord, -): SubmissionSummary => { - return { - id: submission.id, - data: { - ...submission.data, - total: sumRecordsCount(submission.data), - }, - dictionary: submission.dictionary, - dictionaryCategory: submission.dictionaryCategory, - errors: { - ...submission.errors, - total: sumRecordsCount(submission.errors ?? {}), - }, organization: submission.organization, status: submission.status, createdAt: _.toString(submission.createdAt?.toISOString()), @@ -689,104 +640,6 @@ export const pluralizeSchemaName = (schemaName: string) => { return plur(schemaName); }; -export const removeItemsFromSubmission = ( - submissionData: SubmissionData, - filter: { actionType: SubmissionActionType; entityName: string; index: number | null }, -): SubmissionData => { - const filteredSubmissionData = _.cloneDeep(submissionData); - switch (filter.actionType) { - case SUBMISSION_ACTION_TYPE.Values.INSERTS: - if (submissionData.inserts) { - const filteredInserts = Object.entries(submissionData.inserts).reduce>( - (acc, [insertsEntityName, insertsSubmissionData]) => { - if (insertsEntityName === filter.entityName && filter.index == null) { - // remove this whole entity - return acc; - } else if (insertsEntityName === filter.entityName && filter.index != null) { - // remove an item on records based on it's index - const filteredRecords = insertsSubmissionData.records.filter( - (_, recordIndex) => recordIndex !== filter.index, - ); - if (filteredRecords.length > 0) { - acc[insertsEntityName] = { - batchName: insertsSubmissionData.batchName, - records: filteredRecords, - }; - } - } else { - acc[insertsEntityName] = insertsSubmissionData; - } - - return acc; - }, - {}, - ); - if (Object.keys(filteredInserts).length === 0) { - delete filteredSubmissionData.inserts; - } else { - filteredSubmissionData.inserts = filteredInserts; - } - } - break; - case SUBMISSION_ACTION_TYPE.Values.UPDATES: - if (submissionData.updates) { - const filteredUpdates = Object.entries(submissionData.updates).reduce>( - (acc, [updatesEntityName, updatesSubmissionData]) => { - if (updatesEntityName === filter.entityName && filter.index == null) { - // remove this whole entity - return acc; - } else if (updatesEntityName === filter.entityName && filter.index != null) { - // remove an item on records based on it's index - const filteredRecords = updatesSubmissionData.filter((_, recordIndex) => recordIndex !== filter.index); - if (filteredRecords.length > 0) { - acc[updatesEntityName] = filteredRecords; - } - } else { - acc[updatesEntityName] = updatesSubmissionData; - } - - return acc; - }, - {}, - ); - if (Object.keys(filteredUpdates).length === 0) { - delete filteredSubmissionData.updates; - } else { - filteredSubmissionData.updates = filteredUpdates; - } - } - break; - case SUBMISSION_ACTION_TYPE.Values.DELETES: - if (submissionData.deletes) { - const filteredDeletes = Object.entries(submissionData.deletes).reduce>( - (acc, [deletesEntityName, deletesSubmissionData]) => { - if (deletesEntityName === filter.entityName && filter.index == null) { - // remove this whole entity - return acc; - } else if (deletesEntityName === filter.entityName && filter.index != null) { - // remove an item on records based on it's index - const filteredRecords = deletesSubmissionData.filter((_, recordIndex) => recordIndex !== filter.index); - if (filteredRecords.length > 0) { - acc[deletesEntityName] = filteredRecords; - } - } else { - acc[deletesEntityName] = deletesSubmissionData; - } - return acc; - }, - {}, - ); - if (Object.keys(filteredDeletes).length === 0) { - delete filteredSubmissionData.deletes; - } else { - filteredSubmissionData.deletes = filteredDeletes; - } - } - break; - } - return filteredSubmissionData; -}; - /** * Processes the `foundDependentUpdates` array and segregates the updates based on * whether they involve ID fields (dependent fields) or non-ID fields. @@ -838,22 +691,27 @@ export const segregateFieldChangeRecords = ( */ export const submissionInsertDataFromFiles = async ( fileSchemaMap: FileSchemaMap, -): Promise> => { - const output: Record = {}; +): Promise> => { + const output: Record = {}; for (const [entityName, { files, schema }] of Object.entries(fileSchemaMap)) { - for (const file of files) { - const outputEntityValue = output[schema.name] ?? { - batchName: entityName, - records: [], - }; - const parsedFileData = await readTextFile(file, schema); - // TODO: This doesn't handle parsedFileData.errors, when present. + if (files.length === 0) { + continue; + } + const firstFile = files[0]; + if (!firstFile) { + continue; + } - outputEntityValue.records.push(...parsedFileData.records); + const outputEntityValue = output[entityName] ?? { + batchName: firstFile.originalname, + records: [], + }; - output[schema.name] = outputEntityValue; - } + const parsedRecordsByIndex = await parseEntityFiles(files, schema); + parsedRecordsByIndex.forEach((records) => outputEntityValue.records.push(...records)); + + output[entityName] = outputEntityValue; } return output; @@ -887,9 +745,9 @@ export const parseToSchema = (schema: Schema) => (record: Record return parsedRecord.data.record; }; -export const parseSubmissionActionTypes = (values: unknown): SubmissionActionType[] => { +export const parseSubmissionActionTypes = (values: unknown): SubmissionRecordActionType[] => { return asArray(values || []) .map((value) => value.toString().toUpperCase()) .filter(isSubmissionActionTypeValid) - .map((value) => SUBMISSION_ACTION_TYPE.parse(value)); + .map((value) => SUBMISSION_RECORD_ACTION_TYPE.parse(value)); }; diff --git a/packages/data-provider/src/utils/submittedDataUtils.ts b/packages/data-provider/src/utils/submittedDataUtils.ts index bcd42478..5ac8e880 100644 --- a/packages/data-provider/src/utils/submittedDataUtils.ts +++ b/packages/data-provider/src/utils/submittedDataUtils.ts @@ -10,6 +10,7 @@ import { type DataDiff, NewSubmittedData, type SubmissionDeleteData, + type SubmissionInsertData, type SubmissionUpdateData, SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -237,8 +238,8 @@ export const groupSchemaDataByEntityName = (data: { * Edits each record that is marked to be edited on the Submission * @param {object} params * @param {SubmittedData[] | undefined} params.submittedData An array of `SubmittedData` objects to be transformed. - * @param {Record} params.editSubmittedData An Array of `SubmittedData` objects to be updated - * @param {Rnumber} params.submissionId The ID of the Active Submission + * @param {SubmissionRecordWithEntityName[]} params.editSubmittedData An Array of `SubmittedData` objects to be updated + * @param {number} params.submissionId The ID of the Active Submission * @returns {Record} */ export const mapAndMergeSubmittedDataToRecordReferences = ({ @@ -247,26 +248,20 @@ export const mapAndMergeSubmittedDataToRecordReferences = ({ submissionId, }: { submittedData?: SubmittedData[]; - editSubmittedData?: Record; + editSubmittedData?: SubmissionUpdateRecordWithEntityName[]; submissionId: number; }): Record => { if (!submittedData) { return {}; } return submittedData.reduce>((acc, entityData) => { - const entityEditData = editSubmittedData?.[entityData.entityName]; - const foundRecordToUpdateIndex = entityEditData - ? entityEditData.findIndex((item) => item.systemId === entityData.systemId) - : -1; - let record: DataRecordReference; - if (entityEditData && foundRecordToUpdateIndex >= 0) { - const recordToUpdate = entityEditData[foundRecordToUpdateIndex]; - - if (!recordToUpdate) { - return acc; - } + const recordToUpdate = editSubmittedData?.find( + (item) => item.entityName === entityData.entityName && item.data.systemId === entityData.systemId, + ); - const newDataToUpdate = updateEntityData(entityData.data, recordToUpdate); + let record: DataRecordReference; + if (recordToUpdate) { + const newDataToUpdate = updateEntityData(entityData.data, recordToUpdate.data); record = { dataRecord: newDataToUpdate, @@ -274,7 +269,7 @@ export const mapAndMergeSubmittedDataToRecordReferences = ({ type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, systemId: entityData.systemId, submissionId, - index: foundRecordToUpdateIndex, + recordId: recordToUpdate.recordId, }, }; } else { @@ -349,6 +344,17 @@ export const updateSubmittedDataArray = ( }); }; +export type SubmissionInsertRecordWithEntityName = { + recordId: number; + entityName: string; + data: SubmissionInsertData; +}; +export type SubmissionUpdateRecordWithEntityName = { + recordId: number; + entityName: string; + data: SubmissionUpdateData; +}; + /** * Updates the entity data based on the provided update request. * It removes old keys, filters out undefined values from the new data, diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 57e0dd65..d0324667 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -12,9 +12,7 @@ import { type DataDiff, type Dictionary, NewSubmittedData, - SubmissionData, type SubmissionDeleteData, - type SubmissionErrors, type SubmissionUpdateData, type SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -169,6 +167,15 @@ export type { Schema, SchemasDictionary }; export const SUBMISSION_ACTION_TYPE = z.enum(['INSERTS', 'UPDATES', 'DELETES']); export type SubmissionActionType = z.infer; +/** + * Enum matching Submission Record state in database + */ +export const SUBMISSION_RECORD_STATE = z.enum(['RECEIVED', 'VALID', 'INVALID']); +export type SubmissionRecordState = z.infer; + +export const SUBMISSION_RECORD_ACTION_TYPE = z.enum(['INSERT', 'UPDATE', 'DELETE']); +export type SubmissionRecordActionType = z.infer; + /** * File upload validation error types */ @@ -253,85 +260,52 @@ export type PaginationOptions = { export type DataInsertsSubmissionSummary = { batchName: string; recordsCount: number; + errors: number; }; export type DataUpdatesSubmissionSummary = { + batchName: string; recordsCount: number; + errors: number; }; export type DataDeletesSubmissionSummary = { recordsCount: number; -}; - -export type DataErrorsSubmissionSummary = { - recordsCount: number; -}; - -/** - * Response type for Get Submission by Submission ID endpoint - */ -export type SubmissionDetailsResponse = { - id: number; - data: SubmissionData; - dictionary: DictionarySummary; - dictionaryCategory: CategorySummary; - errors: SubmissionErrors; - organization: string; - status: SubmissionStatus; - createdAt: string; - createdBy: string; - updatedAt: string; - updatedBy: string; + errors: number; }; export type SubmissionDataSummary = { - inserts?: Record; - updates?: Record; + inserts?: Record; + updates?: Record; deletes?: Record; }; -export type SubmissionErrorsSummary = { - inserts?: Record; - updates?: Record; - deletes?: Record; +export type SubmissionDataSummaryWithTotal = SubmissionDataSummary & { + totalRecords: number; + errors: number; }; /** * Shortened version of the Submission record that omits the data changes and error details * in favour of the count of records changed and errors for each entity type. */ -export type SubmissionSummary = Omit & { - data: SubmissionDataSummary & { total: number }; -} & { - errors: SubmissionErrorsSummary & { total: number }; +export type SubmissionSummary = SubmissionWithDictionaryAndCategoryRepositoryRecord & { + data: SubmissionDataSummaryWithTotal; }; -/** - * Retrieve Submission object with data summary from repository - */ -export type SubmissionDataSummaryRepositoryRecord = { - id: number; - data: SubmissionDataSummary; - dictionary: DictionarySummary; - dictionaryCategory: CategorySummary; - errors: SubmissionErrorsSummary | null; - organization: string; - status: SubmissionStatus; - createdAt: Date | null; - createdBy: string | null; - updatedAt: Date | null; - updatedBy: string | null; +export type SubmissionSummaryResponse = Omit & { + createdAt: string; + updatedAt: string; }; /** - * Retrieve Submission object with data details from repository + * Retrieve Submission object with Dictionary and Category from repository */ -export type SubmissionDataDetailsRepositoryRecord = { + +export type SubmissionWithDictionaryAndCategoryRepositoryRecord = { id: number; - data: SubmissionData; dictionary: DictionarySummary; dictionaryCategory: CategorySummary; - errors: SubmissionErrors | null; organization: string; status: SubmissionStatus; createdAt: Date | null; @@ -445,13 +419,13 @@ export interface SubmittedDataReference { } export interface NewSubmittedDataReference { - index: number; + recordId: number; submissionId: number; type: typeof MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA; } export interface EditSubmittedDataReference { - index: number; + recordId: number; systemId?: string; submissionId: number; type: typeof MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA; diff --git a/packages/data-provider/src/workers/commitSubmissionWorker.ts b/packages/data-provider/src/workers/commitSubmissionWorker.ts index 982e486f..11fe687f 100644 --- a/packages/data-provider/src/workers/commitSubmissionWorker.ts +++ b/packages/data-provider/src/workers/commitSubmissionWorker.ts @@ -1,9 +1,14 @@ -import type { SubmissionUpdateData } from '@overture-stack/lyric-data-model/models'; +import type { + SubmissionDeleteData, + SubmissionInsertData, + SubmissionUpdateData, +} from '@overture-stack/lyric-data-model/models'; import systemIdGenerator from '../external/systemIdGenerator.js'; import createSubmissionRepository from '../repository/activeSubmissionRepository.js'; import createCategoryRepository from '../repository/categoryRepository.js'; -import submittedRepository from '../repository/submittedRepository.js'; +import createSubmissionRecordsRepository from '../repository/submissionRecordsRepository.js'; +import createSubmittedRepository from '../repository/submittedRepository.js'; import submissionProcessorFactory from '../services/submission/submissionProcessor.js'; import type { ResultOnCommit } from '../utils/types.js'; import { SUBMISSION_STATUS } from '../utils/types.js'; @@ -23,12 +28,13 @@ export const processCommitSubmission = async (message: CommitWorkerInput): Promi const submissionRepo = createSubmissionRepository(dependencies); const categoryRepo = createCategoryRepository(dependencies); - const submittedDataRepo = submittedRepository(dependencies); + const submittedDataRepo = createSubmittedRepository(dependencies); + const submissionRecordsRepo = createSubmissionRecordsRepository(dependencies); const submissionProcessor = submissionProcessorFactory.create(dependencies); // Fetch submission - const submission = await submissionRepo.getSubmissionDetailsById(submissionId); + const submission = await submissionRepo.getSubmissionById(submissionId); if (!submission) { throw new Error(`Submission '${submissionId}' not found`); } @@ -54,39 +60,65 @@ export const processCommitSubmission = async (message: CommitWorkerInput): Promi const { generateIdentifier } = systemIdGenerator(dependencies); + const recordsToInsert = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, { + actionTypes: ['INSERT'], + }); + // Build inserts for validation - const insertsToValidate = submission.data?.inserts - ? Object.entries(submission.data.inserts).flatMap(([entityName, submissionData]) => { - return submissionData.records.map((record) => ({ - data: record, - dictionaryCategoryId: categoryId, - entityName, - isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise - organization: submission.organization, - originalSchemaId: currentDictionary.id, - systemId: generateIdentifier(entityName, record), - createdBy: username, - })); - }) - : []; - - const deleteDataArray = submission.data?.deletes - ? Object.entries(submission.data.deletes).flatMap(([_entityName, submissionDeleteData]) => { - return submissionDeleteData; - }) - : []; - - const updateDataArray = - submission.data?.updates && - Object.entries(submission.data.updates).reduce>( - (acc, [_entityName, submissionUpdateData]) => { - submissionUpdateData.forEach((record) => { - acc[record.systemId] = record; - }); - return acc; - }, - {}, - ); + const insertsToValidate = recordsToInsert + .filter( + ( + record, + ): record is (typeof recordsToInsert)[number] & { + actionType: 'INSERT'; + data: SubmissionInsertData; + } => record.actionType === 'INSERT', + ) + .map(({ entityName, data }) => { + return { + data, + dictionaryCategoryId: categoryId, + entityName, + isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise + organization: submission.organization, + originalSchemaId: currentDictionary.id, + systemId: generateIdentifier(entityName, data), + createdBy: username, + }; + }); + + const recordsToDelete = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, { + actionTypes: ['DELETE'], + }); + + const deleteDataArray = recordsToDelete + .filter( + ( + record, + ): record is (typeof recordsToDelete)[number] & { + actionType: 'DELETE'; + data: SubmissionDeleteData; + } => record.actionType === 'DELETE', + ) + .map(({ data }) => data); + + const recordsToUpdate = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, { + actionTypes: ['UPDATE'], + }); + + const updatesBySystemId = recordsToUpdate + .filter( + ( + record, + ): record is (typeof recordsToUpdate)[number] & { + actionType: 'UPDATE'; + data: SubmissionUpdateData; + } => record.actionType === 'UPDATE', + ) + .reduce>((acc, { data }) => { + acc[data.systemId] = data; + return acc; + }, {}); try { return await submissionProcessor.performCommitSubmissionAsync({ @@ -94,7 +126,7 @@ export const processCommitSubmission = async (message: CommitWorkerInput): Promi inserts: insertsToValidate, submittedData: submittedDataToValidate, deletes: deleteDataArray, - updates: updateDataArray, + updates: updatesBySystemId, }, submissionId: submission.id, dictionary: currentDictionary, From f30237630123c498311e958bd84b046bd11add93 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 10 Aug 2026 11:05:48 -0400 Subject: [PATCH 03/20] fix unit tests --- .../src/utils/submissionUtils.ts | 30 +- packages/data-provider/test/assertions.ts | 10 + .../groupSchemaErrorsByEntity.spec.ts | 45 +- .../mapInsertDataToRecordReferences.spec.ts | 124 +++--- .../mergeAndReferenceEntityData.spec.ts | 127 +++--- .../submission/mergeDeleteRecords.spec.ts | 106 ----- .../submission/mergeInsertsRecords.spec.ts | 46 -- .../submission/parseRecordsToInsert.spec.ts | 37 +- .../parseSubmissionResponse.spec.ts | 152 ------- .../parseSubmissionSummaryResponse.spec.ts | 67 +-- .../removeItemsFromSubmission.spec.ts | 392 ------------------ 11 files changed, 244 insertions(+), 892 deletions(-) create mode 100644 packages/data-provider/test/assertions.ts delete mode 100644 packages/data-provider/test/unit/utils/submission/mergeDeleteRecords.spec.ts delete mode 100644 packages/data-provider/test/unit/utils/submission/mergeInsertsRecords.spec.ts delete mode 100644 packages/data-provider/test/unit/utils/submission/parseSubmissionResponse.spec.ts delete mode 100644 packages/data-provider/test/unit/utils/submission/removeItemsFromSubmission.spec.ts diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index 5d04e4dc..9f944095 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -457,15 +457,27 @@ export const groupSchemaErrorsByEntity = (input: { export const mapInsertDataToRecordReferences = ( activeSubmissionId: number, activeSubmissionInsertDataEntities: SubmissionInsertRecordWithEntityName[], -): DataRecordReference[] => { - return activeSubmissionInsertDataEntities.map((submissionInsertData) => ({ - dataRecord: submissionInsertData.data, - reference: { - submissionId: activeSubmissionId, - type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - recordId: submissionInsertData.recordId, +): Record => { + return activeSubmissionInsertDataEntities.reduce>( + (acc, submissionInsertData) => { + const entityName = submissionInsertData.entityName; + let entityRecords = acc[entityName]; + if (!entityRecords) { + entityRecords = []; + acc[entityName] = entityRecords; + } + entityRecords.push({ + dataRecord: submissionInsertData.data, + reference: { + submissionId: activeSubmissionId, + type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, + recordId: submissionInsertData.recordId, + }, + }); + return acc; }, - })); + {}, + ); }; /** @@ -554,6 +566,7 @@ export const mergeAndReferenceEntityData = ({ editSubmittedData: dataToUpdate, submissionId, }); + console.log(`Submitted Data with Reference: ${JSON.stringify(submittedDataWithRef, null, 2)}`); const dataToInsert: SubmissionInsertRecordWithEntityName[] = submissionData .filter( @@ -571,6 +584,7 @@ export const mergeAndReferenceEntityData = ({ })); const insertDataWithRef = dataToInsert.length > 0 ? mapInsertDataToRecordReferences(submissionId, dataToInsert) : {}; + console.log(`Insert Data with Reference: ${JSON.stringify(insertDataWithRef, null, 2)}`); // This object will merge existing data + new data for validation (Submitted data + active Submission) return _.mergeWith(submittedDataWithRef, insertDataWithRef, (objValue, srcValue) => { diff --git a/packages/data-provider/test/assertions.ts b/packages/data-provider/test/assertions.ts new file mode 100644 index 00000000..4b493b03 --- /dev/null +++ b/packages/data-provider/test/assertions.ts @@ -0,0 +1,10 @@ +import { expect } from 'chai'; + +/** + * Asserts that the value is not null or undefined. + * This lets TypeScript know the value is of type T after this check. + * @param value The value to check for existence. + */ +export function assertExists(value: T | null | undefined): asserts value is T { + expect(value).to.exist; +} diff --git a/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts b/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts index 28097c56..700846dd 100644 --- a/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts +++ b/packages/data-provider/test/unit/utils/submission/groupSchemaErrorsByEntity.spec.ts @@ -5,6 +5,7 @@ import type { DictionaryValidationError, TestResult } from '@overture-stack/lect import { groupSchemaErrorsByEntity } from '../../../../src/utils/submissionUtils.js'; import { type DataRecordReference, MERGE_REFERENCE_TYPE } from '../../../../src/utils/types.js'; +import { assertExists } from '../../../assertions.js'; describe('Submission Utils - Group validation errors by entity', () => { it('retuns empty object when there is no data being processed', () => { @@ -71,7 +72,7 @@ describe('Submission Utils - Group validation errors by entity', () => { { dataRecord: { title: 'XYZ123' }, reference: { - index: 12, + recordId: 12, submissionId: 23, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, }, @@ -79,7 +80,7 @@ describe('Submission Utils - Group validation errors by entity', () => { { dataRecord: { sex_at_birth: 'Homme' }, reference: { - index: 12, + recordId: 13, submissionId: 23, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, }, @@ -89,11 +90,13 @@ describe('Submission Utils - Group validation errors by entity', () => { const response = groupSchemaErrorsByEntity({ resultValidation, dataValidated }); expect(Object.keys(response)).to.eql(['inserts']); + assertExists(response['inserts']); expect(Object.keys(response['inserts'])).to.eql(['sports']); + assertExists(response['inserts']['sports']); expect(response['inserts']['sports'].length).to.eq(2); expect(response['inserts']['sports']).to.eql([ - { fieldName: 'systemId', reason: 'UNRECOGNIZED_FIELD', fieldValue: '', index: 12 }, - { fieldName: 'sex_at_birth', reason: 'UNRECOGNIZED_FIELD', fieldValue: 'Homme', index: 12 }, + { errors: [{ fieldName: 'systemId', reason: 'UNRECOGNIZED_FIELD', fieldValue: '' }], recordId: 12 }, + { errors: [{ fieldName: 'sex_at_birth', reason: 'UNRECOGNIZED_FIELD', fieldValue: 'Homme' }], recordId: 13 }, ]); }); it('retuns errors found on the Submission updates', () => { @@ -123,7 +126,7 @@ describe('Submission Utils - Group validation errors by entity', () => { { dataRecord: { title: 'XYZ123' }, reference: { - index: 12, + recordId: 12, submissionId: 23, type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, }, @@ -131,7 +134,7 @@ describe('Submission Utils - Group validation errors by entity', () => { { dataRecord: { sex_at_birth: 'Homme' }, reference: { - index: 12, + recordId: 13, submissionId: 23, type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, }, @@ -141,22 +144,32 @@ describe('Submission Utils - Group validation errors by entity', () => { const response = groupSchemaErrorsByEntity({ resultValidation, dataValidated }); expect(Object.keys(response)).to.eql(['updates']); + assertExists(response['updates']); expect(Object.keys(response['updates'])).to.eql(['sports']); + assertExists(response['updates']['sports']); expect(response['updates']['sports'].length).to.eq(2); expect(response['updates']['sports']).to.eql([ { - errors: [], - index: 12, - reason: 'INVALID_BY_RESTRICTION', - fieldName: 'systemId', - fieldValue: '', + errors: [ + { + reason: 'INVALID_BY_RESTRICTION', + fieldName: 'systemId', + fieldValue: '', + errors: [], + }, + ], + recordId: 12, }, { - errors: [], - index: 12, - reason: 'INVALID_BY_RESTRICTION', - fieldName: 'sex_at_birth', - fieldValue: '', + errors: [ + { + reason: 'INVALID_BY_RESTRICTION', + fieldName: 'sex_at_birth', + fieldValue: '', + errors: [], + }, + ], + recordId: 13, }, ]); }); diff --git a/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts b/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts index 94773ff4..dfe2c30b 100644 --- a/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts +++ b/packages/data-provider/test/unit/utils/submission/mapInsertDataToRecordReferences.spec.ts @@ -1,28 +1,32 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models'; - +import type { SubmissionInsertRecordWithEntityName } from '../../../../index.js'; import { mapInsertDataToRecordReferences } from '../../../../src/utils/submissionUtils.js'; import { MERGE_REFERENCE_TYPE } from '../../../../src/utils/types.js'; +import { assertExists } from '../../../assertions.js'; describe('Submission Utils - Transforms inserts from the Submission object into a Record grouped by entityName', () => { it('should return an object grouped by entity name with 2 records', () => { - const submissionInsertData: SubmissionInsertData = { - batchName: 'cars.tsv', - records: [ - { + const insertDataEntity: SubmissionInsertRecordWithEntityName[] = [ + { + data: { name: 'Lamborghini Murcielago', }, - { + entityName: 'cars', + recordId: 100, + }, + { + data: { name: 'Lamborghini Gallardo', }, - ], - }; + entityName: 'cars', + recordId: 101, + }, + ]; - const response = mapInsertDataToRecordReferences(100, { cars: submissionInsertData }); - expect(Object.keys(response)).to.eql(['cars']); - expect(response['cars'].length).to.eq(2); + const response = mapInsertDataToRecordReferences(100, insertDataEntity); + expect(Object.keys(response).length).to.eq(1); expect(response['cars']).to.eql([ { dataRecord: { @@ -31,7 +35,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into reference: { submissionId: 100, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - index: 0, + recordId: 100, }, }, { @@ -41,41 +45,39 @@ describe('Submission Utils - Transforms inserts from the Submission object into reference: { submissionId: 100, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - index: 1, + recordId: 101, }, }, ]); }); - it('should return 2 objects grouped by entity names with 2 records each one', () => { - const submissionInsertDataCars: SubmissionInsertData = { - batchName: 'cars.tsv', - records: [ - { - name: 'Lamborghini Murcielago', - }, - { - name: 'Lamborghini Gallardo', - }, - ], - }; - - const submissionInsertDataAnimals: SubmissionInsertData = { - batchName: 'animals.tsv', - records: [ - { - name: 'Cat', - }, - { - name: 'Dog', - }, - ], - }; + it('should return an array of 4 record references', () => { + const submissionInsertRecords: SubmissionInsertRecordWithEntityName[] = [ + { + data: { name: 'Lamborghini Murcielago' }, + entityName: 'cars', + recordId: 100, + }, + { + data: { name: 'Lamborghini Gallardo' }, + entityName: 'cars', + recordId: 101, + }, + { + data: { name: 'Cat' }, + entityName: 'animals', + recordId: 102, + }, + { + data: { name: 'Dog' }, + entityName: 'animals', + recordId: 103, + }, + ]; - const response = mapInsertDataToRecordReferences(100, { - cars: submissionInsertDataCars, - animals: submissionInsertDataAnimals, - }); - expect(Object.keys(response)).to.eql(['cars', 'animals']); + const response = mapInsertDataToRecordReferences(100, submissionInsertRecords); + expect(Object.keys(response).length).to.eq(2); + assertExists(response['cars']); + assertExists(response['animals']); expect(response['cars'].length).to.eq(2); expect(response['animals'].length).to.eq(2); expect(response['cars']).to.eql([ @@ -86,7 +88,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into reference: { submissionId: 100, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - index: 0, + recordId: 100, }, }, { @@ -96,7 +98,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into reference: { submissionId: 100, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - index: 1, + recordId: 101, }, }, ]); @@ -108,7 +110,7 @@ describe('Submission Utils - Transforms inserts from the Submission object into reference: { submissionId: 100, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - index: 0, + recordId: 102, }, }, { @@ -118,35 +120,15 @@ describe('Submission Utils - Transforms inserts from the Submission object into reference: { submissionId: 100, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, - index: 1, + recordId: 103, }, }, ]); }); - it('should return an objects grouped by entity names with zero records', () => { - const submissionInsertDataFruits: SubmissionInsertData = { - batchName: 'fruit.tsv', - records: [], - }; - - const response = mapInsertDataToRecordReferences(101, { - fruit: submissionInsertDataFruits, - }); - expect(Object.keys(response)).to.eql(['fruit']); - expect(response['fruit'].length).to.eq(0); - expect(response['fruit']).to.eql([]); - }); - it('should return an empty object', () => { - const submissionInsertDataFruits: SubmissionInsertData = { - batchName: '', - records: [], - }; + it('should return an empty array', () => { + const submissionInsertRecords: SubmissionInsertRecordWithEntityName[] = []; - const response = mapInsertDataToRecordReferences(103, { - '': submissionInsertDataFruits, - }); - expect(Object.keys(response)).to.eql(['']); - expect(response[''].length).to.eq(0); - expect(response['']).to.eql([]); + const response = mapInsertDataToRecordReferences(103, submissionInsertRecords); + expect(Object.keys(response).length).to.eql(0); }); }); diff --git a/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts index 6ef32ab2..41aa0f78 100644 --- a/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts +++ b/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts @@ -1,20 +1,20 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import type { Submission, SubmissionData, SubmittedData } from '@overture-stack/lyric-data-model/models'; +import type { Submission, SubmittedData } from '@overture-stack/lyric-data-model/models'; +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; import { mergeAndReferenceEntityData } from '../../../../src/utils/submissionUtils.js'; import { MERGE_REFERENCE_TYPE, SUBMISSION_STATUS } from '../../../../src/utils/types.js'; +import { assertExists } from '../../../assertions.js'; describe('Submission Utils - Combine Active Submission and the Submitted Data with reference', () => { const todaysDate = new Date(); it('returns only SubmittedData data when Submission doesnt contain data', () => { const originalSubmission: Submission = { id: 2, - data: {}, dictionaryId: 14, dictionaryCategoryId: 20, - errors: {}, organization: 'zoo', status: SUBMISSION_STATUS.OPEN, createdAt: todaysDate, @@ -22,7 +22,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi updatedAt: null, updatedBy: null, }; - const submissionData: SubmissionData = {}; + const submissionData: SubmissionRecordWithEntityName[] = []; const submittedData: SubmittedData[] = [ { id: 5, @@ -47,6 +47,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi }); expect(Object.keys(response).length).to.eq(1); expect(Object.keys(response)).to.eql(['animals']); + assertExists(response['animals']); expect(response['animals'].length).eq(1); expect(response['animals']).eql([ { @@ -62,10 +63,8 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi it('returns combination of SubmittedData and Submission insert data', () => { const originalSubmission: Submission = { id: 2, - data: {}, dictionaryId: 14, dictionaryCategoryId: 20, - errors: {}, organization: 'zoo', status: SUBMISSION_STATUS.OPEN, createdAt: todaysDate, @@ -73,17 +72,26 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi updatedAt: null, updatedBy: null, }; - const submissionData: SubmissionData = { - inserts: { - animals: { - batchName: 'animals.tsv', - records: [ - { name: 'elephant', color: 'gray' }, - { name: 'beaver', color: 'brown' }, - ], - }, + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'INSERT', + entityName: 'animals', + id: 8, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'elephant', color: 'gray' }, }, - }; + { + actionType: 'INSERT', + entityName: 'animals', + id: 9, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'beaver', color: 'brown' }, + }, + ]; const submittedData: SubmittedData[] = [ { id: 5, @@ -106,8 +114,10 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi submissionData, submittedData, }); + console.log('response', response); expect(Object.keys(response).length).to.eq(1); expect(Object.keys(response)).to.eql(['animals']); + assertExists(response['animals']); expect(response['animals'].length).eq(3); expect(response['animals']).eql([ { @@ -121,16 +131,16 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi { dataRecord: { name: 'elephant', color: 'gray' }, reference: { - index: 0, - submissionId: 2, + recordId: 8, + submissionId: originalSubmission.id, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, }, }, { dataRecord: { name: 'beaver', color: 'brown' }, reference: { - index: 1, - submissionId: 2, + recordId: 9, + submissionId: originalSubmission.id, type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA, }, }, @@ -139,10 +149,8 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi it('returns combination of SubmittedData and Submission update data', () => { const originalSubmission: Submission = { id: 2, - data: {}, dictionaryId: 14, dictionaryCategoryId: 20, - errors: {}, organization: 'zoo', status: SUBMISSION_STATUS.OPEN, createdAt: todaysDate, @@ -150,14 +158,26 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi updatedAt: null, updatedBy: null, }; - const submissionData: SubmissionData = { - updates: { - animals: [ - { systemId: 'TGR1425', old: { color: 'yellow' }, new: { color: 'orange' } }, - { systemId: 'BR8912', old: { color: 'black' }, new: { color: 'brown' } }, - ], + const submissionRecords: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, }, - }; + { + actionType: 'UPDATE', + entityName: 'animals', + id: 11, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'BR8912', new: { color: 'brown' }, old: { color: 'black' } }, + }, + ]; const submittedData: SubmittedData[] = [ { id: 5, @@ -192,11 +212,12 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi ]; const response = mergeAndReferenceEntityData({ submissionId: originalSubmission.id, - submissionData, + submissionData: submissionRecords, submittedData, }); expect(Object.keys(response).length).to.eq(1); expect(Object.keys(response)).to.eql(['animals']); + assertExists(response['animals']); expect(response['animals'].length).eq(2); expect(response['animals']).eql([ { @@ -204,7 +225,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi reference: { systemId: 'TGR1425', submissionId: 2, - index: 0, + recordId: 10, type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, }, }, @@ -213,7 +234,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi reference: { systemId: 'BR8912', submissionId: 2, - index: 1, + recordId: 11, type: MERGE_REFERENCE_TYPE.EDIT_SUBMITTED_DATA, }, }, @@ -222,10 +243,8 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi it('returns combination of SubmittedData and Submission delete data', () => { const originalSubmission: Submission = { id: 2, - data: {}, dictionaryId: 14, dictionaryCategoryId: 20, - errors: {}, organization: 'zoo', status: SUBMISSION_STATUS.OPEN, createdAt: todaysDate, @@ -233,26 +252,26 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi updatedAt: null, updatedBy: null, }; - const submissionData: SubmissionData = { - deletes: { - animals: [ - { - systemId: 'TGR1425', - data: { name: 'tiger', color: 'yellow' }, - entityName: 'animals', - isValid: true, - organization: 'zoo', - }, - { - systemId: 'BR8912', - data: { name: 'bear', color: 'black' }, - entityName: 'animals', - isValid: true, - organization: 'zoo', - }, - ], + const submissionRecords: SubmissionRecordWithEntityName[] = [ + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' }, }, - }; + { + actionType: 'DELETE', + entityName: 'animals', + id: 13, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' }, + }, + ]; const submittedData: SubmittedData[] = [ { id: 5, @@ -287,7 +306,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi ]; const response = mergeAndReferenceEntityData({ submissionId: originalSubmission.id, - submissionData, + submissionData: submissionRecords, submittedData, }); expect(Object.keys(response).length).to.eq(0); diff --git a/packages/data-provider/test/unit/utils/submission/mergeDeleteRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeDeleteRecords.spec.ts deleted file mode 100644 index 3ea06c0d..00000000 --- a/packages/data-provider/test/unit/utils/submission/mergeDeleteRecords.spec.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import type { SubmissionDeleteData } from '@overture-stack/lyric-data-model/models'; - -import { mergeDeleteRecords } from '../../../../src/utils/submissionUtils.js'; - -describe('Submission Utils - Merge multiple Submission delete records', () => { - it('should return an object with 2 records within the same key', () => { - const deletes1: Record = { - food: [ - { data: { name: 'pizza' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PZ8900' }, - ], - }; - const deletes2: Record = { - food: [ - { - data: { name: 'pizza' }, - entityName: 'hamburger', - isValid: true, - organization: 'kitchen', - systemId: 'HG1234', - }, - ], - }; - const response = mergeDeleteRecords(deletes1, deletes2); - expect(Object.keys(response).length).eq(1); - expect(Object.keys(response)[0]).eql('food'); - expect(response['food'].length).eq(2); - expect(response['food']).eql([ - { data: { name: 'pizza' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PZ8900' }, - { - data: { name: 'pizza' }, - entityName: 'hamburger', - isValid: true, - organization: 'kitchen', - systemId: 'HG1234', - }, - ]); - }); - it('should return an object with 2 records with different key', () => { - const deletes1: Record = { - food: [ - { data: { name: 'pizza' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PZ8900' }, - ], - }; - const deletes2: Record = { - animal: [ - { - data: { name: 'lion' }, - entityName: 'animal', - isValid: false, - organization: 'zoo', - systemId: 'LN5566', - }, - ], - }; - const response = mergeDeleteRecords(deletes1, deletes2); - expect(Object.keys(response).length).eq(2); - expect(Object.keys(response)).eql(['food', 'animal']); - expect(response['food'].length).eq(1); - expect(response['animal'].length).eq(1); - expect(response['food'][0]).eql({ - data: { name: 'pizza' }, - entityName: 'food', - isValid: true, - organization: 'kitchen', - systemId: 'PZ8900', - }); - expect(response['animal'][0]).eql({ - data: { name: 'lion' }, - entityName: 'animal', - isValid: false, - organization: 'zoo', - systemId: 'LN5566', - }); - }); - it('should avoid duplication and return an object with 1 record', () => { - const deletes1: Record = { - food: [ - { data: { name: 'Paella' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PAE344' }, - ], - }; - const deletes2: Record = { - food: [ - { data: { name: 'Paella' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PAE344' }, - ], - }; - const deletes3: Record = { - food: [ - { data: { name: 'Paella' }, entityName: 'food', isValid: true, organization: 'kitchen', systemId: 'PAE344' }, - ], - }; - const response = mergeDeleteRecords(deletes1, deletes2, deletes3); - expect(Object.keys(response).length).eq(1); - expect(Object.keys(response)).eql(['food']); - expect(response['food'].length).eq(1); - expect(response['food'][0]).eql({ - data: { name: 'Paella' }, - entityName: 'food', - isValid: true, - organization: 'kitchen', - systemId: 'PAE344', - }); - }); -}); diff --git a/packages/data-provider/test/unit/utils/submission/mergeInsertsRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeInsertsRecords.spec.ts deleted file mode 100644 index 9f1c6665..00000000 --- a/packages/data-provider/test/unit/utils/submission/mergeInsertsRecords.spec.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import type { SubmissionInsertData } from '@overture-stack/lyric-data-model/models'; - -import { mergeInsertsRecords } from '../../../../src/utils/submissionUtils.js'; - -describe('Submission Utils - Merge multiple Submission insert records', () => { - it('should return a record object with one key and merged array items', () => { - const obj1: Record = { - sports: { batchName: 'sports.tsv', records: [{ title: 'footbal' }] }, - }; - const obj2: Record = { - sports: { batchName: 'sports', records: [{ title: 'basketball' }] }, - }; - const result = mergeInsertsRecords(obj1, obj2); - expect(Object.keys(result).length).to.eq(1); - expect(result['sports'].records.length).eql(2); - }); - - it('should return a record object with two different keys', () => { - const obj1: Record = { - food: { batchName: 'food.tsv', records: [{ title: 'apple' }] }, - }; - const obj2: Record = { - sports: { batchName: 'sports', records: [{ title: 'basketball' }] }, - }; - const result = mergeInsertsRecords(obj1, obj2); - expect(Object.keys(result).length).to.eq(2); - expect(result['sports'].records.length).eql(1); - expect(result['food'].records.length).eql(1); - }); - - it('should return a record object with one key and merged array items without duplication', () => { - const obj1: Record = { - sports: { batchName: 'sports.tsv', records: [{ title: 'snowboarding' }] }, - }; - const obj2: Record = { - sports: { batchName: 'sports.csv', records: [{ title: 'snowboarding' }] }, - }; - const result = mergeInsertsRecords(obj1, obj2); - expect(Object.keys(result).length).to.eq(1); - expect(result['sports'].records.length).to.eq(1); - expect(result['sports'].records[0]).eql({ title: 'snowboarding' }); - }); -}); diff --git a/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts b/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts index 90cfbadc..417e33fe 100644 --- a/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts +++ b/packages/data-provider/test/unit/utils/submission/parseRecordsToInsert.spec.ts @@ -44,21 +44,15 @@ describe('parseRecordsToInsert', () => { const result = parseRecordsToInsert(records, inventoryDictionary); // Fields properly formatted with the corresponding data type based on Dictionary - const expectedResult: Record = { - user: { - batchName: 'user', - records: [ - { id: 1, name: 'Alice', birthYear: 2000, hasAllergies: false }, - { id: 2, name: 'Pedro', birthYear: 1990, hasAllergies: true }, - ], - }, - product: { - batchName: 'product', - records: [ - { id: 101, name: 'Laptop' }, - { id: 102, name: 'Monitor' }, - ], - }, + const expectedResult: Record = { + user: [ + { id: 1, name: 'Alice', birthYear: 2000, hasAllergies: false }, + { id: 2, name: 'Pedro', birthYear: 1990, hasAllergies: true }, + ], + product: [ + { id: 101, name: 'Laptop' }, + { id: 102, name: 'Monitor' }, + ], }; expect(Object.keys(result).length).to.eql(2); @@ -83,14 +77,11 @@ describe('parseRecordsToInsert', () => { ], }; - const expectedResult: Record = { - product: { - batchName: 'product', - records: [ - { id: 101, name: 'Laptop' }, - { id: 102, name: 'Monitor' }, - ], - }, + const expectedResult: Record = { + product: [ + { id: 101, name: 'Laptop' }, + { id: 102, name: 'Monitor' }, + ], }; const result = parseRecordsToInsert(records, inventoryDictionary); diff --git a/packages/data-provider/test/unit/utils/submission/parseSubmissionResponse.spec.ts b/packages/data-provider/test/unit/utils/submission/parseSubmissionResponse.spec.ts deleted file mode 100644 index dc1f96e8..00000000 --- a/packages/data-provider/test/unit/utils/submission/parseSubmissionResponse.spec.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import type { DataRecord } from '@overture-stack/lectern-client'; - -import { createBatchResponse } from '../../../../src/utils/submissionResponseParser.js'; -import { createSubmissionDetailsResponse } from '../../../../src/utils/submissionUtils.js'; -import { SUBMISSION_STATUS, type SubmissionDataDetailsRepositoryRecord } from '../../../../src/utils/types.js'; - -describe('Submission Utils - Parse a Submisison object to a response format', () => { - const todaysDate = new Date(); - it('return a Submission response with no data', () => { - const submissionRepositoryRecord: SubmissionDataDetailsRepositoryRecord = { - id: 2, - data: {}, - dictionary: { name: 'books', version: '1' }, - dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, - organization: 'oicr', - status: SUBMISSION_STATUS.OPEN, - createdAt: todaysDate, - createdBy: 'me', - updatedAt: null, - updatedBy: null, - }; - const response = createSubmissionDetailsResponse(submissionRepositoryRecord); - expect(response).to.eql({ - id: 2, - data: {}, - dictionary: { name: 'books', version: '1' }, - dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, - organization: 'oicr', - status: SUBMISSION_STATUS.OPEN, - createdAt: todaysDate.toISOString(), - createdBy: 'me', - updatedAt: '', - updatedBy: '', - }); - }); - it('return a Submission response format with insert, update and delete data', () => { - const submissionRepositoryRecord: SubmissionDataDetailsRepositoryRecord = { - id: 2, - data: { - inserts: { - books: { - batchName: 'books.tsv', - records: [ - { - title: 'abc', - }, - ], - }, - }, - updates: { - books: [ - { - systemId: 'QWE987', - new: { title: 'The Little Prince' }, - old: { title: 'the little prince' }, - }, - ], - }, - deletes: { - books: [ - { - systemId: 'ZXC678', - entityName: 'books', - organization: 'oicr', - isValid: true, - data: { title: 'batman' }, - }, - ], - }, - }, - dictionary: { name: 'books', version: '1.1' }, - dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, - organization: 'oicr', - status: SUBMISSION_STATUS.OPEN, - createdAt: todaysDate, - createdBy: 'me', - updatedAt: null, - updatedBy: null, - }; - const response = createSubmissionDetailsResponse(submissionRepositoryRecord); - expect(response).to.eql({ - id: 2, - data: { - inserts: { - books: { - batchName: 'books.tsv', - records: [ - { - title: 'abc', - }, - ], - }, - }, - updates: { - books: [ - { - systemId: 'QWE987', - new: { title: 'The Little Prince' }, - old: { title: 'the little prince' }, - }, - ], - }, - deletes: { - books: [ - { - systemId: 'ZXC678', - entityName: 'books', - organization: 'oicr', - isValid: true, - data: { title: 'batman' }, - }, - ], - }, - }, - dictionary: { name: 'books', version: '1.1' }, - dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, - organization: 'oicr', - status: SUBMISSION_STATUS.OPEN, - createdAt: todaysDate.toISOString(), - createdBy: 'me', - updatedAt: '', - updatedBy: '', - }); - }); - - describe('Create Submission Insert Response', () => { - it('should return a Submission Insert Response with records', () => { - const records: DataRecord[] = [ - { id: 1, name: 'ABC' }, - { id: 2, name: 'XYZ' }, - ]; - const result = createBatchResponse('sample', records); - expect(result.batchName).eql('sample'); - expect(result.records.length).eql(2); - expect(result.records).eql([ - { id: 1, name: 'ABC' }, - { id: 2, name: 'XYZ' }, - ]); - }); - it('should return a Submission Insert Response with no records', () => { - const result = createBatchResponse('sample', []); - expect(result).eql({ batchName: 'sample', records: [] }); - }); - }); -}); diff --git a/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts b/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts index fc18301a..02ca89ec 100644 --- a/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts +++ b/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts @@ -2,17 +2,19 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; import { createSubmissionSummaryResponse } from '../../../../src/utils/submissionUtils.js'; -import { SUBMISSION_STATUS, type SubmissionDataSummaryRepositoryRecord } from '../../../../src/utils/types.js'; +import { SUBMISSION_STATUS, type SubmissionSummary } from '../../../../src/utils/types.js'; describe('Submission Utils - Parse a Submission object to a Summary of the Active Submission', () => { const todaysDate = new Date(); it('should return a Summary without any data ', () => { - const submissionDataSummaryRepositoryRecord: SubmissionDataSummaryRepositoryRecord = { + const submissionDataSummaryRepositoryRecord: SubmissionSummary = { id: 4, - data: {}, + data: { + totalRecords: 0, + errors: 0, + }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: null, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate, @@ -24,11 +26,11 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ expect(response).to.eql({ id: 4, data: { - total: 0, + totalRecords: 0, + errors: 0, }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: { total: 0 }, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate.toISOString(), @@ -38,29 +40,38 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ }); }); it('should return a Summary with insert, update and delete data ', () => { - const submissionDataSummaryRepositoryRecord: SubmissionDataSummaryRepositoryRecord = { + const submissionDataSummaryRepositoryRecord: SubmissionSummary = { id: 3, data: { inserts: { - books: { - batchName: 'books.tsv', - recordsCount: 1, - }, + books: [ + { + batchName: 'books.tsv', + recordsCount: 1, + errors: 0, + }, + ], }, updates: { - books: { - recordsCount: 1, - }, + books: [ + { + batchName: 'books.tsv', + recordsCount: 1, + errors: 0, + }, + ], }, deletes: { books: { recordsCount: 1, + errors: 0, }, }, + totalRecords: 3, + errors: 0, }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate, @@ -73,26 +84,34 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ id: 3, data: { inserts: { - books: { - batchName: 'books.tsv', - recordsCount: 1, - }, + books: [ + { + batchName: 'books.tsv', + recordsCount: 1, + errors: 0, + }, + ], }, updates: { - books: { - recordsCount: 1, - }, + books: [ + { + batchName: 'books.tsv', + recordsCount: 1, + errors: 0, + }, + ], }, deletes: { books: { recordsCount: 1, + errors: 0, }, }, - total: 3, + totalRecords: 3, + errors: 0, }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: { total: 0 }, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate.toISOString(), diff --git a/packages/data-provider/test/unit/utils/submission/removeItemsFromSubmission.spec.ts b/packages/data-provider/test/unit/utils/submission/removeItemsFromSubmission.spec.ts deleted file mode 100644 index b3eb0cb8..00000000 --- a/packages/data-provider/test/unit/utils/submission/removeItemsFromSubmission.spec.ts +++ /dev/null @@ -1,392 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import type { SubmissionData } from '@overture-stack/lyric-data-model/models'; - -import { removeItemsFromSubmission } from '../../../../src/utils/submissionUtils.js'; -import { SUBMISSION_ACTION_TYPE, type SubmissionActionType } from '../../../../src/utils/types.js'; - -describe('Submission Utils - Removes items from submission based on filter parameters', () => { - const insertDataSubmission: SubmissionData = { - inserts: { - sports: { - batchName: 'sports.tsv', - records: [ - { - name: 'Breakdance', - }, - { - name: 'Skateboarding', - }, - ], - }, - food: { - batchName: 'food.xml', - records: [ - { - name: 'Poutine', - }, - ], - }, - }, - }; - - const updateDataSubmission: SubmissionData = { - updates: { - sports: [ - { - systemId: 'QWE987', - new: { name: 'Basketball 3X3' }, - old: { name: 'Basketball' }, - }, - { - systemId: 'SWI321', - new: { name: 'Swimming' }, - old: { name: 'swiming' }, - }, - ], - food: [ - { - systemId: 'PTO456', - new: { name: 'Potato' }, - old: { name: 'potahto' }, - }, - ], - }, - }; - - const deleteDataSubmission: SubmissionData = { - deletes: { - sports: [ - { - systemId: 'ZXC678', - entityName: 'sports', - organization: 'olimpics', - isValid: true, - data: { name: 'Baseball' }, - }, - { - systemId: 'SFT098', - entityName: 'sports', - organization: 'olimpics', - isValid: true, - data: { name: 'Softball' }, - }, - ], - food: [ - { - systemId: 'EGG789', - entityName: 'food', - organization: 'olimpics', - isValid: true, - data: { name: 'Eggpplant' }, - }, - ], - }, - }; - it('should return an empty response when Submission is empty', () => { - const submissionData: SubmissionData = {}; - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS, - entityName: 'sports', - index: 0, - }; - const response = removeItemsFromSubmission(submissionData, filter); - expect(response).to.eql({}); - }); - it('should returm intact SubmissionData when filter doesnt find anything', () => { - const fullSubmissionData: SubmissionData = { - ...insertDataSubmission, - ...updateDataSubmission, - ...deleteDataSubmission, - }; - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS, - entityName: 'animals', // item doesn't belong in the submission - index: 0, - }; - const response = removeItemsFromSubmission(fullSubmissionData, filter); - expect(response).to.eql({ - inserts: { - sports: { - batchName: 'sports.tsv', - records: [ - { - name: 'Breakdance', - }, - { - name: 'Skateboarding', - }, - ], - }, - food: { - batchName: 'food.xml', - records: [ - { - name: 'Poutine', - }, - ], - }, - }, - updates: { - sports: [ - { - systemId: 'QWE987', - new: { name: 'Basketball 3X3' }, - old: { name: 'Basketball' }, - }, - { - systemId: 'SWI321', - new: { name: 'Swimming' }, - old: { name: 'swiming' }, - }, - ], - food: [ - { - systemId: 'PTO456', - new: { name: 'Potato' }, - old: { name: 'potahto' }, - }, - ], - }, - deletes: { - sports: [ - { - systemId: 'ZXC678', - entityName: 'sports', - organization: 'olimpics', - isValid: true, - data: { name: 'Baseball' }, - }, - { - systemId: 'SFT098', - entityName: 'sports', - organization: 'olimpics', - isValid: true, - data: { name: 'Softball' }, - }, - ], - food: [ - { - systemId: 'EGG789', - entityName: 'food', - organization: 'olimpics', - isValid: true, - data: { name: 'Eggpplant' }, - }, - ], - }, - }); - }); - it('should remove whole inserts object from SubmissionData', () => { - const insertOnseSubmission: SubmissionData = { - inserts: { - sports: { - batchName: 'sports.tsv', - records: [ - { - name: 'Snowboarding', - }, - ], - }, - }, - }; - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS, - entityName: 'sports', - index: null, - }; - const response = removeItemsFromSubmission(insertOnseSubmission, filter); - expect(response).to.eql({}); - }); - it('should remove whole updates object from SubmissionData', () => { - const updateOneSubmission: SubmissionData = { - updates: { - sports: [ - { - systemId: 'QWE987', - new: { name: 'Basketball 3X3' }, - old: { name: 'Basketball' }, - }, - ], - }, - }; - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.UPDATES, - entityName: 'sports', - index: null, - }; - const response = removeItemsFromSubmission(updateOneSubmission, filter); - expect(response).to.eql({}); - }); - it('should remove whole deletes object from SubmissionData', () => { - const deleteOneSubmission: SubmissionData = { - deletes: { - sports: [ - { - systemId: 'ZXC678', - entityName: 'sports', - organization: 'olimpics', - isValid: true, - data: { name: 'Baseball' }, - }, - ], - }, - }; - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.DELETES, - entityName: 'sports', - index: null, - }; - const response = removeItemsFromSubmission(deleteOneSubmission, filter); - expect(response).to.eql({}); - }); - it('should remove one item from inserts object on SubmissionData', () => { - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS, - entityName: 'sports', - index: 1, - }; - const response = removeItemsFromSubmission(insertDataSubmission, filter); - expect(response).to.eql({ - inserts: { - sports: { - batchName: 'sports.tsv', - records: [ - { - name: 'Breakdance', - }, - ], - }, - food: { - batchName: 'food.xml', - records: [ - { - name: 'Poutine', - }, - ], - }, - }, - }); - }); - it('should remove one item from updates object on SubmissionData', () => { - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.UPDATES, - entityName: 'sports', - index: 1, - }; - const response = removeItemsFromSubmission(updateDataSubmission, filter); - expect(response).to.eql({ - updates: { - sports: [ - { - systemId: 'QWE987', - new: { name: 'Basketball 3X3' }, - old: { name: 'Basketball' }, - }, - ], - food: [ - { - systemId: 'PTO456', - new: { name: 'Potato' }, - old: { name: 'potahto' }, - }, - ], - }, - }); - }); - it('should remove one item from deletes object on SubmissionData', () => { - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.DELETES, - entityName: 'sports', - index: 1, - }; - const response = removeItemsFromSubmission(deleteDataSubmission, filter); - expect(response).to.eql({ - deletes: { - sports: [ - { - systemId: 'ZXC678', - entityName: 'sports', - organization: 'olimpics', - isValid: true, - data: { name: 'Baseball' }, - }, - ], - food: [ - { - systemId: 'EGG789', - entityName: 'food', - organization: 'olimpics', - isValid: true, - data: { name: 'Eggpplant' }, - }, - ], - }, - }); - }); - it('should remove inserts if no items are left on the records array', () => { - const insertOnseSubmission: SubmissionData = { - inserts: { - sports: { - batchName: 'sports.tsv', - records: [ - { - name: 'Snowboarding', - }, - ], - }, - }, - }; - - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.INSERTS, - entityName: 'sports', - index: 0, - }; - const response = removeItemsFromSubmission(insertOnseSubmission, filter); - expect(response).to.eql({}); - }); - it('should remove updates if no items are left on the array', () => { - const updateOneSubmission: SubmissionData = { - updates: { - sports: [ - { - systemId: 'QWE987', - new: { name: 'Basketball 3X3' }, - old: { name: 'Basketball' }, - }, - ], - }, - }; - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.UPDATES, - entityName: 'sports', - index: 0, - }; - const response = removeItemsFromSubmission(updateOneSubmission, filter); - expect(response).to.eql({}); - }); - it('should remove deletes if no items are left on the array', () => { - const deleteOneSubmission: SubmissionData = { - deletes: { - sports: [ - { - systemId: 'ZXC678', - entityName: 'sports', - organization: 'olimpics', - isValid: true, - data: { name: 'Baseball' }, - }, - ], - }, - }; - const filter: { actionType: SubmissionActionType; entityName: string; index: number | null } = { - actionType: SUBMISSION_ACTION_TYPE.Values.DELETES, - entityName: 'sports', - index: 0, - }; - const response = removeItemsFromSubmission(deleteOneSubmission, filter); - expect(response).to.eql({}); - }); -}); From 59076b4d8ecae84db4a032caf57fcfa46592df2b Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 10 Aug 2026 11:48:59 -0400 Subject: [PATCH 04/20] fix migration transaction --- .../migrations/0015_CUSTOM_migrate_submission_records.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data-model/migrations/0015_CUSTOM_migrate_submission_records.sql b/packages/data-model/migrations/0015_CUSTOM_migrate_submission_records.sql index af344a61..fb3bc059 100644 --- a/packages/data-model/migrations/0015_CUSTOM_migrate_submission_records.sql +++ b/packages/data-model/migrations/0015_CUSTOM_migrate_submission_records.sql @@ -186,7 +186,7 @@ SELECT er.action_type::submission_record_type, ae.error_data, CASE - WHEN er.submission_status IN ('OPEN', 'VALIDATING') THEN 'RECEIVED'::submission_record_state + WHEN er.submission_status::text IN ('OPEN', 'VALIDATING') THEN 'RECEIVED'::submission_record_state WHEN er.submission_status = 'INVALID' OR ae.error_data IS NOT NULL THEN 'INVALID'::submission_record_state ELSE 'VALID'::submission_record_state END AS state From 430e989fe8b45716a08a1597b028c5260b7bebf4 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Mon, 10 Aug 2026 12:09:17 -0400 Subject: [PATCH 05/20] fix integration tests --- .../src/utils/submissionUtils.ts | 2 - ...sionRouter-submitFiles-persistence.spec.ts | 103 ++++++++++-------- .../mergeAndReferenceEntityData.spec.ts | 2 +- 3 files changed, 56 insertions(+), 51 deletions(-) diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index 9f944095..605ffa4a 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -566,7 +566,6 @@ export const mergeAndReferenceEntityData = ({ editSubmittedData: dataToUpdate, submissionId, }); - console.log(`Submitted Data with Reference: ${JSON.stringify(submittedDataWithRef, null, 2)}`); const dataToInsert: SubmissionInsertRecordWithEntityName[] = submissionData .filter( @@ -584,7 +583,6 @@ export const mergeAndReferenceEntityData = ({ })); const insertDataWithRef = dataToInsert.length > 0 ? mapInsertDataToRecordReferences(submissionId, dataToInsert) : {}; - console.log(`Insert Data with Reference: ${JSON.stringify(insertDataWithRef, null, 2)}`); // This object will merge existing data + new data for validation (Submitted data + active Submission) return _.mergeWith(submittedDataWithRef, insertDataWithRef, (objValue, srcValue) => { diff --git a/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts b/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts index 721f4127..fa131ce4 100644 --- a/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts +++ b/packages/data-provider/test/integration/routers/submission/submissionRouter-submitFiles-persistence.spec.ts @@ -5,6 +5,7 @@ import supertest from 'supertest'; import submissionProcessorFactory from '../../../../src/services/submission/submissionProcessor.js'; import { createTsvFileContent } from '../../../fixtures/createTsvContent.js'; import { dictionarySportsData } from '../../../fixtures/dictionarySchemasTestData.js'; +import { assertExists } from '../../assertions.js'; import { createLyricProvider, type LyricProvider } from '../../dependencies/lyricProvider.js'; import { createTestApp } from '../../dependencies/testServer.js'; import { getContainers } from '../../globalSetup.js'; @@ -28,7 +29,7 @@ const waitForSubmissionToStopValidating = async ({ let submission; do { await sleep(delayMs); - submission = await lyricProvider.repositories.submission.getActiveSubmissionSummary({ + submission = await lyricProvider.repositories.submission.getActiveSubmission({ categoryId, username: '', organization, @@ -100,46 +101,46 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D it('should save submitted file records to the active submission', async () => { const sportTsv = createTsvFileContent(['sport_id', 'name'], [['1', 'Soccer']]); - await app.post(`/category/${categoryId}/files?organization=testOrg`).attach('files', sportTsv, 'sport.tsv'); + const submitResponse = await app + .post(`/category/${categoryId}/files?organization=testOrg`) + .attach('files', sportTsv, 'sport.tsv'); await pendingAsyncWork; - const submission = await lyricProvider.repositories.submission.getActiveSubmissionDetails({ - categoryId, - username: '', - organization: 'testOrg', - }); + const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId( + submitResponse.body.submissionId, + ); - expect(submission).to.exist; - expect(submission!.data.inserts).to.have.property('sport'); - expect(submission!.data.inserts!['sport'].records).to.have.length(1); - expect(submission!.data.inserts!['sport'].records[0]).to.include({ sport_id: '1', name: 'Soccer' }); + expect(submissionRecords.length).to.eq(1); + assertExists(submissionRecords[0]); + expect(submissionRecords[0].entityName).to.eql('sport'); + expect(submissionRecords[0].actionType).to.eql('INSERT'); + expect(submissionRecords[0].data).to.eql({ sport_id: '1', name: 'Soccer' }); }); it('should save records for each entity when multiple files are submitted', async () => { const sportTsv = createTsvFileContent(['sport_id', 'name'], [['1', 'Soccer']]); const teamTsv = createTsvFileContent(['team_id', 'sport_id', 'name'], [['1', '1', 'Team A']]); - await app + const submitResponse = await app .post(`/category/${categoryId}/files?organization=testOrg`) .attach('files', sportTsv, 'sport.tsv') .attach('files', teamTsv, 'team.tsv'); await pendingAsyncWork; - const submission = await lyricProvider.repositories.submission.getActiveSubmissionDetails({ - categoryId, - username: '', - organization: 'testOrg', - }); - - expect(submission).to.exist; - expect(submission!.data.inserts).to.have.property('sport'); - expect(submission!.data.inserts!['sport'].records).to.have.length(1); - expect(submission!.data.inserts!['sport'].records[0]).to.include({ sport_id: '1', name: 'Soccer' }); - expect(submission!.data.inserts).to.have.property('team'); - expect(submission!.data.inserts!['team'].records).to.have.length(1); - expect(submission!.data.inserts!['team'].records[0]).to.include({ team_id: '1', sport_id: '1', name: 'Team A' }); + const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId( + submitResponse.body.submissionId, + ); + + expect(submissionRecords).to.exist; + expect(submissionRecords.length).to.eq(2); + expect(submissionRecords.map((record) => record.entityName)).to.eql(['sport', 'team']); + expect(submissionRecords.map((record) => record.actionType)).to.eql(['INSERT', 'INSERT']); + expect(submissionRecords.map((record) => record.data)).to.eql([ + { sport_id: '1', name: 'Soccer' }, + { team_id: '1', sport_id: '1', name: 'Team A' }, + ]); }); it('should merge records from multiple files for the same entity into a single batch', async () => { @@ -150,7 +151,7 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D { filename: 'sports_batch2.tsv', entity: 'sport' }, ]); - await app + const submitResponse = await app .post(`/category/${categoryId}/files?organization=testOrg`) .attach('files', batch1, 'sports_batch1.tsv') .attach('files', batch2, 'sports_batch2.tsv') @@ -158,15 +159,18 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D await pendingAsyncWork; - const submission = await lyricProvider.repositories.submission.getActiveSubmissionDetails({ - categoryId, - username: '', - organization: 'testOrg', - }); - - expect(submission).to.exist; - expect(submission!.data.inserts).to.have.property('sport'); - expect(submission!.data.inserts!['sport'].records).to.have.length(2); + const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId( + submitResponse.body.submissionId, + ); + + expect(submissionRecords).to.exist; + expect(submissionRecords.length).to.eq(2); + expect(submissionRecords.map((record) => record.entityName)).to.eql(['sport', 'sport']); + expect(submissionRecords.map((record) => record.actionType)).to.eql(['INSERT', 'INSERT']); + expect(submissionRecords.map((record) => record.data)).to.eql([ + { sport_id: '1', name: 'Soccer' }, + { sport_id: '2', name: 'Basketball' }, + ]); }); it('should accumulate records across sequential submissions to the same active submission', async () => { @@ -189,7 +193,9 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D expect(resultFirstSubmission).to.exist; expect(resultFirstSubmission!.status).to.equal('VALID'); - await app.post(`/category/${categoryId}/files?organization=${organization}`).attach('files', teamTsv, 'team.tsv'); + const submitResponse = await app + .post(`/category/${categoryId}/files?organization=${organization}`) + .attach('files', teamTsv, 'team.tsv'); await pendingAsyncWork; const resultFinalSubmission = await waitForSubmissionToStopValidating({ @@ -200,19 +206,20 @@ describe('Integration - Submission Router - POST /category/:categoryId/files - D delayMs: 500, }); - expect(resultFinalSubmission).to.exist; - expect(resultFinalSubmission!.status).to.equal('VALID'); + assertExists(resultFinalSubmission); + expect(resultFinalSubmission.status).to.equal('VALID'); - const submissionDetails = await lyricProvider.repositories.submission.getActiveSubmissionDetails({ - categoryId, - username: '', - organization, - }); + const submissionRecords = await lyricProvider.repositories.submissionRecords.getBySubmissionId( + submitResponse.body.submissionId, + ); - expect(submissionDetails).to.exist; - expect(submissionDetails!.data.inserts).to.have.property('sport'); - expect(submissionDetails!.data.inserts!['sport'].records).to.have.length(1); - expect(submissionDetails!.data.inserts).to.have.property('team'); - expect(submissionDetails!.data.inserts!['team'].records).to.have.length(1); + expect(submissionRecords).to.exist; + expect(submissionRecords.length).to.eq(2); + expect(submissionRecords.map((record) => record.entityName)).to.eql(['sport', 'team']); + expect(submissionRecords.map((record) => record.actionType)).to.eql(['INSERT', 'INSERT']); + expect(submissionRecords.map((record) => record.data)).to.eql([ + { sport_id: '1', name: 'Soccer' }, + { team_id: '1', sport_id: '1', name: 'Team A' }, + ]); }); }); diff --git a/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts index 41aa0f78..681318f6 100644 --- a/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts +++ b/packages/data-provider/test/unit/utils/submission/mergeAndReferenceEntityData.spec.ts @@ -114,7 +114,7 @@ describe('Submission Utils - Combine Active Submission and the Submitted Data wi submissionData, submittedData, }); - console.log('response', response); + expect(Object.keys(response).length).to.eq(1); expect(Object.keys(response)).to.eql(['animals']); assertExists(response['animals']); From bd93529e5e20062705f7c2387373cbe5aefb461b Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 11 Aug 2026 09:11:23 -0400 Subject: [PATCH 06/20] code refactor updated tsdocs, removed unused code, added missing db transactions --- .../repository/activeSubmissionRepository.ts | 2 +- .../submission/submissionProcessor.ts | 193 ++++++++---------- .../services/submittedData/submmittedData.ts | 57 +++--- .../src/utils/submissionUtils.ts | 35 +--- .../src/utils/submittedDataUtils.ts | 13 +- packages/data-provider/src/utils/types.ts | 12 ++ .../filterUpdatesFromDeletes.spec.ts | 128 ------------ 7 files changed, 139 insertions(+), 301 deletions(-) delete mode 100644 packages/data-provider/test/unit/utils/submission/filterUpdatesFromDeletes.spec.ts diff --git a/packages/data-provider/src/repository/activeSubmissionRepository.ts b/packages/data-provider/src/repository/activeSubmissionRepository.ts index 5ed31abd..a5434886 100644 --- a/packages/data-provider/src/repository/activeSubmissionRepository.ts +++ b/packages/data-provider/src/repository/activeSubmissionRepository.ts @@ -47,7 +47,7 @@ const activeSubmissionRepository = (dependencies: BaseDependencies) => { } as const satisfies Record; /** - * Normalizes a queried record's `dictionaryCategory.alias` from the DB's `string | null` + * Normalizes a queried record's `dictionaryCategory.alias` from the DB's `string | null` * to the public `CategorySummary` contract's `string | undefined` (omitted, not null, when unset). */ const withAliasNormalized = (record: T) => diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index b8eb6243..9b46d4a0 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -1,4 +1,3 @@ -import bytes from 'bytes'; import * as _ from 'lodash-es'; import type { DataRecord, DictionaryValidationRecordErrorDetails, Schema } from '@overture-stack/lectern-client'; @@ -20,6 +19,7 @@ import createSubmissionRecordsRepository from '../../repository/submissionRecord import createSubmittedDataRepository from '../../repository/submittedRepository.js'; import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/dictionarySchemaRelations.js'; import { BadRequest } from '../../utils/errors.js'; +import { formatByteSize, getSizeInBytes } from '../../utils/fileUtils.js'; import { convertRecordToString } from '../../utils/formatUtils.js'; import { parseRecordsToInsert } from '../../utils/recordsParser.js'; import { @@ -639,32 +639,6 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { dictionaryRelations, ); - // const submissionRecordsToUpdate = await submissionRecordsRepository.getBySubmissionId(submissionId, undefined, { - // actionTypes: ['UPDATE'], - // }); - - // const formattedSubmissionRecordsToUpdate = submissionRecordsToUpdate - // .filter( - // ( - // record, - // ): record is (typeof submissionRecordsToUpdate)[number] & { - // actionType: 'UPDATE'; - // data: SubmissionUpdateData; - // } => record.actionType === 'UPDATE', - // ) - // .reduce>((acc, record) => { - // if (!acc[record.entityName]) { - // acc[record.entityName] = []; - // } - - // (acc[record.entityName] ?? []).push({ - // systemId: record.data.systemId.toString(), - // old: record.data.old ?? {}, - // new: record.data.new ?? {}, - // }); - // return acc; - // }, {}); - // Aggegates all Update changes on Submission // Note: We do not include records involving primary ID fields changes in here. We would rather do a DELETE and an INSERT const updatedActiveSubmissionData: Record = mergeUpdatesBySystemId( @@ -676,24 +650,12 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { // Creates insert and delete records based on primary ID field change records. const additions = await handleIdFieldChanges(idFieldChangeRecord); - // // Merge Active Submission Inserts with Edit generated new Inserts - // const mergedInserts = mergeInsertsRecords(submission.data.inserts ?? {}, additions.inserts); - - // // Merge Active Submission Deletes with Edit generated new Deletes - // const mergedDeletes = mergeDeleteRecords(submission.data.deletes ?? {}, additions.deletes); - - // // filter out delete records found on update records - // const filteredDeletes = filterDeletesFromUpdates(mergedDeletes, updatedActiveSubmissionData); - // Updating the Submission with the new data and 'VALIDATING' status before validation starts await update(submission.id, { updatedBy: username, status: 'VALIDATING', }); - // TODO insert new data into submissionRecordsRepository, remove the code above to merge - // insert new file for submission - const entityNames: Set = new Set([ ...Object.keys(additions.inserts), ...Object.keys(additions.deletes), @@ -701,46 +663,52 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { ]); for (const entityName of entityNames) { - const savedFileId = await submissionFilesRepository.save({ - entityName: entityName, - fileName: `${Date.now()}.json`, - fileSize: 0, - submissionId: submission.id, - }); - - if (updatedActiveSubmissionData[entityName]) { - await submissionRecordsRepository.saveManyForFile( - savedFileId, - updatedActiveSubmissionData[entityName].map((record) => ({ - actionType: 'UPDATE', - data: record, - state: 'RECEIVED', - fileId: savedFileId, - })), + dependencies.db.transaction(async (tx) => { + const savedFileId = await submissionFilesRepository.save( + { + entityName: entityName, + fileName: `${Date.now()}.json`, + fileSize: getSizeInBytes(JSON.stringify(recordsParsed)), + submissionId: submission.id, + }, + tx, ); - } - if (additions.inserts[entityName]) { - await submissionRecordsRepository.saveManyForFile( - savedFileId, - additions.inserts[entityName].map((record) => ({ - actionType: 'INSERT', - data: record, - state: 'RECEIVED', - fileId: savedFileId, - })), - ); - } - if (additions.deletes[entityName]) { - await submissionRecordsRepository.saveManyForFile( - savedFileId, - additions.deletes[entityName]?.map((record) => ({ - actionType: 'DELETE', - data: record, - state: 'RECEIVED', - })), - ); - } + if (updatedActiveSubmissionData[entityName]) { + await submissionRecordsRepository.saveManyForFile( + savedFileId, + updatedActiveSubmissionData[entityName].map((record) => ({ + actionType: 'UPDATE', + data: record, + state: 'RECEIVED', + })), + tx, + ); + } + + if (additions.inserts[entityName]) { + await submissionRecordsRepository.saveManyForFile( + savedFileId, + additions.inserts[entityName].map((record) => ({ + actionType: 'INSERT', + data: record, + state: 'RECEIVED', + })), + tx, + ); + } + if (additions.deletes[entityName]) { + await submissionRecordsRepository.saveManyForFile( + savedFileId, + additions.deletes[entityName]?.map((record) => ({ + actionType: 'DELETE', + data: record, + state: 'RECEIVED', + })), + tx, + ); + } + }); } // Perform Schema Data validation in a worker thread @@ -802,8 +770,8 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { Object.entries(insertRecords).map(async ([entityName, entityRecords]) => { const savedFileId = await submissionFilesRepository.save({ entityName, - fileName: `insert-${entityName}-${Date.now()}.json`, // default file name for adding JSON records - fileSize: JSON.stringify(entityRecords).length, + fileName: `${Date.now()}.json`, // default file name for adding JSON records + fileSize: getSizeInBytes(JSON.stringify(entityRecords)), submissionId, }); await submissionRecordsRepository.saveManyForFile( @@ -846,46 +814,57 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { schemaErrors: SubmissionErrors; }): Promise => { const { dictionaryId, idActiveSubmission, schemaErrors } = input; - const { update } = submissionRepository; const newStatusSubmission = Object.keys(schemaErrors).length > 0 ? SUBMISSION_STATUS.INVALID : SUBMISSION_STATUS.VALID; - // Update with new data - const updatedActiveSubmissionId = await update(idActiveSubmission, { - status: newStatusSubmission, - dictionaryId: dictionaryId, - }); - const invalidRecords = Object.values(schemaErrors).flatMap((entityErrors) => - Object.values(entityErrors).flatMap((recordErrors) => - recordErrors.map(({ recordId, errors }) => ({ - id: recordId, - errors, - })), - ), - ); + dependencies.db.transaction(async (tx) => { + // Update with new data + const updatedActiveSubmissionId = await submissionRepository.update( + idActiveSubmission, + { + status: newStatusSubmission, + dictionaryId: dictionaryId, + }, + tx, + ); - const submissionRecords = await submissionRecordsRepository.getBySubmissionId(idActiveSubmission); - const recordsWithoutError = submissionRecords - .filter((record) => !invalidRecords.some((invalidRecord) => invalidRecord.id === record.id)) - .map((record) => record.id); + const invalidRecords = Object.values(schemaErrors).flatMap((entityErrors) => + Object.values(entityErrors).flatMap((recordErrors) => + recordErrors.map(({ recordId, errors }) => ({ + id: recordId, + errors, + })), + ), + ); - submissionRecordsRepository.updateValidationState({ - invalidRecords, - validRecordIds: recordsWithoutError, + const submissionRecords = await submissionRecordsRepository.getBySubmissionId(idActiveSubmission); + const recordsWithoutError = submissionRecords + .filter((record) => !invalidRecords.some((invalidRecord) => invalidRecord.id === record.id)) + .map((record) => record.id); + + // Update records with validation state, marking records with errors as 'INVALID' and records without errors as 'VALID' + await submissionRecordsRepository.updateValidationState( + { + invalidRecords, + validRecordIds: recordsWithoutError, + }, + tx, + ); + logger.info( + LOG_MODULE, + `Updated Active submission '${updatedActiveSubmissionId}' with status '${newStatusSubmission}'`, + ); + return updatedActiveSubmissionId; }); - logger.info( - LOG_MODULE, - `Updated Active submission '${updatedActiveSubmissionId}' with status '${newStatusSubmission}'`, - ); - return updatedActiveSubmissionId; + return 0; }; const logFileResult = (result: FileParseResult) => { if (result.status === 'error') { logger.error(LOG_MODULE, `Failed to parse file`, { fileName: result.fileName, - fileSize: bytes.format(result.fileSize, { decimalPlaces: 2 }), + fileSize: formatByteSize(result.fileSize, 'MB', 2), entityName: result.entityName, error: result.streamError, }); @@ -893,7 +872,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { // Log field names and line numbers only — not field values (OWASP A03). logger.warn(LOG_MODULE, `File parsed with schema validation issues`, { fileName: result.fileName, - fileSize: bytes.format(result.fileSize, { decimalPlaces: 2 }), + fileSize: formatByteSize(result.fileSize, 'MB', 2), entityName: result.entityName, errorCount: result.parseErrors.length, issues: result.parseErrors.slice(0, 10).map((e) => ({ diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index 3102ec58..013177b0 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -12,6 +12,7 @@ import submittedRepository from '../../repository/submittedRepository.js'; import { convertSqonToQuery } from '../../utils/convertSqonToQuery.js'; import { getDictionarySchemaRelations } from '../../utils/dictionarySchemaRelations.js'; import { InternalServerError, StatusConflict } from '../../utils/errors.js'; +import { getSizeInBytes } from '../../utils/fileUtils.js'; import type { PaginatedResult } from '../../utils/result.js'; import { fetchDataErrorResponse, @@ -132,30 +133,40 @@ const submittedData = (dependencies: BaseDependencies) => { // but for now just insert the new records const entitiesToProcess = Object.keys(recordsToDeleteMap); - // Updating the Submission with the new data and 'VALIDATING' status before validation starts - await udpateSubmission(activeSubmissionId, { - updatedBy: username, - status: 'VALIDATING', - }); + dependencies.db.transaction(async (tx) => { + // Updating the Submission with the new data and 'VALIDATING' status before validation starts + await udpateSubmission( + activeSubmissionId, + { + updatedBy: username, + status: 'VALIDATING', + }, + tx, + ); - await Promise.all( - Object.entries(recordsToDeleteMap).map(async ([entityName, entityRecords]) => { - const savedFileId = await submissionFilesRepository.save({ - entityName, - fileName: `update-${entityName}-${Date.now()}.json`, // default file name for adding JSON records - fileSize: JSON.stringify(entityRecords).length, - submissionId: activeSubmissionId, - }); - await submissionRecordsRepository.saveManyForFile( - savedFileId, - entityRecords.map((record) => ({ - actionType: 'DELETE', - data: record, - state: 'RECEIVED', - })), - ); - }), - ); + await Promise.all( + Object.entries(recordsToDeleteMap).map(async ([entityName, entityRecords]) => { + const savedFileId = await submissionFilesRepository.save( + { + entityName, + fileName: `${Date.now()}.json`, + fileSize: getSizeInBytes(JSON.stringify(entityRecords)), + submissionId: activeSubmissionId, + }, + tx, + ); + await submissionRecordsRepository.saveManyForFile( + savedFileId, + entityRecords.map((record) => ({ + actionType: 'DELETE', + data: record, + state: 'RECEIVED', + })), + tx, + ); + }), + ); + }); // Perform Schema Data validation in a worker thread dependencies.workerPool.dataValidation({ submissionId: activeSubmissionId }); diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index abc12499..ffab3356 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -27,12 +27,7 @@ import { getSchemaFieldNames } from './dictionaryUtils.js'; import { readHeaders, readTextFile } from './fileUtils.js'; import { asArray } from './formatUtils.js'; import type { FilenameEntityPair } from './schemas.js'; -import { - groupErrorsByIndex, - mapAndMergeSubmittedDataToRecordReferences, - type SubmissionInsertRecordWithEntityName, - type SubmissionUpdateRecordWithEntityName, -} from './submittedDataUtils.js'; +import { groupErrorsByIndex, mapAndMergeSubmittedDataToRecordReferences } from './submittedDataUtils.js'; import { BATCH_ERROR_TYPE, type BatchError, @@ -43,10 +38,12 @@ import { type NewSubmittedDataReference, SUBMISSION_RECORD_ACTION_TYPE, SUBMISSION_STATUS, + type SubmissionInsertRecordWithEntityName, type SubmissionRecordActionType, type SubmissionStatus, type SubmissionSummary, type SubmissionSummaryResponse, + type SubmissionUpdateRecordWithEntityName, SubmittedDataReference, } from './types.js'; @@ -264,27 +261,6 @@ export const filterRecordsByConflicts = ( }, {}); }; -/** - * Filters updates from the provided `submissionUpdateData` based on conflicts found in the `submissionDeleteData`. - * Conflicts are determined by matching the `systemId` of the items in both records. - * - * @param submissionUpdateData - A record containing arrays of `SubmissionUpdateData` to be filtered. - * @param submissionDeleteData - A record containing arrays of `SubmissionDeleteData` that defines the conflicts. - * @returns A filtered record of `SubmissionUpdateData[]` where no items conflict with those in `submissionDeleteData`. - * TODO: do not delete this function, this MIGHT be usefull in the future to avoid duplicates between submission records - */ -export const filterUpdatesFromDeletes = ( - submissionUpdateData: Record, - submissionDeleteData: Record, -): Record => { - return filterRecordsByConflicts( - submissionUpdateData, - submissionDeleteData, - (itemToUpdate) => itemToUpdate.systemId, - (itemToDelete) => itemToDelete.systemId, - ); -}; - /** * Filters deletes from the provided `submissionDeleteData` based on conflicts found in the `submissionUpdateData`. * Conflicts are determined by matching the `systemId` of the items in both records. @@ -408,7 +384,7 @@ export const groupSchemaErrorsByEntity = (input: { * The result mapping is used to perform the cross schema validation * @param {number} activeSubmissionId * @param {SubmissionInsertRecordWithEntityName[]} activeSubmissionInsertDataEntities - * @returns {DataRecordReference[]} + * @returns {Record} */ export const mapInsertDataToRecordReferences = ( activeSubmissionId: number, @@ -478,7 +454,6 @@ export const mapGroupedUpdateSubmissionData = ({ * Then, the Schema Data is extracted and mapped with its internal reference ID. * The returned Object is a collection of the raw Schema Data with it's reference ID grouped by entity name. * @param {number} submissionId ID of the Active Submission - * @param {Object} submissionData * @param {SubmissionRecordWithEntityName[]} submissionData The Active Submission data * @param {SubmittedData[]} submittedData An array of Submitted Data * @returns {Record} @@ -585,7 +560,7 @@ export const mergeUpdatesBySystemId = ( }; /** - * Utility to convert Dates to ISO string for a SubmissionSummaryResponse + * Utility to convert a raw Submission record to a Response type * @param {SubmissionSummary} submission * @returns {SubmissionSummaryResponse} */ diff --git a/packages/data-provider/src/utils/submittedDataUtils.ts b/packages/data-provider/src/utils/submittedDataUtils.ts index 5ac8e880..c460ff99 100644 --- a/packages/data-provider/src/utils/submittedDataUtils.ts +++ b/packages/data-provider/src/utils/submittedDataUtils.ts @@ -10,7 +10,6 @@ import { type DataDiff, NewSubmittedData, type SubmissionDeleteData, - type SubmissionInsertData, type SubmissionUpdateData, SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -22,6 +21,7 @@ import { MERGE_REFERENCE_TYPE, type MutableDataDiff, type MutableDataRecord, + type SubmissionUpdateRecordWithEntityName, VIEW_TYPE, type ViewType, } from './types.js'; @@ -344,17 +344,6 @@ export const updateSubmittedDataArray = ( }); }; -export type SubmissionInsertRecordWithEntityName = { - recordId: number; - entityName: string; - data: SubmissionInsertData; -}; -export type SubmissionUpdateRecordWithEntityName = { - recordId: number; - entityName: string; - data: SubmissionUpdateData; -}; - /** * Updates the entity data based on the provided update request. * It removes old keys, filters out undefined values from the new data, diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index fe7f296b..10f1bf93 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -13,6 +13,7 @@ import { type Dictionary, NewSubmittedData, type SubmissionDeleteData, + type SubmissionInsertData, type SubmissionUpdateData, type SubmittedData, } from '@overture-stack/lyric-data-model/models'; @@ -459,6 +460,17 @@ export interface DataRecordNested { [key: string]: DataRecordValue | DataRecordNested | DataRecordNested[]; } +export type SubmissionInsertRecordWithEntityName = { + recordId: number; + entityName: string; + data: SubmissionInsertData; +}; +export type SubmissionUpdateRecordWithEntityName = { + recordId: number; + entityName: string; + data: SubmissionUpdateData; +}; + /** * Keys of an object type as a union * diff --git a/packages/data-provider/test/unit/utils/submission/filterUpdatesFromDeletes.spec.ts b/packages/data-provider/test/unit/utils/submission/filterUpdatesFromDeletes.spec.ts deleted file mode 100644 index 883c2807..00000000 --- a/packages/data-provider/test/unit/utils/submission/filterUpdatesFromDeletes.spec.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import type { SubmissionDeleteData, SubmissionUpdateData } from '@overture-stack/lyric-data-model/models'; - -import { filterUpdatesFromDeletes } from '../../../../src/utils/submissionUtils.js'; - -describe('Submission Utils - Remove conflicts on Submission with records to be updated', () => { - it('should remove 1 matching record from the updates records', () => { - const submissionUpdateData: Record = { - cars: [ - { systemId: 'AAA111', new: { name: 'lamborghini Huracan' }, old: { name: 'Lambo' } }, - { systemId: 'BBB222', new: { name: 'Volkswagen Beettle' }, old: { name: 'Beettle' } }, - ], - }; - - const submissionDeleteData: Record = { - cars: [ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ], - }; - const result = filterUpdatesFromDeletes(submissionUpdateData, submissionDeleteData); - expect(Object.keys(result)).to.eql(['cars']); - expect(result['cars'].length).to.eq(1); - expect(result['cars'][0]).to.eql({ - systemId: 'BBB222', - new: { name: 'Volkswagen Beettle' }, - old: { name: 'Beettle' }, - }); - }); - it('should not remove any record if there is no matching Id', () => { - const submissionUpdateData: Record = { - cars: [ - { systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } }, - { systemId: 'DDDD444', new: { name: 'Audi' }, old: { name: 'Q5' } }, - ], - }; - - const submissionDeleteData: Record = { - cars: [ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ], - }; - - const result = filterUpdatesFromDeletes(submissionUpdateData, submissionDeleteData); - expect(Object.keys(result)).to.eql(['cars']); - expect(result['cars'].length).to.eq(2); - expect(result['cars']).to.eql([ - { systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } }, - { systemId: 'DDDD444', new: { name: 'Audi' }, old: { name: 'Q5' } }, - ]); - }); - it('should not remove any record if there is no matching entity name', () => { - const submissionUpdateData: Record = { - food: [ - { systemId: 'PTT123', new: { name: 'Potato' }, old: { name: 'Tomato' } }, - { systemId: 'SPNCH889', new: { name: 'Spinach' }, old: { name: 'Lettuce' } }, - ], - }; - - const submissionDeleteData: Record = { - cars: [ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ], - }; - const result = filterUpdatesFromDeletes(submissionUpdateData, submissionDeleteData); - expect(Object.keys(result)).to.eql(['food']); - expect(result['food'].length).to.eq(2); - expect(result['food']).to.eql([ - { systemId: 'PTT123', new: { name: 'Potato' }, old: { name: 'Tomato' } }, - { systemId: 'SPNCH889', new: { name: 'Spinach' }, old: { name: 'Lettuce' } }, - ]); - }); - it('should return empty object when passing an empty object', () => { - const submissionDeleteData: Record = { - cars: [ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ], - }; - const result = filterUpdatesFromDeletes({}, submissionDeleteData); - expect(Object.keys(result).length).to.eq(0); - }); -}); From aa765ad69746225aeaa75201b91469ec26255a38 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 11 Aug 2026 11:41:08 -0400 Subject: [PATCH 07/20] utils improvements --- .../submission/submissionProcessor.ts | 80 ++++++++++--------- .../services/submittedData/submmittedData.ts | 2 +- packages/data-provider/src/utils/fileUtils.ts | 9 ++- .../test/unit/utils/result.spec.ts | 25 ++++++ .../utils/submissionResponseParser.spec.ts | 71 ++++++++++++++++ 5 files changed, 149 insertions(+), 38 deletions(-) create mode 100644 packages/data-provider/test/unit/utils/result.spec.ts create mode 100644 packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index 9b46d4a0..f199e0ef 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -19,7 +19,7 @@ import createSubmissionRecordsRepository from '../../repository/submissionRecord import createSubmittedDataRepository from '../../repository/submittedRepository.js'; import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/dictionarySchemaRelations.js'; import { BadRequest } from '../../utils/errors.js'; -import { formatByteSize, getSizeInBytes } from '../../utils/fileUtils.js'; +import { formatByteSize, genericSubmissionFileName, getSizeInBytes } from '../../utils/fileUtils.js'; import { convertRecordToString } from '../../utils/formatUtils.js'; import { parseRecordsToInsert } from '../../utils/recordsParser.js'; import { @@ -651,23 +651,27 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const additions = await handleIdFieldChanges(idFieldChangeRecord); // Updating the Submission with the new data and 'VALIDATING' status before validation starts - await update(submission.id, { - updatedBy: username, - status: 'VALIDATING', - }); + await dependencies.db.transaction(async (tx) => { + await update( + submission.id, + { + updatedBy: username, + status: 'VALIDATING', + }, + tx, + ); - const entityNames: Set = new Set([ - ...Object.keys(additions.inserts), - ...Object.keys(additions.deletes), - ...Object.keys(updatedActiveSubmissionData), - ]); + const entityNames: Set = new Set([ + ...Object.keys(additions.inserts), + ...Object.keys(additions.deletes), + ...Object.keys(updatedActiveSubmissionData), + ]); - for (const entityName of entityNames) { - dependencies.db.transaction(async (tx) => { + for (const entityName of entityNames) { const savedFileId = await submissionFilesRepository.save( { entityName: entityName, - fileName: `${Date.now()}.json`, + fileName: genericSubmissionFileName(), fileSize: getSizeInBytes(JSON.stringify(recordsParsed)), submissionId: submission.id, }, @@ -708,8 +712,8 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { tx, ); } - }); - } + } + }); // Perform Schema Data validation in a worker thread dependencies.workerPool.dataValidation({ submissionId: submission.id }); @@ -770,7 +774,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { Object.entries(insertRecords).map(async ([entityName, entityRecords]) => { const savedFileId = await submissionFilesRepository.save({ entityName, - fileName: `${Date.now()}.json`, // default file name for adding JSON records + fileName: genericSubmissionFileName(), fileSize: getSizeInBytes(JSON.stringify(entityRecords)), submissionId, }); @@ -817,7 +821,7 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const newStatusSubmission = Object.keys(schemaErrors).length > 0 ? SUBMISSION_STATUS.INVALID : SUBMISSION_STATUS.VALID; - dependencies.db.transaction(async (tx) => { + await dependencies.db.transaction(async (tx) => { // Update with new data const updatedActiveSubmissionId = await submissionRepository.update( idActiveSubmission, @@ -898,25 +902,29 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const fileResult: FileParseResult[] = []; try { - // Updating the Submission with the new data and 'VALIDATING' status before validation starts - await submissionRepository.update(submissionId, { - updatedBy: username, - status: 'VALIDATING', - }); + await dependencies.db.transaction(async (tx) => { + // Updating the Submission with the new data and 'VALIDATING' status before validation starts + await submissionRepository.update( + submissionId, + { + updatedBy: username, + status: 'VALIDATING', + }, + tx, + ); - // Parse file data — each file is isolated; a failure on one does not block others. - const parsingFileDataResult = await submissionInsertDataFromFiles(fileSchemaMap); + // Parse file data — each file is isolated; a failure on one does not block others. + const parsingFileDataResult = await submissionInsertDataFromFiles(fileSchemaMap); - for (const fileProcessed of parsingFileDataResult) { - logFileResult(fileProcessed.fileResult); - const { - data, - fileResult: { entityName, fileName, fileSize, status }, - } = fileProcessed; - fileResult.push(fileProcessed.fileResult); + for (const fileProcessed of parsingFileDataResult) { + logFileResult(fileProcessed.fileResult); + const { + data, + fileResult: { entityName, fileName, fileSize, status }, + } = fileProcessed; + fileResult.push(fileProcessed.fileResult); - if (status === 'ok') { - dependencies.db.transaction(async (tx) => { + if (status === 'ok') { const fileId = await submissionFilesRepository.save( { entityName, @@ -936,12 +944,12 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { })), tx, ); - }); + } } - } + }); // Perform Schema Data validation in a worker thread - dependencies.workerPool.dataValidation({ submissionId: submissionId }); + dependencies.workerPool.dataValidation({ submissionId }); } catch (error) { logger.error(LOG_MODULE, `Error processing submitted files`, { error: error instanceof Error ? error.message : String(error), diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index 013177b0..92556114 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -133,7 +133,7 @@ const submittedData = (dependencies: BaseDependencies) => { // but for now just insert the new records const entitiesToProcess = Object.keys(recordsToDeleteMap); - dependencies.db.transaction(async (tx) => { + await dependencies.db.transaction(async (tx) => { // Updating the Submission with the new data and 'VALIDATING' status before validation starts await udpateSubmission( activeSubmissionId, diff --git a/packages/data-provider/src/utils/fileUtils.ts b/packages/data-provider/src/utils/fileUtils.ts index 7998d170..9e46826f 100644 --- a/packages/data-provider/src/utils/fileUtils.ts +++ b/packages/data-provider/src/utils/fileUtils.ts @@ -13,7 +13,7 @@ import { } from '@overture-stack/lectern-client'; import { getSubmittedFileType } from '../services/submission/submissionFile.js'; -import { failure, success, type Result } from './result.js'; +import { failure, type Result, success } from './result.js'; import { BATCH_ERROR_TYPE, type BatchError } from './types.js'; export const SUPPORTED_FILE_EXTENSIONS = z.enum(['tsv', 'csv']); @@ -125,6 +125,13 @@ function formatForExcelCompatibility(data: string) { .trim(); } +/** + * Generates a generic file name for submission files with `.json` extension. + * Based on the current date and time in ISO format. + * @returns + */ +export const genericSubmissionFileName = () => `submission-${new Date().toISOString()}.json`; + export function getSizeInBytes(size: string | number): number { // Parse the string value into an integer in bytes. // If value is a number it is assumed is in bytes. diff --git a/packages/data-provider/test/unit/utils/result.spec.ts b/packages/data-provider/test/unit/utils/result.spec.ts new file mode 100644 index 00000000..261551ee --- /dev/null +++ b/packages/data-provider/test/unit/utils/result.spec.ts @@ -0,0 +1,25 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { failure, success } from '../../../src/utils/result.js'; + +describe('Result', () => { + describe('success', () => { + it('should wrap data in a success result', () => { + const result = success({ id: 1, name: 'sport' }); + expect(result).to.eql({ success: true, data: { id: 1, name: 'sport' } }); + }); + }); + + describe('failure', () => { + it('should wrap data in a failure result', () => { + const result = failure('something went wrong'); + expect(result).to.eql({ success: false, data: 'something went wrong' }); + }); + + it('should support a non-string failure data type', () => { + const result = failure({ code: 'NOT_FOUND' }); + expect(result).to.eql({ success: false, data: { code: 'NOT_FOUND' } }); + }); + }); +}); diff --git a/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts b/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts new file mode 100644 index 00000000..056e99bb --- /dev/null +++ b/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts @@ -0,0 +1,71 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { RecordsSummaryRepository } from '../../../src/repository/submissionRecordsRepository.js'; +import { buildDataSummary } from '../../../src/utils/submissionResponseParser.js'; + +describe('buildDataSummary', () => { + it('should return an empty summary when given no rows', () => { + const result = buildDataSummary([]); + + expect(result).to.eql({ + inserts: {}, + updates: {}, + deletes: {}, + totalRecords: 0, + errors: 0, + }); + }); + + it('should group insert and update rows by entity name and accumulate totals', () => { + const rows: RecordsSummaryRepository[] = [ + { actionType: 'INSERT', entityName: 'sport', totalRecords: 3, batchName: 'batch1.tsv', errors: 0 }, + { actionType: 'INSERT', entityName: 'sport', totalRecords: 2, batchName: 'batch2.tsv', errors: 1 }, + { actionType: 'UPDATE', entityName: 'player', totalRecords: 5, batchName: 'batch3.tsv', errors: 0 }, + ]; + + const result = buildDataSummary(rows); + + expect(result).to.eql({ + inserts: { + sport: [ + { batchName: 'batch1.tsv', recordsCount: 3, errors: 0 }, + { batchName: 'batch2.tsv', recordsCount: 2, errors: 1 }, + ], + }, + updates: { + player: [{ batchName: 'batch3.tsv', recordsCount: 5, errors: 0 }], + }, + deletes: {}, + totalRecords: 10, + errors: 1, + }); + }); + + it('should aggregate delete rows for the same entity instead of listing them individually', () => { + const rows: RecordsSummaryRepository[] = [ + { actionType: 'DELETE', entityName: 'sport', totalRecords: 2, batchName: 'batch1.tsv', errors: 0 }, + { actionType: 'DELETE', entityName: 'sport', totalRecords: 1, batchName: 'batch2.tsv', errors: 1 }, + ]; + + const result = buildDataSummary(rows); + + expect(result.deletes).to.eql({ + sport: { recordsCount: 3, errors: 1 }, + }); + expect(result.totalRecords).to.eq(3); + expect(result.errors).to.eq(1); + }); + + it('should default a missing batchName to an empty string', () => { + const rows: RecordsSummaryRepository[] = [ + { actionType: 'INSERT', entityName: 'sport', totalRecords: 1, errors: 0 }, + ]; + + const result = buildDataSummary(rows); + + expect(result.inserts).to.eql({ + sport: [{ batchName: '', recordsCount: 1, errors: 0 }], + }); + }); +}); From 5679af7acbb0ecfa850ec4cef07658cabe24f0e3 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Tue, 11 Aug 2026 13:42:09 -0400 Subject: [PATCH 08/20] filter submission records by action Type --- .../services/submission/submissionService.ts | 3 +- .../src/utils/submissionResponseParser.ts | 86 +++++++++----- .../src/utils/submissionUtils.ts | 107 +++++++++--------- .../src/workers/commitSubmissionWorker.ts | 67 ++++------- .../createSubmissionInsertRecords.spec.ts | 71 ++++++++++++ .../createSubmissionUpdateRecords.spec.ts | 71 ++++++++++++ .../isDeleteSubmissionRecord.spec.ts | 49 ++++++++ .../isInsertSubmissionRecord.spec.ts | 49 ++++++++ .../isUpdateSubmissionRecord.spec.ts | 49 ++++++++ .../parseSubmissionSummaryResponse.spec.ts | 2 +- 10 files changed, 424 insertions(+), 130 deletions(-) create mode 100644 packages/data-provider/test/unit/utils/submission/createSubmissionInsertRecords.spec.ts create mode 100644 packages/data-provider/test/unit/utils/submission/createSubmissionUpdateRecords.spec.ts create mode 100644 packages/data-provider/test/unit/utils/submission/isDeleteSubmissionRecord.spec.ts create mode 100644 packages/data-provider/test/unit/utils/submission/isInsertSubmissionRecord.spec.ts create mode 100644 packages/data-provider/test/unit/utils/submission/isUpdateSubmissionRecord.spec.ts diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts index 55669119..49f640ec 100644 --- a/packages/data-provider/src/services/submission/submissionService.ts +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -15,10 +15,9 @@ import { getSchemaByName } from '../../utils/dictionaryUtils.js'; import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; import type { PaginatedResult } from '../../utils/result.js'; import type { FilenameEntityPair } from '../../utils/schemas.js'; -import { buildDataSummary } from '../../utils/submissionResponseParser.js'; +import { buildDataSummary, createSubmissionSummaryResponse } from '../../utils/submissionResponseParser.js'; import { checkEntityFieldNames, - createSubmissionSummaryResponse, type FileParseResult, isSubmissionActive, resolveFileEntities, diff --git a/packages/data-provider/src/utils/submissionResponseParser.ts b/packages/data-provider/src/utils/submissionResponseParser.ts index 087dbb33..0129958c 100644 --- a/packages/data-provider/src/utils/submissionResponseParser.ts +++ b/packages/data-provider/src/utils/submissionResponseParser.ts @@ -1,48 +1,78 @@ +import * as _ from 'lodash-es'; + import type { RecordsSummaryRepository } from '../repository/submissionRecordsRepository.js'; -import { type SubmissionDataSummaryWithTotal } from './types.js'; +import { + type DataDeletesSubmissionSummary, + type DataInsertsSubmissionSummary, + type DataUpdatesSubmissionSummary, + type SubmissionDataSummaryWithTotal, + type SubmissionSummary, + type SubmissionSummaryResponse, +} from './types.js'; // This function accepts a raw array of submission records from the database and builds a summary response. export const buildDataSummary = (rows: RecordsSummaryRepository[]): SubmissionDataSummaryWithTotal => { - const initialSummary: SubmissionDataSummaryWithTotal = { - inserts: {}, - updates: {}, - deletes: {}, - totalRecords: 0, - errors: 0, - }; + const inserts: Record = {}; + const updates: Record = {}; + const deletes: Record = {}; + let totalRecords = 0; + let errors = 0; - return rows.reduce((acc, row) => { - const { actionType, entityName, totalRecords, batchName, errors } = row; - const summaryItem = { batchName: batchName ?? '', recordsCount: totalRecords, errors }; + for (const row of rows) { + const { actionType, entityName, totalRecords: rowTotalRecords, batchName, errors: rowErrors } = row; + const summaryItem = { batchName: batchName ?? '', recordsCount: rowTotalRecords, errors: rowErrors }; - acc.totalRecords += totalRecords; - acc.errors += errors; + totalRecords += rowTotalRecords; + errors += rowErrors; switch (actionType) { case 'INSERT': { - const entitySummaries = acc.inserts ?? {}; - entitySummaries[entityName] = entitySummaries[entityName] ?? []; - entitySummaries[entityName].push(summaryItem); - acc.inserts = entitySummaries; + const entitySummaries = inserts[entityName] ?? []; + entitySummaries.push(summaryItem); + inserts[entityName] = entitySummaries; break; } case 'UPDATE': { - const entitySummaries = acc.updates ?? {}; - entitySummaries[entityName] = entitySummaries[entityName] ?? []; - entitySummaries[entityName].push(summaryItem); - acc.updates = entitySummaries; + const entitySummaries = updates[entityName] ?? []; + entitySummaries.push(summaryItem); + updates[entityName] = entitySummaries; break; } case 'DELETE': { - const entitySummaries = acc.deletes ?? {}; - entitySummaries[entityName] = entitySummaries[entityName] ?? { recordsCount: 0, errors: 0 }; - entitySummaries[entityName].recordsCount += totalRecords; - entitySummaries[entityName].errors += errors; - acc.deletes = entitySummaries; + const entitySummary = deletes[entityName] ?? { recordsCount: 0, errors: 0 }; + entitySummary.recordsCount += rowTotalRecords; + entitySummary.errors += rowErrors; + deletes[entityName] = entitySummary; break; } } + } + + return { + inserts, + updates, + deletes, + totalRecords, + errors, + }; +}; - return acc; - }, initialSummary); +/** + * Utility to convert a raw Submission record to a Response type + * @param {SubmissionSummary} submission + * @returns {SubmissionSummaryResponse} + */ +export const createSubmissionSummaryResponse = (submission: SubmissionSummary): SubmissionSummaryResponse => { + return { + id: submission.id, + data: submission.data, + dictionary: submission.dictionary, + dictionaryCategory: submission.dictionaryCategory, + organization: submission.organization, + status: submission.status, + createdAt: _.toString(submission.createdAt?.toISOString()), + createdBy: _.toString(submission.createdBy), + updatedAt: _.toString(submission.updatedAt?.toISOString()), + updatedBy: _.toString(submission.updatedBy), + }; }; diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index ffab3356..f1b765a0 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -41,8 +41,6 @@ import { type SubmissionInsertRecordWithEntityName, type SubmissionRecordActionType, type SubmissionStatus, - type SubmissionSummary, - type SubmissionSummaryResponse, type SubmissionUpdateRecordWithEntityName, SubmittedDataReference, } from './types.js'; @@ -449,6 +447,57 @@ export const mapGroupedUpdateSubmissionData = ({ ); }; +export const isUpdateSubmissionRecord = ( + item: SubmissionRecordWithEntityName, +): item is SubmissionRecordWithEntityName & { + actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE; + data: SubmissionUpdateData; +} => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE; + +export const isInsertSubmissionRecord = ( + item: SubmissionRecordWithEntityName, +): item is SubmissionRecordWithEntityName & { + actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT; + data: SubmissionInsertData; +} => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT; + +export const isDeleteSubmissionRecord = ( + item: SubmissionRecordWithEntityName, +): item is SubmissionRecordWithEntityName & { + actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.DELETE; + data: SubmissionDeleteData; +} => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.DELETE; + +export const createSubmissionUpdateRecords = ( + submissionData: SubmissionRecordWithEntityName[], +): SubmissionUpdateRecordWithEntityName[] => { + return submissionData.reduce((acc, item) => { + if (isUpdateSubmissionRecord(item)) { + acc.push({ + recordId: item.id, + entityName: item.entityName, + data: item.data, + }); + } + return acc; + }, []); +}; + +export const createSubmissionInsertRecords = ( + submissionData: SubmissionRecordWithEntityName[], +): SubmissionInsertRecordWithEntityName[] => { + return submissionData.reduce((acc, item) => { + if (isInsertSubmissionRecord(item)) { + acc.push({ + recordId: item.id, + entityName: item.entityName, + data: item.data, + }); + } + return acc; + }, []); +}; + /** * Combines **Active Submission** and the **Submitted Data** recevied as arguments. * Then, the Schema Data is extracted and mapped with its internal reference ID. @@ -467,9 +516,7 @@ export const mergeAndReferenceEntityData = ({ submissionData: SubmissionRecordWithEntityName[]; submittedData: SubmittedData[]; }): Record => { - const systemsIdsToRemove = submissionData - .filter((item) => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.DELETE) - .map((item) => item.data.systemId); + const systemsIdsToRemove = submissionData.filter(isDeleteSubmissionRecord).map((item) => item.data.systemId); // Exclude items that are marked for deletion const submittedDataFiltered = @@ -477,20 +524,7 @@ export const mergeAndReferenceEntityData = ({ ? submittedData.filter(({ systemId }) => !systemsIdsToRemove.includes(systemId)) : submittedData; - const dataToUpdate: SubmissionUpdateRecordWithEntityName[] = submissionData - .filter( - ( - item, - ): item is SubmissionRecordWithEntityName & { - actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE; - data: SubmissionUpdateData; - } => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.UPDATE, - ) - .map((item) => ({ - recordId: item.id, - entityName: item.entityName, - data: item.data, - })); + const dataToUpdate = createSubmissionUpdateRecords(submissionData); const submittedDataWithRef = mapAndMergeSubmittedDataToRecordReferences({ submittedData: submittedDataFiltered, @@ -498,20 +532,7 @@ export const mergeAndReferenceEntityData = ({ submissionId, }); - const dataToInsert: SubmissionInsertRecordWithEntityName[] = submissionData - .filter( - ( - item, - ): item is SubmissionRecordWithEntityName & { - actionType: typeof SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT; - data: SubmissionInsertData; - } => item.actionType === SUBMISSION_RECORD_ACTION_TYPE.Values.INSERT, - ) - .map((item) => ({ - recordId: item.id, - entityName: item.entityName, - data: item.data, - })); + const dataToInsert = createSubmissionInsertRecords(submissionData); const insertDataWithRef = dataToInsert.length > 0 ? mapInsertDataToRecordReferences(submissionId, dataToInsert) : {}; @@ -559,26 +580,6 @@ export const mergeUpdatesBySystemId = ( return result; }; -/** - * Utility to convert a raw Submission record to a Response type - * @param {SubmissionSummary} submission - * @returns {SubmissionSummaryResponse} - */ -export const createSubmissionSummaryResponse = (submission: SubmissionSummary): SubmissionSummaryResponse => { - return { - id: submission.id, - data: submission.data, - dictionary: submission.dictionary, - dictionaryCategory: submission.dictionaryCategory, - organization: submission.organization, - status: submission.status, - createdAt: _.toString(submission.createdAt?.toISOString()), - createdBy: _.toString(submission.createdBy), - updatedAt: _.toString(submission.updatedAt?.toISOString()), - updatedBy: _.toString(submission.updatedBy), - }; -}; - export const pluralizeSchemaName = (schemaName: string) => { return plur(schemaName); }; diff --git a/packages/data-provider/src/workers/commitSubmissionWorker.ts b/packages/data-provider/src/workers/commitSubmissionWorker.ts index 11fe687f..a6249471 100644 --- a/packages/data-provider/src/workers/commitSubmissionWorker.ts +++ b/packages/data-provider/src/workers/commitSubmissionWorker.ts @@ -1,8 +1,4 @@ -import type { - SubmissionDeleteData, - SubmissionInsertData, - SubmissionUpdateData, -} from '@overture-stack/lyric-data-model/models'; +import type { SubmissionUpdateData } from '@overture-stack/lyric-data-model/models'; import systemIdGenerator from '../external/systemIdGenerator.js'; import createSubmissionRepository from '../repository/activeSubmissionRepository.js'; @@ -10,8 +6,12 @@ import createCategoryRepository from '../repository/categoryRepository.js'; import createSubmissionRecordsRepository from '../repository/submissionRecordsRepository.js'; import createSubmittedRepository from '../repository/submittedRepository.js'; import submissionProcessorFactory from '../services/submission/submissionProcessor.js'; -import type { ResultOnCommit } from '../utils/types.js'; -import { SUBMISSION_STATUS } from '../utils/types.js'; +import { + isDeleteSubmissionRecord, + isInsertSubmissionRecord, + isUpdateSubmissionRecord, +} from '../utils/submissionUtils.js'; +import { type ResultOnCommit, SUBMISSION_STATUS } from '../utils/types.js'; import type { CommitWorkerInput } from './types.js'; import { getWorkerDependencies } from './workerContext.js'; @@ -65,56 +65,31 @@ export const processCommitSubmission = async (message: CommitWorkerInput): Promi }); // Build inserts for validation - const insertsToValidate = recordsToInsert - .filter( - ( - record, - ): record is (typeof recordsToInsert)[number] & { - actionType: 'INSERT'; - data: SubmissionInsertData; - } => record.actionType === 'INSERT', - ) - .map(({ entityName, data }) => { - return { - data, - dictionaryCategoryId: categoryId, - entityName, - isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise - organization: submission.organization, - originalSchemaId: currentDictionary.id, - systemId: generateIdentifier(entityName, data), - createdBy: username, - }; - }); + const insertsToValidate = recordsToInsert.filter(isInsertSubmissionRecord).map(({ entityName, data }) => { + return { + data, + dictionaryCategoryId: categoryId, + entityName, + isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise + organization: submission.organization, + originalSchemaId: currentDictionary.id, + systemId: generateIdentifier(entityName, data), + createdBy: username, + }; + }); const recordsToDelete = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, { actionTypes: ['DELETE'], }); - const deleteDataArray = recordsToDelete - .filter( - ( - record, - ): record is (typeof recordsToDelete)[number] & { - actionType: 'DELETE'; - data: SubmissionDeleteData; - } => record.actionType === 'DELETE', - ) - .map(({ data }) => data); + const deleteDataArray = recordsToDelete.filter(isDeleteSubmissionRecord).map(({ data }) => data); const recordsToUpdate = await submissionRecordsRepo.getBySubmissionId(submissionId, undefined, { actionTypes: ['UPDATE'], }); const updatesBySystemId = recordsToUpdate - .filter( - ( - record, - ): record is (typeof recordsToUpdate)[number] & { - actionType: 'UPDATE'; - data: SubmissionUpdateData; - } => record.actionType === 'UPDATE', - ) + .filter(isUpdateSubmissionRecord) .reduce>((acc, { data }) => { acc[data.systemId] = data; return acc; diff --git a/packages/data-provider/test/unit/utils/submission/createSubmissionInsertRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/createSubmissionInsertRecords.spec.ts new file mode 100644 index 00000000..ea6e310d --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/createSubmissionInsertRecords.spec.ts @@ -0,0 +1,71 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; +import { createSubmissionInsertRecords } from '../../../../src/utils/submissionUtils.js'; + +describe('createSubmissionInsertRecords', () => { + it('should map insert records to SubmissionInsertRecordWithEntityName, dropping non-insert records', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'INSERT', + entityName: 'animals', + id: 8, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'elephant', color: 'gray' }, + }, + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' }, + }, + ]; + + const result = createSubmissionInsertRecords(submissionData); + + expect(result).to.eql([ + { + recordId: 8, + entityName: 'animals', + data: { name: 'elephant', color: 'gray' }, + }, + ]); + }); + + it('should return an empty array when there are no insert records', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + ]; + + const result = createSubmissionInsertRecords(submissionData); + + expect(result).to.eql([]); + }); + + it('should return an empty array when given no records', () => { + expect(createSubmissionInsertRecords([])).to.eql([]); + }); +}); diff --git a/packages/data-provider/test/unit/utils/submission/createSubmissionUpdateRecords.spec.ts b/packages/data-provider/test/unit/utils/submission/createSubmissionUpdateRecords.spec.ts new file mode 100644 index 00000000..49b3f703 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/createSubmissionUpdateRecords.spec.ts @@ -0,0 +1,71 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; +import { createSubmissionUpdateRecords } from '../../../../src/utils/submissionUtils.js'; + +describe('createSubmissionUpdateRecords', () => { + it('should map update records to SubmissionUpdateRecordWithEntityName, dropping non-update records', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'INSERT', + entityName: 'animals', + id: 8, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'elephant', color: 'gray' }, + }, + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' }, + }, + ]; + + const result = createSubmissionUpdateRecords(submissionData); + + expect(result).to.eql([ + { + recordId: 10, + entityName: 'animals', + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + ]); + }); + + it('should return an empty array when there are no update records', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'INSERT', + entityName: 'animals', + id: 8, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'elephant', color: 'gray' }, + }, + ]; + + const result = createSubmissionUpdateRecords(submissionData); + + expect(result).to.eql([]); + }); + + it('should return an empty array when given no records', () => { + expect(createSubmissionUpdateRecords([])).to.eql([]); + }); +}); diff --git a/packages/data-provider/test/unit/utils/submission/isDeleteSubmissionRecord.spec.ts b/packages/data-provider/test/unit/utils/submission/isDeleteSubmissionRecord.spec.ts new file mode 100644 index 00000000..dd960bb6 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/isDeleteSubmissionRecord.spec.ts @@ -0,0 +1,49 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; +import { isDeleteSubmissionRecord } from '../../../../src/utils/submissionUtils.js'; + +describe('isDeleteSubmissionRecord', () => { + it('should return true when actionType is DELETE', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'DELETE', + entityName: 'animals', + id: 1, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' }, + }; + + expect(isDeleteSubmissionRecord(record)).to.be.true; + }); + + it('should return false when actionType is INSERT', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'INSERT', + entityName: 'animals', + id: 2, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'elephant', color: 'gray' }, + }; + + expect(isDeleteSubmissionRecord(record)).to.be.false; + }); + + it('should return false when actionType is UPDATE', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'UPDATE', + entityName: 'animals', + id: 3, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }; + + expect(isDeleteSubmissionRecord(record)).to.be.false; + }); +}); diff --git a/packages/data-provider/test/unit/utils/submission/isInsertSubmissionRecord.spec.ts b/packages/data-provider/test/unit/utils/submission/isInsertSubmissionRecord.spec.ts new file mode 100644 index 00000000..effe8dc4 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/isInsertSubmissionRecord.spec.ts @@ -0,0 +1,49 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; +import { isInsertSubmissionRecord } from '../../../../src/utils/submissionUtils.js'; + +describe('isInsertSubmissionRecord', () => { + it('should return true when actionType is INSERT', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'INSERT', + entityName: 'animals', + id: 1, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'elephant', color: 'gray' }, + }; + + expect(isInsertSubmissionRecord(record)).to.be.true; + }); + + it('should return false when actionType is UPDATE', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'UPDATE', + entityName: 'animals', + id: 2, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }; + + expect(isInsertSubmissionRecord(record)).to.be.false; + }); + + it('should return false when actionType is DELETE', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'DELETE', + entityName: 'animals', + id: 3, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' }, + }; + + expect(isInsertSubmissionRecord(record)).to.be.false; + }); +}); diff --git a/packages/data-provider/test/unit/utils/submission/isUpdateSubmissionRecord.spec.ts b/packages/data-provider/test/unit/utils/submission/isUpdateSubmissionRecord.spec.ts new file mode 100644 index 00000000..45a69342 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/isUpdateSubmissionRecord.spec.ts @@ -0,0 +1,49 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; +import { isUpdateSubmissionRecord } from '../../../../src/utils/submissionUtils.js'; + +describe('isUpdateSubmissionRecord', () => { + it('should return true when actionType is UPDATE', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'UPDATE', + entityName: 'animals', + id: 1, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }; + + expect(isUpdateSubmissionRecord(record)).to.be.true; + }); + + it('should return false when actionType is INSERT', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'INSERT', + entityName: 'animals', + id: 2, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'elephant', color: 'gray' }, + }; + + expect(isUpdateSubmissionRecord(record)).to.be.false; + }); + + it('should return false when actionType is DELETE', () => { + const record: SubmissionRecordWithEntityName = { + actionType: 'DELETE', + entityName: 'animals', + id: 3, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' }, + }; + + expect(isUpdateSubmissionRecord(record)).to.be.false; + }); +}); diff --git a/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts b/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts index 02ca89ec..2adb27ea 100644 --- a/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts +++ b/packages/data-provider/test/unit/utils/submission/parseSubmissionSummaryResponse.spec.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { describe, it } from 'mocha'; -import { createSubmissionSummaryResponse } from '../../../../src/utils/submissionUtils.js'; +import { createSubmissionSummaryResponse } from '../../../../src/utils/submissionResponseParser.js'; import { SUBMISSION_STATUS, type SubmissionSummary } from '../../../../src/utils/types.js'; describe('Submission Utils - Parse a Submission object to a Summary of the Active Submission', () => { From 3431f4b6b1fbdb096ca72fadd13fb4f1a8fe048f Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 12 Aug 2026 18:10:38 -0400 Subject: [PATCH 09/20] solve conflicts matching system ids --- .dev/tech-debt.md | 18 ++ .../src/models/submission_records.ts | 20 +- .../submission/submissionProcessor.ts | 28 ++- .../src/utils/submissionUtils.ts | 192 +++++++++++----- ...tractRecordIdsFromSubmissionErrors.spec.ts | 30 +++ .../filterDeletesFromUpdates.spec.ts | 136 ----------- .../findUpdateDeleteConflicts.spec.ts | 216 ++++++++++++++++++ .../submission/mergeSubmissionErrors.spec.ts | 49 ++++ 8 files changed, 490 insertions(+), 199 deletions(-) create mode 100644 packages/data-provider/test/unit/utils/submission/extractRecordIdsFromSubmissionErrors.spec.ts delete mode 100644 packages/data-provider/test/unit/utils/submission/filterDeletesFromUpdates.spec.ts create mode 100644 packages/data-provider/test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts create mode 100644 packages/data-provider/test/unit/utils/submission/mergeSubmissionErrors.spec.ts diff --git a/.dev/tech-debt.md b/.dev/tech-debt.md index cdecf677..bdd4421f 100644 --- a/.dev/tech-debt.md +++ b/.dev/tech-debt.md @@ -53,8 +53,26 @@ context: `dictionary_categories.id` is a plain auto-increment `serial`, used dir --- +### Staging a DELETE submission record never checks for or merges with an existing pending record for the same systemId +standalone: yes +context: `submmittedData.ts::deleteSubmittedDataBySystemId` (132-133) has its own in-code TODO: "get the existing entity names for a submission and merge with the new ones to avoid duplicates, but for now just insert the new records." Two sequential calls (e.g. edit systemId X, then delete systemId X, before committing) can leave both an UPDATE row and a DELETE row for X in `submission_records` simultaneously, with no DB constraint preventing it either (no unique index on systemId in `packages/data-model/src/models/submission_records.ts:46-69`). Partially safety-netted as of 2026-08-12: `performDataValidation` now detects this exact conflict and marks both rows INVALID before validation runs (see Resolved section), so the conflict no longer silently drops one side — but the staging-time TODO itself is still unaddressed, so the submission still has to go back through an edit/delete cycle to resolve it rather than being prevented at write time. + +### Duplicate UPDATE submission records for the same systemId collapse via undefined last-write-wins at commit time +standalone: yes +context: `commitSubmissionWorker.ts::processCommitSubmission` (91-96) reduces UPDATE rows into `Record` keyed by systemId via plain object assignment, and the underlying query (`submissionRecordsRepository.ts::getByFileIds`, 38-64) has no `ORDER BY`. If two UPDATE rows exist for the same systemId, which one wins is whatever order Postgres happens to return, not an application-defined order. Distinct from the UPDATE/DELETE conflict resolved 2026-08-12 below — this is UPDATE-vs-UPDATE, not yet addressed. Fix: add an explicit `ORDER BY` (e.g. by id/createdAt) or reconcile duplicate updates for the same systemId earlier, in the staging path. + +### No integration tests exist for the submission edit, delete, or commit endpoints — let alone mixed insert+update+delete scenarios +standalone: yes +context: `packages/data-provider/test/integration/routers/submission/` only covers `submissionRouter-submit*` (insert-only file/JSON uploads). There is no integration spec for `editSubmittedData`, `deleteSubmittedDataBySystemId`, or the commit flow (`performCommitSubmissionAsync`/`commitSubmissionWorker.ts`) at all, despite the test infra (`test/integration/dependencies/containers.ts`) already running a real Postgres container capable of exercising the real commit transaction. The new UPDATE/DELETE conflict resolution added 2026-08-12 has unit coverage only (`test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts` and friends) — still no HTTP+DB-level test proving a submission with a staged conflict actually ends up INVALID end-to-end. Fix: add integration specs that stage inserts+updates+deletes in the same active submission (including same-systemId collisions) and assert the committed `submitted_data` table end state and the active submission's status/record states. + ## Resolved +### UPDATE/DELETE conflict on the same systemId was silently dropped instead of surfaced as an error +resolved: 2026-08-12, added explicit conflict detection ahead of dictionary validation. `findUpdateDeleteConflicts` (`packages/data-provider/src/utils/submissionUtils.ts`) scans staged submission records for entityName+systemId pairs with both an UPDATE and a DELETE, before `performDataValidation` (`submissionProcessor.ts`) runs `validateSchemas`. Conflicting records are excluded from validation, both sides are marked `INVALID` with a new `CONFLICTING_ACTION` error (`RecordErrorActionConflict`, added to `SubmissionRecordError` in `@overture-stack/lyric-data-model`), the active submission is marked `INVALID`, and the scenario is `logger.error`-logged. Previously this was only prevented by incidental JS array-filtering order with no error surfaced (see the still-open items above for what remains: the staging-time TODO, and integration coverage). + +### `filterDeletesFromUpdates`/`filterRecordsByConflicts` removed as dead code +resolved: 2026-08-12, same session as the fix above. Once `findUpdateDeleteConflicts` became the real, wired-in conflict detector, the unused `filterRecordsByConflicts`/`filterDeletesFromUpdates` pair (`submissionUtils.ts`, previously lines 234-280) had no remaining reason to exist — deleted both functions and their orphaned spec (`test/unit/utils/submission/filterDeletesFromUpdates.spec.ts`). Confirmed via repo-wide grep that nothing else referenced them before removing. + ### Kafka publish tracking: no unit tests for `createPublishTracker` diff --git a/packages/data-model/src/models/submission_records.ts b/packages/data-model/src/models/submission_records.ts index c09b07b2..d6d91e95 100644 --- a/packages/data-model/src/models/submission_records.ts +++ b/packages/data-model/src/models/submission_records.ts @@ -41,7 +41,25 @@ export type UnrecognizedValueReason = { export type RecordErrorInvalidValue = FieldDetails & UnrecognizedValueReason; -export type SubmissionRecordError = DictionaryValidationRecordErrorDetails | RecordErrorInvalidValue; +export type ConflictingActionReason = { + reason: 'CONFLICTING_ACTION'; +}; + +/** + * Raised when a record's `systemId` has both an UPDATE and a DELETE staged in the same + * Active Submission. `conflictingActionType` names the *other* action type this record + * conflicts with, so both sides of the conflict can be reported independently. + */ +export type RecordErrorActionConflict = ConflictingActionReason & { + systemId: string; + conflictingActionType: 'UPDATE' | 'DELETE'; + message: string; +}; + +export type SubmissionRecordError = + | DictionaryValidationRecordErrorDetails + | RecordErrorInvalidValue + | RecordErrorActionConflict; export const submissionRecords = pgTable( 'submission_records', diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index f199e0ef..0c45e9e9 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -23,14 +23,17 @@ import { formatByteSize, genericSubmissionFileName, getSizeInBytes } from '../.. import { convertRecordToString } from '../../utils/formatUtils.js'; import { parseRecordsToInsert } from '../../utils/recordsParser.js'; import { + extractRecordIdsFromSubmissionErrors, extractSchemaDataFromMergedDataRecords, type FileParseResult, filterRelationsForPrimaryIdUpdate, findInvalidRecordErrorsBySchemaName, + findUpdateDeleteConflicts, groupSchemaErrorsByEntity, isSubmissionActive, mapGroupedUpdateSubmissionData, mergeAndReferenceEntityData, + mergeSubmissionErrors, mergeUpdatesBySystemId, parseToSchema, segregateFieldChangeRecords, @@ -522,10 +525,29 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const submissionRecords = await submissionRecordsRepository.getBySubmissionId(submissionId); + // Detect records where the same systemId has both an UPDATE and a DELETE staged, before + // running dictionary validation. Both sides of a conflict are rejected explicitly instead + // of letting one action silently win. + const conflictErrors = findUpdateDeleteConflicts(submissionRecords); + const conflictingRecordIds = extractRecordIdsFromSubmissionErrors(conflictErrors); + + if (conflictingRecordIds.size > 0) { + logger.error( + LOG_MODULE, + `Detected '${conflictingRecordIds.size}' Submission Record(s) with conflicting UPDATE/DELETE actions on the same systemId in Submission '${submissionId}'`, + JSON.stringify(conflictErrors), + ); + } + + // Exclude conflicting records from validation; neither side of a conflict should be applied + const nonConflictingSubmissionRecords = conflictingRecordIds.size + ? submissionRecords.filter((record) => !conflictingRecordIds.has(record.id)) + : submissionRecords; + // Merge Submitted Data with Active Submission keepping reference of each record ID const dataMergedByEntityName = mergeAndReferenceEntityData({ submissionId, - submissionData: submissionRecords, + submissionData: nonConflictingSubmissionRecords, submittedData, }); @@ -536,11 +558,13 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const resultValidation = validateSchemas(currentDictionary, crossSchemasDataToValidate); // Collect errors of the Active Submission - const submissionSchemaErrors = groupSchemaErrorsByEntity({ + const schemaValidationErrors = groupSchemaErrorsByEntity({ resultValidation, dataValidated: dataMergedByEntityName, }); + const submissionSchemaErrors = mergeSubmissionErrors(conflictErrors, schemaValidationErrors); + if (_.isEmpty(submissionSchemaErrors)) { logger.info(LOG_MODULE, `No error found on data submission`); } else { diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index f1b765a0..19b9086e 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -12,6 +12,7 @@ import { validate, } from '@overture-stack/lectern-client'; import { + type RecordErrorActionConflict, type SubmissionDeleteData, type SubmissionInsertData, type SubmissionRecordError, @@ -219,66 +220,6 @@ export const findInvalidRecordErrorsBySchemaName = ( : []; }; -/** - * Generalized function to filter out conflicting records between two data sets based on `systemId`. - * - * This function can be used to either filter updates from deletes or deletes from updates, depending on the provided parameters. - * It removes records from the `sourceData` that have a matching `systemId` in the `conflictData`. - * - * @param sourceData - A record of the primary data (e.g., updates or deletes) to be filtered, grouped by entity name. - * @param conflictData - A record of data that might conflict (e.g., deletes or updates), grouped by entity name. - * @param entitySelector - A function to select the `systemId` from the source records. - * @param conflictSelector - A function to select the `systemId` from the conflict records. - * @returns A record of filtered source data, excluding records that conflict based on `systemId`. - */ -export const filterRecordsByConflicts = ( - sourceData: Record, - conflictData: Record, - entitySelector: (item: SourceData) => string, - conflictSelector: (item: ConflictData) => string, -): Record => { - return Object.entries(sourceData).reduce>((acc, [entityName, sourceItems]) => { - const conflicts = conflictData[entityName]; - - if (conflicts) { - // Create a Set of systemIds from conflict records for faster lookup - const conflictIdsSet = new Set(conflicts.map(conflictSelector)); - - // Filter source data that does not have a matching systemId in the conflict set - const filteredValues = sourceItems.filter((item) => !conflictIdsSet.has(entitySelector(item))); - - if (filteredValues.length > 0) { - acc[entityName] = filteredValues; - } - } else { - // If no conflicts, keep the source data as is - acc[entityName] = sourceItems; - } - - return acc; - }, {}); -}; - -/** - * Filters deletes from the provided `submissionDeleteData` based on conflicts found in the `submissionUpdateData`. - * Conflicts are determined by matching the `systemId` of the items in both records. - * - * @param submissionDeleteData - A record containing arrays of `SubmissionDeleteData` to be filtered. - * @param submissionUpdateData - A record containing arrays of `SubmissionUpdateData` that defines the conflicts. - * @returns A filtered record of `SubmissionDeleteData[]` where no items conflict with those in `submissionUpdateData`. - */ -export const filterDeletesFromUpdates = ( - submissionDeleteData: Record, - submissionUpdateData: Record, -): Record => { - return filterRecordsByConflicts( - submissionDeleteData, - submissionUpdateData, - (itemToDelete) => itemToDelete.systemId, - (itemToUpdate) => itemToUpdate.systemId, - ); -}; - /** * Returns a filter to query the database used to find dependents records when the update record involves changes of an primary ID field * @@ -376,6 +317,137 @@ export const groupSchemaErrorsByEntity = (input: { return submissionSchemaErrors; }; +/** + * Scans the Active Submission for `systemId`s (scoped per entity) that have both an UPDATE and a + * DELETE record staged at the same time. This is meant to run *before* dictionary validation: + * detecting the conflict up front lets both conflicting records be rejected explicitly, instead of + * one action silently winning based on array-filtering order later in the validation/merge pipeline. + * @param {SubmissionRecordWithEntityName[]} submissionData The Active Submission data + * @returns {SubmissionErrors} Conflict errors under the 'updates' and 'deletes' buckets, grouped by entity name + */ +export const findUpdateDeleteConflicts = (submissionData: SubmissionRecordWithEntityName[]): SubmissionErrors => { + const updatesByEntity = new Map>(); + const deletesByEntity = new Map>(); + + const trackRecordId = ( + bucket: Map>, + entityName: string, + systemId: string, + recordId: number, + ) => { + const bySystemId = bucket.get(entityName) ?? new Map(); + bySystemId.set(systemId, [...(bySystemId.get(systemId) ?? []), recordId]); + bucket.set(entityName, bySystemId); + }; + + submissionData.forEach((record) => { + if (isUpdateSubmissionRecord(record)) { + trackRecordId(updatesByEntity, record.entityName, record.data.systemId, record.id); + } else if (isDeleteSubmissionRecord(record)) { + trackRecordId(deletesByEntity, record.entityName, record.data.systemId, record.id); + } + }); + + const conflictErrorFor = ( + systemId: string, + conflictingActionType: RecordErrorActionConflict['conflictingActionType'], + ): RecordErrorActionConflict => ({ + reason: 'CONFLICTING_ACTION', + systemId, + conflictingActionType, + message: `Record with systemId '${systemId}' has both an UPDATE and a DELETE staged in the same Active Submission`, + }); + + const conflictErrors: SubmissionErrors = {}; + + updatesByEntity.forEach((updateSystemIds, entityName) => { + const deleteSystemIds = deletesByEntity.get(entityName); + if (!deleteSystemIds) { + return; + } + + updateSystemIds.forEach((updateRecordIds, systemId) => { + const deleteRecordIds = deleteSystemIds.get(systemId); + if (!deleteRecordIds) { + return; + } + + conflictErrors.updates ??= {}; + conflictErrors.updates[entityName] = [ + ...(conflictErrors.updates[entityName] ?? []), + ...updateRecordIds.map((recordId) => ({ recordId, errors: [conflictErrorFor(systemId, 'DELETE')] })), + ]; + + conflictErrors.deletes ??= {}; + conflictErrors.deletes[entityName] = [ + ...(conflictErrors.deletes[entityName] ?? []), + ...deleteRecordIds.map((recordId) => ({ recordId, errors: [conflictErrorFor(systemId, 'UPDATE')] })), + ]; + }); + }); + + return conflictErrors; +}; + +/** + * Collects every `recordId` referenced across all buckets of a `SubmissionErrors` object. + * @param {SubmissionErrors} errors + * @returns {Set} + */ +export const extractRecordIdsFromSubmissionErrors = (errors: SubmissionErrors): Set => { + const recordIds = new Set(); + for (const entities of Object.values(errors)) { + if (!entities) { + continue; + } + for (const records of Object.values(entities)) { + records.forEach(({ recordId }) => recordIds.add(recordId)); + } + } + return recordIds; +}; + +/** + * Merges two `SubmissionErrors` objects together, concatenating each entity's error array + * bucket-by-bucket instead of overwriting it. + * @param {SubmissionErrors} a + * @param {SubmissionErrors} b + * @returns {SubmissionErrors} + */ +export const mergeSubmissionErrors = (a: SubmissionErrors, b: SubmissionErrors): SubmissionErrors => { + const mergeBucket = ( + bucketA?: Record, + bucketB?: Record, + ): Record | undefined => { + if (!bucketA && !bucketB) { + return undefined; + } + const merged: Record = { ...bucketA }; + for (const [entityName, records] of Object.entries(bucketB ?? {})) { + merged[entityName] = [...(merged[entityName] ?? []), ...records]; + } + return merged; + }; + + // Only set a bucket key when it actually has content — callers rely on `Object.keys(...).length` + // (and `_.isEmpty`) to detect the "no errors" case, so an always-present `undefined` value would + // make every submission look like it has errors. + const merged: SubmissionErrors = {}; + const inserts = mergeBucket(a.inserts, b.inserts); + if (inserts) { + merged.inserts = inserts; + } + const updates = mergeBucket(a.updates, b.updates); + if (updates) { + merged.updates = updates; + } + const deletes = mergeBucket(a.deletes, b.deletes); + if (deletes) { + merged.deletes = deletes; + } + return merged; +}; + /** * This function extracts the Schema Data from the Active Submission * and maps it to it's original reference Id diff --git a/packages/data-provider/test/unit/utils/submission/extractRecordIdsFromSubmissionErrors.spec.ts b/packages/data-provider/test/unit/utils/submission/extractRecordIdsFromSubmissionErrors.spec.ts new file mode 100644 index 00000000..2c6bf626 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/extractRecordIdsFromSubmissionErrors.spec.ts @@ -0,0 +1,30 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { extractRecordIdsFromSubmissionErrors, type SubmissionErrors } from '../../../../src/utils/submissionUtils.js'; + +describe('Submission Utils - Extract Record Ids From Submission Errors', () => { + it('returns an empty Set when there are no errors', () => { + const response = extractRecordIdsFromSubmissionErrors({}); + expect(response.size).to.eq(0); + }); + + it('collects recordIds across every bucket and entity', () => { + const errors: SubmissionErrors = { + inserts: { plants: [{ recordId: 1, errors: [] }] }, + updates: { animals: [{ recordId: 2, errors: [] }, { recordId: 3, errors: [] }] }, + deletes: { animals: [{ recordId: 4, errors: [] }] }, + }; + const response = extractRecordIdsFromSubmissionErrors(errors); + expect([...response]).to.have.members([1, 2, 3, 4]); + }); + + it('deduplicates a recordId that appears more than once', () => { + const errors: SubmissionErrors = { + updates: { animals: [{ recordId: 5, errors: [] }] }, + deletes: { animals: [{ recordId: 5, errors: [] }] }, + }; + const response = extractRecordIdsFromSubmissionErrors(errors); + expect([...response]).to.eql([5]); + }); +}); diff --git a/packages/data-provider/test/unit/utils/submission/filterDeletesFromUpdates.spec.ts b/packages/data-provider/test/unit/utils/submission/filterDeletesFromUpdates.spec.ts deleted file mode 100644 index 7c4194c4..00000000 --- a/packages/data-provider/test/unit/utils/submission/filterDeletesFromUpdates.spec.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { expect } from 'chai'; -import { describe, it } from 'mocha'; - -import type { SubmissionDeleteData, SubmissionUpdateData } from '@overture-stack/lyric-data-model/models'; - -import { filterDeletesFromUpdates } from '../../../../src/utils/submissionUtils.js'; - -describe('Submission Utils - Remove conflicts on Submission with records to be delete', () => { - it('should remove 1 matching record from the Delete records', () => { - const submissionDeleteData: Record = { - cars: [ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ], - }; - - const submissionUpdateData: Record = { - cars: [{ systemId: 'AAA111', new: { name: 'lamborghini' }, old: { name: 'Lambo' } }], - }; - const result = filterDeletesFromUpdates(submissionDeleteData, submissionUpdateData); - expect(Object.keys(result)).to.eql(['cars']); - expect(result['cars'].length).to.eq(1); - expect(result['cars'][0]).to.eql({ - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }); - }); - it('should not remove any record if there is no matching Id', () => { - const submissionDeleteData: Record = { - cars: [ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ], - }; - - const submissionUpdateData: Record = { - cars: [{ systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } }], - }; - const result = filterDeletesFromUpdates(submissionDeleteData, submissionUpdateData); - expect(Object.keys(result)).to.eql(['cars']); - expect(result['cars'].length).to.eq(2); - expect(result['cars']).to.eql([ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ]); - }); - it('should not remove any record if there is no matching entity name', () => { - const submissionDeleteData: Record = { - cars: [ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ], - }; - - const submissionUpdateData: Record = { - food: [{ systemId: 'PTT123', new: { name: 'Potato' }, old: { name: 'Tomato' } }], - }; - const result = filterDeletesFromUpdates(submissionDeleteData, submissionUpdateData); - expect(Object.keys(result)).to.eql(['cars']); - expect(result['cars'].length).to.eq(2); - expect(result['cars']).to.eql([ - { - systemId: 'AAA111', - data: { name: 'Lambo' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - { - systemId: 'BBB222', - data: { name: 'Beettle' }, - entityName: 'cars', - isValid: true, - organization: 'myCollection', - }, - ]); - }); - it('should return empty object when passing an empty object', () => { - const submissionUpdateData: Record = { - cars: [{ systemId: 'CCC333', new: { name: 'Volkswagen' }, old: { name: 'VW' } }], - }; - const result = filterDeletesFromUpdates({}, submissionUpdateData); - expect(Object.keys(result).length).to.eq(0); - }); -}); diff --git a/packages/data-provider/test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts b/packages/data-provider/test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts new file mode 100644 index 00000000..a3a521b0 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts @@ -0,0 +1,216 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; +import { findUpdateDeleteConflicts } from '../../../../src/utils/submissionUtils.js'; + +describe('Submission Utils - Find Update/Delete Conflicts', () => { + it('returns no conflicts when there are no Submission records', () => { + const response = findUpdateDeleteConflicts([]); + expect(response).eql({}); + }); + + it('returns no conflicts when UPDATE and DELETE records target different systemIds', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'DELETE', + entityName: 'animals', + id: 11, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'BR8912', data: { name: 'bear', color: 'black' }, isValid: true, organization: 'zoo' }, + }, + ]; + const response = findUpdateDeleteConflicts(submissionData); + expect(response).eql({}); + }); + + it('returns no conflicts when the matching systemId belongs to a different entity', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'DELETE', + entityName: 'zookeepers', + id: 11, + fileId: 2, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'someone' }, isValid: true, organization: 'zoo' }, + }, + ]; + const response = findUpdateDeleteConflicts(submissionData); + expect(response).eql({}); + }); + + it('ignores INSERT records and does not treat them as part of a conflict', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'INSERT', + entityName: 'animals', + id: 9, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { name: 'beaver', color: 'brown' }, + }, + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + ]; + const response = findUpdateDeleteConflicts(submissionData); + expect(response).eql({}); + }); + + it('flags both the UPDATE and DELETE record when they share a systemId in the same entity', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' }, + }, + ]; + const response = findUpdateDeleteConflicts(submissionData); + expect(response).eql({ + updates: { + animals: [ + { + recordId: 10, + errors: [ + { + reason: 'CONFLICTING_ACTION', + systemId: 'TGR1425', + conflictingActionType: 'DELETE', + message: + "Record with systemId 'TGR1425' has both an UPDATE and a DELETE staged in the same Active Submission", + }, + ], + }, + ], + }, + deletes: { + animals: [ + { + recordId: 12, + errors: [ + { + reason: 'CONFLICTING_ACTION', + systemId: 'TGR1425', + conflictingActionType: 'UPDATE', + message: + "Record with systemId 'TGR1425' has both an UPDATE and a DELETE staged in the same Active Submission", + }, + ], + }, + ], + }, + }); + }); + + it('flags every UPDATE and DELETE row when more than one row exists for the same systemId', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'UPDATE', + entityName: 'animals', + id: 20, + fileId: 2, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { weight: '200kg' }, old: { weight: '190kg' } }, + }, + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' }, + }, + ]; + const response = findUpdateDeleteConflicts(submissionData); + expect(response.updates?.['animals']?.map((record) => record.recordId)).to.have.members([10, 20]); + expect(response.deletes?.['animals']?.map((record) => record.recordId)).to.eql([12]); + }); + + it('only flags the entities/systemIds that actually conflict, leaving others untouched', () => { + const submissionData: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', data: { name: 'tiger', color: 'yellow' }, isValid: true, organization: 'zoo' }, + }, + { + actionType: 'UPDATE', + entityName: 'animals', + id: 11, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'BR8912', new: { color: 'brown' }, old: { color: 'black' } }, + }, + ]; + const response = findUpdateDeleteConflicts(submissionData); + expect(Object.keys(response.updates ?? {})).to.eql(['animals']); + expect(response.updates?.['animals']?.map((record) => record.recordId)).to.eql([10]); + expect(response.deletes?.['animals']?.map((record) => record.recordId)).to.eql([12]); + }); +}); diff --git a/packages/data-provider/test/unit/utils/submission/mergeSubmissionErrors.spec.ts b/packages/data-provider/test/unit/utils/submission/mergeSubmissionErrors.spec.ts new file mode 100644 index 00000000..3b500300 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/mergeSubmissionErrors.spec.ts @@ -0,0 +1,49 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { mergeSubmissionErrors, type SubmissionErrors } from '../../../../src/utils/submissionUtils.js'; + +describe('Submission Utils - Merge Submission Errors', () => { + it('returns an empty object when both inputs are empty', () => { + const response = mergeSubmissionErrors({}, {}); + expect(response).eql({}); + }); + + it('returns the empty object without setting keys for buckets neither input has', () => { + const response = mergeSubmissionErrors({}, {}); + expect(Object.keys(response)).to.eql([]); + }); + + it('returns the other input unchanged when one side is empty', () => { + const a: SubmissionErrors = { updates: { animals: [{ recordId: 1, errors: [] }] } }; + const response = mergeSubmissionErrors(a, {}); + expect(response).eql(a); + }); + + it('concatenates entity error arrays instead of overwriting them', () => { + const a: SubmissionErrors = { + updates: { animals: [{ recordId: 1, errors: [] }] }, + }; + const b: SubmissionErrors = { + updates: { animals: [{ recordId: 2, errors: [] }] }, + }; + const response = mergeSubmissionErrors(a, b); + expect(response.updates?.['animals']?.map((record) => record.recordId)).to.eql([1, 2]); + }); + + it('merges different buckets and different entities independently', () => { + const a: SubmissionErrors = { + updates: { animals: [{ recordId: 1, errors: [] }] }, + deletes: { animals: [{ recordId: 2, errors: [] }] }, + }; + const b: SubmissionErrors = { + inserts: { plants: [{ recordId: 3, errors: [] }] }, + updates: { plants: [{ recordId: 4, errors: [] }] }, + }; + const response = mergeSubmissionErrors(a, b); + expect(response.updates?.['animals']?.map((record) => record.recordId)).to.eql([1]); + expect(response.updates?.['plants']?.map((record) => record.recordId)).to.eql([4]); + expect(response.deletes?.['animals']?.map((record) => record.recordId)).to.eql([2]); + expect(response.inserts?.['plants']?.map((record) => record.recordId)).to.eql([3]); + }); +}); From 61e6dd4e9b5107b138671bb7100fac72baa635bf Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 12 Aug 2026 18:30:55 -0400 Subject: [PATCH 10/20] retrieve records ordered by id --- .dev/tech-debt.md | 7 +++---- .../src/repository/submissionRecordsRepository.ts | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.dev/tech-debt.md b/.dev/tech-debt.md index bdd4421f..60f5dfe0 100644 --- a/.dev/tech-debt.md +++ b/.dev/tech-debt.md @@ -57,16 +57,15 @@ context: `dictionary_categories.id` is a plain auto-increment `serial`, used dir standalone: yes context: `submmittedData.ts::deleteSubmittedDataBySystemId` (132-133) has its own in-code TODO: "get the existing entity names for a submission and merge with the new ones to avoid duplicates, but for now just insert the new records." Two sequential calls (e.g. edit systemId X, then delete systemId X, before committing) can leave both an UPDATE row and a DELETE row for X in `submission_records` simultaneously, with no DB constraint preventing it either (no unique index on systemId in `packages/data-model/src/models/submission_records.ts:46-69`). Partially safety-netted as of 2026-08-12: `performDataValidation` now detects this exact conflict and marks both rows INVALID before validation runs (see Resolved section), so the conflict no longer silently drops one side — but the staging-time TODO itself is still unaddressed, so the submission still has to go back through an edit/delete cycle to resolve it rather than being prevented at write time. -### Duplicate UPDATE submission records for the same systemId collapse via undefined last-write-wins at commit time -standalone: yes -context: `commitSubmissionWorker.ts::processCommitSubmission` (91-96) reduces UPDATE rows into `Record` keyed by systemId via plain object assignment, and the underlying query (`submissionRecordsRepository.ts::getByFileIds`, 38-64) has no `ORDER BY`. If two UPDATE rows exist for the same systemId, which one wins is whatever order Postgres happens to return, not an application-defined order. Distinct from the UPDATE/DELETE conflict resolved 2026-08-12 below — this is UPDATE-vs-UPDATE, not yet addressed. Fix: add an explicit `ORDER BY` (e.g. by id/createdAt) or reconcile duplicate updates for the same systemId earlier, in the staging path. - ### No integration tests exist for the submission edit, delete, or commit endpoints — let alone mixed insert+update+delete scenarios standalone: yes context: `packages/data-provider/test/integration/routers/submission/` only covers `submissionRouter-submit*` (insert-only file/JSON uploads). There is no integration spec for `editSubmittedData`, `deleteSubmittedDataBySystemId`, or the commit flow (`performCommitSubmissionAsync`/`commitSubmissionWorker.ts`) at all, despite the test infra (`test/integration/dependencies/containers.ts`) already running a real Postgres container capable of exercising the real commit transaction. The new UPDATE/DELETE conflict resolution added 2026-08-12 has unit coverage only (`test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts` and friends) — still no HTTP+DB-level test proving a submission with a staged conflict actually ends up INVALID end-to-end. Fix: add integration specs that stage inserts+updates+deletes in the same active submission (including same-systemId collisions) and assert the committed `submitted_data` table end state and the active submission's status/record states. ## Resolved +### Duplicate UPDATE submission records for the same systemId collapsed via undefined last-write-wins at commit time +resolved: 2026-08-12, added a deterministic `ORDER BY` instead of relying on unspecified Postgres row order. `submissionRecordsRepository.ts::getByFileIds` now does `.orderBy(submissionRecords.id)` (ascending) — `id` is a `serial` primary key, monotonic with insertion order, so `commitSubmissionWorker.ts`'s `record[systemId] = record` reduction now deterministically keeps the most recently inserted UPDATE row for a given systemId instead of whichever row Postgres happened to scan last. Considered adding a `created_at` timestamp column instead, but `id` already gives the same ordering guarantee without a schema migration, so that was dropped in favour of the simpler fix. + ### UPDATE/DELETE conflict on the same systemId was silently dropped instead of surfaced as an error resolved: 2026-08-12, added explicit conflict detection ahead of dictionary validation. `findUpdateDeleteConflicts` (`packages/data-provider/src/utils/submissionUtils.ts`) scans staged submission records for entityName+systemId pairs with both an UPDATE and a DELETE, before `performDataValidation` (`submissionProcessor.ts`) runs `validateSchemas`. Conflicting records are excluded from validation, both sides are marked `INVALID` with a new `CONFLICTING_ACTION` error (`RecordErrorActionConflict`, added to `SubmissionRecordError` in `@overture-stack/lyric-data-model`), the active submission is marked `INVALID`, and the scenario is `logger.error`-logged. Previously this was only prevented by incidental JS array-filtering order with no error surfaced (see the still-open items above for what remains: the staging-time TODO, and integration coverage). diff --git a/packages/data-provider/src/repository/submissionRecordsRepository.ts b/packages/data-provider/src/repository/submissionRecordsRepository.ts index 5579d0f2..1e988304 100644 --- a/packages/data-provider/src/repository/submissionRecordsRepository.ts +++ b/packages/data-provider/src/repository/submissionRecordsRepository.ts @@ -54,7 +54,8 @@ const submissionRecordsRepository = (dependencies: BaseDependencies) => { filterOptions?.actionTypes ? inArray(submissionRecords.actionType, filterOptions.actionTypes) : undefined, filterOptions?.states ? inArray(submissionRecords.state, filterOptions.states) : undefined, ), - ); + ) + .orderBy(submissionRecords.id); if (paginationOptions) { query.limit(paginationOptions.pageSize).offset((paginationOptions.page - 1) * paginationOptions.pageSize); From 31d90204277200b7048ceb7db230069524783e5a Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 12 Aug 2026 18:55:58 -0400 Subject: [PATCH 11/20] detect conflicts during delete submitted data --- .dev/tech-debt.md | 11 +- .../services/submittedData/submmittedData.ts | 51 +++++- .../src/utils/submissionUtils.ts | 69 ++++++++ .../resolveDeleteStagingConflicts.spec.ts | 157 ++++++++++++++++++ 4 files changed, 280 insertions(+), 8 deletions(-) create mode 100644 packages/data-provider/test/unit/utils/submission/resolveDeleteStagingConflicts.spec.ts diff --git a/.dev/tech-debt.md b/.dev/tech-debt.md index 60f5dfe0..54a8ddf4 100644 --- a/.dev/tech-debt.md +++ b/.dev/tech-debt.md @@ -53,16 +53,19 @@ context: `dictionary_categories.id` is a plain auto-increment `serial`, used dir --- -### Staging a DELETE submission record never checks for or merges with an existing pending record for the same systemId -standalone: yes -context: `submmittedData.ts::deleteSubmittedDataBySystemId` (132-133) has its own in-code TODO: "get the existing entity names for a submission and merge with the new ones to avoid duplicates, but for now just insert the new records." Two sequential calls (e.g. edit systemId X, then delete systemId X, before committing) can leave both an UPDATE row and a DELETE row for X in `submission_records` simultaneously, with no DB constraint preventing it either (no unique index on systemId in `packages/data-model/src/models/submission_records.ts:46-69`). Partially safety-netted as of 2026-08-12: `performDataValidation` now detects this exact conflict and marks both rows INVALID before validation runs (see Resolved section), so the conflict no longer silently drops one side — but the staging-time TODO itself is still unaddressed, so the submission still has to go back through an edit/delete cycle to resolve it rather than being prevented at write time. - ### No integration tests exist for the submission edit, delete, or commit endpoints — let alone mixed insert+update+delete scenarios standalone: yes context: `packages/data-provider/test/integration/routers/submission/` only covers `submissionRouter-submit*` (insert-only file/JSON uploads). There is no integration spec for `editSubmittedData`, `deleteSubmittedDataBySystemId`, or the commit flow (`performCommitSubmissionAsync`/`commitSubmissionWorker.ts`) at all, despite the test infra (`test/integration/dependencies/containers.ts`) already running a real Postgres container capable of exercising the real commit transaction. The new UPDATE/DELETE conflict resolution added 2026-08-12 has unit coverage only (`test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts` and friends) — still no HTTP+DB-level test proving a submission with a staged conflict actually ends up INVALID end-to-end. Fix: add integration specs that stage inserts+updates+deletes in the same active submission (including same-systemId collisions) and assert the committed `submitted_data` table end state and the active submission's status/record states. +### Flaky integration test: dictionary migration force-retry intermittently returns 409 instead of 200 +standalone: yes +context: `test/integration/routers/dictionary/dictionaryMigration.spec.ts:135` ("should retry migration when force registering the same dictionary version after a failed migration") registers a migration, then immediately overwrites its status to `FAILED` via a direct repository call (`lyricProvider.repositories.migration.update`), then force-retries and asserts `200`. But `initiateMigration` (`packages/data-provider/src/services/migrationService.ts:170-236`) hands the actual migration off to a background worker (`dependencies.workerPool.dictionaryMigration(...)`, fired without awaiting) and returns immediately — so the test's manual status override can race with the worker's own status transition, and the retry assertion intermittently sees `409` instead of `200`. Observed 2026-08-12 across multiple full integration-suite runs this session (unrelated to the submission-records work being tested at the time); pass counts fluctuated run to run (364/374/379 passing), consistent with a pre-existing race rather than a regression. Fix: have the test wait for the migration to reach a terminal status (poll/await worker completion) before mutating it directly, instead of assuming the async worker has already finished. + ## Resolved +### Staging a DELETE submission record never checked for or merged with an existing pending record for the same systemId +resolved: 2026-08-12, added a staging-time check instead of letting one action override the other silently. `resolveDeleteStagingConflicts` (`packages/data-provider/src/utils/submissionUtils.ts`) is now called from `deleteSubmittedDataBySystemId` (`submmittedData.ts`) before any new DELETE record is inserted: it fetches the Active Submission's existing UPDATE/DELETE records and cross-checks them against the systemIds about to be deleted (the target record plus its dependents). A systemId with a pending UPDATE is now rejected outright (`INVALID_SUBMISSION` response, no record staged) — consistent with the "both sides invalid" policy used for the same conflict at validation time — instead of quietly reaching `performDataValidation` later. A systemId that already has a pending DELETE is treated as a duplicate and skipped rather than inserted a second time, addressing the original in-code TODO directly. Previously the function always blindly inserted new DELETE rows regardless of what was already staged. + ### Duplicate UPDATE submission records for the same systemId collapsed via undefined last-write-wins at commit time resolved: 2026-08-12, added a deterministic `ORDER BY` instead of relying on unspecified Postgres row order. `submissionRecordsRepository.ts::getByFileIds` now does `.orderBy(submissionRecords.id)` (ascending) — `id` is a `serial` primary key, monotonic with insertion order, so `commitSubmissionWorker.ts`'s `record[systemId] = record` reduction now deterministically keeps the most recently inserted UPDATE row for a given systemId instead of whichever row Postgres happened to scan last. Considered adding a `created_at` timestamp column instead, but `id` already gives the same ordering guarantee without a schema migration, so that was dropped in favour of the simpler fix. diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index 92556114..917467a2 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -14,6 +14,7 @@ import { getDictionarySchemaRelations } from '../../utils/dictionarySchemaRelati import { InternalServerError, StatusConflict } from '../../utils/errors.js'; import { getSizeInBytes } from '../../utils/fileUtils.js'; import type { PaginatedResult } from '../../utils/result.js'; +import { resolveDeleteStagingConflicts } from '../../utils/submissionUtils.js'; import { fetchDataErrorResponse, getEntityNamesFromFilterOptions, @@ -129,9 +130,51 @@ const submittedData = (dependencies: BaseDependencies) => { throw error; } - // TODO: get the existing entity names for a submission and merge with the new ones to avoid duplicates, - // but for now just insert the new records - const entitiesToProcess = Object.keys(recordsToDeleteMap); + // Check what the Active Submission already has pending for these systemIds before staging + // anything new: a pending UPDATE is a conflict (reject, consistent with how the same + // conflict is handled at validation time), a pending DELETE is a duplicate (skip it). + const existingSubmissionRecords = await submissionRecordsRepository.getBySubmissionId( + activeSubmissionId, + undefined, + { + actionTypes: ['UPDATE', 'DELETE'], + }, + ); + + const { filteredRecordsToDeleteMap, conflictingSystemIds, duplicateSystemIds } = resolveDeleteStagingConflicts( + recordsToDeleteMap, + existingSubmissionRecords, + ); + + if (conflictingSystemIds.length > 0) { + logger.error( + LOG_MODULE, + `Cannot delete system ID(s) '${conflictingSystemIds.join(', ')}' on Submission '${activeSubmissionId}': a pending update already exists for the same system ID`, + ); + return { + status: ACTIVE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: `System ID(s) '${conflictingSystemIds.join(', ')}' already have a pending update staged on Submission '${activeSubmissionId}'. Resolve the conflicting update before deleting.`, + inProcessEntities: [], + }; + } + + if (duplicateSystemIds.length > 0) { + logger.info( + LOG_MODULE, + `System ID(s) '${duplicateSystemIds.join(', ')}' are already staged for deletion on Submission '${activeSubmissionId}', skipping duplicate`, + ); + } + + const entitiesToProcess = Object.keys(filteredRecordsToDeleteMap); + + if (entitiesToProcess.length === 0) { + return { + status: ACTIVE_SUBMISSION_STATUS.PROCESSING, + description: 'All requested records are already staged for deletion on the Active Submission', + submissionId: activeSubmissionId.toString(), + inProcessEntities: [], + }; + } await dependencies.db.transaction(async (tx) => { // Updating the Submission with the new data and 'VALIDATING' status before validation starts @@ -145,7 +188,7 @@ const submittedData = (dependencies: BaseDependencies) => { ); await Promise.all( - Object.entries(recordsToDeleteMap).map(async ([entityName, entityRecords]) => { + Object.entries(filteredRecordsToDeleteMap).map(async ([entityName, entityRecords]) => { const savedFileId = await submissionFilesRepository.save( { entityName, diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index 19b9086e..0c91b032 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -389,6 +389,75 @@ export const findUpdateDeleteConflicts = (submissionData: SubmissionRecordWithEn return conflictErrors; }; +export type DeleteStagingConflicts = { + /** `recordsToDeleteMap` with systemIds that already have a pending DELETE removed, so they aren't staged twice */ + filteredRecordsToDeleteMap: Record; + /** systemIds that already have a pending UPDATE staged for the same entity in the Active Submission */ + conflictingSystemIds: string[]; + /** systemIds that already have a pending DELETE staged for the same entity — skipped instead of duplicated */ + duplicateSystemIds: string[]; +}; + +/** + * Checks systemIds about to be staged for deletion against what the Active Submission already has + * pending for the same entity, before a new DELETE record is ever inserted. A systemId with a + * pending UPDATE is reported as a conflict — the caller should reject the delete rather than + * silently letting one action override the other, consistent with how `findUpdateDeleteConflicts` + * treats the same conflict at validation time. A systemId that already has a pending DELETE is + * treated as a duplicate and dropped from the result, instead of inserting a second DELETE record. + * @param {Record} recordsToDeleteMap New deletes, grouped by entity name + * @param {SubmissionRecordWithEntityName[]} existingSubmissionRecords The Active Submission's current UPDATE/DELETE records + * @returns {DeleteStagingConflicts} + */ +export const resolveDeleteStagingConflicts = ( + recordsToDeleteMap: Record, + existingSubmissionRecords: SubmissionRecordWithEntityName[], +): DeleteStagingConflicts => { + const existingUpdateSystemIds = new Map>(); + const existingDeleteSystemIds = new Map>(); + + const trackSystemId = (bucket: Map>, entityName: string, systemId: string) => { + const systemIds = bucket.get(entityName) ?? new Set(); + systemIds.add(systemId); + bucket.set(entityName, systemIds); + }; + + existingSubmissionRecords.forEach((record) => { + if (isUpdateSubmissionRecord(record)) { + trackSystemId(existingUpdateSystemIds, record.entityName, record.data.systemId); + } else if (isDeleteSubmissionRecord(record)) { + trackSystemId(existingDeleteSystemIds, record.entityName, record.data.systemId); + } + }); + + const conflictingSystemIds: string[] = []; + const duplicateSystemIds: string[] = []; + const filteredRecordsToDeleteMap: Record = {}; + + Object.entries(recordsToDeleteMap).forEach(([entityName, records]) => { + const conflictingUpdateIds = existingUpdateSystemIds.get(entityName); + const duplicateDeleteIds = existingDeleteSystemIds.get(entityName); + + const recordsToKeep = records.filter((record) => { + if (conflictingUpdateIds?.has(record.systemId)) { + conflictingSystemIds.push(record.systemId); + return false; + } + if (duplicateDeleteIds?.has(record.systemId)) { + duplicateSystemIds.push(record.systemId); + return false; + } + return true; + }); + + if (recordsToKeep.length > 0) { + filteredRecordsToDeleteMap[entityName] = recordsToKeep; + } + }); + + return { filteredRecordsToDeleteMap, conflictingSystemIds, duplicateSystemIds }; +}; + /** * Collects every `recordId` referenced across all buckets of a `SubmissionErrors` object. * @param {SubmissionErrors} errors diff --git a/packages/data-provider/test/unit/utils/submission/resolveDeleteStagingConflicts.spec.ts b/packages/data-provider/test/unit/utils/submission/resolveDeleteStagingConflicts.spec.ts new file mode 100644 index 00000000..215aba44 --- /dev/null +++ b/packages/data-provider/test/unit/utils/submission/resolveDeleteStagingConflicts.spec.ts @@ -0,0 +1,157 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import type { SubmissionDeleteData } from '@overture-stack/lyric-data-model/models'; + +import type { SubmissionRecordWithEntityName } from '../../../../src/repository/submissionRecordsRepository.js'; +import { resolveDeleteStagingConflicts } from '../../../../src/utils/submissionUtils.js'; + +const deleteRecord = (systemId: string): SubmissionDeleteData => ({ + systemId, + data: { name: 'tiger', color: 'yellow' }, + isValid: true, + organization: 'zoo', +}); + +describe('Submission Utils - Resolve Delete Staging Conflicts', () => { + it('keeps every record and reports no conflicts/duplicates when there are no existing records', () => { + const recordsToDeleteMap: Record = { + animals: [deleteRecord('TGR1425')], + }; + const response = resolveDeleteStagingConflicts(recordsToDeleteMap, []); + expect(response).to.eql({ + filteredRecordsToDeleteMap: recordsToDeleteMap, + conflictingSystemIds: [], + duplicateSystemIds: [], + }); + }); + + it('reports a conflict and excludes the record when a pending UPDATE exists for the same entity+systemId', () => { + const recordsToDeleteMap: Record = { + animals: [deleteRecord('TGR1425')], + }; + const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + ]; + const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords); + expect(response).to.eql({ + filteredRecordsToDeleteMap: {}, + conflictingSystemIds: ['TGR1425'], + duplicateSystemIds: [], + }); + }); + + it('reports a duplicate and excludes the record when a pending DELETE already exists for the same entity+systemId', () => { + const recordsToDeleteMap: Record = { + animals: [deleteRecord('TGR1425')], + }; + const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [ + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: deleteRecord('TGR1425'), + }, + ]; + const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords); + expect(response).to.eql({ + filteredRecordsToDeleteMap: {}, + conflictingSystemIds: [], + duplicateSystemIds: ['TGR1425'], + }); + }); + + it('does not treat a matching systemId in a different entity as a conflict or duplicate', () => { + const recordsToDeleteMap: Record = { + animals: [deleteRecord('TGR1425')], + }; + const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'zookeepers', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { name: 'someone' }, old: { name: 'someone else' } }, + }, + { + actionType: 'DELETE', + entityName: 'zookeepers', + id: 11, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: deleteRecord('TGR1425'), + }, + ]; + const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords); + expect(response).to.eql({ + filteredRecordsToDeleteMap: recordsToDeleteMap, + conflictingSystemIds: [], + duplicateSystemIds: [], + }); + }); + + it('keeps clean records while filtering out conflicting and duplicate ones within the same entity', () => { + const recordsToDeleteMap: Record = { + animals: [deleteRecord('TGR1425'), deleteRecord('BR8912'), deleteRecord('ZBR001')], + }; + const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + { + actionType: 'DELETE', + entityName: 'animals', + id: 12, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: deleteRecord('BR8912'), + }, + ]; + const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords); + expect(response.conflictingSystemIds).to.eql(['TGR1425']); + expect(response.duplicateSystemIds).to.eql(['BR8912']); + expect(response.filteredRecordsToDeleteMap['animals']?.map((record) => record.systemId)).to.eql(['ZBR001']); + }); + + it('omits an entity entirely from the filtered map once every one of its records is filtered out', () => { + const recordsToDeleteMap: Record = { + animals: [deleteRecord('TGR1425')], + plants: [deleteRecord('OAK001')], + }; + const existingSubmissionRecords: SubmissionRecordWithEntityName[] = [ + { + actionType: 'UPDATE', + entityName: 'animals', + id: 10, + fileId: 1, + state: 'RECEIVED', + errors: [], + data: { systemId: 'TGR1425', new: { color: 'orange' }, old: { color: 'yellow' } }, + }, + ]; + const response = resolveDeleteStagingConflicts(recordsToDeleteMap, existingSubmissionRecords); + expect(Object.keys(response.filteredRecordsToDeleteMap)).to.eql(['plants']); + expect(response.filteredRecordsToDeleteMap['plants']?.map((record) => record.systemId)).to.eql(['OAK001']); + }); +}); From c8c98b674fe7decee6102d6f1f4776e5a5f18c18 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 12 Aug 2026 19:10:50 -0400 Subject: [PATCH 12/20] fix flaky dictionary migration integration test --- .dev/tech-debt.md | 7 +++--- .../dictionary/dictionaryMigration.spec.ts | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.dev/tech-debt.md b/.dev/tech-debt.md index 54a8ddf4..be033b4e 100644 --- a/.dev/tech-debt.md +++ b/.dev/tech-debt.md @@ -57,12 +57,11 @@ context: `dictionary_categories.id` is a plain auto-increment `serial`, used dir standalone: yes context: `packages/data-provider/test/integration/routers/submission/` only covers `submissionRouter-submit*` (insert-only file/JSON uploads). There is no integration spec for `editSubmittedData`, `deleteSubmittedDataBySystemId`, or the commit flow (`performCommitSubmissionAsync`/`commitSubmissionWorker.ts`) at all, despite the test infra (`test/integration/dependencies/containers.ts`) already running a real Postgres container capable of exercising the real commit transaction. The new UPDATE/DELETE conflict resolution added 2026-08-12 has unit coverage only (`test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts` and friends) — still no HTTP+DB-level test proving a submission with a staged conflict actually ends up INVALID end-to-end. Fix: add integration specs that stage inserts+updates+deletes in the same active submission (including same-systemId collisions) and assert the committed `submitted_data` table end state and the active submission's status/record states. -### Flaky integration test: dictionary migration force-retry intermittently returns 409 instead of 200 -standalone: yes -context: `test/integration/routers/dictionary/dictionaryMigration.spec.ts:135` ("should retry migration when force registering the same dictionary version after a failed migration") registers a migration, then immediately overwrites its status to `FAILED` via a direct repository call (`lyricProvider.repositories.migration.update`), then force-retries and asserts `200`. But `initiateMigration` (`packages/data-provider/src/services/migrationService.ts:170-236`) hands the actual migration off to a background worker (`dependencies.workerPool.dictionaryMigration(...)`, fired without awaiting) and returns immediately — so the test's manual status override can race with the worker's own status transition, and the retry assertion intermittently sees `409` instead of `200`. Observed 2026-08-12 across multiple full integration-suite runs this session (unrelated to the submission-records work being tested at the time); pass counts fluctuated run to run (364/374/379 passing), consistent with a pre-existing race rather than a regression. Fix: have the test wait for the migration to reach a terminal status (poll/await worker completion) before mutating it directly, instead of assuming the async worker has already finished. - ## Resolved +### Flaky integration test: dictionary migration force-retry intermittently returned 409 instead of 200 +resolved: 2026-08-12, made the test wait for the background migration worker to reach a terminal status before mutating it directly, removing the race. `dictionaryMigration.spec.ts`'s "should retry migration..." test now calls a `waitForMigrationToFinish` helper (same polling pattern already used in `dictionaryMigrationData.spec.ts`) right before overwriting the migration's status to `FAILED`, instead of assuming `initiateMigration`'s fire-and-forget worker had already finished. Verified with 3 consecutive full integration-suite runs, 380/380 passing each time, no failures. + ### Staging a DELETE submission record never checked for or merged with an existing pending record for the same systemId resolved: 2026-08-12, added a staging-time check instead of letting one action override the other silently. `resolveDeleteStagingConflicts` (`packages/data-provider/src/utils/submissionUtils.ts`) is now called from `deleteSubmittedDataBySystemId` (`submmittedData.ts`) before any new DELETE record is inserted: it fetches the Active Submission's existing UPDATE/DELETE records and cross-checks them against the systemIds about to be deleted (the target record plus its dependents). A systemId with a pending UPDATE is now rejected outright (`INVALID_SUBMISSION` response, no record staged) — consistent with the "both sides invalid" policy used for the same conflict at validation time — instead of quietly reaching `performDataValidation` later. A systemId that already has a pending DELETE is treated as a duplicate and skipped rather than inserted a second time, addressing the original in-code TODO directly. Previously the function always blindly inserted new DELETE rows regardless of what was already staged. diff --git a/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts b/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts index d01ded51..6f783e5c 100644 --- a/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts +++ b/packages/data-provider/test/integration/routers/dictionary/dictionaryMigration.spec.ts @@ -14,6 +14,8 @@ import { VALID_DICTIONARY_VERSION, } from './fixtures.js'; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + describe('Integration - Dictionary Migration', () => { let appDictionary: supertest.Agent; let appMigration: supertest.Agent; @@ -43,6 +45,23 @@ describe('Integration - Dictionary Migration', () => { const registerDictionary = async (payload: RegisterPayload, force = false) => appDictionary.post(`/register${force ? '?force=true' : ''}`).send(payload); + // The migration itself runs in a background worker (fired without being awaited by + // `registerDictionary`), so a test that mutates a migration's status directly must first wait + // for that worker to reach a terminal status — otherwise the manual override races with the + // worker's own status transition. + const waitForMigrationToFinish = async (migrationId: number, maxRetries = 20, delayMs = 300) => { + let response = await appMigration.get(`/${migrationId}`); + let attempts = 0; + + while (response.body.status === 'IN_PROGRESS' && attempts < maxRetries) { + await sleep(delayMs); + response = await appMigration.get(`/${migrationId}`); + attempts += 1; + } + + return response; + }; + before(async () => { schemaServiceUrl = getContainers().providerConfig.schemaService.url; lyricProvider = await createLyricProvider(getContainers().providerConfig); @@ -160,6 +179,10 @@ describe('Integration - Dictionary Migration', () => { const migrationId = migrationResponse.body.migrationId; + // Wait for the background migration worker to finish before overriding its status below, + // otherwise the worker's own status write can race with this test's manual override. + await waitForMigrationToFinish(migrationId); + // Making the migration fail by force registering the same new version again await lyricProvider.repositories.migration.update(migrationId, { status: 'FAILED', From 9bbde93a4361a140c5beface287a105833bc9594 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 14 Aug 2026 09:19:59 -0400 Subject: [PATCH 13/20] fix filter by action types --- apps/server/swagger/schemas.yml | 2 +- apps/server/swagger/submission-api.yml | 6 +++--- .../data-provider/src/controllers/submissionController.ts | 4 ++-- .../src/repository/submissionRecordsRepository.ts | 4 +++- packages/data-provider/src/utils/auditUtils.ts | 4 ++-- packages/data-provider/src/utils/types.ts | 6 ------ packages/data-provider/src/workers/workerPoolManager.ts | 4 ++++ 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 0e18592a..72046796 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -95,7 +95,7 @@ components: type: type: string description: Type of action - enum: ['INSERTS', 'UPDATES', 'DELETES'] + enum: ['INSERT', 'UPDATE', 'DELETE'] entity: type: string description: Name of the entity diff --git a/apps/server/swagger/submission-api.yml b/apps/server/swagger/submission-api.yml index ccafed1b..6b9106d4 100644 --- a/apps/server/swagger/submission-api.yml +++ b/apps/server/swagger/submission-api.yml @@ -56,7 +56,7 @@ /submission/{submissionId}/details: get: - summary: Fetch Submission Data records. Sorted in their original file order and grouped by `inserts`, `updates`, and `deletes`. + summary: Fetch Submission Data records. Sorted in their original file order and grouped by `insert`, `update`, and `delete`. tags: - Submission parameters: @@ -67,9 +67,9 @@ type: array items: type: string - enum: [inserts, updates, deletes] + enum: [insert, update, delete] uniqueItems: true - description: Filters the Submission Data records by action type. Valid values are `inserts`, `updates`, and `deletes` (case insensitive). If not provided, all action types are returned. + description: Filters the Submission Data records by action type. Valid values are `insert`, `update`, and `delete` (case insensitive). If not provided, all action types are returned. - name: entityNames in: query schema: diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts index 17ceecab..621b9d75 100644 --- a/packages/data-provider/src/controllers/submissionController.ts +++ b/packages/data-provider/src/controllers/submissionController.ts @@ -30,7 +30,7 @@ import { BATCH_ERROR_TYPE, BatchError, type PaginatedResponse, - SUBMISSION_ACTION_TYPE, + SUBMISSION_RECORD_ACTION_TYPE, type SubmissionSummaryResponse, } from '../utils/types.js'; @@ -296,7 +296,7 @@ const controller = ({ const submissionId = Number(req.params.submissionId); const entityNames = asArray(req.query.entityNames || []); - const actionTypes = parseSubmissionActionTypes(req.query.actionTypes || SUBMISSION_ACTION_TYPE.options); + const actionTypes = parseSubmissionActionTypes(req.query.actionTypes || SUBMISSION_RECORD_ACTION_TYPE.options); // query params const page = parseInt(String(req.query.page)) || DEFAULT_PAGE; diff --git a/packages/data-provider/src/repository/submissionRecordsRepository.ts b/packages/data-provider/src/repository/submissionRecordsRepository.ts index 1e988304..df7a04b9 100644 --- a/packages/data-provider/src/repository/submissionRecordsRepository.ts +++ b/packages/data-provider/src/repository/submissionRecordsRepository.ts @@ -163,7 +163,9 @@ const submissionRecordsRepository = (dependencies: BaseDependencies) => { .where( and( eq(submissionFiles.submissionId, submissionId), - filterOptions?.entityNames ? inArray(submissionFiles.entityName, filterOptions.entityNames) : undefined, + filterOptions?.entityNames?.length + ? inArray(submissionFiles.entityName, filterOptions.entityNames) + : undefined, ), ); return await getByFileIds( diff --git a/packages/data-provider/src/utils/auditUtils.ts b/packages/data-provider/src/utils/auditUtils.ts index aaad3324..c69f50d4 100644 --- a/packages/data-provider/src/utils/auditUtils.ts +++ b/packages/data-provider/src/utils/auditUtils.ts @@ -5,7 +5,7 @@ import { AuditAction, AuditDataResponse, AuditRepositoryRecord, - SUBMISSION_ACTION_TYPE, + SUBMISSION_RECORD_ACTION_TYPE, } from './types.js'; /** @@ -18,7 +18,7 @@ export const isAuditEventValid = (value: unknown): boolean => typeof value === 'string' && AUDIT_ACTION.safeParse(value.toUpperCase()).success; export const isSubmissionActionTypeValid = (value: unknown): boolean => - typeof value === 'string' && SUBMISSION_ACTION_TYPE.safeParse(value.toUpperCase()).success; + typeof value === 'string' && SUBMISSION_RECORD_ACTION_TYPE.safeParse(value.toUpperCase()).success; /** * Convert a value string into it's Audit event type if it matches. diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 10f1bf93..8e02fb56 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -169,12 +169,6 @@ export type MigrationAuditRecord = Omit; - /** Action field included in each Kafka message emitted after a successful commit. */ export const KAFKA_ACTION = zod.enum(['delete', 'insert', 'update']); export type KafkaAction = zod.infer; diff --git a/packages/data-provider/src/workers/workerPoolManager.ts b/packages/data-provider/src/workers/workerPoolManager.ts index d02e55bf..9809a725 100644 --- a/packages/data-provider/src/workers/workerPoolManager.ts +++ b/packages/data-provider/src/workers/workerPoolManager.ts @@ -133,6 +133,10 @@ export const createWorkerPool = (configData: AppConfig, options?: CreateWorkerPo } }, terminate: async (): Promise => { + // Wait for the worker's startup handshake to settle before terminating the pool. + // Terminating while the child process is still registering closes the IPC channel + // out from under its own `process.send()`, which crashes the process with EPIPE. + await readyProxy.catch(() => undefined); await pool.terminate(); }, }; From 9342c16bc662d13da46c400928540512f3a8f6c9 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 14 Aug 2026 09:35:32 -0400 Subject: [PATCH 14/20] Update tech-debt.md --- .dev/tech-debt.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.dev/tech-debt.md b/.dev/tech-debt.md index be033b4e..386f8623 100644 --- a/.dev/tech-debt.md +++ b/.dev/tech-debt.md @@ -57,6 +57,30 @@ context: `dictionary_categories.id` is a plain auto-increment `serial`, used dir standalone: yes context: `packages/data-provider/test/integration/routers/submission/` only covers `submissionRouter-submit*` (insert-only file/JSON uploads). There is no integration spec for `editSubmittedData`, `deleteSubmittedDataBySystemId`, or the commit flow (`performCommitSubmissionAsync`/`commitSubmissionWorker.ts`) at all, despite the test infra (`test/integration/dependencies/containers.ts`) already running a real Postgres container capable of exercising the real commit transaction. The new UPDATE/DELETE conflict resolution added 2026-08-12 has unit coverage only (`test/unit/utils/submission/findUpdateDeleteConflicts.spec.ts` and friends) — still no HTTP+DB-level test proving a submission with a staged conflict actually ends up INVALID end-to-end. Fix: add integration specs that stage inserts+updates+deletes in the same active submission (including same-systemId collisions) and assert the committed `submitted_data` table end state and the active submission's status/record states. +### `GET /submission/:submissionId/details` can't filter records by state +standalone: yes +context: `submissionController.ts::getSubmissionDetailsById` (294-321) and `submissionService.ts::getSubmissionDetailsById` (326-372) only accept `entityNames`/`actionTypes` filters (`submissionDetailsRequestSchema`, `schemas.ts:339-351`), then call `submissionRecordsRepository.getBySubmissionId` (365-369) without a `states` filter. The repository itself already supports it — `getBySubmissionId`/`getByFileIds` (`submissionRecordsRepository.ts`) both accept `filterOptions.states?: SubmissionRecordState[]` and `SUBMISSION_RECORD_STATE` (`types.ts`) already enumerates `RECEIVED`/`VALID`/`INVALID` — so this is a plumbing gap, not a missing capability. Fix: add a `states`/`state` query param to `submissionDetailsRequestSchema`, thread it through `getSubmissionDetailsById`'s `filterOptions` in both the controller and service, and pass it to `submissionRecordsRepository.getBySubmissionId`. + +### API to download the originally uploaded file +standalone: yes +context: No original file bytes are stored anywhere today. Uploads land in a multer temp path (`dest: '/tmp'`, `submissionRouter.ts:26`), get streamed and parsed by `collectRows` (`fileUtils.ts:62-80`), and the temp file is deleted immediately after parsing (`fileUtils.ts:78`, `fs.unlink`). `submission_files` (`packages/data-model/src/models/submission_files.ts:7-17`) only keeps metadata (`fileName`, `entityName`, `fileSize`); `submission_records` only keeps parsed row data (jsonb), not the raw file. Needs a scope decision before implementation: (a) persist the raw uploaded bytes somewhere (object store or DB blob) at upload time going forward — adds storage/retention cost and doesn't help for submissions already committed under the old behavior, or (b) reconstruct a file from the stored parsed rows — lossy, won't reproduce original column order, formatting, or any extra/ignored columns, and "recreate" may not satisfy whatever this is needed for (audit, re-upload, external sharing). No download/`Content-Disposition` endpoint exists for submission files today; the only precedent for that response pattern in the codebase is `dictionaryController.ts:74-87` (zips dictionary templates, unrelated data). + +### Download error report for a file +standalone: no +context: Depends on the storage/identity decision above and needs its own format decision. Per-record errors already exist as `SubmissionRecordError[]` in `submission_records.errors` (jsonb, `packages/data-model/src/models/submission_records.ts:59-73`), and a file already has a stable identity distinct from `entityName` (`submission_files.id`, `submission_records.fileId`) — so per-file error retrieval is possible today via `getBySubmissionId`/`getByFileIds` (`submissionRecordsRepository.ts:34-63,149-177`), just not exposed as a dedicated download endpoint. Open question flagged by the requirement itself: return the raw per-record `errors` JSON as-is, or design a purpose-built report format (e.g. row/field/message table)? Note that line numbers are computed at parse time (`fileUtils.ts:104`, `+1 for header row, +1 for 1-based line numbers`) but aren't currently persisted into the stored `errors` — worth carrying through if the report should reference original file line numbers. + +### Download error report as a zip for all files in a submission +standalone: no +context: Depends on the single-file error report above being defined first — this is just "download that report N times, zipped." `jszip` is already a dependency (`package.json:48`) and already used for exactly this response shape (`zip.generateAsync` + `Content-Disposition: attachment` + `application/zip`) in `dictionaryController.ts:74-87`, so no new library or pattern work is needed once the per-file report format exists. + +### List submissions endpoint needs sorting and filtering +standalone: yes +context: `GET /submission/category/:categoryId` (`submissionController.ts:233-276`) only accepts `onlyActive`, `organization`, `username` (`submissionsByCategoryRequestSchema`, `schemas.ts:318-329`), plus `page`/`pageSize`. Sort order is hardcoded to `desc(submissions.createdAt)` (`activeSubmissionRepository.ts:216`) with no sort param at all. The mandatory requirement (reverse creation-date sort) already matches today's fixed default — the gap is that it isn't documented as a stable, guaranteed default, and there's no way to choose anything else. Nice-to-have filters not yet supported: study/category beyond the path param, date range (created/updated), creator, contributing users, statuses. `auditRepository.ts` already has a directly reusable pattern for both pieces: date-range filtering (lines 76-81, `lt`/`gt` on `createdAt` against `startDate`/`endDate`) and configurable-direction `orderBy` (line 126), driven by `AuditFilterOptions` (`types.ts:89-98`) — worth modeling the new filter options after that rather than inventing a new shape. Swagger (`submission-api.yml:144-171`) will need the new params documented too. + +### `GET /submission/:submissionId` needs richer per-file details +standalone: yes +context: The response (`SubmissionSummaryResponse`, `types.ts:302-305`, built by `createSubmissionSummaryResponse`/`submissionResponseParser.ts:65-78`) groups `inserts`/`updates` by `entityName`, each entry only exposing `batchName` (the original filename), `recordsCount`, and `errors` (a count, not detail) — see `submissionResponseParser.ts:14-58` and the source rows shape in `submissionRecordsRepository.ts:20-26,188-208`. Missing relative to the ask: no `fileId` in the response (so a client can't correlate a listed file to a future per-file download/error-report endpoint, above); no file size, even though `submission_files.fileSize` is already stored in the DB (`submission_files.ts:16`) and just isn't selected by `getRecordsSummaryBySubmissionId` (`submissionRecordsRepository.ts:188-197`); no rolled-up per-file validation status (only an error count — the per-record `state` enum `RECEIVED`/`VALID`/`INVALID` exists at `submission_records.ts:12` but isn't aggregated to file level); `deletes` has no per-file breakdown at all in the current model (`DataDeletesSubmissionSummary` is not batched by file). Also found in passing: swagger's `SubmissionDetailsResult` schema (`schemas.yml:87-111`) documents a shape that doesn't match what `/submission/:submissionId/details` actually returns (`SubmissionRecordWithEntityName[]`, i.e. `{id, actionType, state, fileId, data, errors, entityName}[]`) — worth correcting alongside this work since both touch the same response family. + ## Resolved ### Flaky integration test: dictionary migration force-retry intermittently returned 409 instead of 200 From b809f0357da714e1eee969a1da42e92ba0070967 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 14 Aug 2026 10:10:50 -0400 Subject: [PATCH 15/20] await tx insert submission recrods --- .../submission/submissionProcessor.ts | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/data-provider/src/services/submission/submissionProcessor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts index 0c45e9e9..b0ef49c0 100644 --- a/packages/data-provider/src/services/submission/submissionProcessor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -795,22 +795,28 @@ const createSubmissionProcessor = (dependencies: BaseDependencies) => { const insertRecords = parseRecordsToInsert(records, schemasDictionary); await Promise.all( - Object.entries(insertRecords).map(async ([entityName, entityRecords]) => { - const savedFileId = await submissionFilesRepository.save({ - entityName, - fileName: genericSubmissionFileName(), - fileSize: getSizeInBytes(JSON.stringify(entityRecords)), - submissionId, - }); - await submissionRecordsRepository.saveManyForFile( - savedFileId, - entityRecords.map((record) => ({ - actionType: 'INSERT', - data: record, - state: 'RECEIVED', - })), - ); - }), + Object.entries(insertRecords).map(([entityName, entityRecords]) => + dependencies.db.transaction(async (tx) => { + const savedFileId = await submissionFilesRepository.save( + { + entityName, + fileName: genericSubmissionFileName(), + fileSize: getSizeInBytes(JSON.stringify(entityRecords)), + submissionId, + }, + tx, + ); + await submissionRecordsRepository.saveManyForFile( + savedFileId, + entityRecords.map((record) => ({ + actionType: 'INSERT', + data: record, + state: 'RECEIVED', + })), + tx, + ); + }), + ), ); // Perform Schema Data validation in a worker thread From 5bfd1a22f97b2e4cb0fcd89e44375eb0778ca472 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Fri, 14 Aug 2026 12:04:25 -0400 Subject: [PATCH 16/20] get submission records by file id --- apps/server/swagger/submission-api.yml | 9 +++++++-- .../src/controllers/submissionController.ts | 7 ++----- .../src/repository/submissionRecordsRepository.ts | 13 ++++++++++++- .../data-provider/src/routers/submissionRouter.ts | 4 ++-- .../src/services/submission/submissionService.ts | 2 +- packages/data-provider/src/utils/schemas.ts | 2 ++ 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/apps/server/swagger/submission-api.yml b/apps/server/swagger/submission-api.yml index 6b9106d4..8c0bb3fd 100644 --- a/apps/server/swagger/submission-api.yml +++ b/apps/server/swagger/submission-api.yml @@ -54,7 +54,7 @@ 503: $ref: '#/components/responses/ServiceUnavailableError' -/submission/{submissionId}/details: +/submission/{submissionId}/data: get: summary: Fetch Submission Data records. Sorted in their original file order and grouped by `insert`, `update`, and `delete`. tags: @@ -78,6 +78,12 @@ type: string uniqueItems: true description: Filters the Submission Data records by entity name. Must match names listed in the Submission Summary endpoint. If not provided, all entity names are returned. + - name: fileId + in: query + required: false + schema: + type: integer + description: An optional query parameter used to specify the file ID within the submission to be retrieved. - $ref: '#/components/parameters/query/Page' - $ref: '#/components/parameters/query/PageSize' responses: @@ -98,7 +104,6 @@ 503: $ref: '#/components/responses/ServiceUnavailableError' -/submission/{submissionId}/data: delete: summary: Clear Active Submission by entity name tags: diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts index 621b9d75..e2ea9bf0 100644 --- a/packages/data-provider/src/controllers/submissionController.ts +++ b/packages/data-provider/src/controllers/submissionController.ts @@ -295,6 +295,7 @@ const controller = ({ try { const submissionId = Number(req.params.submissionId); const entityNames = asArray(req.query.entityNames || []); + const fileId = req.query.fileId ? parseInt(req.query.fileId) : undefined; const actionTypes = parseSubmissionActionTypes(req.query.actionTypes || SUBMISSION_RECORD_ACTION_TYPE.options); @@ -307,13 +308,9 @@ const controller = ({ const submission = await submissionService.getSubmissionDetailsById({ submissionId, paginationOptions: { page, pageSize }, - filterOptions: { entityNames, actionTypes }, + filterOptions: { entityNames, actionTypes, fileId }, }); - if (isEmpty(submission)) { - throw new NotFound('Submission not found'); - } - return res.status(200).json(submission); } catch (error) { next(error); diff --git a/packages/data-provider/src/repository/submissionRecordsRepository.ts b/packages/data-provider/src/repository/submissionRecordsRepository.ts index df7a04b9..82c106a4 100644 --- a/packages/data-provider/src/repository/submissionRecordsRepository.ts +++ b/packages/data-provider/src/repository/submissionRecordsRepository.ts @@ -52,7 +52,7 @@ const submissionRecordsRepository = (dependencies: BaseDependencies) => { and( inArray(submissionRecords.fileId, fileIds), filterOptions?.actionTypes ? inArray(submissionRecords.actionType, filterOptions.actionTypes) : undefined, - filterOptions?.states ? inArray(submissionRecords.state, filterOptions.states) : undefined, + filterOptions?.states?.length ? inArray(submissionRecords.state, filterOptions.states) : undefined, ), ) .orderBy(submissionRecords.id); @@ -154,6 +154,7 @@ const submissionRecordsRepository = (dependencies: BaseDependencies) => { actionTypes?: SubmissionRecordActionType[]; states?: SubmissionRecordState[]; entityNames?: string[]; + fileId?: number; }, ): Promise => { try { @@ -166,8 +167,18 @@ const submissionRecordsRepository = (dependencies: BaseDependencies) => { filterOptions?.entityNames?.length ? inArray(submissionFiles.entityName, filterOptions.entityNames) : undefined, + filterOptions?.fileId ? eq(submissionFiles.id, filterOptions.fileId) : undefined, ), ); + + if (submissionFileIds.length === 0) { + logger.info( + LOG_MODULE, + `No submission files found for submissionId '${submissionId}' with the provided filter options.`, + ); + return []; + } + return await getByFileIds( submissionFileIds.map((file) => file.id), paginationOptions, diff --git a/packages/data-provider/src/routers/submissionRouter.ts b/packages/data-provider/src/routers/submissionRouter.ts index 2f8c99fd..a522d142 100644 --- a/packages/data-provider/src/routers/submissionRouter.ts +++ b/packages/data-provider/src/routers/submissionRouter.ts @@ -71,10 +71,10 @@ const router = ({ router.get('/:submissionId', submissionController.getSubmissionById); - router.get('/:submissionId/details', submissionController.getSubmissionDetailsById); - router.delete('/:submissionId', submissionController.delete); + router.get('/:submissionId/data', submissionController.getSubmissionDetailsById); + router.delete('/:submissionId/data', submissionController.deleteByRecordIdOrFileId); router.get('/category/:categoryId', submissionController.getSubmissionsByCategory); diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts index 49f640ec..8986e987 100644 --- a/packages/data-provider/src/services/submission/submissionService.ts +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -330,7 +330,7 @@ const submissionService = (dependencies: BaseDependencies) => { }: { submissionId: number; paginationOptions: PaginationOptions; - filterOptions: { entityNames: string[]; actionTypes: SubmissionRecordActionType[] }; + filterOptions: { entityNames: string[]; actionTypes: SubmissionRecordActionType[]; fileId?: number }; }): Promise => { const submission = await submissionRepository.getSubmissionById(submissionId); if (!submission) { diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts index fabbeaa1..66edfb49 100644 --- a/packages/data-provider/src/utils/schemas.ts +++ b/packages/data-provider/src/utils/schemas.ts @@ -334,6 +334,7 @@ export const submissionByIdRequestSchema: RequestValidation Date: Fri, 14 Aug 2026 12:04:40 -0400 Subject: [PATCH 17/20] fix file name on delete records --- .../src/services/submittedData/submmittedData.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index 917467a2..c2bcadd8 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -12,7 +12,7 @@ import submittedRepository from '../../repository/submittedRepository.js'; import { convertSqonToQuery } from '../../utils/convertSqonToQuery.js'; import { getDictionarySchemaRelations } from '../../utils/dictionarySchemaRelations.js'; import { InternalServerError, StatusConflict } from '../../utils/errors.js'; -import { getSizeInBytes } from '../../utils/fileUtils.js'; +import { genericSubmissionFileName, getSizeInBytes } from '../../utils/fileUtils.js'; import type { PaginatedResult } from '../../utils/result.js'; import { resolveDeleteStagingConflicts } from '../../utils/submissionUtils.js'; import { @@ -192,7 +192,7 @@ const submittedData = (dependencies: BaseDependencies) => { const savedFileId = await submissionFilesRepository.save( { entityName, - fileName: `${Date.now()}.json`, + fileName: genericSubmissionFileName(), fileSize: getSizeInBytes(JSON.stringify(entityRecords)), submissionId: activeSubmissionId, }, From 38fdf50416c00d903402f032e0473b00b09fbf27 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 27 Aug 2026 11:24:53 -0400 Subject: [PATCH 18/20] Update README.md --- packages/data-model/docs/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/data-model/docs/README.md b/packages/data-model/docs/README.md index 622a07e5..6729bb79 100644 --- a/packages/data-model/docs/README.md +++ b/packages/data-model/docs/README.md @@ -39,14 +39,10 @@ Stores data submissions, each associated with a dictionary category and dictiona Key fields in the submissions table include: -- `data`: This field contains the actual submission data stored as a JSON object. The use of JSON allows for flexible and dynamic data structures, accommodating varying submission formats while preserving the integrity of the information. - - `dictionary_category_id`: This field establishes a link to the corresponding dictionary category, which defines the schema against which the submission data will be validated. By referencing the dictionary category, the submission ensures compliance with the rules and structures outlined in the associated dictionary. - `dictionary_id`: This field links the submission directly to the specific dictionary version being utilized for validation. This connection ensures that the submission is aligned with the correct schema version, enabling consistent validation and data integrity. -- `errors`: This field captures any validation errors encountered during the submission process. It allows for detailed tracking of issues, providing insights into why certain data may not conform to the expected schema. - - `organization`: This field indicates the organization responsible for the submission. This contextual information is essential for data management and auditing purposes, allowing for the tracking of submissions by different entities. - `status`: This field represents the current state of the submission and can include values such as open, valid, invalid, closed, or committed. The status helps to monitor the submission's lifecycle, ensuring that users can easily identify which submissions are pending, validated, or finalized. From d34774ecb03ae131477ba9bc939efb5fb7418127 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 27 Aug 2026 14:32:55 -0400 Subject: [PATCH 19/20] adding fileId to submission summary --- apps/server/swagger/schemas.yml | 6 ++++++ .../repository/submissionRecordsRepository.ts | 15 ++++++++++---- .../src/utils/submissionResponseParser.ts | 4 ++-- packages/data-provider/src/utils/types.ts | 8 +++++--- .../utils/submissionResponseParser.spec.ts | 20 +++++++++---------- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 72046796..84ee7624 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -127,6 +127,12 @@ components: batchName: type: string description: Original filename of the submission + errors: + type: number + description: Number of errors in the submission + fileId: + type: string + description: ID of the file in the submission recordsCount: type: number description: Number Of Records diff --git a/packages/data-provider/src/repository/submissionRecordsRepository.ts b/packages/data-provider/src/repository/submissionRecordsRepository.ts index 82c106a4..dfcca583 100644 --- a/packages/data-provider/src/repository/submissionRecordsRepository.ts +++ b/packages/data-provider/src/repository/submissionRecordsRepository.ts @@ -21,10 +21,11 @@ export type SubmissionRecordWithEntityName = SubmissionRecord & { entityName: st // Raw data returned from the database export type RecordsSummaryRepository = { actionType: SubmissionRecordActionType; - entityName: string; - totalRecords: number; batchName?: string; + entityName: string; errors: number; + fileId: number; + totalRecords: number; }; const submissionRecordsRepository = (dependencies: BaseDependencies) => { @@ -200,13 +201,19 @@ const submissionRecordsRepository = (dependencies: BaseDependencies) => { actionType: submissionRecords.actionType, batchName: submissionFiles.fileName, entityName: submissionFiles.entityName, - totalRecords: count(), errors: count(submissionRecords.errors), + fileId: submissionFiles.id, + totalRecords: count(), }) .from(submissionRecords) .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id)) .where(eq(submissionFiles.submissionId, submissionId)) - .groupBy(submissionRecords.actionType, submissionFiles.fileName, submissionFiles.entityName); + .groupBy( + submissionFiles.id, + submissionRecords.actionType, + submissionFiles.entityName, + submissionFiles.fileName, + ); return submissionFileRecords; } catch (error) { diff --git a/packages/data-provider/src/utils/submissionResponseParser.ts b/packages/data-provider/src/utils/submissionResponseParser.ts index 0129958c..4192c880 100644 --- a/packages/data-provider/src/utils/submissionResponseParser.ts +++ b/packages/data-provider/src/utils/submissionResponseParser.ts @@ -19,8 +19,8 @@ export const buildDataSummary = (rows: RecordsSummaryRepository[]): SubmissionDa let errors = 0; for (const row of rows) { - const { actionType, entityName, totalRecords: rowTotalRecords, batchName, errors: rowErrors } = row; - const summaryItem = { batchName: batchName ?? '', recordsCount: rowTotalRecords, errors: rowErrors }; + const { actionType, entityName, totalRecords: rowTotalRecords, batchName, errors: rowErrors, fileId } = row; + const summaryItem = { batchName: batchName ?? '', errors: rowErrors, fileId, recordsCount: rowTotalRecords }; totalRecords += rowTotalRecords; errors += rowErrors; diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 8e02fb56..369d9970 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -265,19 +265,21 @@ export type PaginationOptions = { export type DataInsertsSubmissionSummary = { batchName: string; - recordsCount: number; errors: number; + fileId: number; + recordsCount: number; }; export type DataUpdatesSubmissionSummary = { batchName: string; - recordsCount: number; errors: number; + fileId: number; + recordsCount: number; }; export type DataDeletesSubmissionSummary = { - recordsCount: number; errors: number; + recordsCount: number; }; export type SubmissionDataSummary = { diff --git a/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts b/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts index 056e99bb..0edd5368 100644 --- a/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts +++ b/packages/data-provider/test/unit/utils/submissionResponseParser.spec.ts @@ -19,9 +19,9 @@ describe('buildDataSummary', () => { it('should group insert and update rows by entity name and accumulate totals', () => { const rows: RecordsSummaryRepository[] = [ - { actionType: 'INSERT', entityName: 'sport', totalRecords: 3, batchName: 'batch1.tsv', errors: 0 }, - { actionType: 'INSERT', entityName: 'sport', totalRecords: 2, batchName: 'batch2.tsv', errors: 1 }, - { actionType: 'UPDATE', entityName: 'player', totalRecords: 5, batchName: 'batch3.tsv', errors: 0 }, + { actionType: 'INSERT', entityName: 'sport', totalRecords: 3, batchName: 'batch1.tsv', errors: 0, fileId: 1 }, + { actionType: 'INSERT', entityName: 'sport', totalRecords: 2, batchName: 'batch2.tsv', errors: 1, fileId: 2 }, + { actionType: 'UPDATE', entityName: 'player', totalRecords: 5, batchName: 'batch3.tsv', errors: 0, fileId: 3 }, ]; const result = buildDataSummary(rows); @@ -29,12 +29,12 @@ describe('buildDataSummary', () => { expect(result).to.eql({ inserts: { sport: [ - { batchName: 'batch1.tsv', recordsCount: 3, errors: 0 }, - { batchName: 'batch2.tsv', recordsCount: 2, errors: 1 }, + { batchName: 'batch1.tsv', recordsCount: 3, errors: 0, fileId: 1 }, + { batchName: 'batch2.tsv', recordsCount: 2, errors: 1, fileId: 2 }, ], }, updates: { - player: [{ batchName: 'batch3.tsv', recordsCount: 5, errors: 0 }], + player: [{ batchName: 'batch3.tsv', recordsCount: 5, errors: 0, fileId: 3 }], }, deletes: {}, totalRecords: 10, @@ -44,8 +44,8 @@ describe('buildDataSummary', () => { it('should aggregate delete rows for the same entity instead of listing them individually', () => { const rows: RecordsSummaryRepository[] = [ - { actionType: 'DELETE', entityName: 'sport', totalRecords: 2, batchName: 'batch1.tsv', errors: 0 }, - { actionType: 'DELETE', entityName: 'sport', totalRecords: 1, batchName: 'batch2.tsv', errors: 1 }, + { actionType: 'DELETE', entityName: 'sport', totalRecords: 2, batchName: 'batch1.tsv', errors: 0, fileId: 1 }, + { actionType: 'DELETE', entityName: 'sport', totalRecords: 1, batchName: 'batch2.tsv', errors: 1, fileId: 2 }, ]; const result = buildDataSummary(rows); @@ -59,13 +59,13 @@ describe('buildDataSummary', () => { it('should default a missing batchName to an empty string', () => { const rows: RecordsSummaryRepository[] = [ - { actionType: 'INSERT', entityName: 'sport', totalRecords: 1, errors: 0 }, + { actionType: 'INSERT', entityName: 'sport', totalRecords: 1, errors: 0, fileId: 1 }, ]; const result = buildDataSummary(rows); expect(result.inserts).to.eql({ - sport: [{ batchName: '', recordsCount: 1, errors: 0 }], + sport: [{ batchName: '', recordsCount: 1, errors: 0, fileId: 1 }], }); }); }); From 563881f691282ef158b77e31814e94fcc76a0052 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Thu, 27 Aug 2026 14:51:27 -0400 Subject: [PATCH 20/20] submission details swagger definition --- apps/server/swagger/schemas.yml | 50 ++++++++++--------- .../repository/submissionRecordsRepository.ts | 8 +-- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 84ee7624..5b36df03 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -85,30 +85,34 @@ components: $ref: '#/components/schemas/SubmissionResult' SubmissionDetailsResult: - type: object - properties: - data: - type: array - items: + type: array + items: + type: object + properties: + actionType: + type: string + description: Type of action + enum: ['INSERT', 'UPDATE', 'DELETE'] + data: type: object - properties: - type: - type: string - description: Type of action - enum: ['INSERT', 'UPDATE', 'DELETE'] - entity: - type: string - description: Name of the entity - value: - type: object - description: Content of the record in JSON format - index: - type: number - description: Index of the record in the submission - errors: - type: array - items: - $ref: '#/components/schemas/ValidationError' + description: Content of the record in JSON format + entityName: + type: string + description: Name of the entity + errors: + type: array + items: + $ref: '#/components/schemas/ValidationError' + fileId: + type: string + description: ID of the file in the submission + id: + type: number + description: ID of the record in the submission + state: + type: string + description: State of the record in the submission + enum: ['VALID', 'INVALID', 'RECEIVED'] SubmissionResult: type: object diff --git a/packages/data-provider/src/repository/submissionRecordsRepository.ts b/packages/data-provider/src/repository/submissionRecordsRepository.ts index dfcca583..eeec6d7b 100644 --- a/packages/data-provider/src/repository/submissionRecordsRepository.ts +++ b/packages/data-provider/src/repository/submissionRecordsRepository.ts @@ -39,13 +39,13 @@ const submissionRecordsRepository = (dependencies: BaseDependencies) => { ): Promise => { const query = db .select({ - id: submissionRecords.id, actionType: submissionRecords.actionType, - state: submissionRecords.state, - fileId: submissionRecords.fileId, data: submissionRecords.data, - errors: submissionRecords.errors, entityName: submissionFiles.entityName, + errors: submissionRecords.errors, + fileId: submissionRecords.fileId, + id: submissionRecords.id, + state: submissionRecords.state, }) .from(submissionRecords) .innerJoin(submissionFiles, eq(submissionRecords.fileId, submissionFiles.id))