diff --git a/src/data/MetadataD2Repository.ts b/src/data/MetadataD2Repository.ts index 508000d4..63f1aa67 100644 --- a/src/data/MetadataD2Repository.ts +++ b/src/data/MetadataD2Repository.ts @@ -42,7 +42,7 @@ export class MetadataD2Repository implements MetadataRepository { const pageSize = 100_000; const res$ = this.api.get<{ pager: Pager } & { [K in string]: D2User[] }>(`/${options.model}`, { - fields: "id,name,userCredentials[username]", + fields: "id,name,userCredentials[username],*", pageSize: pageSize, page: options.page, }); @@ -54,6 +54,7 @@ export class MetadataD2Repository implements MetadataRepository { ...user, model: options.model, code: user.userCredentials.username, + additionalFields: { ...user }, }) ) .value(); @@ -65,7 +66,7 @@ export class MetadataD2Repository implements MetadataRepository { const res$ = this.api.get<{ pager: Pager } & { [K in string]: BasicD2Object[] }>( `/${getPluralModel(options.model)}`, { - fields: "id,name,code", + fields: "id,name,code,*", pageSize: pageSize, page: options.page, } @@ -73,7 +74,9 @@ export class MetadataD2Repository implements MetadataRepository { const res = await res$.getData(); const objects = _(res[options.model]) - .map((obj): MetadataObject => ({ ...obj, model: options.model })) + .map( + (obj): MetadataObject => ({ ...obj, model: options.model, additionalFields: { ...obj } }) + ) .value(); return { objects: objects, pager: res.pager }; diff --git a/src/domain/entities/MetadataObject.ts b/src/domain/entities/MetadataObject.ts index 47f5a521..e40409ca 100644 --- a/src/domain/entities/MetadataObject.ts +++ b/src/domain/entities/MetadataObject.ts @@ -9,6 +9,7 @@ export interface MetadataObject { id: Id; name: string; code: Maybe; + additionalFields?: Record; } export interface MetadataObjectWithTranslations extends MetadataObject { diff --git a/src/domain/entities/MetadataValidationResult.ts b/src/domain/entities/MetadataValidationResult.ts new file mode 100644 index 00000000..c036801a --- /dev/null +++ b/src/domain/entities/MetadataValidationResult.ts @@ -0,0 +1,43 @@ +import { Id } from "./Base"; +import { MetadataModel, MetadataObject } from "./MetadataObject"; + +export type Source = { type: "main" } | { type: "replica"; index: number }; + +export type ExclusiveMetadataItem = { + object: MetadataObject; + source: Source; +}; + +export type MetadataValidationResult = { + model: MetadataModel; + exclusive: ExclusiveMetadataItem[]; +}; + +export type DiscrepancyMetadata = { + model: MetadataModel; + id: Id; + mainObject: MetadataObject; + replicaObject: MetadataObject; + replicaIndex: number; + differingFields: string[]; +}; + +export type DiscrepancyValidationResult = { + model: MetadataModel; + items: DiscrepancyMetadata[]; +}; + +export const METADATA_PROPERTIES_TO_IGNORE = [ + "url", + "href", + "externalAccess", + "publicAccess", + "userGroupAccesses", + "userAccesses", + "sharing", + "access", + "lastUpdatedBy", + "createdBy", + "code", + "lastUpdated", +]; diff --git a/src/domain/usecases/SyncMetadataUseCase.ts b/src/domain/usecases/SyncMetadataUseCase.ts new file mode 100644 index 00000000..11709836 --- /dev/null +++ b/src/domain/usecases/SyncMetadataUseCase.ts @@ -0,0 +1,249 @@ +import _ from "lodash"; + +import { Maybe } from "utils/ts-utils"; +import { Async } from "domain/entities/Async"; +import { MetadataModel, MetadataObject } from "domain/entities/MetadataObject"; +import { MetadataRepository } from "domain/repositories/MetadataRepository"; +import { + ExclusiveMetadataItem, + MetadataValidationResult, + DiscrepancyMetadata, + DiscrepancyValidationResult, + METADATA_PROPERTIES_TO_IGNORE, +} from "domain/entities/MetadataValidationResult"; + +export class SyncMetadataUseCase { + constructor( + private metadataRepositoryMain: MetadataRepository, + private metadataReplicaRepositories: MetadataRepository[] + ) {} + + async execute(options: UseCaseOptions): Async { + const { modelsToCheck } = options; + return this.validateMetadataByModel(modelsToCheck); + } + + private async validateMetadataByModel(modelsToCheck: MetadataModel[]): Async { + const metadataReport: SyncMetadataReport = { + exclusiveMetadata: [], + metadataWithCodeDiscrepancies: [], + metadataWithPropertiesDiscrepancies: [], + }; + for (const model of modelsToCheck) { + const mainObjects = await this.getObjects({ model, server: this.metadataRepositoryMain }); + + const replicaObjects = await Promise.all( + this.metadataReplicaRepositories.map(replicaServer => + this.getObjects({ model, server: replicaServer }) + ) + ); + + const exclusiveMetadata = this.findExclusiveMetadataWithSource(mainObjects, replicaObjects); + + const metadataWithCodeDiscrepancies = this.findCodeDiscrepanciesForModel( + model, + mainObjects, + replicaObjects + ); + + const metadataWithPropertiesDiscrepancies = this.findFieldDiscrepanciesForModel( + model, + mainObjects, + replicaObjects + ); + + metadataReport.exclusiveMetadata.push({ + model, + exclusive: exclusiveMetadata, + }); + metadataReport.metadataWithCodeDiscrepancies.push({ + model, + items: metadataWithCodeDiscrepancies, + }); + metadataReport.metadataWithPropertiesDiscrepancies.push({ + model, + items: metadataWithPropertiesDiscrepancies, + }); + } + + return metadataReport; + } + + /* + Metadata objects that exist only in one of the instances (based on IDs). + */ + private findExclusiveMetadataWithSource( + main: MetadataObject[], + replicas: MetadataObject[][] + ): ExclusiveMetadataItem[] { + const metadataWithSource = [ + ...main.map( + (metadataObject): ExclusiveMetadataItem => ({ + object: metadataObject, + source: { type: "main" }, + }) + ), + ...replicas.flatMap((list, replicaIndex) => + list.map( + (metadataObject): ExclusiveMetadataItem => ({ + object: metadataObject, + source: { type: "replica", index: replicaIndex }, + }) + ) + ), + ]; + + const metadataGroupedById = _.groupBy(metadataWithSource, item => item.object.id); + const exclusiveMetadata = _.pickBy(metadataGroupedById, group => group.length === 1); + + return _(exclusiveMetadata) + .map(group => group[0]) + .compact() + .value(); + } + + private findCodeDiscrepanciesAgainstMain( + model: MetadataModel, + mainList: MetadataObject[], + replicaList: MetadataObject[], + replicaIdx: number + ): DiscrepancyMetadata[] { + const mainById = _(mainList) + .filter(mainObj => mainObj.model === model) + .keyBy(x => x.id) + .value(); + + return replicaList + .filter(replicaObj => replicaObj.model === model) + .filter(replicaObj => { + const mainObj = mainById[replicaObj.id]; + if (!mainObj) return false; + + const codeMain = mainObj.code ?? ""; + const codeReplica = replicaObj.code ?? ""; + return codeMain !== codeReplica; + }) + .map(replicaObj => { + const mainObj = mainById[replicaObj.id]!; + return { + model, + id: replicaObj.id, + mainObject: mainObj, + replicaObject: replicaObj, + replicaIndex: replicaIdx, + differingFields: ["code"], + }; + }); + } + + /* + Detect objects with the same ID but different codes to flag discrepancies. + */ + private findCodeDiscrepanciesForModel( + model: MetadataModel, + mainList: MetadataObject[], + replicaLists: MetadataObject[][] + ): DiscrepancyMetadata[] { + return replicaLists.flatMap((replicaList, idx) => + this.findCodeDiscrepanciesAgainstMain(model, mainList, replicaList, idx) + ); + } + + private compareFields(mainObj: MetadataObject, replicaObj: MetadataObject): string[] { + const diffs: string[] = []; + + const mainAdd = mainObj.additionalFields ?? {}; + const replicaAdd = replicaObj.additionalFields ?? {}; + + const allKeys = new Set([...Object.keys(mainAdd), ...Object.keys(replicaAdd)]); + + for (const key of allKeys) { + if (METADATA_PROPERTIES_TO_IGNORE.includes(key)) { + continue; + } + const mainVal = mainAdd[key]; + const replicaVal = replicaAdd[key]; + if (!_.isEqual(mainVal, replicaVal)) { + diffs.push(key); + } + } + return diffs; + } + + /* + Detect objects with the same ID but different in certain fields + */ + private findFieldDiscrepanciesAgainstMain( + model: MetadataModel, + mainList: MetadataObject[], + replicaList: MetadataObject[], + replicaIdx: number + ): DiscrepancyMetadata[] { + const mainById = _.keyBy(mainList, "id"); + + return _(replicaList) + .map((replicaObj): Maybe => { + const mainObj = mainById[replicaObj.id]; + if (!mainObj) return undefined; + + const differingFields = this.compareFields(mainObj, replicaObj); + if (differingFields.length === 0) return undefined; + + return { + model, + id: replicaObj.id, + mainObject: mainObj, + replicaObject: replicaObj, + replicaIndex: replicaIdx, + differingFields, + }; + }) + .compact() + .value(); + } + + private findFieldDiscrepanciesForModel( + model: MetadataModel, + mainList: MetadataObject[], + replicaLists: MetadataObject[][] + ): DiscrepancyMetadata[] { + return replicaLists.flatMap((replicaList, idx) => + this.findFieldDiscrepanciesAgainstMain(model, mainList, replicaList, idx) + ); + } + + private async getObjects(options: { + model: MetadataModel; + server: MetadataRepository; + }): Async { + const { model, server } = options; + const allObjects: MetadataObject[] = []; + + let page = 1; + while (true) { + const { objects, pager } = await server.getPaginated({ + model: model, + page: page, + }); + allObjects.push(...objects.map(obj => ({ ...obj }))); + + if (pager.page >= pager.pageCount) break; + page++; + } + + return allObjects; + } +} + +type UseCaseOptions = { modelsToCheck: string[] }; + +type MainAndReplicaObjects = { + mainObjects: MetadataObject[]; + replicaObjects: MetadataObject[][]; +}; + +export type SyncMetadataReport = { + exclusiveMetadata: MetadataValidationResult[]; + metadataWithCodeDiscrepancies: DiscrepancyValidationResult[]; + metadataWithPropertiesDiscrepancies: DiscrepancyValidationResult[]; +}; diff --git a/src/domain/usecases/__tests__/sync-metadata/fixtures.data.ts b/src/domain/usecases/__tests__/sync-metadata/fixtures.data.ts new file mode 100644 index 00000000..18f5ecd3 --- /dev/null +++ b/src/domain/usecases/__tests__/sync-metadata/fixtures.data.ts @@ -0,0 +1,122 @@ +import { MetadataModel, MetadataObject } from "domain/entities/MetadataObject"; + +export function buildMetadataObject(model: MetadataModel, data?: Partial): MetadataObject { + return { + id: "id", + name: "name", + code: "code", + model, + ...data, + }; +} + +const dataElementOneFakeData = buildMetadataObject("dataElements", { + id: "de_id_1", + name: "Data Element 1", + code: "DE1", +}); + +const dataElementTwoFakeData = buildMetadataObject("dataElements", { + id: "de_id_2", + name: "Data Element 2", + code: "DE2", +}); + +const indicatorOneFakeData = buildMetadataObject("indicators", { + id: "ind_id_1", + name: "Indicator 1", + code: "IND1", +}); + +const indicatorTwoFakeData = buildMetadataObject("indicators", { + id: "ind_id_2", + name: "Indicator 2", + code: "IND2", +}); + +export const metadataServerFakeData = [ + dataElementOneFakeData, + dataElementTwoFakeData, + indicatorOneFakeData, + indicatorTwoFakeData, +]; + +export const replicaOneFakeData = [ + dataElementOneFakeData, + dataElementTwoFakeData, + indicatorOneFakeData, + indicatorTwoFakeData, + buildMetadataObject("indicators", { + id: "ind_id_exclusive", + name: "Indicator exclusive", + code: "IND_EXCLUSIVE_2", + }), +]; + +export const replicaTwoFakeData = [ + buildMetadataObject("dataElements", { id: "de_id_3", name: "Data Element 3", code: "DE3" }), + dataElementTwoFakeData, + buildMetadataObject("indicators", { id: "ind_id_3", name: "Indicator 3", code: "IND3" }), + indicatorTwoFakeData, +]; + +export const mainDataWithCodeDiscrepancies = [ + buildMetadataObject("dataSets", { id: "ds_id_1", name: "DataSet 1", code: "DS_1" }), + buildMetadataObject("dataSets", { id: "ds_id_2", name: "DataSet 2", code: "DS_2" }), + buildMetadataObject("dataSets", { id: "ds_id_3", name: "DataSet 3", code: "DS_3" }), + buildMetadataObject("categories", { id: "cat_id_1", name: "Category 1", code: "cat_1" }), + buildMetadataObject("categories", { id: "cat_id_2", name: "Category 2", code: "cat_2" }), +]; + +export const replicaDataWithCodeDiscrepancies = [ + buildMetadataObject("dataSets", { id: "ds_id_1", name: "DataSet 1", code: undefined }), + buildMetadataObject("dataSets", { id: "ds_id_2", name: "DataSet 2", code: "DS_2" }), + buildMetadataObject("dataSets", { id: "ds_id_3", name: "DataSet 3", code: "DS_THREE_OTHER_CODE" }), + buildMetadataObject("categories", { id: "cat_id_1", name: "Category 1", code: "cat_replica_1" }), +]; + +export const replicaTwoDataWithCodeDiscrepancies = [ + buildMetadataObject("dataSets", { id: "ds_id_2", name: "DataSet 2", code: "DS_TWO_REPLICA_TWO" }), +]; + +export const mainDataWithPropertiesDiscrepancies = [ + buildMetadataObject("indicators", { + id: "ind_p_1", + name: "Indicator 1", + code: "IP_1", + additionalFields: { dataSets: ["ds_id1"], numerator: "de1+de2" }, + }), + buildMetadataObject("indicators", { + id: "ind_p_2", + name: "Indicator 2", + code: "IP_2", + additionalFields: { dataSets: ["ds_id_2", "ds_id_4"], numerator: "de4+de6" }, + }), + buildMetadataObject("indicators", { + id: "ind_p_3", + name: "Indicator 3", + code: "IP_3", + additionalFields: { name: "Indicator 3", dataSets: [], numerator: "" }, + }), +]; + +export const replicaOneDataWithPropertiesDiscrepancies = [ + buildMetadataObject("indicators", { + id: "ind_p_1", + name: "Indicator 1", + code: "IP_1", + additionalFields: { dataSets: ["ds_id1"], numerator: "de1+de2" }, + }), + buildMetadataObject("indicators", { + id: "ind_p_2", + name: "Indicator 2 Replica One", + code: "IP_2", + additionalFields: { name: "Indicator 2 Replica One", dataSets: ["ds_id_4"], numerator: "de15+de20" }, + }), + buildMetadataObject("indicators", { + id: "ind_p_3", + name: "Indicator 3", + code: "IP_3", + additionalFields: { name: "Indicator 3", dataSets: [], numerator: "" }, + }), +]; diff --git a/src/scripts/cli.ts b/src/scripts/cli.ts index b2863928..5bf77905 100644 --- a/src/scripts/cli.ts +++ b/src/scripts/cli.ts @@ -19,6 +19,7 @@ import * as enrollments from "./commands/enrollments"; import * as options from "./commands/options"; import * as data from "./commands/data/data"; import * as categoryOptionCombos from "./commands/categoryOptionCombos"; +import * as metadata from "./commands/metadata"; export function runCli() { const cliSubcommands = subcommands({ @@ -42,6 +43,7 @@ export function runCli() { options: options.getCommand(), data: data.getCommand(), categoryOptionCombos: categoryOptionCombos.getCommand(), + metadata: metadata.getCommand(), }, }); diff --git a/src/scripts/commands/metadata.ts b/src/scripts/commands/metadata.ts new file mode 100644 index 00000000..c3613916 --- /dev/null +++ b/src/scripts/commands/metadata.ts @@ -0,0 +1,6 @@ +import { subcommands } from "cmd-ts"; +import { syncMetadata } from "./metadata/sync"; + +export function getCommand() { + return subcommands({ name: "metadata", cmds: { sync: syncMetadata } }); +} diff --git a/src/scripts/commands/metadata/SyncReport.ts b/src/scripts/commands/metadata/SyncReport.ts new file mode 100644 index 00000000..e2d8d844 --- /dev/null +++ b/src/scripts/commands/metadata/SyncReport.ts @@ -0,0 +1,137 @@ +import { SyncMetadataReport } from "domain/usecases/SyncMetadataUseCase"; +import { createArrayCsvStringifier } from "csv-writer"; +import { writeFileSync } from "fs"; +import logger from "utils/log"; + +export class SyncReport { + public generateCsvReports(syncReport: SyncMetadataReport): void { + const metadataExclusiveCsvContent = this.generateExclusiveMetadataCsv(syncReport); + this.generateCsvFileAndLog( + metadataExclusiveCsvContent, + "exclusive_metadata.csv", + "Exclusive metadata CSV report" + ); + writeFileSync( + "exclusive_metadata.json", + JSON.stringify(syncReport.exclusiveMetadata, null, 2), + "utf-8" + ); + + const metadataDiscrepanciesCsvContent = this.generateDiscrepanciesMetadataCsv(syncReport); + + this.generateCsvFileAndLog( + metadataDiscrepanciesCsvContent, + "discrepancies_metadata.csv", + "Discrepancies metadata CSV report" + ); + writeFileSync( + "discrepancies_metadata.json", + JSON.stringify(syncReport.metadataWithCodeDiscrepancies, null, 2), + "utf-8" + ); + + const metadataPropertiesDiscrepanciesCsvContent = + this.generatePropertiesDiscrepanciesMetadataCsv(syncReport); + + this.generateCsvFileAndLog( + metadataPropertiesDiscrepanciesCsvContent, + "properties_discrepancies_metadata.csv", + "Properties discrepancies metadata CSV report" + ); + writeFileSync( + "properties_discrepancies_metadata.json", + JSON.stringify(syncReport.metadataWithPropertiesDiscrepancies, null, 2), + "utf-8" + ); + } + + private generatePropertiesDiscrepanciesMetadataCsv(syncReport: SyncMetadataReport): string { + const propertiesDiscrepanciesCsvStringifier = createArrayCsvStringifier({ + header: [ + "model", + "id", + "Code in Metadata Server", + "Code in Replica Server", + "Replica Server Number", + "Diff Fields", + ], + }); + const header = propertiesDiscrepanciesCsvStringifier.getHeaderString(); + const rows = syncReport.metadataWithPropertiesDiscrepancies.flatMap(result => + result.items + .map(item => + propertiesDiscrepanciesCsvStringifier.stringifyRecords([ + [ + item.model, + item.id, + item.mainObject.code, + item.replicaObject.code ?? "", + item.replicaIndex + 1, + item.differingFields.join("-"), + ], + ]) + ) + .join("") + ); + return [header, ...rows].join(""); + } + + private generateCsvFileAndLog(csvContent: string, fileName: string, message: string): void { + if (csvContent) { + writeFileSync(fileName, csvContent, { encoding: "utf8" }); + } + logger.info(`${message} generated: ${fileName}`); + } + + private generateDiscrepanciesMetadataCsv(syncReport: SyncMetadataReport): string { + const discrepanciesCsvStringifier = createArrayCsvStringifier({ + header: [ + "model", + "id", + "Code in Metadata Server", + "Code in Replica Server", + "Replica Server Number", + ], + }); + const header = discrepanciesCsvStringifier.getHeaderString(); + const rows = syncReport.metadataWithCodeDiscrepancies.flatMap(result => + result.items + .map(item => + discrepanciesCsvStringifier.stringifyRecords([ + [ + item.model, + item.id, + item.mainObject.code, + item.replicaObject.code ?? "", + item.replicaIndex + 1, + ], + ]) + ) + .join("") + ); + return [header, ...rows].join(""); + } + + private generateExclusiveMetadataCsv(syncReport: SyncMetadataReport): string { + const exclusiveCsvStringifier = createArrayCsvStringifier({ + header: ["model", "id", "name", "code", "server"], + }); + const header = exclusiveCsvStringifier.getHeaderString(); + const rows = syncReport.exclusiveMetadata.flatMap(result => + result.exclusive + .map(item => + exclusiveCsvStringifier.stringifyRecords([ + [ + item.object.model, + item.object.id, + item.object.name, + item.object.code ?? "", + item.source.type === "replica" ? `Replica ${item.source.index + 1}` : "metadata", + ], + ]) + ) + .join("") + ); + return [header, ...rows].join(""); + } +} diff --git a/src/scripts/commands/metadata/sync.ts b/src/scripts/commands/metadata/sync.ts new file mode 100644 index 00000000..d0731bb0 --- /dev/null +++ b/src/scripts/commands/metadata/sync.ts @@ -0,0 +1,64 @@ +import _ from "lodash"; +import { command, string, option } from "cmd-ts"; +import { buildAuthFromString, getD2ApiFromArgs, StringsSeparatedByCommas } from "scripts/common"; +import { MetadataD2Repository } from "data/MetadataD2Repository"; +import { readFileSync } from "fs"; +import { SyncMetadataUseCase } from "domain/usecases/SyncMetadataUseCase"; +import { SyncReport } from "./SyncReport"; + +type MetadataServer = { + url: string; + auth: string; + isMain: boolean; +}; + +function getRepositoriesFromJsonFile(jsonFilePath: string) { + const serverContentFile = readFileSync(jsonFilePath, "utf8"); + const { servers } = JSON.parse(serverContentFile) as unknown as { servers: MetadataServer[] }; + const mainServers = servers.filter(server => server.isMain); + const mainServer = mainServers[0]; + if (mainServers.length !== 1 || !mainServer) + throw new Error( + "Only one server can be the main one. Set isMain: true to the server you want to be the main one" + ); + + return { + mainMetadataRepository: new MetadataD2Repository( + getD2ApiFromArgs({ url: mainServer.url, auth: buildAuthFromString(mainServer.auth) }) + ), + repositories: servers + .filter(server => !server.isMain) + .map( + server => + new MetadataD2Repository( + getD2ApiFromArgs({ url: server.url, auth: buildAuthFromString(server.auth) }) + ) + ), + }; +} + +export const syncMetadata = command({ + name: "sync", + description: "Sync metadata between DHIS2 instances", + args: { + modelsToCheck: option({ + type: StringsSeparatedByCommas, + long: "check-models", + description: "DHIS2 models, comma-separated (dataSets, organisationUnits, users, ...)", + }), + serverConfig: option({ + type: string, + long: "server-config", + description: "Path to the JSON file with server configurations", + }), + }, + handler: async args => { + const metadataReposFromFile = getRepositoriesFromJsonFile(args.serverConfig); + const report = await new SyncMetadataUseCase( + metadataReposFromFile.mainMetadataRepository, + metadataReposFromFile.repositories + ).execute({ modelsToCheck: args.modelsToCheck }); + + new SyncReport().generateCsvReports(report); + }, +}); diff --git a/src/scripts/common.ts b/src/scripts/common.ts index b8c732e1..a2646620 100644 --- a/src/scripts/common.ts +++ b/src/scripts/common.ts @@ -130,9 +130,7 @@ export const IdsSeparatedByCommas: Type = { export const AuthString: Type = { async from(str) { - const [username, password] = str.split(":"); - if (!username || !password) throw new Error(`Invalid pair: ${str} (expected USERNAME:PASSWORD)`); - return { username, password }; + return buildAuthFromString(str); }, }; @@ -197,3 +195,9 @@ export const MetadataDate: Type = { } }, }; + +export const buildAuthFromString = (str: string): Auth => { + const [username, password] = str.split(":"); + if (!username || !password) throw new Error(`Invalid pair: ${str} (expected USERNAME:PASSWORD)`); + return { username, password }; +};