diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index cfc7e2b..5ad49b4 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -3,7 +3,7 @@ name: publish docker image on: push: branches: - - 'main' + - main tags: - v* @@ -27,7 +27,6 @@ jobs: push: runs-on: ubuntu-latest - if: github.event_name == 'push' steps: - name: Checkout diff --git a/apps/web/src/lib/server/appConfig/index.ts b/apps/web/src/lib/server/appConfig/index.ts index 352c030..76817ad 100644 --- a/apps/web/src/lib/server/appConfig/index.ts +++ b/apps/web/src/lib/server/appConfig/index.ts @@ -30,8 +30,9 @@ export function getAllSymbologyConfigs() { export function getClientSideSymbologyConfigs() { const allSymbologyConfigs = getAllSymbologyConfigs(); return (allSymbologyConfigs ?? []).map((symbologyConfig: SingleApiSymbolConfig) => { - const { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid } = symbologyConfig; - return { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid }; + const { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid, title } = + symbologyConfig; + return { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid, title }; }); } diff --git a/apps/web/src/routes/configs/+page.svelte b/apps/web/src/routes/configs/+page.svelte index e006828..74daeb0 100644 --- a/apps/web/src/routes/configs/+page.svelte +++ b/apps/web/src/routes/configs/+page.svelte @@ -85,6 +85,21 @@
+
+ +
+ + +
+
diff --git a/apps/web/src/routes/configs/utils.ts b/apps/web/src/routes/configs/utils.ts index 9f51104..716b8d2 100644 --- a/apps/web/src/routes/configs/utils.ts +++ b/apps/web/src/routes/configs/utils.ts @@ -12,6 +12,7 @@ export enum PriorityLevel { export interface FormFields { uuid: string; + title?: string; baseUrl: string; regFormId: string; visitFormId: string; @@ -39,6 +40,7 @@ export const defaultPriorityErrorValues = { export const initialValues: FormFields = { uuid: '', + title: '', baseUrl: '', regFormId: '', visitFormId: '', @@ -49,6 +51,7 @@ export const initialValues: FormFields = { export const configValidationSchema = yup.object().shape({ uuid: yup.string(), + title: yup.string(), baseUrl: yup.string().required('Base Url is required'), regFormId: yup.string().required('Geo point registration form is required'), visitFormId: yup.string().required('Visit form field is required'), @@ -92,8 +95,10 @@ export const configValidationSchema = yup.object().shape({ }); export const generateFilledData = (formFields: FormFields) => { - const { baseUrl, regFormId, visitFormId, apiToken, symbolConfig, schedule, uuid } = formFields; + const { baseUrl, regFormId, visitFormId, apiToken, symbolConfig, schedule, uuid, title } = + formFields; return { + title, baseUrl, regFormId, visitFormId, @@ -107,6 +112,7 @@ export const generateFilledData = (formFields: FormFields) => { export const getInitialValues = (data?: WebConfig): FormFields => { if (data) { return { + title: data.title, uuid: data.uuid, baseUrl: data.baseUrl, regFormId: data.regFormId, diff --git a/apps/web/src/routes/workflows/+page.server.ts b/apps/web/src/routes/workflows/+page.server.ts index 77511d7..5a53a78 100644 --- a/apps/web/src/routes/workflows/+page.server.ts +++ b/apps/web/src/routes/workflows/+page.server.ts @@ -1,5 +1,4 @@ import { getClientSideSymbologyConfigs, pipelineController } from '$lib/server/appConfig'; -import { getLastPipelineMetricForConfig } from '$lib/server/logger/configMetrics'; import type { ConfigRunner } from '@onaio/symbology-calc-core'; /** @type {import('./$types').PageLoad} */ @@ -7,14 +6,12 @@ export function load() { const configs = getClientSideSymbologyConfigs(); const ConfigsWithMetrics = configs.map((config) => { const configId = config.uuid; - const metricForThisConfig = getLastPipelineMetricForConfig(configId); const pipeLineRunner = pipelineController.getPipelines(configId) as ConfigRunner; const isRunning = pipeLineRunner?.isRunning(); return { ...config, - metric: metricForThisConfig, isRunning, - invalidityErrror: pipeLineRunner.invalidError + invalidityErrror: pipeLineRunner?.invalidError ?? null }; }); return { diff --git a/apps/web/src/routes/workflows/+page.svelte b/apps/web/src/routes/workflows/+page.svelte index f74a916..e408fc6 100644 --- a/apps/web/src/routes/workflows/+page.svelte +++ b/apps/web/src/routes/workflows/+page.svelte @@ -1,67 +1,14 @@ - - {#if data.configs.length === 0}
- +
No Pipeline configurations were detected. @@ -70,120 +17,34 @@
{:else}
- - {#each data.configs as config, idex} - {@const { tableHeaders, tableRows, colorsColSpan } = parseForTable(config)} - {@const metric = config.metric} -
-
- - - -
-
- {#if config.invalidityErrror !== null} -
-
- {config.invalidityErrror} -
-
- {/if} -
-
API Base url
-
{config.baseUrl}
-
Registration form Id
-
{config.regFormId}
-
Visit form Id
-
{config.visitFormId}
-
-
- - - - {#each tableHeaders as header, i} - {#if i === tableHeaders.length - 1} - - {:else} - - {/if} - {/each} - - - - {#each tableRows as row} - - {#each range(colorsColSpan + (tableHeaders.length - 1)) as idx} - {@const thisElement = row[idx]} - {#if thisElement === undefined} - - {:else} - - {/if} - {/each} - - {/each} - -
{header}{header}
- {:else if Array.isArray(thisElement)} - {thisElement[0]}{thisElement}
- Schedule:{convertCronToHuman(config.schedule)} -
-
-
Metrics for the last run
- {#if metric === undefined} -
-
- No previous run information was found for this Pipeline. -
-
- {:else} - {#if config.isRunning} - Pipeline is currently running. - {/if} -
-
Started
-
- {metric?.startTime ? formatTimestamp(metric?.startTime) : ' - '} -
-
Ended
-
- {metric?.endTime ? formatTimestamp(metric?.endTime) : ' - '} -
-
Total no. of facilities
-
{metric?.totalSubmissions ?? ' - '}
-
No. of facilities evaluated
-
{metric?.evaluated}
-
No. of registration submissions modified
-
{metric?.modified}
-
No. of registration submissions not modified due to error
-
{metric?.notModifiedWithError}
-
No. of registration submissions not modified without error
-
{metric?.notModifiedWithoutError}
-
- {/if} -
-
- {/each} + + + + + + + + + + + + + + {#each data.configs as config, idex} + + + + + + + + {/each} + + +
TitleBaseUrlfacility Reg Form IdVisit Form IdSchedule
+ + {config.title} + + {config.baseUrl}{config.regFormId}{config.visitFormId}{convertCronToHuman(config.schedule)}
{/if} - - diff --git a/apps/web/src/routes/workflows/reports/[slug]/+page.server.ts b/apps/web/src/routes/workflows/reports/[slug]/+page.server.ts new file mode 100644 index 0000000..eeb7b44 --- /dev/null +++ b/apps/web/src/routes/workflows/reports/[slug]/+page.server.ts @@ -0,0 +1,24 @@ +import { getClientSideSymbologyConfigs, pipelineController } from '$lib/server/appConfig'; +import { getLastPipelineMetricForConfig } from '$lib/server/logger/configMetrics'; +import type { ConfigRunner } from '@onaio/symbology-calc-core'; + +/** @type {import('./$types').PageLoad} */ +export function load({params}) { + const uuid = params.slug ?? ''; + const configs = getClientSideSymbologyConfigs().filter(config => config.uuid === uuid); + const ConfigsWithMetrics = configs.map((config) => { + const configId = config.uuid; + const metricForThisConfig = getLastPipelineMetricForConfig(configId); + const pipeLineRunner = pipelineController.getPipelines(configId) as ConfigRunner; + const isRunning = pipeLineRunner?.isRunning(); + return { + ...config, + metric: metricForThisConfig, + isRunning, + invalidityErrror: pipeLineRunner?.invalidError ?? null + }; + }); + return { + configs: ConfigsWithMetrics + }; +} diff --git a/apps/web/src/routes/workflows/reports/[slug]/+page.svelte b/apps/web/src/routes/workflows/reports/[slug]/+page.svelte new file mode 100644 index 0000000..e17cf0e --- /dev/null +++ b/apps/web/src/routes/workflows/reports/[slug]/+page.svelte @@ -0,0 +1,323 @@ + + + + +{#if data.configs.length === 0} +
+ +
+
+ No Pipeline configurations were detected. +
+
+
+{:else} +
+ + {#each data.configs as config, idex} + {@const { tableHeaders, tableRows, colorsColSpan } = parseForTable(config)} + {@const metric = config.metric} +
+
+ + + +
+
+ {#if config.invalidityErrror !== null} +
+
+ {config.invalidityErrror} +
+
+ {/if} +
+
Pipeline name
+
{config.title}
+
API Base url
+
{config.baseUrl}
+
Registration form Id
+
{config.regFormId}
+
Visit form Id
+
{config.visitFormId}
+
+
+ + + + {#each tableHeaders as header, i} + {#if i === tableHeaders.length - 1} + + {:else} + + {/if} + {/each} + + + + {#each tableRows as row} + + {#each range(colorsColSpan + (tableHeaders.length - 1)) as idx} + {@const thisElement = row[idx]} + {#if thisElement === undefined} + + {:else} + + {/if} + {/each} + + {/each} + +
{header}{header}
+ {:else if Array.isArray(thisElement)} + {thisElement[0]}{thisElement}
+ Schedule:{convertCronToHuman(config.schedule)} +
+
+
Metrics for the last run
+ {#if metric === undefined} +
+
+ No previous run information was found for this Pipeline. +
+
+ {:else} + {#if config.isRunning} + Pipeline is currently running. + {/if} +
+
pipeline ran for
+
+ {formatTriggerDuration(metric.trigger.from, metric.trigger.to, config.isRunning)} +
+
Pipeline triggered via
+
{metric.trigger.by}
+ +
Total no. of facilities
+
{metric?.totalFacilities ?? ' - '}
+ +
No. of facilities evaluated
+
{metric?.facilitiesEvaluated.total}
+ +
No. of facilities NOT evaluated
+
{metric?.facilitiesNotEvaluated.total}
+
+ +
+
+

+ Report for facilities Evaluated. ({metric.facilitiesEvaluated.total}) +

+

Modified.({metric.facilitiesEvaluated.modified.total})

+ + + + + + + + + + {#each Object.entries(getModifiedColors(metric)) as row} + + + + + + {/each} + + + + + + +
ChangecolorNo. of facilities
Marker color changed to    {row[1]}
Total + {metric.facilitiesEvaluated.modified.total} +
+ +

Not Modified.({metric.facilitiesEvaluated.notModified.total})

+ + + + + + + + + + {#each Object.entries(getNotModifiedReasons(metric)) as row} + + + + + + {/each} + + + + + + +
Reason codeReason descriptionNo. of facilities
{row[0]} {row[1].description}{row[1].total}
Total + {metric.facilitiesEvaluated.notModified.total} +
+ + +
+
+ +
+
+

+ Report for facilities NOT Evaluated. ({metric.facilitiesNotEvaluated.total}) +

+ + + + + + + + + + + {#each Object.entries(getNotEvaluatedReasons(metric)) as row} + + + + + + {/each} + + + + + + + +
Reason codeReason descriptionNo. of facilities
{row[0]} {row[1].description}{row[1].total}
Total + {metric.facilitiesNotEvaluated.total} +
+ + +
+
+ {/if} +
+
+ {/each} +
+{/if} + + diff --git a/apps/web/src/routes/workflows/reports/[slug]/utils.ts b/apps/web/src/routes/workflows/reports/[slug]/utils.ts new file mode 100644 index 0000000..9e53adc --- /dev/null +++ b/apps/web/src/routes/workflows/reports/[slug]/utils.ts @@ -0,0 +1,60 @@ +import type { ClientSideSingleSymbolConfig } from '$lib/shared/types'; + +export function parseForTable(singleConfig: ClientSideSingleSymbolConfig) { + const tableHeaders = [ + 'Priority Level', + 'Required visits frequency (days)', + '# days passed required visit' + ]; + const tableRows: (string | number | [number, string])[][] = []; + let colorsColSpan = 0; + (singleConfig.symbolConfig ?? []).forEach( + ({ priorityLevel, frequency, symbologyOnOverflow }, index) => { + if (tableRows[index] === undefined) { + tableRows[index] = []; + } + const symbologyOnOverFlow = symbologyOnOverflow.slice() ?? []; + // sort in ascending order + tableRows[index].push(priorityLevel); + tableRows[index].push(frequency); + const orderedSymbologyOnOverflow = symbologyOnOverFlow.sort( + (a, b) => a.overFlowDays - b.overFlowDays + ); + orderedSymbologyOnOverflow.forEach(({ overFlowDays, color }, idx) => { + tableRows[index].push([overFlowDays, color]); + const span = idx + 1; + if (span > colorsColSpan) { + colorsColSpan = span; + } + }); + } + ); + + return { tableHeaders, tableRows, colorsColSpan }; +} + +/** creates a human readable date time string + * @param timeStamp - time as timestamp to be converted. + */ +export function formatTimestamp(timeStamp: number) { + return new Date(timeStamp).toLocaleString(); +} + + +export function formatTriggerDuration(start?: number, end?: number, isRunning=false){ + if (start && end){ + return `${formatTimestamp(start)} to ${formatTimestamp(end)} (${((end - start) / 60000).toFixed(0)} mins)` + } + if(!start){ + return `Unable to determine when pipeline was started` + } + if(!end){ + if(isRunning){ + const runningFor = ((Date.now() - start) / 60000).toFixed(0) + return `Started at: ${formatTimestamp(start)}; (Running for ${runningFor} mins)` + } + else{ + return `Started at: ${formatTimestamp(start)}; (Ran cancelled before completion)` + } + } +} \ No newline at end of file diff --git a/apps/web/src/routes/workflows/utils.ts b/apps/web/src/routes/workflows/utils.ts index 5d8e93d..d179aaa 100644 --- a/apps/web/src/routes/workflows/utils.ts +++ b/apps/web/src/routes/workflows/utils.ts @@ -1,38 +1,5 @@ -import type { ClientSideSingleSymbolConfig } from '$lib/shared/types'; import cronstrue from 'cronstrue'; -export function parseForTable(singleConfig: ClientSideSingleSymbolConfig) { - const tableHeaders = [ - 'Priority Level', - 'Required visits frequency (days)', - '# days passed required visit' - ]; - const tableRows: (string | number | [number, string])[][] = []; - let colorsColSpan = 0; - (singleConfig.symbolConfig ?? []).forEach( - ({ priorityLevel, frequency, symbologyOnOverflow }, index) => { - if (tableRows[index] === undefined) { - tableRows[index] = []; - } - const symbologyOnOverFlow = symbologyOnOverflow.slice() ?? []; - // sort in ascending order - tableRows[index].push(priorityLevel); - tableRows[index].push(frequency); - const orderedSymbologyOnOverflow = symbologyOnOverFlow.sort( - (a, b) => a.overFlowDays - b.overFlowDays - ); - orderedSymbologyOnOverflow.forEach(({ overFlowDays, color }, idx) => { - tableRows[index].push([overFlowDays, color]); - const span = idx + 1; - if (span > colorsColSpan) { - colorsColSpan = span; - } - }); - } - ); - - return { tableHeaders, tableRows, colorsColSpan }; -} /** Converts a cron syntax string to huma readable string * @param cronString - cron-like syntax string. @@ -48,10 +15,3 @@ export function convertCronToHuman(cronString: string) { return ''; } } - -/** creates a human readable date time string - * @param timeStamp - time as timestamp to be converted. - */ -export function formatTimestamp(timeStamp: number) { - return new Date(timeStamp).toLocaleString(); -} diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index cd37609..f9e9fca 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -4,6 +4,9 @@ export const editSubmissionEndpoint = 'api/v1/submissions' as const; // field accessor names export const markerColorAccessor = 'marker-color'; -export const dateOfVisitAccessor = 'date_of_visit'; +export const dateOfVisitAccessor = 'endtime'; export const priorityLevelAccessor = 'priority_level'; export const numOfSubmissionsAccessor = 'num_of_submissions'; + +// magic strings +export const AbortErrorName = 'AbortError'; diff --git a/packages/core/src/evaluator/configRunner.ts b/packages/core/src/evaluator/configRunner.ts index 3de27ad..1bfc163 100644 --- a/packages/core/src/evaluator/configRunner.ts +++ b/packages/core/src/evaluator/configRunner.ts @@ -11,7 +11,21 @@ import { import cron from 'node-cron'; import { createMetricFactory } from './helpers/utils'; import { transformFacility } from './helpers/utils'; -import { Sig, Result } from '../helpers/Result'; +import { + Result, + UNKNOWN_TRANSFORM_FACILITY_ERROR, + ResultCodes, + UNKNOWN_SUCCESS_REASON, + EVALUATION_ABORTED, + SuccessResultDetail, + FailureResultDetail +} from '../helpers/Result'; +import { ReportMetric, TriggeredBy } from './metricReporter'; + +/** Config Runner: + * Create an object that has different properties that respectively represent a type of metric category. + * when there is an action of the specific category, add count to the necessary method. + */ /** * Represents a single config, Contains actions that pertain to the execution @@ -27,6 +41,8 @@ export class ConfigRunner { private abortController; /** stores validity of config */ public invalidError: string | null = null; + /** metric category store for each run */ + // private reporter: ReportMetric; constructor(config: Config) { this.config = config; @@ -36,10 +52,11 @@ export class ConfigRunner { } catch (err: unknown) { this.invalidError = (err as Error).message; } + // this.reporter = new ReportMetric(config.uuid); } /** Runs the pipeline, generator that yields metric information regarding the current run */ - async *transformGenerator() { + private async *transformGenerator(triggeredVia: TriggeredBy) { const config = this.config; const { regFormId, @@ -49,26 +66,16 @@ export class ConfigRunner { baseUrl, apiToken, uuid: configId, - regFormSubmissionChunks: facilityProcessingChunks + regFormSubmissionChunks: facilityProcessingChunks, + editSubmissionChunks: facilityEditChunks } = config; const regFormSubmissionChunks = facilityProcessingChunks ?? 1000; + const editSubmissionsChunks = facilityEditChunks ?? 100; + const reporter = new ReportMetric(configId); + reporter.updateStart(triggeredVia); - const startTime = Date.now(); - const createMetric = createMetricFactory(startTime, configId); - let evaluatedSubmissions = 0; - let notModifiedWithError = 0; - let notModifiedWithoutError = 0; - let modified = 0; let totalRegFormSubmissions = 0; - - // allows us to continously and progressively get reports on number of submissions evaluated. - yield createMetric( - evaluatedSubmissions, - notModifiedWithoutError, - notModifiedWithError, - modified, - totalRegFormSubmissions - ); + yield reporter.generateJsonReport(); const service = new OnaApiService(baseUrl, apiToken, logger, this.abortController); const colorDecider = colorDeciderFactory(symbolConfig, logger); @@ -76,18 +83,12 @@ export class ConfigRunner { abortableBlock: { const regForm = await service.fetchSingleForm(regFormId); if (regForm.isFailure) { - yield createMetric( - evaluatedSubmissions, - notModifiedWithoutError, - notModifiedWithError, - modified, - totalRegFormSubmissions, - true - ); + yield reporter.generateJsonReport(true); return; } const regFormSubmissionsNum = regForm.getValue()[numOfSubmissionsAccessor]; totalRegFormSubmissions = regFormSubmissionsNum; + reporter.updateTotalFacilities(totalRegFormSubmissions); // fetches submissions for the first form. const regFormSubmissionsIterator = service.fetchPaginatedFormSubmissionsGenerator( @@ -99,8 +100,16 @@ export class ConfigRunner { for await (const regFormSubmissionsResult of regFormSubmissionsIterator) { if (regFormSubmissionsResult.isFailure) { - if (regFormSubmissionsResult.errorCode === Sig.ABORT_EVALUATION) { + if (regFormSubmissionsResult.detail?.code === EVALUATION_ABORTED) { + // TODO - reporter can report that the pipeline was cancelled. + // this.updateMetric(EVALUATION_ABORTED, 0) break abortableBlock; + } else { + const { code, recsAffected } = (regFormSubmissionsResult.detail ?? + {}) as FailureResultDetail; + const sanitizedCode = code ?? UNKNOWN_TRANSFORM_FACILITY_ERROR; + // TODO - we are still updating metric reports as sideEffects in the code. + reporter.updateFacilitiesNotEvaluated(sanitizedCode, recsAffected ?? 0); } continue; } @@ -108,7 +117,8 @@ export class ConfigRunner { const regFormSubmissions = regFormSubmissionsResult.getValue(); const updateRegFormSubmissionsPromises = (regFormSubmissions as RegFormSubmission[]).map( (regFormSubmission) => async () => { - const modificationStatus = await transformFacility( + reporter.updateFacilitiesEvaluated(); + const transformFacilityResult = await transformFacility( service, regFormSubmission, regFormId, @@ -116,24 +126,41 @@ export class ConfigRunner { colorDecider, logger ); - const { modified: moded, error: modError } = modificationStatus; - evaluatedSubmissions++; - if (moded) { - modified++; + + let resultCode = transformFacilityResult.detail?.code; + + // TODO - refactor all this. + if (transformFacilityResult.isFailure) { + if (!resultCode) resultCode = UNKNOWN_TRANSFORM_FACILITY_ERROR; + reporter.updateEvaluatedNotModified(resultCode); + } else if ( + transformFacilityResult.isSuccess && + (transformFacilityResult.detail as SuccessResultDetail)?.colorChange === undefined + ) { + if (!resultCode) { + resultCode = UNKNOWN_SUCCESS_REASON; + } + reporter.updateEvaluatedNotModified(resultCode); } else { - modError ? notModifiedWithError++ : notModifiedWithoutError++; + if (!resultCode) { + resultCode = UNKNOWN_SUCCESS_REASON; + } + reporter.updateEvaluatedModified( + resultCode, + (transformFacilityResult.detail as SuccessResultDetail)?.colorChange as string + ); } } ); let cursor = 0; - const postChunks = 100; while (cursor <= updateRegFormSubmissionsPromises.length) { - const end = cursor + postChunks; + const end = cursor + editSubmissionsChunks; const chunksToSend = updateRegFormSubmissionsPromises.slice(cursor, end); - cursor = cursor + postChunks; + cursor = cursor + editSubmissionsChunks; await Promise.allSettled(chunksToSend.map((x) => x())); } + yield reporter.generateJsonReport(); } } logger?.( @@ -141,20 +168,13 @@ export class ConfigRunner { `Finished form pair {regFormId: ${config.regFormId}, visitFormId: ${config.visitFormId}}` ) ); - yield createMetric( - evaluatedSubmissions, - notModifiedWithoutError, - notModifiedWithError, - modified, - totalRegFormSubmissions, - true - ); + yield reporter.generateJsonReport(true); } /** Wrapper around the transform generator, collates the metrics and calls a callback that * inverts the control of writing the metric information to the configs writeMetric method. */ - async transform() { + async transform(triggeredVia: TriggeredBy = 'schedule') { const config = this.config; if (this.invalidError) { return Result.fail(`Configuration is not valid, ${this.invalidError}`); @@ -166,8 +186,8 @@ export class ConfigRunner { } else { this.running = true; let finalMetric; - for await (const metric of this.transformGenerator()) { - WriteMetric(metric); + for await (const metric of this.transformGenerator(triggeredVia)) { + WriteMetric(metric as any); finalMetric = metric; } this.running = false; diff --git a/packages/core/src/evaluator/helpers/utils.ts b/packages/core/src/evaluator/helpers/utils.ts index 6d63e78..f62afed 100644 --- a/packages/core/src/evaluator/helpers/utils.ts +++ b/packages/core/src/evaluator/helpers/utils.ts @@ -7,7 +7,7 @@ import { computeTimeToNow, createInfoLog } from '../../helpers/utils'; -import { Result } from '../../helpers/Result'; +import { MAKER_COLOR_UPDATED, NO_SYMBOLOGY_CHANGE_NEEDED, Result } from '../../helpers/Result'; /** Given a facility record, fetches its most recent visit, evaluates the marker color * and edits the facility record with the marker color @@ -26,8 +26,8 @@ export async function transformFacility( colorDecider: ReturnType, logger?: LogFn ) { - let modifificationStatus: { modified: boolean; error: string | null } = - createModificationStatus(false); + // let modifificationStatus: { modified: boolean; error: string | null } = + // createModificationStatus(false); const facilityId = regFormSubmission._id; const mostRecentVisitResult = await getMostRecentVisitDateForFacility( @@ -36,51 +36,52 @@ export async function transformFacility( visitFormId, logger ); + if (mostRecentVisitResult.isFailure) { - modifificationStatus = createModificationStatus(false, mostRecentVisitResult); - return modifificationStatus; + // modifificationStatus = createModificationStatus(false, mostRecentVisitResult); + // return modifificationStatus; + return Result.bubbleFailure(mostRecentVisitResult); } const mostRecentVisitDate = mostRecentVisitResult.getValue(); const timeDifference = computeTimeToNow(mostRecentVisitDate); - const color = colorDecider(timeDifference, regFormSubmission); - if (color) { - if (regFormSubmission[markerColorAccessor] === color) { - modifificationStatus = createModificationStatus(false); - logger?.( - createInfoLog( - `facility _id: ${facilityId} submission already has the correct color, no action needed` - ) - ); + const colorResult = colorDecider(timeDifference, regFormSubmission) as Result; + if (colorResult.isFailure) { + return Result.bubbleFailure(colorResult); + } + const color = colorResult.getValue(); + + if (regFormSubmission[markerColorAccessor] === color) { + logger?.( + createInfoLog( + `facility _id: ${facilityId} submission already has the correct color, no action needed` + ) + ); + return Result.ok(undefined, NO_SYMBOLOGY_CHANGE_NEEDED); + } else { + const uploadMarkerResult = await upLoadMarkerColor( + service, + regFormId, + regFormSubmission, + color + ); + if (uploadMarkerResult.isFailure) { + return Result.bubbleFailure(uploadMarkerResult); } else { - const uploadMarkerResult = await upLoadMarkerColor( - service, - regFormId, - regFormSubmission, - color - ); - if (uploadMarkerResult.isFailure) { - modifificationStatus = createModificationStatus(false, uploadMarkerResult); - } else { - modifificationStatus = createModificationStatus(true); - } + return Result.ok(undefined, { code: MAKER_COLOR_UPDATED, colorChange: color }); } - } else { - const coloringResult = Result.fail('Unable to determine color to assign'); - modifificationStatus = createModificationStatus(false, coloringResult); } - return modifificationStatus; } -/** Factory function that creates a status representing if/why facility was modified - * @param modified - whether facility was modified. - * @param result - why facility was or not modified. - */ -const createModificationStatus = (modified: boolean, result?: Result) => { - return { - modified, - error: result ? result.error : null - }; -}; +// /** Factory function that creates a status representing if/why facility was modified +// * @param modified - whether facility was modified. +// * @param result - why facility was or not modified. +// */ +// const createModificationStatus = (modified: boolean, result?: Result) => { +// return { +// modified, +// error: result ? result.error : null +// }; +// }; /** creates a function that abstracts creating metric objects. Metric objects represent the * intermediary or final status of a running pipeline. @@ -106,3 +107,20 @@ export const createMetricFactory = totalSubmissions } as Metric; }; + +// /** types of categories that we start with: +// * 1. no priority level - ECODE001 +// * 2. No visit submissions - ECODE002 +// * 3. Network error after retries. - ECODE003 +// * 4. cancelled. - ECODE004 +// * 5. editedWithColor - This is dynamic depends on the assigned color. SCODE_ +// * +// */ + +// // I guess we can use a class or a vanilla object to track this changes. +// const metricCategories = { +// ECODE001: 0, +// ECODE002: 0, +// ECODE003: 0, +// ECODE004: 0 +// } diff --git a/packages/core/src/evaluator/metricReporter.ts b/packages/core/src/evaluator/metricReporter.ts new file mode 100644 index 0000000..4d43151 --- /dev/null +++ b/packages/core/src/evaluator/metricReporter.ts @@ -0,0 +1,145 @@ +import { ResultCodes, ResultCodingsDescriptions } from '../helpers/Result'; + +export type TriggeredBy = 'schedule' | 'manual'; + +export class ReportMetric { + private triggeredBy?: TriggeredBy; + private triggeredStart?: number; + private triggeredEnd?: number; + private totalFacilities?: number; + private facilitiesEvaluated = 0; + private facilitiesNotEvaluated: Record = {}; + private facilitiesEvaluatedModified: Record> = {}; + private facilitiesEvaluatedNotModified: Record = {}; + private configId: string; + + constructor(configId: string) { + this.configId = configId; + } + + public updateStart(triggeredBy: TriggeredBy = 'schedule') { + this.triggeredBy = triggeredBy; + this.triggeredStart = Date.now(); + } + + public updateEnd() { + this.triggeredEnd = Date.now(); + } + + private updateMetric( + fieldName: 'facilitiesEvaluatedNotModified', + resultCode: ResultCodes | string, + value?: number + ) { + const fieldAccessor = fieldName; + if (this[fieldAccessor][resultCode] === undefined) { + this[fieldAccessor][resultCode] = 0; + } + if (value) { + this[fieldAccessor][resultCode] = value; + } else { + this[fieldAccessor][resultCode]++; + } + } + + public updateTotalFacilities(value: number) { + this.totalFacilities = value; + } + + public updateFacilitiesEvaluated() { + this.facilitiesEvaluated += 1; + } + + public updateFacilitiesNotEvaluated(code: ResultCodes | string, value: number) { + this.facilitiesNotEvaluated[code] = value; + } + + public updateEvaluatedNotModified(resultCode: ResultCodes | string, value?: number) { + this.updateMetric('facilitiesEvaluatedNotModified', resultCode, value); + } + + public updateEvaluatedModified(resultCode: ResultCodes | string, color: string, value?: number) { + if (this.facilitiesEvaluatedModified[resultCode] === undefined) { + this.facilitiesEvaluatedModified[resultCode] = {}; + } + if (this.facilitiesEvaluatedModified[resultCode][color] === undefined) { + this.facilitiesEvaluatedModified[resultCode][color] = 0; + } + this.facilitiesEvaluatedModified[resultCode][color]++; + } + + public generateJsonReport(closeReport = false) { + if (closeReport) { + this.updateEnd(); + } + + const modified = Object.values(this.facilitiesEvaluatedModified).reduce( + (acc, value) => { + for (const [key, val] of Object.entries(value)){ + acc.total += val; + acc[key] = val + } + return acc + }, + { total: 0 } as any + ); + + const notModified = Object.entries(this.facilitiesEvaluatedNotModified).reduce((acc, [key, value]) => { + acc.total += value; + acc[key] = { + total: value, + description: ResultCodingsDescriptions[key] as string + } + return acc + }, { total: 0 } as any) + + + const notEvaluated = Object.entries(this.facilitiesNotEvaluated).reduce((acc, [key, value]) => { + acc.total += value; + acc[key] = { + total: value, + description: ResultCodingsDescriptions[key] as string + } + return acc; + }, {total: 0} as any) + + return { + configId: this.configId, + trigger: { + by: this.triggeredBy, + from: this.triggeredStart, + to: this.triggeredEnd, + tookMills: + this.triggeredEnd && this.triggeredStart + ? ((this.triggeredEnd - this.triggeredStart)) + : undefined + }, + totalFacilities: this.totalFacilities, + totalFacilitiesEvaluated: this.facilitiesEvaluated, + facilitiesEvaluated: { + total: modified.total + notModified.total, + modified: { + ...modified + }, + notModified: { + ...notModified + } + }, + facilitiesNotEvaluated: { + ...notEvaluated + } + }; + } +} + +export interface JSONMetricReport { + configId: string; + trigger: { + by: TriggeredBy; + from: number; + to?: number; + tookMills?: number; + }; +} + +/** Needs some re-achitecturing */ diff --git a/packages/core/src/evaluator/pipelinesController.ts b/packages/core/src/evaluator/pipelinesController.ts index 2c661c8..3a852ca 100644 --- a/packages/core/src/evaluator/pipelinesController.ts +++ b/packages/core/src/evaluator/pipelinesController.ts @@ -125,7 +125,7 @@ export class PipelinesController { if (!interestingPipeline) { return Result.fail(`Pipeline with config ${configId} was not found`); } - interestingPipeline.transform(); + interestingPipeline.transform('manual'); return Result.ok('Pipeline triggered successfully, running in the background'); } diff --git a/packages/core/src/evaluator/tests/helpers.test.ts b/packages/core/src/evaluator/tests/helpers.test.ts index 4b93191..1698161 100644 --- a/packages/core/src/evaluator/tests/helpers.test.ts +++ b/packages/core/src/evaluator/tests/helpers.test.ts @@ -103,8 +103,14 @@ describe('transform facility tests', () => { ); expect(response).toEqual({ - error: null, - modified: true + _value: undefined, + detail: { + code: 'SCODE1', + colorChange: 'green' + }, + error: undefined, + isFailure: false, + isSuccess: true }); }); @@ -166,7 +172,16 @@ describe('transform facility tests', () => { logger ); - expect(response.modified).toBeFalsy(); + expect(response).toEqual({ + _value: undefined, + detail: { + code: 'ECODE3', + recsAffected: 0 + }, + error: '400: {"message":"error"}: Network request failed.', + isFailure: true, + isSuccess: false + }); expect(response.error).toEqual('400: {"message":"error"}: Network request failed.'); }); @@ -221,8 +236,14 @@ describe('transform facility tests', () => { controller.abort(); await response.then((value) => { expect(value).toEqual({ - error: 'aborted: AbortError: The user aborted a request..', - modified: false + _value: undefined, + detail: { + code: 'ECODE3', + recsAffected: 0 + }, + error: 'AbortError: The user aborted a request..', + isFailure: true, + isSuccess: false }); }); }); diff --git a/packages/core/src/evaluator/tests/loadTest.ts/mockServer.ts b/packages/core/src/evaluator/tests/loadTest.ts/mockServer.ts index 01dd58a..b09d59a 100644 --- a/packages/core/src/evaluator/tests/loadTest.ts/mockServer.ts +++ b/packages/core/src/evaluator/tests/loadTest.ts/mockServer.ts @@ -56,7 +56,7 @@ function generateSingleVisitSub(facilitId: number, geoLocation = {}) { 'meta/instanceID': `uuid:${uuid}`, 'meta/deprecatedID': `uuid:${datatype.uuid()}`, _geolocation: geoLocation, - date_of_visit: formatVisitDate(dateOfVisit) + endtime: formatVisitDate(dateOfVisit) }; } diff --git a/packages/core/src/evaluator/tests/pipelinesController.test.ts b/packages/core/src/evaluator/tests/pipelinesController.test.ts index 4c1699d..ed32ae9 100644 --- a/packages/core/src/evaluator/tests/pipelinesController.test.ts +++ b/packages/core/src/evaluator/tests/pipelinesController.test.ts @@ -7,7 +7,24 @@ import { form3623Submissions, form3624Submissions } from './fixtures/fixtures'; -import { logCalls } from './fixtures/logCalls'; + +const scheduler = typeof setImmediate === 'function' ? setImmediate : setTimeout; + +function flushPromises() { + return new Promise(function (resolve) { + scheduler(resolve); + }); +} + +const jestConsole = console; + +beforeEach(() => { + global.console = require('console'); +}); + +afterEach(() => { + global.console = jestConsole; +}); // eslint-disable-next-line @typescript-eslint/no-var-requires const nock = require('nock'); @@ -64,7 +81,7 @@ it('works correctly nominal case', async () => { page_size: 1, page: 1, query: `{"facility": ${facilityId}}`, - sort: '{"date_of_visit": -1}' + sort: '{"endtime": -1}' }) .reply( 200, @@ -119,18 +136,24 @@ it('works correctly nominal case', async () => { const metric = await response; expect(runner.isRunning()).toBeFalsy(); - expect(loggerMock.mock.calls).toEqual(logCalls); + // expect(loggerMock.mock.calls).toEqual(logCalls); expect(metric.getValue()).toEqual({ configId: 'uuid', - endTime: 1673275673342, - evaluated: 10, - modified: 8, - notModifiedWithError: 2, - notModifiedWithoutError: 0, - startTime: 1673275673342, - totalSubmissions: 10 + facilitiesEvaluated: { + modified: { red: 8, total: 8 }, + notModified: { + ECODE2: { description: 'Facility does not have a priority level', total: 2 }, + total: 2 + }, + total: 10 + }, + facilitiesNotEvaluated: { total: 0 }, + totalFacilities: 10, + totalFacilitiesEvaluated: 10, + trigger: { by: 'schedule', from: 1673275673342, to: 1673275673342, tookMills: 0 } }); + await flushPromises(); expect(nock.pendingMocks()).toEqual([]); }); @@ -139,7 +162,7 @@ it('error when fetching the registration form', async () => { const configs = createConfigs(loggerMock); // mock fetched firstform - nock(configs.baseUrl).get(`/${formEndpoint}/3623`).replyWithError('Could not find form with id'); + nock(configs.baseUrl).get(`/${formEndpoint}/3623`).reply(400, 'Could not find form with id'); const pipelinesController = new PipelinesController(() => [configs]); const configRunner = pipelinesController.getPipelines(configs.uuid) as ConfigRunner; @@ -153,7 +176,7 @@ it('error when fetching the registration form', async () => { { level: 'error', message: - 'Operation to fetch form: 3623, failed with err: Error: system: FetchError: request to https://test-api.ona.io/api/v1/forms/3623 failed, reason: Could not find form with id.' + 'Operation to fetch form: 3623, failed with err: Error: 400: Could not find form with id: Network request failed.' } ] ]); diff --git a/packages/core/src/helpers/Result.ts b/packages/core/src/helpers/Result.ts index 48d68bf..56ab0c9 100644 --- a/packages/core/src/helpers/Result.ts +++ b/packages/core/src/helpers/Result.ts @@ -1,7 +1,87 @@ -// Error Codes: -export enum Sig { - ABORT_EVALUATION = 'abort_evaluation' -} +// Result Codes: - known static codes that can be used when reporting the +// result of an operation. + +// # error - associated. +export const EVALUATION_ABORTED = 'ECODE1' as const; +export const MISSING_PRIORITY_LEVEL = 'ECODE2' as const; +export const NETWORK_ERROR = 'ECODE3' as const; +export const UNRECOGNIZED_PRIORITY_LEVEL = 'ECODE5' as const; +export const UNKNOWN_TRANSFORM_FACILITY_ERROR = 'ECODE6' as const; + +export type ErrorResultCodes = + | typeof EVALUATION_ABORTED + | typeof MISSING_PRIORITY_LEVEL + | typeof NETWORK_ERROR + | typeof UNRECOGNIZED_PRIORITY_LEVEL + | typeof UNKNOWN_TRANSFORM_FACILITY_ERROR; + +// # warning - associated +export const NO_VISIT_SUBMISSIONS = 'WCODE1' as const; + +export type WarningResultCodes = typeof NO_VISIT_SUBMISSIONS; + +// # Info associated +export const NO_SYMBOLOGY_CHANGE_NEEDED = 'ICODE1' as const; + +export type InfoResultCodes = typeof NO_SYMBOLOGY_CHANGE_NEEDED; + +// # success associated +export const MAKER_COLOR_UPDATED = 'SCODE1'; +export const UNKNOWN_SUCCESS_REASON = 'SCODE2'; + +export type SuccessResultCodes = typeof UNKNOWN_SUCCESS_REASON | typeof MAKER_COLOR_UPDATED; + +export const ResultCodings = { + evaluationAborted: { + code: EVALUATION_ABORTED, + description: 'Evaluation was cancelled' + }, + missingPriorityLevel: { + code: MISSING_PRIORITY_LEVEL, + description: 'Facility does not have a priority level' + }, + networkError: { + code: NETWORK_ERROR, + description: 'Request failed due to an unrecoverable network error' + }, + noVisitSubmissions: { + code: NO_VISIT_SUBMISSIONS, + description: 'Facility do not have visit submissions' + }, + unrecognizedPriorityLevel: { + code: UNRECOGNIZED_PRIORITY_LEVEL, + description: 'Facility has an invalid priority level' + }, + noSymblologyChangeNeeded: { + code: NO_SYMBOLOGY_CHANGE_NEEDED, + description: 'Facitlity already has the correct symbology marker color' + }, + unknownTransformFacilityError: { + code: UNKNOWN_TRANSFORM_FACILITY_ERROR, + description: 'Reason for result status is unknown' + }, + markerColorUpdated: { + code: MAKER_COLOR_UPDATED, + description: "The marker color was successfully updated," + } +}; + +export const ResultCodingsDescriptions = Object.values(ResultCodings).reduce((acc, value) => { + return { + ...acc, + [value.code]: value.description + }; +}, {}) as any; + +export type ResultCodes = + | ErrorResultCodes + | InfoResultCodes + | SuccessResultCodes + | WarningResultCodes; + +export type SuccessResultDetail = { code?: ResultCodes | string; colorChange?: string }; +export type FailureResultDetail = { code?: ResultCodes | string; recsAffected?: number }; +export type ResultDetail = SuccessResultDetail | FailureResultDetail; /** This is a generic interface that describes the output (or ... Result) of * a function. @@ -15,10 +95,10 @@ export class Result { public isSuccess: boolean; public isFailure: boolean; public error: string; - public errorCode?: Sig; + public detail?: ResultDetail; private _value: T; - private constructor(isSuccess: boolean, error?: string, value?: T, sig?: Sig) { + private constructor(isSuccess: boolean, error?: string, value?: T, detail?: ResultDetail) { if (isSuccess && error) { throw new Error(`InvalidOperation: A result cannot be successful and contain an error`); @@ -34,7 +114,7 @@ export class Result { this.error = error!; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion this._value = value!; - this.errorCode = sig; + this.detail = detail; Object.freeze(this); } @@ -47,11 +127,31 @@ export class Result { return this._value; } - public static ok(value?: U): Result { - return new Result(true, undefined, value); + public static ok(value?: U, detail?: ResultCodes | string | ResultDetail): Result { + let resultDetail; + if (typeof detail === 'string') { + resultDetail = { code: detail }; + } else { + resultDetail = detail; + } + return new Result(true, undefined, value, resultDetail); } - public static fail(error: string, code?: Sig): Result { - return new Result(false, error, undefined, code); + public static fail(error: string, detail?: ResultDetail | ResultCodes): Result { + let resultDetail; + if (typeof detail === 'string') { + resultDetail = { code: detail }; + } else { + resultDetail = detail; + } + return new Result(false, error, undefined, resultDetail); + } + + public static bubbleFailure(result: Result): Result { + // result should be a failure. + if (!result.isFailure) { + throw new Error('Invalid operation: you can only buble up a failed result'); + } + return this.fail(result.error, result.detail); } } diff --git a/packages/core/src/helpers/tests/utils.test.ts b/packages/core/src/helpers/tests/utils.test.ts index a1d9494..e3cbccf 100644 --- a/packages/core/src/helpers/tests/utils.test.ts +++ b/packages/core/src/helpers/tests/utils.test.ts @@ -31,41 +31,41 @@ describe('colorDecider', () => { it('Marks very high priotiry facilities correctly', () => { const submission = form3623Submissions[0] as RegFormSubmission; - expect(colorDecider(0, submission)).toEqual('green'); - expect(colorDecider(2, submission)).toEqual('green'); - expect(colorDecider(3, submission)).toEqual('green'); - expect(colorDecider(4, submission)).toEqual('yellow'); - expect(colorDecider(5, submission)).toEqual('red'); - expect(colorDecider(99999, submission)).toEqual('red'); + expect(colorDecider(0, submission).getValue()).toEqual('green'); + expect(colorDecider(2, submission).getValue()).toEqual('green'); + expect(colorDecider(3, submission).getValue()).toEqual('green'); + expect(colorDecider(4, submission).getValue()).toEqual('yellow'); + expect(colorDecider(5, submission).getValue()).toEqual('red'); + expect(colorDecider(99999, submission).getValue()).toEqual('red'); }); it('Marks high priotiry facilities correctly', () => { const submission = form3623Submissions[1] as RegFormSubmission; - expect(colorDecider(0, submission)).toEqual('green'); - expect(colorDecider(6, submission)).toEqual('green'); - expect(colorDecider(7, submission)).toEqual('green'); - expect(colorDecider(8, submission)).toEqual('red'); - expect(colorDecider(9, submission)).toEqual('red'); - expect(colorDecider(99999, submission)).toEqual('red'); + expect(colorDecider(0, submission).getValue()).toEqual('green'); + expect(colorDecider(6, submission).getValue()).toEqual('green'); + expect(colorDecider(7, submission).getValue()).toEqual('green'); + expect(colorDecider(8, submission).getValue()).toEqual('red'); + expect(colorDecider(9, submission).getValue()).toEqual('red'); + expect(colorDecider(99999, submission).getValue()).toEqual('red'); }); it('Marks medium priotiry facilities correctly', () => { const submission = form3623Submissions[3] as RegFormSubmission; - expect(colorDecider(0, submission)).toEqual('green'); - expect(colorDecider(13, submission)).toEqual('green'); - expect(colorDecider(14, submission)).toEqual('green'); - expect(colorDecider(15, submission)).toEqual('red'); - expect(colorDecider(16, submission)).toEqual('red'); - expect(colorDecider(99999, submission)).toEqual('red'); + expect(colorDecider(0, submission).getValue()).toEqual('green'); + expect(colorDecider(13, submission).getValue()).toEqual('green'); + expect(colorDecider(14, submission).getValue()).toEqual('green'); + expect(colorDecider(15, submission).getValue()).toEqual('red'); + expect(colorDecider(16, submission).getValue()).toEqual('red'); + expect(colorDecider(99999, submission).getValue()).toEqual('red'); }); it('Marks low priotiry facilities correctly', () => { const submission = form3623Submissions[2] as RegFormSubmission; - expect(colorDecider(0, submission)).toEqual('green'); - expect(colorDecider(29, submission)).toEqual('green'); - expect(colorDecider(30, submission)).toEqual('green'); - expect(colorDecider(31, submission)).toEqual('red'); - expect(colorDecider(32, submission)).toEqual('red'); - expect(colorDecider(99999, submission)).toEqual('red'); + expect(colorDecider(0, submission).getValue()).toEqual('green'); + expect(colorDecider(29, submission).getValue()).toEqual('green'); + expect(colorDecider(30, submission).getValue()).toEqual('green'); + expect(colorDecider(31, submission).getValue()).toEqual('red'); + expect(colorDecider(32, submission).getValue()).toEqual('red'); + expect(colorDecider(99999, submission).getValue()).toEqual('red'); }); }); diff --git a/packages/core/src/helpers/types.ts b/packages/core/src/helpers/types.ts index c789d29..97ee0c1 100644 --- a/packages/core/src/helpers/types.ts +++ b/packages/core/src/helpers/types.ts @@ -22,6 +22,8 @@ export type CronTabString = string; export interface Config { // an id: helps with managing the configs uuid: string; + // title: human readable string that identifies config + title?: string; // id for form used to register the geo points regFormId: string; // id for form used by Health workers to visit added geopoints @@ -38,6 +40,8 @@ export interface Config { schedule: CronTabString; // how many registration form submissions to process at a time. regFormSubmissionChunks?: number; + // out of regFormSubmissionChunk how many should be posted/edited at a time + editSubmissionChunks?: number; // store metric; progress information regarding a running pipeline or the last run of an pipeline writeMetric: WriteMetric; } @@ -99,7 +103,7 @@ export interface RegFormSubmission extends BaseFormSubmission { } export interface VisitFormSubmission extends BaseFormSubmission { - date_of_visit: string; + endtime: string; } export interface Form { diff --git a/packages/core/src/helpers/utils.ts b/packages/core/src/helpers/utils.ts index 71e6162..ca6be31 100644 --- a/packages/core/src/helpers/utils.ts +++ b/packages/core/src/helpers/utils.ts @@ -15,7 +15,7 @@ import * as yup from 'yup'; import nodeCron from 'node-cron'; import { dateOfVisitAccessor, priorityLevelAccessor } from '../constants'; import { OnaApiService } from '../services/onaApi/services'; -import { Result } from './Result'; +import { MISSING_PRIORITY_LEVEL, NO_VISIT_SUBMISSIONS, Result, UNRECOGNIZED_PRIORITY_LEVEL } from './Result'; export const createInfoLog = (message: string) => ({ level: LogMessageLevels.INFO, message }); export const createWarnLog = (message: string) => ({ level: LogMessageLevels.WARN, message }); @@ -48,12 +48,18 @@ export const colorDeciderFactory = (symbolConfig: SymbologyConfig, logger?: LogF if (!thisFacilityPriority) { logger?.(createWarnLog(`facility _id: ${submission._id} does not have a priority_level`)); - return; + return Result.fail("Missing priority level", MISSING_PRIORITY_LEVEL); } // TODO - risky coupling. const symbologyConfigByPriorityLevel = keyBy(orderedSymbologyConfig, 'priorityLevel'); const symbologyConfig = symbologyConfigByPriorityLevel[thisFacilityPriority]; + // TODO - when priority_level is unrecognized. - do we need to also report facilities affected by this errors + if(symbologyConfig === undefined){ + logger?.(createWarnLog(`facility _id: ${submission._id} as priority_level ${thisFacilityPriority}: Unrecognized priority level`)) + return Result.fail("Unrecognized priority level", UNRECOGNIZED_PRIORITY_LEVEL) + } + const overflowsConfig = symbologyConfig.symbologyOnOverflow; let colorChoice = overflowsConfig[overflowsConfig.length - 1].color; @@ -65,7 +71,7 @@ export const colorDeciderFactory = (symbolConfig: SymbologyConfig, logger?: LogF break; } } - return colorChoice; + return Result.ok(colorChoice); }; return colorDecider; @@ -73,6 +79,7 @@ export const colorDeciderFactory = (symbolConfig: SymbologyConfig, logger?: LogF export const configValidationSchema = yup.object().shape({ uuid: yup.string().required('Config does not have an identifier'), + title: yup.string(), baseUrl: yup.string().required('Base Url is required'), regFormId: yup.string().required('Geo point registration form is required'), visitFormId: yup.string().required('Visit form field is required'), @@ -110,6 +117,7 @@ export async function getMostRecentVisitDateForFacility( visitFormId: string, logger?: LogFn ) { + // can run into an error, // can yield an empty result. const query = { @@ -121,17 +129,19 @@ export async function getMostRecentVisitDateForFacility( const formSubmissionIterator = service.fetchPaginatedFormSubmissionsGenerator(visitFormId, 1, query, 1); - const visitSubmissionsResult = (await formSubmissionIterator - .next() - .then((res) => res.value)) as Result; - + const visitSubmissionsResult = (await formSubmissionIterator + .next() + .then((res) => res.value)) as Result; + if (visitSubmissionsResult.isFailure) { logger?.( createErrorLog( `Operation to fetch submission for facility: ${facilityId} failed with error: ${visitSubmissionsResult.error}` ) ); - return Result.fail(visitSubmissionsResult.error); + + // TODO - can we create a result from another result; can add method passResult. + return Result.bubbleFailure(visitSubmissionsResult); } const visitSubmissions = visitSubmissionsResult.getValue(); @@ -148,7 +158,7 @@ export async function getMostRecentVisitDateForFacility( return Result.ok(dateOfVisit); } else { logger?.(createWarnLog(`facility _id: ${facilityId} has no visit submissions`)); - return Result.ok(); + return Result.ok(undefined, NO_VISIT_SUBMISSIONS); } } diff --git a/packages/core/src/services/onaApi/services.ts b/packages/core/src/services/onaApi/services.ts index f0fcfc9..f8b5c6a 100644 --- a/packages/core/src/services/onaApi/services.ts +++ b/packages/core/src/services/onaApi/services.ts @@ -1,5 +1,6 @@ import { flatMap } from 'lodash-es'; import { + AbortErrorName, editSubmissionEndpoint, formEndpoint, markerColorAccessor, @@ -9,10 +10,11 @@ import { v4 } from 'uuid'; import { BaseFormSubmission, Color, Form, LogFn, RegFormSubmission } from '../../helpers/types'; import { createErrorLog, createInfoLog, createVerboseLog } from '../../helpers/utils'; import fetchRetry, { RequestInitWithRetry } from 'fetch-retry'; -import { Result } from '../../helpers/Result'; +import { NETWORK_ERROR, Result } from '../../helpers/Result'; const persistentFetch = fetchRetry(fetch); +// TODO - move Result codes reporting into custom fetch as well. /** Wrapper around the default fetch function. Adds a retyr mechanism based on exponential backof * @param input - url or a request object representing the request to be made * @param init - fetch options. @@ -21,36 +23,36 @@ export const customFetch = async (input: RequestInfo, init?: RequestInit, logger // The exponential backoff strategy can be hardcoded, should it be left to the calling function. // post requests are not idempotent const numOfRetries = 10; - const delayConstant = 500; //ms + const delayConstant = 15000; //ms const requestOptionsWithRetry: RequestInitWithRetry = { ...init, retries: numOfRetries, retryOn: function (_, error, response) { let retry = false; const method = init?.method ?? 'GET'; - if (error) { + if (error && error.name !== AbortErrorName) { retry = method === 'GET'; } if (response) { const status = response?.status; // retry on all server side error http codes. - retry = status >= 500 && status < 600; + retry = (status >= 500 && status < 600) || ([429].includes(status)); } if (retry) { const msg = response - ? `Retrying request; request respondend with status: ${response?.status}` - : 'Retrying request, Request does not have a response'; + ? `Retrying request ${input}; request respondend with status: ${response?.status}` + : `Retrying request ${input}, Request does not have a response`; logger?.(createVerboseLog(msg)); } return retry; }, retryDelay: function (attempt) { - return Math.pow(2, attempt) * delayConstant; + return attempt * delayConstant; } }; const response = await persistentFetch(input, requestOptionsWithRetry).catch((err) => { - throw Error(`${err.name}: ${err.message}.`); + throw Error(`${response.status}: ${err.name}: ${err.message}.`); }); if (response?.ok) { return response; @@ -104,7 +106,7 @@ export class OnaApiService { this.logger?.( createErrorLog(`Operation to fetch form: ${formId}, failed with err: ${err}`) ); - return Result.fail(err); + return Result.fail(err, NETWORK_ERROR); }); } @@ -156,12 +158,12 @@ export class OnaApiService { getSubmissionsPath: string = submittedDataEndpoint ) { const fullSubmissionsUrl = `${this.baseUrl}/${getSubmissionsPath}/${formId}`; - let page = 1; + let page = 0; do { const query = { page_size: `${pageSize}`, - page: `${page}`, + page: `${page + 1}`, ...extraQueryObj }; const sParams = new URLSearchParams(query); @@ -189,7 +191,11 @@ export class OnaApiService { `Unable to fetch submissions for form id: ${formId} page: ${paginatedSubmissionsUrl} with err : ${err.message}` ) ); - return Result.fail(err.message); + let recsAffected = pageSize; + if((totalSubmissions - (page * pageSize)) < pageSize )[ + recsAffected = totalSubmissions - (page * pageSize) + ] + return Result.fail(err.message, {code: NETWORK_ERROR, recsAffected, }); }); } while (page * pageSize <= totalSubmissions); } @@ -243,7 +249,7 @@ export class OnaApiService { `Failed to edit sumbission with _id: ${submissionPayload._id} for form with id: ${formId} with err: ${err.message}` ) ); - return Result.fail(err); + return Result.fail(err, NETWORK_ERROR); }); } }