Merge v1.0.1-rc4.1/v1.1.0 (#12)

* Fix pipelineRunner can be undefined * Do not retry requests that were aborted * Add config for how many submissions should be posted at a time * Update docker publish gh action * Add title to config * Rough work around extracting metrics reporting to its own module * Update workflows ui to include deidcated report view * Fix regressions in package core * Drop log reporting threshhold for console transport to silly * Change visit form date-of-visit field to endtime Lay ground work to enable configuring the form field names via configs * Revert "Change visit form date-of-visit field to endtime" This reverts commit 259d178. * Modify date_of_visit field to endtime * Fix bug in colorDecider util, update backoff mechanism
onaio · Dec 15, 2023 · 26e51b9 · 26e51b9
1 parent 30c5289
commit 26e51b9
Show file tree

Hide file tree

Showing 23 changed files with 979 additions and 383 deletions.
diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml
@@ -3,7 +3,7 @@ name: publish docker image
 on:
   push:
     branches:
-      - 'main'
+      - main
     tags:
       - v*
 
@@ -27,7 +27,6 @@ jobs:
 
   push:
     runs-on: ubuntu-latest
-    if: github.event_name == 'push'
     steps:
 
       - name: Checkout

diff --git a/apps/web/src/lib/server/appConfig/index.ts b/apps/web/src/lib/server/appConfig/index.ts
@@ -30,8 +30,9 @@ export function getAllSymbologyConfigs() {
 export function getClientSideSymbologyConfigs() {
 	const allSymbologyConfigs = getAllSymbologyConfigs();
 	return (allSymbologyConfigs ?? []).map((symbologyConfig: SingleApiSymbolConfig) => {
-		const { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid } = symbologyConfig;
-		return { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid };
+		const { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid, title } =
+			symbologyConfig;
+		return { baseUrl, visitFormId, regFormId, symbolConfig, schedule, uuid, title };
 	});
 }
 

diff --git a/apps/web/src/routes/configs/+page.svelte b/apps/web/src/routes/configs/+page.svelte
@@ -85,6 +85,21 @@
 <div class="row g-3 mx-auto">
 	<div class="col-sm-6 col-md-8">
 		<form class="form" on:submit={handleSubmit}>
+			<div class="form-group row mb-2">
+				<label for="regFormId" class="col-sm-3">Name</label>
+				<div class="col-sm-9">
+					<input
+						type="text"
+						id="title"
+						class="form-control"
+						name={`title`}
+						on:change={handleChange}
+						on:blur={handleChange}
+						bind:value={$form.title}
+					/>
+					<ErrorMessage {errors} name="title" />
+				</div>
+			</div>
 			<div class="form-group row mb-2">
 				<label for="regFormId" class="col-sm-3">Registration Form Id</label>
 				<div class="col-sm-9">

diff --git a/apps/web/src/routes/configs/utils.ts b/apps/web/src/routes/configs/utils.ts
@@ -12,6 +12,7 @@ export enum PriorityLevel {
 
 export interface FormFields {
 	uuid: string;
+	title?: string;
 	baseUrl: string;
 	regFormId: string;
 	visitFormId: string;
@@ -39,6 +40,7 @@ export const defaultPriorityErrorValues = {
 
 export const initialValues: FormFields = {
 	uuid: '',
+	title: '',
 	baseUrl: '',
 	regFormId: '',
 	visitFormId: '',
@@ -49,6 +51,7 @@ export const initialValues: FormFields = {
 
 export const configValidationSchema = yup.object().shape({
 	uuid: yup.string(),
+	title: yup.string(),
 	baseUrl: yup.string().required('Base Url is required'),
 	regFormId: yup.string().required('Geo point registration form is required'),
 	visitFormId: yup.string().required('Visit form field is required'),
@@ -92,8 +95,10 @@ export const configValidationSchema = yup.object().shape({
 });
 
 export const generateFilledData = (formFields: FormFields) => {
-	const { baseUrl, regFormId, visitFormId, apiToken, symbolConfig, schedule, uuid } = formFields;
+	const { baseUrl, regFormId, visitFormId, apiToken, symbolConfig, schedule, uuid, title } =
+		formFields;
 	return {
+		title,
 		baseUrl,
 		regFormId,
 		visitFormId,
@@ -107,6 +112,7 @@ export const generateFilledData = (formFields: FormFields) => {
 export const getInitialValues = (data?: WebConfig): FormFields => {
 	if (data) {
 		return {
+			title: data.title,
 			uuid: data.uuid,
 			baseUrl: data.baseUrl,
 			regFormId: data.regFormId,

diff --git a/apps/web/src/routes/workflows/+page.server.ts b/apps/web/src/routes/workflows/+page.server.ts
@@ -1,20 +1,17 @@
 import { getClientSideSymbologyConfigs, pipelineController } from '$lib/server/appConfig';
-import { getLastPipelineMetricForConfig } from '$lib/server/logger/configMetrics';
 import type { ConfigRunner } from '@onaio/symbology-calc-core';
 
 /** @type {import('./$types').PageLoad} */
 export function load() {
 	const configs = getClientSideSymbologyConfigs();
 	const ConfigsWithMetrics = configs.map((config) => {
 		const configId = config.uuid;
-		const metricForThisConfig = getLastPipelineMetricForConfig(configId);
 		const pipeLineRunner = pipelineController.getPipelines(configId) as ConfigRunner;
 		const isRunning = pipeLineRunner?.isRunning();
 		return {
 			...config,
-			metric: metricForThisConfig,
 			isRunning,
-			invalidityErrror: pipeLineRunner.invalidError
+			invalidityErrror: pipeLineRunner?.invalidError ?? null
 		};
 	});
 	return {

diff --git a/apps/web/src/routes/workflows/+page.svelte b/apps/web/src/routes/workflows/+page.svelte
@@ -1,67 +1,14 @@
 <script lang="ts">
 	import type { PageData } from './$types';
-	import { range } from 'lodash-es';
-	import { convertCronToHuman, formatTimestamp, parseForTable } from './utils';
+	import { convertCronToHuman } from './utils';
 	import PageHeader from '$lib/shared/components/PageHeader.svelte';
-	import { goto, invalidate } from '$app/navigation';
-	import { toast } from '@zerodevx/svelte-toast';
 
 	export let data: PageData;
-
-	// callback for manual trigger user action.
-	const manualTrigger = async (uuid: string) => {
-		const sParams = new URLSearchParams({
-			uuid
-		});
-		const fullUrl = `/workflows/run?${sParams.toString()}`;
-		return await fetch(fullUrl)
-			.then((res) => {
-				res.json().then((body) => {
-					toast.push(body.message);
-				});
-			})
-			.catch((err) => {
-				toast.push(err.message);
-			});
-	};
-
-	// callback for edit trigger user action.
-	const editTrigger = async (uuid: string) => {
-		const sParams = new URLSearchParams({
-			uuid
-		});
-		const fullUrl = `/configs?${sParams.toString()}`;
-		goto(fullUrl);
-	};
-
-	// callback for delete trigger user action.
-	const deleteTrigger = async (uuid: string) => {
-		const sParams = new URLSearchParams({
-			uuid
-		});
-		const fullUrl = `/configs?${sParams.toString()}`;
-		return await fetch(fullUrl, {
-			method: 'DELETE'
-		})
-			.then(() => {
-				toast.push('Config deleted.');
-			})
-			.catch((err) => {
-				toast.push(err.message);
-			})
-			.finally(() => {
-				window.location.reload();
-			});
-	};
 </script>
 
-<!-- <svelte:head>
-	<meta http-equiv="refresh" content="5" />
-</svelte:head> -->
-
 {#if data.configs.length === 0}
 	<main>
-		<PageHeader pageTitle="Configured Pipeline list" />
+		<PageHeader pageTitle="Pipeline list" />
 		<div class="card">
 			<div class="card-body">
 				<span class="text-danger">No Pipeline configurations were detected.</span>
@@ -70,120 +17,34 @@
 	</main>
 {:else}
 	<main>
-		<PageHeader pageTitle="Configured Pipeline list" />
-		{#each data.configs as config, idex}
-			{@const { tableHeaders, tableRows, colorsColSpan } = parseForTable(config)}
-			{@const metric = config.metric}
-			<div class="card my-3">
-				<div class="card-header d-flex justify-content-end gap-2">
-					<button
-						on:click={() => manualTrigger(config.uuid)}
-						disabled={config.isRunning}
-						class="btn btn-outline-primary btn-sm"
-						><i class="fas fa-cogs" /> Manually Trigger workflow</button
-					>
-					<button on:click={() => editTrigger(config.uuid)} class="btn btn-outline-primary btn-sm"
-						><i class="fas fa-edit" /> Edit</button
-					>
-					<button on:click={() => deleteTrigger(config.uuid)} class="btn btn-outline-danger btn-sm">
-						<i class="fas fa-trash" /> Delete
-					</button>
-				</div>
-				<div class="card-body">
-					{#if config.invalidityErrror !== null}
-						<div class="card mb-1">
-							<div class="card-body">
-								<span class="text-danger">{config.invalidityErrror}</span>
-							</div>
-						</div>
-					{/if}
-					<dl class="row">
-						<dt class="col-sm-3">API Base url</dt>
-						<dd class="col-sm-9">{config.baseUrl}</dd>
-						<dt class="col-sm-3">Registration form Id</dt>
-						<dd class="col-sm-9">{config.regFormId}</dd>
-						<dt class="col-sm-3">Visit form Id</dt>
-						<dd class="col-sm-9">{config.visitFormId}</dd>
-					</dl>
-					<div class="text-center">
-						<table class="table table-bordered table-sm table-hover text-center">
-							<thead>
-								<tr>
-									{#each tableHeaders as header, i}
-										{#if i === tableHeaders.length - 1}
-											<th scope="col" colspan={colorsColSpan}>{header}</th>
-										{:else}
-											<th scope="col">{header}</th>
-										{/if}
-									{/each}
-								</tr>
-							</thead>
-							<tbody>
-								{#each tableRows as row}
-									<tr>
-										{#each range(colorsColSpan + (tableHeaders.length - 1)) as idx}
-											{@const thisElement = row[idx]}
-											{#if thisElement === undefined}
-												<td />
-											{:else if Array.isArray(thisElement)}
-												<td style={`background-color: ${thisElement[1]}`}
-													><span class="fw-bolder">{thisElement[0]}</span></td
-												>
-											{:else}
-												<td>{thisElement}</td>
-											{/if}
-										{/each}
-									</tr>
-								{/each}
-							</tbody>
-						</table>
-						<span class="card-text d-inline-block me-2 text-muted">Schedule:</span><span
-							class="card-text">{convertCronToHuman(config.schedule)}</span
-						>
-					</div>
-					<hr />
-					<h5>Metrics for the last run</h5>
-					{#if metric === undefined}
-						<div class="card">
-							<div class="card-body">
-								<span class="text-danger"
-									>No previous run information was found for this Pipeline.</span
-								>
-							</div>
-						</div>
-					{:else}
-						{#if config.isRunning}
-							<span class="text-info">Pipeline is currently running.</span>
-						{/if}
-						<dl class="row">
-							<dt class="col-sm-9">Started</dt>
-							<dd class="col-sm-3">
-								{metric?.startTime ? formatTimestamp(metric?.startTime) : ' - '}
-							</dd>
-							<dt class="col-sm-9">Ended</dt>
-							<dd class="col-sm-3">
-								{metric?.endTime ? formatTimestamp(metric?.endTime) : ' - '}
-							</dd>
-							<dt class="col-sm-9">Total no. of facilities</dt>
-							<dd class="col-sm-3">{metric?.totalSubmissions ?? ' - '}</dd>
-							<dt class="col-sm-9">No. of facilities evaluated</dt>
-							<dd class="col-sm-3">{metric?.evaluated}</dd>
-							<dt class="col-sm-9">No. of registration submissions modified</dt>
-							<dd class="col-sm-3">{metric?.modified}</dd>
-							<dt class="col-sm-9">No. of registration submissions not modified due to error</dt>
-							<dd class="col-sm-3">{metric?.notModifiedWithError}</dd>
-							<dt class="col-sm-9">No. of registration submissions not modified without error</dt>
-							<dd class="col-sm-3">{metric?.notModifiedWithoutError}</dd>
-						</dl>
-					{/if}
-				</div>
-			</div>
-		{/each}
+		<PageHeader pageTitle="Pipeline list" />
+
+		<table class="table">
+			<thead>
+				<tr>
+					<th>Title</th>
+					<th>BaseUrl</th>
+					<th>facility Reg Form Id</th>
+					<th>Visit Form Id</th>
+					<th>Schedule</th>
+				</tr>
+			</thead>
+			<tbody>
+				{#each data.configs as config, idex}
+					<tr>
+						<td>
+							<a href={`/workflows/reports/${config.uuid}`}>
+								{config.title}
+							</a>
+						</td>
+						<td>{config.baseUrl}</td>
+						<td>{config.regFormId}</td>
+						<td>{config.visitFormId}</td>
+						<td>{convertCronToHuman(config.schedule)}</td>
+					</tr>
+				{/each}
+				<tr />
+			</tbody>
+		</table>
 	</main>
 {/if}
-
-<style scoped>
-	dt {
-		color: #5e5e5e;
-	}
-</style>
diff --git a/apps/web/src/routes/workflows/reports/[slug]/+page.server.ts b/apps/web/src/routes/workflows/reports/[slug]/+page.server.ts
@@ -0,0 +1,24 @@
+import { getClientSideSymbologyConfigs, pipelineController } from '$lib/server/appConfig';
+import { getLastPipelineMetricForConfig } from '$lib/server/logger/configMetrics';
+import type { ConfigRunner } from '@onaio/symbology-calc-core';
+
+/** @type {import('./$types').PageLoad} */
+export function load({params}) {
+	const uuid = params.slug ?? '';
+	const configs = getClientSideSymbologyConfigs().filter(config => config.uuid === uuid);
+	const ConfigsWithMetrics = configs.map((config) => {
+		const configId = config.uuid;
+		const metricForThisConfig = getLastPipelineMetricForConfig(configId);
+		const pipeLineRunner = pipelineController.getPipelines(configId) as ConfigRunner;
+		const isRunning = pipeLineRunner?.isRunning();
+		return {
+			...config,
+			metric: metricForThisConfig,
+			isRunning,
+			invalidityErrror: pipeLineRunner?.invalidError ?? null
+		};
+	});
+	return {
+		configs: ConfigsWithMetrics
+	};
+}