From 2b332f8d8f012a79b46ddb28d11b15e32d4a2b01 Mon Sep 17 00:00:00 2001 From: Guillaume Bourdat Date: Mon, 18 Nov 2024 10:23:39 +0100 Subject: [PATCH] see #160 : revert agentId allowed to null --- .../dbApi/kysely/createGetCompiledData.ts | 2 +- .../adapters/dbApi/kysely/kysely.database.ts | 2 +- ...3874014_allow-null-for-automated-import.ts | 14 ------ api/src/core/adapters/hal/getHalSoftware.ts | 9 ++-- api/src/core/adapters/hal/halRawSoftware.ts | 48 ++++++++----------- api/src/core/bootstrap.ts | 2 +- api/src/core/ports/CompileData.ts | 2 +- api/src/core/ports/DbApiV2.ts | 4 +- api/src/core/usecases/feedDB.ts | 2 +- .../core/usecases/readWriteSillData/types.ts | 14 +++--- api/src/env.ts | 4 +- .../docker-compose-example/.env.sample | 1 + 12 files changed, 39 insertions(+), 65 deletions(-) delete mode 100644 api/src/core/adapters/dbApi/kysely/migrations/1731423874014_allow-null-for-automated-import.ts diff --git a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts index 6a92b7e1..f3cbf48d 100644 --- a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts +++ b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts @@ -131,7 +131,7 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise }): CompiledData.Software<"private"> => { return { ...stripNullOrUndefinedValues(software), - addedByAgentEmail: addedByAgentId ? agentById[addedByAgentId].email : undefined, + addedByAgentEmail: agentById[addedByAgentId].email, updateTime: new Date(+updateTime).getTime(), referencedSinceTime: new Date(+referencedSinceTime).getTime(), doRespectRgaa, diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index 561ec72a..b8e30ab4 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -120,7 +120,7 @@ type SoftwaresTable = { >; categories: JSONColumnType; generalInfoMd: string | null; - addedByAgentId: number | null; + addedByAgentId: number; logoUrl: string | null; keywords: JSONColumnType; }; diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1731423874014_allow-null-for-automated-import.ts b/api/src/core/adapters/dbApi/kysely/migrations/1731423874014_allow-null-for-automated-import.ts deleted file mode 100644 index 940c24ad..00000000 --- a/api/src/core/adapters/dbApi/kysely/migrations/1731423874014_allow-null-for-automated-import.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Kysely } from 'kysely'; - -export async function up(db: Kysely): Promise { - await db.schema.alterTable("softwares") - .alterColumn("addedByAgentId", (ac) => ac.dropNotNull()) - .execute(); - await db.schema.alterTable("softwares").alterColumn("addedByAgentId", (ac) => ac.dropNotNull()).execute(); -} - -export async function down(db: Kysely): Promise { - await db.schema.alterTable("softwares") - .alterColumn("addedByAgentId", (ac) => ac.setNotNull()) - .execute(); -} \ No newline at end of file diff --git a/api/src/core/adapters/hal/getHalSoftware.ts b/api/src/core/adapters/hal/getHalSoftware.ts index 30646eeb..ac1cfd5e 100644 --- a/api/src/core/adapters/hal/getHalSoftware.ts +++ b/api/src/core/adapters/hal/getHalSoftware.ts @@ -62,13 +62,10 @@ export async function fetchHalSoftwares(): Promise> { // Filter only software who have an swhidId to filter clean data on https://hal.science, TODO remove and set it as an option to be generic const url = `https://api.archives-ouvertes.fr/search/?q=docType_s:SOFTWARE&rows=10000&fl=${halSoftwareFieldsToReturnAsString}&&fq=swhidId_s:["" TO *]`; - const res = await fetch(url).catch(() => undefined); - - console.debug("Hal response status : ", res?.status); - - if (res === undefined) { + const res = await fetch(url).catch((err) => { + console.error(err); throw new HalFetchError(undefined); - } + }); if (res.status === 429) { await new Promise(resolve => setTimeout(resolve, 100)); diff --git a/api/src/core/adapters/hal/halRawSoftware.ts b/api/src/core/adapters/hal/halRawSoftware.ts index 19dc5fff..3ff8b99e 100644 --- a/api/src/core/adapters/hal/halRawSoftware.ts +++ b/api/src/core/adapters/hal/halRawSoftware.ts @@ -20,13 +20,8 @@ const halSoftwareFieldsToReturn: (keyof HalRawSoftware)[] = [ export const halSoftwareFieldsToReturnAsString = halSoftwareFieldsToReturn.join(","); export const rawHalSoftwareToSoftwareExternalData = (halSoftware: HalRawSoftware): SoftwareExternalData => { - let bibliographicReferences = undefined; - try { - bibliographicReferences = parseBibliographicFields(halSoftware.label_bibtex); - } catch (error) { - console.error('HalRawSoftwareToSoftwareForm', error); - } - const license = bibliographicReferences && bibliographicReferences.license ? bibliographicReferences.license.join(", ") : undefined; + const bibliographicReferences = parseBibliographicFields(halSoftware.label_bibtex); + const license = bibliographicReferences?.license.join(", "); const developers = bibliographicReferences && bibliographicReferences.author ? bibliographicReferences.author.map(author => ({ id: author.toLowerCase().split(" ").join("-"), @@ -85,9 +80,9 @@ export type HalRawSoftware = { title_s: string[]; en_title_s?: string[]; fr_title_s?: string[]; - abstract_s?: string[]; // 1030 / 1398 - en_abstract_s?: string[]; // 896 / 1398 - fr_abstract_s?: string[]; // 235 / 1398 + abstract_s?: string[]; + en_abstract_s?: string[]; + fr_abstract_s?: string[]; uri_s: string; openAccess_bool: boolean; docType_s: string; @@ -109,9 +104,9 @@ export type HalRawSoftware = { // es_domainAllCodeLabel_fs: string[]; // eu_domainAllCodeLabel_fs: string[]; // primaryDomain_s: string; - // en_keyword_s?: string[]; // 711 / 1398 - // keyword_s: string[]; // 786 / 1398 - // fr_keyword_s?: string[]; // 184 / 1398 + // en_keyword_s?: string[]; + // keyword_s: string[]; + // fr_keyword_s?: string[]; // authIdFormPerson_s: string[]; // authIdForm_i: number[]; // authLastName_s: string[]; @@ -166,7 +161,7 @@ export type HalRawSoftware = { // contributorFullName_s: string; // contributorIdFullName_fs: string; // contributorFullNameId_fs: string; - // language_s: string[]; // Could use + // language_s: string[]; // halId_s: string; // version_i: number; // status_i: number; @@ -216,28 +211,23 @@ export type HalRawSoftware = { // collCategoryCodeName_fs: string[]; // collNameCode_fs: string[]; // fileMain_s: string; - // files_s: string[]; // Could ontains zip code + // files_s: string[]; // fileType_s: string[]; // _version_: bigint; // dateLastIndexed_tdate: string; // label_xml: string; - // softCodeRepository_s: string[]; // 727 / 1398 - // softDevelopmentStatus_s: string[]; // 715 / 1398 - // softPlatform_s:string[]; // 449 / 1398 - // softProgrammingLanguage_s: string[]; // 929 / 1398 - // softRuntimePlatform_s: string[]; // 195 / 1398 - // softVersion_s: string[]; // 642 / 1398 - // licence_s: string[]; // default licencse ? -> 20 / 1398 + // softCodeRepository_s: string[]; + // softDevelopmentStatus_s: string[]; + // softPlatform_s:string[]; + // softProgrammingLanguage_s: string[]; + // softRuntimePlatform_s: string[]; + // softVersion_s: string[]; + // licence_s: string[]; }; export const HalRawSoftwareToSoftwareForm = (halSoftware: HalRawSoftware): SoftwareFormData => { - let bibliographicReferences = undefined; - try { - bibliographicReferences = parseBibliographicFields(halSoftware.label_bibtex); - } catch (error) { - console.error('HalRawSoftwareToSoftwareForm', error); - } - const license = bibliographicReferences && bibliographicReferences.license ?bibliographicReferences.license.join(", ") : undefined; + const bibliographicReferences = parseBibliographicFields(halSoftware.label_bibtex); + const license = bibliographicReferences?.license.join(", "); // TODO Mapping const formData : SoftwareFormData = { diff --git a/api/src/core/bootstrap.ts b/api/src/core/bootstrap.ts index 29088fe4..98e99800 100644 --- a/api/src/core/bootstrap.ts +++ b/api/src/core/bootstrap.ts @@ -115,7 +115,7 @@ export async function bootstrapCore( await initializeUserApiCache(); } - if (config.feedFromSource) { + if (config.initializeSoftwareFromSource) { if (config.externalSoftwareDataOrigin === 'HAL') { console.log(' ------ Feeding database with HAL software started ------'); const HAL = feedDBfromHAL(dbApi); diff --git a/api/src/core/ports/CompileData.ts b/api/src/core/ports/CompileData.ts index 7bebb061..7f765d75 100644 --- a/api/src/core/ports/CompileData.ts +++ b/api/src/core/ports/CompileData.ts @@ -88,7 +88,7 @@ export namespace CompiledData { }; export type Private = Common & { - addedByAgentEmail: string | undefined; + addedByAgentEmail: string; users: (Pick & Pick)[]; referents: (Pick & diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index 7e3388e6..c7db0ff2 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -24,13 +24,13 @@ export interface SoftwareRepository { params: { formData: SoftwareFormData; externalDataOrigin: ExternalDataOrigin; - } & (WithAgentId | {agentId : undefined}) + } & WithAgentId ) => Promise; update: ( params: { softwareSillId: number; formData: SoftwareFormData; - } & (WithAgentId | {agentId : undefined}) + } & WithAgentId ) => Promise; updateLastExtraDataFetchAt: (params: { softwareId: number }) => Promise; getAll: (filters?: GetSoftwareFilters) => Promise; diff --git a/api/src/core/usecases/feedDB.ts b/api/src/core/usecases/feedDB.ts index 9e76872f..cf5d0351 100644 --- a/api/src/core/usecases/feedDB.ts +++ b/api/src/core/usecases/feedDB.ts @@ -13,7 +13,7 @@ export const feedDBfromHAL : any = (dbApi : DbApiV2) => { return Promise.resolve(soft.id); } else { console.log('Importing HAL : ', software.docid); - return dbApi.software.create({ formData: newSoft, externalDataOrigin: 'HAL', agentId: undefined }); + return dbApi.software.create({ formData: newSoft, externalDataOrigin: 'HAL', agentId: 1 }); // TODO agent id } }); } diff --git a/api/src/core/usecases/readWriteSillData/types.ts b/api/src/core/usecases/readWriteSillData/types.ts index 4eedba86..59132c76 100644 --- a/api/src/core/usecases/readWriteSillData/types.ts +++ b/api/src/core/usecases/readWriteSillData/types.ts @@ -128,17 +128,17 @@ export type SoftwareFormData = { softwareName: string; softwareDescription: string; softwareType: SoftwareType; - externalId: string | undefined; // Id - comptoirDuLibreId: number | undefined; // id on the library comptoir du libre - softwareLicense: string; // or default licence ? + externalId: string | undefined; + comptoirDuLibreId: number | undefined; + softwareLicense: string; softwareMinimalVersion: string; - similarSoftwareExternalDataIds: string[]; // + similarSoftwareExternalDataIds: string[]; softwareLogoUrl: string | undefined; softwareKeywords: string[]; - isPresentInSupportContract: boolean; // ?? - isFromFrenchPublicService: boolean; // Financed and developped by public service - doRespectRgaa: boolean | null; // référentiel général d’amélioration de l’accessibilité + isPresentInSupportContract: boolean; + isFromFrenchPublicService: boolean; + doRespectRgaa: boolean | null; }; export type DeclarationFormData = DeclarationFormData.User | DeclarationFormData.Referent; diff --git a/api/src/env.ts b/api/src/env.ts index 2c56153b..e6646090 100644 --- a/api/src/env.ts +++ b/api/src/env.ts @@ -29,7 +29,7 @@ const zConfiguration = z.object({ "redirectUrl": z.string().optional(), "externalSoftwareDataOrigin": z.enum(["wikidata", "HAL"]).optional(), "databaseUrl": z.string(), - "feedFromSource": z.boolean(), + "initializeSoftwareFromSource": z.boolean(), }); const getJsonConfiguration = () => { @@ -67,7 +67,7 @@ const getJsonConfiguration = () => { "externalSoftwareDataOrigin": process.env.SILL_EXTERNAL_SOFTWARE_DATA_ORIGIN, "redirectUrl": process.env.SILL_REDIRECT_URL, "databaseUrl": process.env.DATABASE_URL, - "feedFromSource": process.env.FEEDFROMSOURCE ? process.env.FEEDFROMSOURCE.toLowerCase() == "true" : false, + "initializeSoftwareFromSource": process.env?.INITSOFTFROMSOURCE?.toLowerCase() === "true", }; }; diff --git a/deployments/docker-compose-example/.env.sample b/deployments/docker-compose-example/.env.sample index 1b928705..c11283d6 100644 --- a/deployments/docker-compose-example/.env.sample +++ b/deployments/docker-compose-example/.env.sample @@ -12,6 +12,7 @@ SILL_GITHUB_TOKEN=xxxxx SILL_API_PORT=3084 SILL_IS_DEV_ENVIRONNEMENT=true SILL_EXTERNAL_SOFTWARE_DATA_ORIGIN=wikidata +INITSOFTFROMSOURCE=false DATABASE_URL=postgresql://sill:pg_password@localhost:5432/sill POSTGRES_DB=sill