Skip to content
This repository has been archived by the owner on May 13, 2024. It is now read-only.

Commit

Permalink
Merge pull request #17 from codegouvfr/fetch-data-from-HAL
Browse files Browse the repository at this point in the history 
fetch data from HAL
  • Loading branch information
@JeromeBu
JeromeBu authored Feb 16, 2024
2 parents 2e7c56d + 9b5aff3 commit fb1bb52
Show file tree
Hide file tree
Showing 27 changed files with 1,324 additions and 205 deletions.
7 changes: 5 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@codegouvfr/sill",
"version": "1.19.3",
"version": "1.20.0",
"description": "The backend of code.gouv.fr/sill",
"repository": {
"type": "git",
Expand All @@ -10,6 +10,7 @@
"types": "dist/lib/index.d.ts",
"scripts": {
"prepare": "[ ! -f .env.local.sh ] && cp .env.sh .env.local.sh",
"test": "vitest",
"build": "tsc",
"start": "./.env.local.sh forever dist/main.js",
"_format": "prettier \"**/*.{ts,tsx,json,md}\"",
Expand Down Expand Up @@ -76,10 +77,12 @@
"superjson": "^1.12.2",
"ts-node": "^10.9.1",
"typescript": "^5.0.4",
"url-join": "^4.0.1"
"url-join": "^4.0.1",
"vitest": "^1.2.2"
},
"dependencies": {
"@octokit/graphql": "^7.0.2",
"@retorquere/bibtex-parser": "^7.0.11",
"@trpc/server": "^10.18.0",
"jwt-decode": "^3.1.2",
"semver": "^7.5.4",
Expand Down
3 changes: 2 additions & 1 deletion scripts/compile-data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import { env } from "../src/env";
},
"githubPersonalAccessTokenForApiRateLimit": env.githubPersonalAccessTokenForApiRateLimit,
"doPerPerformPeriodicalCompilation": false,
"doPerformCacheInitialization": false
"doPerformCacheInitialization": false,
"externalSoftwareDataOrigin": env.externalSoftwareDataOrigin
});

await core.functions.readWriteSillData.manuallyTriggerNonIncrementalCompilation();
Expand Down
8 changes: 4 additions & 4 deletions scripts/migration/software.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ const zSoftwareRow = z.object({
"isFromFrenchPublicService": z.boolean(),
"isPresentInSupportContract": z.boolean(),
"similarSoftwareWikidataIds": z.array(z.string()),
"wikidataId": z.string().optional(),
"externalId": z.string().optional(),
"comptoirDuLibreId": z.number().optional(),
"license": z.string(),
"softwareType": zSoftwareType,
Expand Down Expand Up @@ -119,7 +119,7 @@ fs.writeFileSync(
parentSoftwareWikidataId,
isFromFrenchPublicService,
isPresentInSupportContract,
wikidataId,
externalId,
comptoirDuLibreId,
license,
catalogNumeriqueGouvFrId,
Expand All @@ -129,7 +129,7 @@ fs.writeFileSync(
generalInfoMd,
updateTime,
doRespectRgaa,
similarSoftwareWikidataIds,
similarExternalSoftwareIds,
softwareType,
categories,
addedByAgentEmail,
Expand Down Expand Up @@ -169,7 +169,7 @@ fs.writeFileSync(
generalInfoMd,
updateTime,
doRespectRgaa,
similarSoftwareWikidataIds,
similarExternalSoftwareIds: similarSoftwareWikidataIds,
softwareType,
categories,
addedByAgentEmail,
Expand Down
69 changes: 35 additions & 34 deletions src/core/adapters/compileData.ts
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
import { assert } from "tsafe/assert";
import type { CompileData, CompiledData } from "../ports/CompileData";
import { GetServiceProviders } from "../ports/GetServiceProviders";
import type { GetWikidataSoftware, WikidataSoftware } from "../ports/GetWikidataSoftware";
import type { GetSoftwareExternalData, SoftwareExternalData } from "../ports/GetSoftwareExternalData";
import type { GetCnllPrestatairesSill } from "../ports/GetCnllPrestatairesSill";
import type { GetSoftwareLatestVersion } from "../ports/GetSoftwareLatestVersion";
import type { ComptoirDuLibreApi, ComptoirDuLibre } from "../ports/ComptoirDuLibreApi";
import { exclude } from "tsafe/exclude";
import memoize from "memoizee";

export function createCompileData(params: {
getWikidataSoftware: GetWikidataSoftware;
getSoftwareExternalData: GetSoftwareExternalData;
getCnllPrestatairesSill: GetCnllPrestatairesSill;
comptoirDuLibreApi: ComptoirDuLibreApi;
getSoftwareLatestVersion: GetSoftwareLatestVersion;
getServiceProviders: GetServiceProviders;
}) {
const {
getWikidataSoftware: getWikidataSoftware_params,
getSoftwareExternalData: getSoftwareExternalData_params,
comptoirDuLibreApi,
getCnllPrestatairesSill,
getSoftwareLatestVersion: getSoftwareLatestVersion_params,
Expand Down Expand Up @@ -44,12 +44,12 @@ export function createCompileData(params: {
const { partialSoftwareBySillId } = await (async () => {
const partialSoftwareBySillId: Record<number, CompileData.PartialSoftware> = {};

const getWikidataSoftware = memoize(
const getExternalSoftware = memoize(
async (wikidataId: string) => {
if (getCachedSoftware === undefined) {
getWikidataSoftware_params.clear(wikidataId);
getSoftwareExternalData_params.clear(wikidataId);
}
return await getWikidataSoftware_params(wikidataId);
return await getSoftwareExternalData_params(wikidataId);
},
{ "promise": true }
);
Expand All @@ -71,25 +71,25 @@ export function createCompileData(params: {
for (const {
id: sillId,
name,
wikidataId,
externalId,
comptoirDuLibreId,
parentSoftwareWikidataId,
similarSoftwareWikidataIds
similarSoftwareExternalDataIds
} of softwareRows) {
const cache = getCachedSoftware?.({ "sillSoftwareId": sillId });

if (cache === undefined) {
console.log(`Scrapping the web for info about ${name}`);
}

const wikidataSoftware_prev = cache?.wikidataSoftware;
const externalSoftware_prev = cache?.softwareExternalData;

const wikidataSoftware =
wikidataId === undefined
const externalSoftware =
externalId === undefined
? undefined
: cache?.wikidataSoftware?.wikidataId === wikidataId
? cache.wikidataSoftware
: await getWikidataSoftware(wikidataId);
: cache?.softwareExternalData?.externalId === externalId
? cache.softwareExternalData
: await getExternalSoftware(externalId);

const cdlSoftware_prev = cdlSoftwares.find(({ id }) => cache?.comptoirDuLibreSoftware?.id === id);

Expand All @@ -100,17 +100,17 @@ export function createCompileData(params: {
comptoirDuLibreLogoUrl,
comptoirDuLibreKeywords,
parentWikidataSoftware,
similarWikidataSoftwares,
similarExternalSoftwares,
instances
] = await Promise.all([
(async () => {
const getRepoUrl = (
wikidataSoftware: WikidataSoftware | undefined,
wikidataSoftware: SoftwareExternalData | undefined,
cdlSoftware: ComptoirDuLibre.Software | undefined
) => wikidataSoftware?.sourceUrl ?? cdlSoftware?.external_resources?.repository ?? undefined;

const repoUrl_prev = getRepoUrl(wikidataSoftware_prev, cdlSoftware_prev);
const repoUrl = getRepoUrl(wikidataSoftware, cdlSoftware);
const repoUrl_prev = getRepoUrl(externalSoftware_prev, cdlSoftware_prev);
const repoUrl = getRepoUrl(externalSoftware, cdlSoftware);

if (repoUrl_prev === repoUrl) {
return cache?.latestVersion;
Expand Down Expand Up @@ -145,49 +145,50 @@ export function createCompileData(params: {
return comptoirDuLibreApi.getKeywords({ "comptoirDuLibreId": cdlSoftware.id });
})(),
(async () => {
if (cache?.parentWikidataSoftware?.wikidataId === parentSoftwareWikidataId) {
if (cache?.parentWikidataSoftware?.externalId === parentSoftwareWikidataId) {
return cache?.parentWikidataSoftware;
}

if (parentSoftwareWikidataId === undefined) {
return undefined;
}

const parentWikidataSoftware = await getWikidataSoftware(parentSoftwareWikidataId);
const parentWikidataSoftware = await getExternalSoftware(parentSoftwareWikidataId);

if (parentWikidataSoftware === undefined) {
return undefined;
}

return {
"wikidataId": parentWikidataSoftware.wikidataId,
"externalId": parentWikidataSoftware.externalId,
"label": parentWikidataSoftware.label,
"description": parentWikidataSoftware.description
};
})(),
Promise.all(
similarSoftwareWikidataIds.map(async wikidataId => {
similarSoftwareExternalDataIds.map(async externalId => {
{
const similarSoftware_cache = cache?.similarWikidataSoftwares?.find(
similarSoftware => similarSoftware.wikidataId === wikidataId
const similarSoftware_cache = cache?.similarExternalSoftwares?.find(
similarSoftware => similarSoftware.externalId === externalId
);

if (similarSoftware_cache !== undefined) {
return similarSoftware_cache;
}
}

const wikidataSoftware = await getWikidataSoftware(wikidataId);
const wikidataSoftware = await getExternalSoftware(externalId);

if (wikidataSoftware === undefined) {
return undefined;
}

return {
"wikidataId": wikidataSoftware.wikidataId,
"externalId": wikidataSoftware.externalId,
"label": wikidataSoftware.label,
"description": wikidataSoftware.description,
"isLibreSoftware": wikidataSoftware.isLibreSoftware
"isLibreSoftware": wikidataSoftware.isLibreSoftware,
"externalDataOrigin": wikidataSoftware.externalDataOrigin
};
})
).then(similarWikidataSoftwares => similarWikidataSoftwares.filter(exclude(undefined))),
Expand All @@ -203,21 +204,21 @@ export function createCompileData(params: {
otherSoftwareWikidataIds.map(async wikidataId => {
const otherWikidataSoftware_cache =
instance_cache?.otherWikidataSoftwares.find(
wikidataSoftware => wikidataSoftware.wikidataId === wikidataId
wikidataSoftware => wikidataSoftware.externalId === wikidataId
);

if (otherWikidataSoftware_cache !== undefined) {
return otherWikidataSoftware_cache;
}

const wikidataSoftware = await getWikidataSoftware(wikidataId);
const wikidataSoftware = await getExternalSoftware(wikidataId);

if (wikidataSoftware === undefined) {
return undefined;
}

return {
"wikidataId": wikidataSoftware.wikidataId,
"externalId": wikidataSoftware.externalId,
"label": wikidataSoftware.label,
"description": wikidataSoftware.description
};
Expand All @@ -229,7 +230,7 @@ export function createCompileData(params: {
] as const);

partialSoftwareBySillId[sillId] = {
wikidataSoftware,
softwareExternalData: externalSoftware,
latestVersion,
comptoirDuLibreSoftware:
cdlSoftware === undefined
Expand All @@ -240,7 +241,7 @@ export function createCompileData(params: {
"keywords": comptoirDuLibreKeywords
},
parentWikidataSoftware,
similarWikidataSoftwares,
similarExternalSoftwares,
instances
};
}
Expand Down Expand Up @@ -336,8 +337,8 @@ export function createCompileData(params: {
logoUrl,
keywords,
"serviceProviders": serviceProvidersBySillId[sillId] ?? [],
"wikidataSoftware": partialSoftwareBySillId[sillId].wikidataSoftware,
"similarWikidataSoftwares": partialSoftwareBySillId[sillId].similarWikidataSoftwares,
"softwareExternalData": partialSoftwareBySillId[sillId].softwareExternalData,
"similarExternalSoftwares": partialSoftwareBySillId[sillId].similarExternalSoftwares,
"parentWikidataSoftware": partialSoftwareBySillId[sillId].parentWikidataSoftware,
"comptoirDuLibreSoftware":
comptoirDuLibreId === undefined
Expand Down
15 changes: 8 additions & 7 deletions src/core/adapters/getWikidataSoftware.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@ import { id } from "tsafe/id";
import {
languages,
type Language,
type GetWikidataSoftware,
type WikidataSoftware,
type GetSoftwareExternalData,
type SoftwareExternalData,
type LocalizedString
} from "../ports/GetWikidataSoftware";
} from "../ports/GetSoftwareExternalData";
import type { Entity, DataValue, LocalizedString as WikiDataLocalizedString } from "../../tools/WikidataEntity";
const { resolveLocalizedString } = createResolveLocalizedString({
"currentLanguage": id<Language>("en"),
"fallbackLanguage": "en"
});

export const getWikidataSoftware: GetWikidataSoftware = memoize(
async wikidataId => {
export const getWikidataSoftware: GetSoftwareExternalData = memoize(
async (wikidataId): Promise<SoftwareExternalData | undefined> => {
const { entity } =
(await fetchEntity(wikidataId).catch(error => {
if (error instanceof WikidataFetchError) {
Expand Down Expand Up @@ -57,7 +57,8 @@ export const getWikidataSoftware: GetWikidataSoftware = memoize(
})();

return {
wikidataId,
externalId: wikidataId,
externalDataOrigin: "wikidata",
"label": wikidataSingleLocalizedStringToLocalizedString(entity.labels) ?? {
"en": "No label"
},
Expand Down Expand Up @@ -203,7 +204,7 @@ export const getWikidataSoftware: GetWikidataSoftware = memoize(
"areEquals": same
});

return removeDuplicates<WikidataSoftware["developers"][number]>();
return removeDuplicates<SoftwareExternalData["developers"][number]>();
})()
)
)
Expand Down
7 changes: 4 additions & 3 deletions src/core/adapters/getWikidataSoftwareOptions.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import type { GetWikidataSoftwareOptions } from "../ports/GetWikidataSoftwareOptions";
import type { GetSoftwareExternalDataOptions } from "../ports/GetSoftwareExternalDataOptions";
import fetch from "node-fetch";
import { freeSoftwareLicensesWikidataIds } from "./getWikidataSoftware";

const useAgent = "Socle interministériel de logiciels libres - Ap";

export const getWikidataSoftwareOptions: GetWikidataSoftwareOptions = async ({ queryString, language }) => {
export const getWikidataSoftwareOptions: GetSoftwareExternalDataOptions = async ({ queryString, language }) => {
const results: {
search: {
id: string;
Expand Down Expand Up @@ -33,7 +33,8 @@ export const getWikidataSoftwareOptions: GetWikidataSoftwareOptions = async ({ q
const licensesById = await getLicenses(arr.map(({ id }) => id));

return arr.map(({ id, label, description }) => ({
id,
externalId: id,
externalDataOrigin: "wikidata",
label,
description,
"isLibreSoftware": (() => {
Expand Down
Loading

0 comments on commit fb1bb52

Please sign in to comment.