Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Made a recurring job for the automatic creation of the geojson file that is used for index page as layer. #26

Merged
merged 1 commit into from
Sep 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,087 changes: 1,030 additions & 57 deletions functions/package-lock.json

Large diffs are not rendered by default.

7 changes: 4 additions & 3 deletions functions/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log",
"compile": "tsc",
"debug": "node --inspect node_modules/.bin/functions-framework --source=build/src/ --target=index",
"watch": "concurrently \"npm run compile -- --watch\" \"nodemon --watch ./build/ --exec npm run debug\""
"debug": "node --inspect node_modules/.bin/functions-framework --source=lib/functions/src/ --target=index",
"watch": "concurrently \"npm run compile -- --watch\" \"nodemon --watch ./lib/ --exec npm run debug\""
},
"engines": {
"node": "18"
Expand All @@ -22,7 +22,8 @@
"@google-cloud/storage": "^7.12.1",
"busboy": "^1.6.0",
"firebase-admin": "^12.1.0",
"firebase-functions": "^5.0.0"
"firebase-functions": "^5.1.1",
"functions-framework": "^0.6.10"
},
"devDependencies": {
"@types/busboy": "^1.5.4",
Expand Down
3 changes: 2 additions & 1 deletion functions/src/constants/google-storage-constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@

export const BUCKET_NAME = config().functions.bucket_name;
export const SUBJECTS_COLLECTION_NAME = config().functions.subjects_collection_name;
export const IMAGES_COLLECTION_NAME = config().functions.images_collection_name;
export const IMAGES_COLLECTION_NAME = config().functions.images_collection_name;
export const MAP_DATA_GEOJSON_NAME = config().functions.map_data_geojson_name;
24 changes: 23 additions & 1 deletion functions/src/controllers/dataController.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {Request, Response} from "express";
import {HttpsFunction, https} from "firebase-functions";
import {https, HttpsFunction, pubsub} from "firebase-functions";
import * as logger from "firebase-functions/logger";

import {validateCors} from "../utils/cors-helper";
Expand All @@ -16,4 +16,26 @@ export const getGeoJson: HttpsFunction = https.onRequest(
}
}
);

export const createGeoJson: HttpsFunction = https.onRequest(
async (request: Request, response: Response): Promise<any> => {
response = validateCors(request, response);
try {
await dataService.createMapData();
response.status(204).send();
} catch (error) {
logger.error("Error retrieving file:", error);
response.status(500).send("Error retrieving file: " + error);
}
}
);

export const createGeoJsonJob = pubsub.topic('create-geojson-topic').onPublish(async () => {
try {
logger.log('GeoJSON creation process started.');
await dataService.createMapData();
} catch (error) {
logger.error("Error creating file:", error);
}
});

8 changes: 8 additions & 0 deletions functions/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,16 @@ import * as controllers from "./controllers"
*
* Retrieves GeoJSON data from the database.
* @function getGeoJson
*
* Creates the GeoJSON data manually.
* @function createGeoJson
*
* Sets-up the GeoJSON data automatically via recurring job.
* @function createGeoJsonJob
*/
export const getGeoJson = controllers.dataController.getGeoJson;
export const createGeoJson = controllers.dataController.createGeoJson;
export const createGeoJsonJob = controllers.dataController.createGeoJsonJob;


/**
Expand Down
59 changes: 56 additions & 3 deletions functions/src/services/dataService.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,65 @@
import {storage} from "./externalServices";
import * as logger from "firebase-functions/logger";

import {storage} from "./externalServices";
import {getAllImages} from "./imageService";
import {ImageDocument} from "../../../sharedModels/interfaces";
import {MAP_DATA_GEOJSON_NAME} from "../constants/google-storage-constants";

export async function getMapData(): Promise<Buffer> {

// Fetch GeoJSON data from the bucket
const file = storage.file("TEST-MAP.geojson");
const file = storage.file(MAP_DATA_GEOJSON_NAME);

// Get the file's contents
const [contents] = await file.download();

return contents
}

export async function createMapData(): Promise<void> {

const images: ImageDocument[] = await getAllImages()

// Construct GeoJSON structure
const geoJson = {
type: "FeatureCollection",
features: images
.filter(image => image.geopoint !== null && image.geopoint.latitude != 0 && image.geopoint.longitude != 0)
.map(image => ({
type: "Feature",
geometry: {
type: "Point",
coordinates: [
image.geopoint?.longitude || 0,
image.geopoint?.latitude || 0,
0.0
]
},
properties: {
id: image.id,
imageName: image.imageName,
imgURL: image.imgURL,
nameOfSender: image.nameOfSender,
yearOfImage: image.yearOfImage
}
}))
};

// Convert the GeoJSON object to a string
const geoJsonString = JSON.stringify(geoJson, null, 2);
logger.log(geoJsonString);

// Define the file name and path in the bucket
const fileName = MAP_DATA_GEOJSON_NAME;
const file = storage.file(fileName);

// Upload the GeoJSON string to Google Cloud Storage
try {
await file.save(geoJsonString, {
contentType: 'application/json'
});
logger.log(`GeoJSON file saved to ${fileName} in bucket ${storage.name}.`);
} catch (error) {
logger.error('Error saving GeoJSON to Google Cloud Storage:', error);
}
}
15 changes: 14 additions & 1 deletion functions/src/services/imageService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {storage, firestore} from "./externalServices";
import * as Busboy from "busboy";
import * as logger from "firebase-functions/logger";

import {FileData, FileDataFields, ImageDocumentFS, MapMarker} from "../../../sharedModels/interfaces";
import {FileData, FileDataFields, ImageDocument, ImageDocumentFS, MapMarker} from "../../../sharedModels/interfaces";
import {extractDatesFromText, getFileExtension, removeFileExtension} from "../utils/string-helper";
import {BUCKET_NAME, IMAGES_COLLECTION_NAME} from "../constants/google-storage-constants";
import {subjectService} from "./";
Expand Down Expand Up @@ -90,3 +90,16 @@ function processImages(busboy: Busboy.Busboy, files: FileData[], subjectRef: Doc
async function createImage(image: ImageDocumentFS): Promise<DocumentReference> {
return await firestore.collection(IMAGES_COLLECTION_NAME).add(image);
}

export async function getAllImages(): Promise<ImageDocument[]> {

const documentRefs = await firestore
.collection(IMAGES_COLLECTION_NAME).get();

return documentRefs.docs.map(doc => {
return {
id: doc.id,
...doc.data() as ImageDocumentFS
};
});
}