diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/GoogleCloudStorage.svelte b/packages/builder/src/components/backend/DatasourceNavigator/icons/GoogleCloudStorage.svelte new file mode 100644 index 00000000000..fc1cbd8336c --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/GoogleCloudStorage.svelte @@ -0,0 +1,62 @@ +Icon_24px_CloudStorage_Color diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 88359c27829..cb20e34d630 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -16,6 +16,7 @@ import Firebase from "./Firebase.svelte" import Redis from "./Redis.svelte" import Snowflake from "./Snowflake.svelte" import Custom from "./Custom.svelte" +import GoogleCloudStorage from "./GoogleCloudStorage.svelte" import { integrations } from "stores/builder" import { get } from "svelte/store" @@ -38,6 +39,7 @@ const ICONS = { REDIS: Redis, SNOWFLAKE: Snowflake, CUSTOM: Custom, + GOOGLE_CLOUD: GoogleCloudStorage, } export default ICONS diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index f2d1520878b..d36f6a90ee5 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -1,6 +1,7 @@ import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui" import DataSourceSelect from "./controls/DataSourceSelect/DataSourceSelect.svelte" import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte" +import GoogleCloudStorageDataSourceSelect from "./controls/GoogleCloudStorageDataSourceSelect.svelte" import DataProviderSelect from "./controls/DataProviderSelect.svelte" import ButtonActionEditor from "./controls/ButtonActionEditor/ButtonActionEditor.svelte" import TableSelect from "./controls/TableSelect.svelte" @@ -35,6 +36,7 @@ const componentMap = { radio: RadioGroup, dataSource: DataSourceSelect, "dataSource/s3": S3DataSourceSelect, + "dataSource/googleCloudStorage": GoogleCloudStorageDataSourceSelect, dataProvider: DataProviderSelect, boolean: Checkbox, number: Stepper, @@ -68,6 +70,7 @@ const componentMap = { "field/datetime": FormFieldSelect, "field/attachment": FormFieldSelect, "field/s3": Input, + "field/gcs": Input, "field/link": FormFieldSelect, "field/array": FormFieldSelect, "field/json": FormFieldSelect, diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/GoogleCloudUpload.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/GoogleCloudUpload.svelte new file mode 100644 index 00000000000..2126f59cce5 --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/GoogleCloudUpload.svelte @@ -0,0 +1,33 @@ + + +
+ + diff --git a/packages/builder/src/components/integration/QueryFieldsBuilder.svelte b/packages/builder/src/components/integration/QueryFieldsBuilder.svelte index 340dc7839ca..95b27900027 100644 --- a/packages/builder/src/components/integration/QueryFieldsBuilder.svelte +++ b/packages/builder/src/components/integration/QueryFieldsBuilder.svelte @@ -1,5 +1,5 @@ + + diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte index 0147cbca6e9..591a0f7067b 100644 --- a/packages/client/src/components/app/forms/S3Upload.svelte +++ b/packages/client/src/components/app/forms/S3Upload.svelte @@ -1,7 +1,5 @@ - -
- {#if fieldState} - - {/if} - {#if loading} -
-
- -
- {/if} -
- - - + {onChange} + storeFieldType={"s3upload"} +/> diff --git a/packages/client/src/components/app/forms/StoreUpload.svelte b/packages/client/src/components/app/forms/StoreUpload.svelte new file mode 100644 index 00000000000..fc22f1718cf --- /dev/null +++ b/packages/client/src/components/app/forms/StoreUpload.svelte @@ -0,0 +1,157 @@ + + + +
+ {#if fieldState} + + {/if} + {#if loading} +
+
+ +
+ {/if} +
+ + + diff --git a/packages/client/src/components/app/forms/index.js b/packages/client/src/components/app/forms/index.js index 5804d3a79d7..b5f0372cd2b 100644 --- a/packages/client/src/components/app/forms/index.js +++ b/packages/client/src/components/app/forms/index.js @@ -14,5 +14,6 @@ export { default as passwordfield } from "./PasswordField.svelte" export { default as formstep } from "./FormStep.svelte" export { default as jsonfield } from "./JSONField.svelte" export { default as s3upload } from "./S3Upload.svelte" +export { default as googlecloudupload } from "./GoogleCloudUpload.svelte" export { default as codescanner } from "./CodeScannerField.svelte" export { default as bbreferencefield } from "./BBReferenceField.svelte" diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index 68478b76acf..ce1f0b7d499 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -320,6 +320,17 @@ const updateStateHandler = action => { } } +const googleCloudUploadHandler = async action => { + const { componentId } = action.parameters + if (!componentId) { + return + } + const res = await uploadStore.actions.processFileUpload(componentId) + return { + publicUrl: res?.publicUrl, + } +} + const s3UploadHandler = async action => { const { componentId } = action.parameters if (!componentId) { @@ -412,6 +423,7 @@ const handlerMap = { ["Close Screen Modal"]: closeScreenModalHandler, ["Update State"]: updateStateHandler, ["Upload File to S3"]: s3UploadHandler, + ["Upload File to GoogleCloud"]: googleCloudUploadHandler, ["Export Data"]: exportDataHandler, ["Continue if / Stop if"]: continueIfHandler, ["Show Notification"]: showNotificationHandler, diff --git a/packages/server/package.json b/packages/server/package.json index 97de17eb588..1e5d9c26f4b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -57,6 +57,7 @@ "@bull-board/koa": "5.10.2", "@elastic/elasticsearch": "7.10.0", "@google-cloud/firestore": "6.8.0", + "@google-cloud/storage": "^7.5.0", "@koa/router": "8.0.8", "@socket.io/redis-adapter": "^8.2.1", "airtable": "0.10.1", diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index c718d5f704c..04e95314d73 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -19,12 +19,14 @@ import { utils, configs, BadRequestError, + Duration, } from "@budibase/backend-core" import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types" +import { Storage } from "@google-cloud/storage" import { getAppMigrationVersion, getLatestMigrationId, @@ -293,6 +295,12 @@ export const getSignedUploadURL = async function (ctx: Ctx) { const awsRegion = (datasource?.config?.region || "eu-west-1") as string if (datasource?.source === "S3") { const { bucket, key } = ctx.request.body || {} + + // Ensure we aren't using a custom endpoint + if (datasource?.config?.endpoint) { + ctx.throw(400, "S3 datasources with custom endpoints are not supported") + } + if (!bucket || !key) { ctx.throw(400, "bucket and key values are required") } @@ -310,6 +318,41 @@ export const getSignedUploadURL = async function (ctx: Ctx) { } catch (error: any) { ctx.throw(400, error) } + } else if (datasource?.source === "GOOGLE_CLOUD") { + const { bucket, key } = ctx.request.body || {} + + if (!(bucket && key)) { + ctx.throw(400, "Request requires a destination bucket and file key") + } + + const parsedKey = datasource?.config?.privateKey + .split(String.raw`\n`) + .join("\n") + + publicUrl = `https://storage.cloud.google.com/${bucket}/${key}` + + try { + const storage = new Storage({ + projectId: datasource?.config?.projectId, + scopes: "https://www.googleapis.com/auth/cloud-platform", + credentials: { + client_email: datasource?.config?.clientEmail, + private_key: parsedKey, + }, + }) + // 15 minute default duration. + const [url] = await storage + .bucket(bucket) + .file(key) + .getSignedUrl({ + version: "v4", + action: "write", + expires: Date.now() + Duration.fromMinutes(15).toMs(), + }) + signedUrl = url + } catch (error: any) { + ctx.throw(400, error) + } } ctx.body = { signedUrl, publicUrl } diff --git a/packages/server/src/integrations/googlecloud.ts b/packages/server/src/integrations/googlecloud.ts new file mode 100644 index 00000000000..d122b2d47cd --- /dev/null +++ b/packages/server/src/integrations/googlecloud.ts @@ -0,0 +1,173 @@ +import { + Integration, + QueryType, + IntegrationBase, + DatasourceFieldType, + DatasourceFeature, + ConnectionInfo, +} from "@budibase/types" +import { Storage } from "@google-cloud/storage" + +interface GoogleCloudConfig { + projectId: string + privateKey: string + clientEmail: string +} + +const SCHEMA: Integration = { + docs: "https://googleapis.dev/nodejs/storage/latest", + auth: { + type: "google", + }, + description: + "Google Cloud Storage is an object storage service that offers industry-leading scalability, data availability, security, and performance.", + friendlyName: "GoogleCloud Storage", + type: "Object store", + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, + datasource: { + projectId: { + type: DatasourceFieldType.STRING, + required: true, + }, + privateKey: { + type: DatasourceFieldType.LONGFORM, + required: true, + }, + clientEmail: { + type: DatasourceFieldType.STRING, + required: true, + }, + }, + query: { + create: { + type: QueryType.FIELDS, + fields: { + bucket: { + display: "New Bucket", + type: DatasourceFieldType.STRING, + required: true, + }, + location: { + required: true, + type: DatasourceFieldType.SELECT, + display: "Location", + default: "US", + config: { + options: ["ASIA", "EU", "US"], + }, + }, + storageClass: { + required: true, + type: DatasourceFieldType.SELECT, + display: "Storage Class", + default: "STANDARD", + config: { + options: ["STANDARD", "NEARLINE", "COLDLINE", "ARCHIVE"], + }, + }, + allowPublicAccess: { + type: DatasourceFieldType.BOOLEAN, + display: "Allow Public Access", + default: false, + }, + }, + }, + read: { + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + delete: { + type: QueryType.FIELDS, + fields: { + bucket: { + type: DatasourceFieldType.STRING, + required: true, + }, + delete: { + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + }, +} + +class GoogleCloudIntegration implements IntegrationBase { + private readonly config: GoogleCloudConfig + private storage: any + + constructor(config: GoogleCloudConfig) { + this.config = config + this.storage = new Storage({ + projectId: this.config.projectId, + scopes: "https://www.googleapis.com/auth/cloud-platform", + credentials: { + client_email: this.config.clientEmail, + private_key: this.config.privateKey.split(String.raw`\n`).join("\n"), + }, + }) + } + + async testConnection() { + const response: ConnectionInfo = { + connected: false, + } + try { + await this.connect() + return { connected: true } + } catch (e: any) { + return { + connected: false, + error: e.message as string, + } + } + } + + async connect() { + await this.storage.getBuckets() + } + + async read(query: { bucket: string }) { + const bucket = this.storage.bucket(query.bucket) + // Does support paging. Could possibly use this feature + const [files, queryForPage2] = await bucket.getFiles({ + autoPaginate: false, + }) + return files.map((file: any) => file.metadata) + } + + async delete(query: { bucket: string; delete: string }) { + return await this.storage.bucket(query.bucket).file(query.delete).delete() + } + + async create(query: { + bucket: string + location: string + storageClass: string + allowPublicAccess: boolean + }) { + let [newBucket] = await this.storage.createBucket(query.bucket, { + // http://g.co/cloud/storage/docs/locations#location-mr + // ASIA, EU, US + location: query.location, + // https://cloud.google.com/storage/docs/storage-classes + [query.storageClass]: true, + }) + if (query.allowPublicAccess === true && newBucket) { + await this.storage.bucket(query.bucket).makePublic() + } + return newBucket + } +} + +export default { + schema: SCHEMA, + integration: GoogleCloudIntegration, +} diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 8cbc29251bc..0e28283c26d 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -24,6 +24,7 @@ import { getDatasourcePlugin } from "../utilities/fileSystem" import env from "../environment" import cloneDeep from "lodash/cloneDeep" import sdk from "../sdk" +import googlecloud from "./googlecloud" const DEFINITIONS: Record = { [SourceName.POSTGRES]: postgres.schema, @@ -43,6 +44,7 @@ const DEFINITIONS: Record = { [SourceName.SNOWFLAKE]: snowflake.schema, [SourceName.ORACLE]: undefined, [SourceName.BUDIBASE]: undefined, + [SourceName.GOOGLE_CLOUD]: googlecloud.schema, } type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase @@ -66,6 +68,7 @@ const INTEGRATIONS: Record = [SourceName.SNOWFLAKE]: snowflake.integration, [SourceName.ORACLE]: undefined, [SourceName.BUDIBASE]: undefined, + [SourceName.GOOGLE_CLOUD]: googlecloud.integration, } // optionally add oracle integration if the oracle binary can be installed diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 6e3e25364eb..c93a1724873 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -34,6 +34,7 @@ const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.REDIS]: undefined, [SourceName.SNOWFLAKE]: undefined, [SourceName.BUDIBASE]: undefined, + [SourceName.GOOGLE_CLOUD]: undefined, } export function getSQLClient(datasource: Datasource): SqlClient { diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 6b09959b6c5..18ab9395878 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -57,6 +57,7 @@ export enum SourceName { FIRESTORE = "FIRESTORE", REDIS = "REDIS", SNOWFLAKE = "SNOWFLAKE", + GOOGLE_CLOUD = "GOOGLE_CLOUD", BUDIBASE = "BUDIBASE", }