fix: direct download of extensions (#4206)

.adiorc.js

@@ -36,6 +36,8 @@ module.exports = {
             "https",
             "inspector",
             "node:fs",
+            "node:timers",
+            "node:path",
             "os",
             "path",
             "readline",

packages/aws-sdk/src/client-s3/index.ts

@@ -12,6 +12,7 @@ export {
     HeadObjectCommand,
     HeadObjectOutput,
     ListObjectsOutput,
+    ListObjectsV2Command,
     ListPartsCommand,
     ListPartsCommandOutput,
     ListPartsOutput,

packages/cli-plugin-deploy-pulumi/commands/deploy.js

@@ -4,6 +4,7 @@ const { PackagesBuilder } = require("./buildPackages/PackagesBuilder");
 const pulumiLoginSelectStack = require("./deploy/pulumiLoginSelectStack");
 const executeDeploy = require("./deploy/executeDeploy");
 const executePreview = require("./deploy/executePreview");
+const { setTimeout } = require("node:timers/promises");
 
 module.exports = (params, context) => {
     const command = createPulumiCommand({
@@ -16,6 +17,12 @@ module.exports = (params, context) => {
 
             const hookArgs = { context, env, inputs, projectApplication };
 
+            context.info("Webiny version: %s", context.version);
+            console.log();
+
+            // Just so the version stays on the screen for a second, before the process continues.
+            await setTimeout(1000);
+
             if (build) {
                 await runHook({
                     hook: "hook-before-build",

packages/cli-plugin-scaffold-extensions/package.json

@@ -19,15 +19,16 @@
   },
   "homepage": "https://github.com/webiny/webiny-js#readme",
   "dependencies": {
+    "@webiny/aws-sdk": "0.0.0",
     "@webiny/cli": "0.0.0",
     "@webiny/cli-plugin-scaffold": "0.0.0",
     "@webiny/error": "0.0.0",
     "case": "^1.6.3",
+    "chalk": "^4.1.0",
     "execa": "^5.0.0",
     "glob": "^7.1.2",
     "load-json-file": "^6.2.0",
     "lodash": "^4.17.21",
-    "ncp": "^2.0.0",
     "replace-in-path": "^1.1.0",
     "ts-morph": "^11.0.0",
     "validate-npm-package-name": "^3.0.0",

packages/cli-plugin-scaffold-extensions/src/downloadAndLinkExtension.ts

@@ -0,0 +1,155 @@
+import os from "os";
+import path from "path";
+import fs from "node:fs";
+import fsAsync from "node:fs/promises";
+import { CliCommandScaffoldCallableArgs } from "@webiny/cli-plugin-scaffold/types";
+import { setTimeout } from "node:timers/promises";
+import { WEBINY_DEV_VERSION } from "~/utils/constants";
+import { linkAllExtensions } from "./utils/linkAllExtensions";
+import { Input } from "./types";
+import { downloadFolderFromS3 } from "./downloadAndLinkExtension/downloadFolderFromS3";
+import { setWebinyPackageVersions } from "~/utils/setWebinyPackageVersions";
+import { runYarnInstall } from "@webiny/cli-plugin-scaffold/utils";
+import { getDownloadedExtensionType } from "~/downloadAndLinkExtension/getDownloadedExtensionType";
+import chalk from "chalk";
+import { Extension } from "./extensions/Extension";
+
+const EXTENSIONS_ROOT_FOLDER = "extensions";
+
+const S3_BUCKET_NAME = "webiny-examples";
+const S3_BUCKET_REGION = "us-east-1";
+
+const getVersionFromVersionFolders = async (
+    versionFoldersList: string[],
+    currentWebinyVersion: string
+) => {
+    const availableVersions = versionFoldersList.map(v => v.replace(".x", ".0")).sort();
+
+    let versionToUse = "";
+
+    // When developing Webiny, we want to use the latest version.
+    if (currentWebinyVersion === WEBINY_DEV_VERSION) {
+        versionToUse = availableVersions[availableVersions.length - 1];
+    } else {
+        for (const availableVersion of availableVersions) {
+            if (currentWebinyVersion >= availableVersion) {
+                versionToUse = availableVersion;
+            } else {
+                break;
+            }
+        }
+    }
+
+    return versionToUse.replace(".0", ".x");
+};
+
+export const downloadAndLinkExtension = async ({
+    input,
+    ora,
+    context
+}: CliCommandScaffoldCallableArgs<Input>) => {
+    const currentWebinyVersion = context.version;
+
+    const downloadExtensionSource = input.templateArgs!;
+
+    try {
+        ora.start(`Downloading extension...`);
+
+        const randomId = String(Date.now());
+        const downloadFolderPath = path.join(os.tmpdir(), `wby-ext-${randomId}`);
+
+        await downloadFolderFromS3({
+            bucketName: S3_BUCKET_NAME,
+            bucketRegion: S3_BUCKET_REGION,
+            bucketFolderKey: downloadExtensionSource,
+            downloadFolderPath
+        });
+
+        ora.text = `Copying extension...`;
+        await setTimeout(1000);
+
+        let extensionsFolderToCopyPath = path.join(downloadFolderPath, "extensions");
+
+        // If we have `extensions` folder in the root of the downloaded extension.
+        // it means the example extension is not versioned, and we can just copy it.
+        const extensionsFolderExistsInRoot = fs.existsSync(extensionsFolderToCopyPath);
+        const versionedExtension = !extensionsFolderExistsInRoot;
+
+        if (versionedExtension) {
+            // If we have `x.x.x` folders in the root of the downloaded
+            // extension, we need to find the right version to use.
+
+            // This can be `5.40.x`, `5.41.x`, etc.
+            const versionFolders = await fsAsync.readdir(downloadFolderPath);
+
+            const versionToUse = await getVersionFromVersionFolders(
+                versionFolders,
+                currentWebinyVersion
+            );
+
+            extensionsFolderToCopyPath = path.join(downloadFolderPath, versionToUse, "extensions");
+        }
+
+        await fsAsync.cp(extensionsFolderToCopyPath, EXTENSIONS_ROOT_FOLDER, {
+            recursive: true
+        });
+
+        ora.text = `Linking extension...`;
+
+        // Retrieve extensions folders in the root of the downloaded extension. We use this
+        // later to run additional setup tasks on each extension.
+        const extensionsFolderNames = await fsAsync.readdir(extensionsFolderToCopyPath);
+        const downloadedExtensions: Extension[] = [];
+
+        for (const extensionsFolderName of extensionsFolderNames) {
+            const folderPath = path.join(EXTENSIONS_ROOT_FOLDER, extensionsFolderName);
+            const extensionType = await getDownloadedExtensionType(folderPath);
+
+            downloadedExtensions.push(
+                new Extension({
+                    name: extensionsFolderName,
+                    type: extensionType,
+                    location: folderPath,
+
+                    // We don't care about the package name here.
+                    packageName: extensionsFolderName
+                })
+            );
+        }
+
+        for (const downloadedExtension of downloadedExtensions) {
+            await setWebinyPackageVersions(downloadedExtension, currentWebinyVersion);
+        }
+
+        await linkAllExtensions();
+        await runYarnInstall();
+
+        if (downloadedExtensions.length === 1) {
+            const [downloadedExtension] = downloadedExtensions;
+            ora.succeed(
+                `Extension downloaded in ${context.success.hl(downloadedExtension.getLocation())}.`
+            );
+
+            const nextSteps = downloadedExtension.getNextSteps();
+
+            console.log();
+            console.log(chalk.bold("Next Steps"));
+            nextSteps.forEach(message => {
+                console.log(`‣ ${message}`);
+            });
+        } else {
+            const paths = downloadedExtensions.map(ext => ext.getLocation());
+            ora.succeed(`Extensions downloaded in ${context.success.hl(paths.join(", "))}.`);
+        }
+    } catch (e) {
+        switch (e.code) {
+            case "NO_OBJECTS_FOUND":
+                ora.fail("Could not download extension. Looks like the extension does not exist.");
+                break;
+            default:
+                ora.fail("Could not create extension. Please check the logs below.");
+                console.log();
+                console.log(e);
+        }
+    }
+};

packages/cli-plugin-scaffold-extensions/src/downloadAndLinkExtension/downloadFolderFromS3.ts

@@ -0,0 +1,73 @@
+import { S3Client, ListObjectsV2Command, GetObjectCommand } from "@webiny/aws-sdk/client-s3";
+import fs from "fs";
+import path from "path";
+import { WebinyError } from "@webiny/error";
+
+interface DownloadFolderFromS3Params {
+    bucketName: string;
+    bucketRegion: string;
+    bucketFolderKey: string;
+    downloadFolderPath: string;
+}
+
+export const downloadFolderFromS3 = async (params: DownloadFolderFromS3Params) => {
+    const { bucketName, bucketRegion, bucketFolderKey, downloadFolderPath } = params;
+
+    // Configure the S3 client
+    const s3Client = new S3Client({ region: bucketRegion });
+
+    // List all objects in the specified S3 folder
+    const listObjects = async (bucket: string, folderKey: string) => {
+        const command = new ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: folderKey
+        });
+        const response = await s3Client.send(command);
+        return response.Contents;
+    };
+
+    // Download an individual file from S3
+    const downloadFile = async (bucket: string, key: string, localPath: string) => {
+        const command = new GetObjectCommand({
+            Bucket: bucket,
+            Key: key
+        });
+
+        const response = await s3Client.send(command);
+
+        return new Promise((resolve, reject) => {
+            const fileStream = fs.createWriteStream(localPath);
+            // @ts-expect-error
+            response.Body.pipe(fileStream);
+            // @ts-expect-error
+            response.Body.on("error", reject);
+            fileStream.on("finish", resolve);
+        });
+    };
+
+    const objects = (await listObjects(bucketName, bucketFolderKey)) || [];
+    if (!objects.length) {
+        throw new WebinyError(`No objects found in the specified S3 folder.`, "NO_OBJECTS_FOUND");
+    }
+
+    for (const object of objects) {
+        const s3Key = object.Key!;
+        const relativePath = path.relative(bucketFolderKey, s3Key);
+        const localFilePath = path.join(downloadFolderPath, relativePath);
+
+        if (s3Key.endsWith("/")) {
+            // It's a directory, create it if it doesn't exist.
+            if (!fs.existsSync(localFilePath)) {
+                fs.mkdirSync(localFilePath, { recursive: true });
+            }
+        } else {
+            // It's a file, download it.
+            const localDirPath = path.dirname(localFilePath);
+            if (!fs.existsSync(localDirPath)) {
+                fs.mkdirSync(localDirPath, { recursive: true });
+            }
+
+            await downloadFile(bucketName, s3Key, localFilePath);
+        }
+    }
+};

packages/cli-plugin-scaffold-extensions/src/downloadAndLinkExtension/getDownloadedExtensionType.ts

@@ -0,0 +1,19 @@
+import loadJson from "load-json-file";
+import { PackageJson } from "@webiny/cli-plugin-scaffold/types";
+import path from "node:path";
+
+export const getDownloadedExtensionType = async (downloadedExtensionRootPath: string) => {
+    const pkgJsonPath = path.join(downloadedExtensionRootPath, "package.json");
+    const pkgJson = await loadJson<PackageJson>(pkgJsonPath);
+
+    const keywords = pkgJson.keywords;
+    if (Array.isArray(keywords)) {
+        for (const keyword of keywords) {
+            if (keyword.startsWith("webiny-extension-type:")) {
+                return keyword.replace("webiny-extension-type:", "");
+            }
+        }
+    }
+
+    throw new Error(`Could not determine the extension type from the downloaded extension.`);
+};

packages/cli-plugin-scaffold-extensions/src/extensions/AbstractExtension.ts

@@ -0,0 +1,34 @@
+export interface ExtensionTypeConstructorParams {
+    name: string;
+    type: string;
+    location: string;
+    packageName: string;
+}
+
+export abstract class AbstractExtension {
+    protected params: ExtensionTypeConstructorParams;
+
+    constructor(params: ExtensionTypeConstructorParams) {
+        this.params = params;
+    }
+
+    abstract generate(): Promise<void>;
+
+    abstract getNextSteps(): string[];
+
+    getPackageJsonPath(): string {
+        return `${this.params.location}/package.json`;
+    }
+
+    getLocation(): string {
+        return this.params.location;
+    }
+
+    getPackageName(): string {
+        return this.params.packageName;
+    }
+
+    getName(): string {
+        return this.params.name;
+    }
+}