From bfa1e563733fd4f77a235f018c205df0980ecdc7 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 3 Nov 2024 14:40:40 +0200 Subject: [PATCH 01/45] Add dependency to go.mod and update go.sum --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 445b0e8a0..bd0842dcf 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chainguard-dev/git-urls v1.0.2 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/charmbracelet/x/ansi v0.4.2 // indirect github.com/charmbracelet/x/term v0.2.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect diff --git a/go.sum b/go.sum index cc52dfecc..5a1060077 100644 --- a/go.sum +++ b/go.sum @@ -399,6 +399,8 @@ github.com/charmbracelet/bubbletea v1.1.2 h1:naQXF2laRxyLyil/i7fxdpiz1/k06IKquhm github.com/charmbracelet/bubbletea v1.1.2/go.mod h1:9HIU/hBV24qKjlehyj8z1r/tR9TYTQEag+cWZnuXo8E= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo= github.com/charmbracelet/x/ansi v0.4.2 h1:0JM6Aj/g/KC154/gOP4vfxun0ff6itogDYk41kof+qk= From 95265cbb96dac766d82b95f51d8542f2e15d81c4 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 3 Nov 2024 14:41:03 +0200 Subject: [PATCH 02/45] add pkg installer --- internal/exec/vendor_utils.go | 535 +++++++++++++++++++++------------- 1 file changed, 326 insertions(+), 209 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 973bbd6a5..de096ee27 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -3,6 +3,7 @@ package exec import ( "context" "fmt" + "log" "os" "path" "path/filepath" @@ -10,6 +11,10 @@ import ( "strings" "time" + "github.com/charmbracelet/bubbles/progress" + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" "github.com/hashicorp/go-getter" cp "github.com/otiai10/copy" "github.com/samber/lo" @@ -20,6 +25,170 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) +type pkg struct { + uri string + tempDir string + name string + targetPath string + cliConfig schema.CliConfiguration + s schema.AtmosVendorSource + sourceIsLocalFile bool +} +type model struct { + packages []pkg + index int + width int + height int + spinner spinner.Model + progress progress.Model + done bool +} + +var ( + currentPkgNameStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("211")) + doneStyle = lipgloss.NewStyle().Margin(1, 2) + checkMark = lipgloss.NewStyle().Foreground(lipgloss.Color("42")).SetString("✓") + xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") +) + +func newModel(pkg []pkg) (model, error) { + p := progress.New( + progress.WithDefaultGradient(), + progress.WithWidth(40), + progress.WithoutPercentage(), + ) + s := spinner.New() + s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) + + return model{ + packages: pkg, + spinner: s, + progress: p, + }, nil +} + +func (m model) Init() tea.Cmd { + // Start downloading with the `uri`, package name, and `tempDir` directly from the model + return tea.Batch(downloadAndInstall(m.packages[0]), m.spinner.Tick) +} +func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c", "esc", "q": + return m, tea.Quit + } + case installedPkgMsg: + + pkg := m.packages[m.index] + if msg.err != nil { + u.LogError(pkg.cliConfig, msg.err) + m.done = false + return m, tea.Sequence( + tea.Printf("%s %s", xMark, pkg.name), + tea.Quit, + ) + + } + if m.index >= len(m.packages)-1 { + // Everything's been installed. We're done! + m.done = true + return m, tea.Sequence( + tea.Printf("%s %s", checkMark, pkg.name), // print the last success message + tea.Quit, // exit the program + ) + } + + // Update progress bar + m.index++ + progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) + return m, tea.Batch( + progressCmd, + tea.Printf("%s Successfully pull component %s into %s", checkMark, pkg.name, pkg.targetPath), // print success message above our program + downloadAndInstall(m.packages[m.index]), // download the next package + ) + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + case progress.FrameMsg: + newModel, cmd := m.progress.Update(msg) + if newModel, ok := newModel.(progress.Model); ok { + m.progress = newModel + } + return m, cmd + } + return m, nil +} + +func (m model) View() string { + n := len(m.packages) + w := lipgloss.Width(fmt.Sprintf("%d", n)) + + if m.done { + return doneStyle.Render(fmt.Sprintf("Done! Installed %d components.\n", n)) + } + + pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) + + spin := m.spinner.View() + " " + prog := m.progress.View() + cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) + + pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) + + info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) + + cellsRemaining := max(0, m.width-lipgloss.Width(spin+info+prog+pkgCount)) + gap := strings.Repeat(" ", cellsRemaining) + + return spin + info + gap + prog + pkgCount +} + +type installedPkgMsg struct { + err error + name string +} + +func downloadAndInstall(p pkg) tea.Cmd { + return func() tea.Msg { + // Set up the getter.Client with `tempDir` as destination + defer removeTempDir(p.cliConfig, p.tempDir) + client := &getter.Client{ + Ctx: context.Background(), + Dst: p.tempDir, + Src: p.uri, + Mode: getter.ClientModeAny, + } + + // Download the package + if err := client.Get(); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + err := copyToTarget(p.cliConfig, p.tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri) + if err != nil { + //u.LogError(p.cliConfig, err) + log.Println(err) + } + return installedPkgMsg{ + err: nil, + name: p.name, + } + } +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + // ExecuteVendorPullCommand executes `atmos vendor` commands func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { info, err := processCommandLineArgs("terraform", cmd, args, nil) @@ -175,16 +344,11 @@ func ExecuteAtmosVendorInternal( dryRun bool, ) error { - var tempDir string var err error - var uri string vendorConfigFilePath := path.Dir(vendorConfigFileName) - logMessage := fmt.Sprintf("Processing vendor config file '%s'", vendorConfigFileName) - if len(tags) > 0 { - logMessage = fmt.Sprintf("%s for tags {%s}", logMessage, strings.Join(tags, ", ")) - } - u.LogInfo(cliConfig, logMessage) + logInitialMessage(cliConfig, vendorConfigFileName, tags) + if len(atmosVendorSpec.Sources) == 0 && len(atmosVendorSpec.Imports) == 0 { return fmt.Errorf("either 'spec.sources' or 'spec.imports' (or both) must be defined in the vendor config file '%s'", vendorConfigFileName) } @@ -255,33 +419,14 @@ func ExecuteAtmosVendorInternal( //} // Process sources + var packages []pkg for indexSource, s := range sources { - // If `--component` is specified, and it's not equal to this component, skip this component - if component != "" && s.Component != component { - continue - } - - // If `--tags` list is specified, and it does not contain any tags defined in this component, skip this component - // https://github.com/samber/lo?tab=readme-ov-file#intersect - if len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0 { + if shouldSkipSource(s, component, tags) { continue } - if s.File == "" { - s.File = vendorConfigFileName - } - - if s.Source == "" { - return fmt.Errorf("'source' must be specified in 'sources' in the vendor config file '%s'", - s.File, - ) - } - - if len(s.Targets) == 0 { - return fmt.Errorf("'targets' must be specified for the source '%s' in the vendor config file '%s'", - s.Source, - s.File, - ) + if err := validateSourceFields(s, vendorConfigFileName); err != nil { + return err } tmplData := struct { @@ -289,206 +434,62 @@ func ExecuteAtmosVendorInternal( Version string }{s.Component, s.Version} - // Parse 'source' template - uri, err = ProcessTmpl(fmt.Sprintf("source-%d", indexSource), s.Source, tmplData, false) + uri, err := generateSourceURI(s, tmplData, indexSource) if err != nil { return err } - useOciScheme := false - useLocalFileSystem := false - sourceIsLocalFile := false - - // Check if `uri` uses the `oci://` scheme (to download the source from an OCI-compatible registry). - if strings.HasPrefix(uri, "oci://") { - useOciScheme = true - uri = strings.TrimPrefix(uri, "oci://") - } - - if !useOciScheme { - if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, uri); err == nil { - uri = absPath - useLocalFileSystem = true - - if u.FileExists(uri) { - sourceIsLocalFile = true - } - } - } - - // Iterate over the targets + useOciScheme, useLocalFileSystem, sourceIsLocalFile := determineSourceType(uri, vendorConfigFilePath) + // Process each target within the source for indexTarget, tgt := range s.Targets { - var target string - // Parse 'target' template - target, err = ProcessTmpl(fmt.Sprintf("target-%d-%d", indexSource, indexTarget), tgt, tmplData, false) + target, err := ProcessTmpl(fmt.Sprintf("target-%d-%d", indexSource, indexTarget), tgt, tmplData, false) if err != nil { return err } - targetPath := path.Join(vendorConfigFilePath, target) - - if s.Component != "" { - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources for the component '%s' from '%s' into '%s'", - s.Component, - uri, - targetPath, - )) - } else { - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources from '%s' into '%s'", - uri, - targetPath, - )) - } - if dryRun { + logPullingAction(cliConfig, s.Component, uri, targetPath) return nil } - - // Create temp folder - // We are using a temp folder for the following reasons: - // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) - // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder - tempDir, err = os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - - defer removeTempDir(cliConfig, tempDir) - - // Download the source into the temp directory if useOciScheme { - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(cliConfig, uri, tempDir) + logPullingAction(cliConfig, s.Component, uri, targetPath) + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { return err } - } else if useLocalFileSystem { - copyOptions := cp.Options{ - PreserveTimes: false, - PreserveOwner: false, - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - if sourceIsLocalFile { - tempDir = path.Join(tempDir, filepath.Base(uri)) - } - - if err = cp.Copy(uri, tempDir, copyOptions); err != nil { - return err - } - } else { - // Use `go-getter` to download the sources into the temp directory - // When cloning from the root of a repo w/o using modules (sub-paths), `go-getter` does the following: - // - If the destination directory does not exist, it creates it and runs `git init` - // - If the destination directory exists, it should be an already initialized Git repository (otherwise an error will be thrown) - // For more details, refer to - // - https://github.com/hashicorp/go-getter/issues/114 - // - https://github.com/hashicorp/go-getter?tab=readme-ov-file#subdirectories - // We add the `uri` to the already created `tempDir` so it does not exist to allow `go-getter` to create - // and correctly initialize it - tempDir = path.Join(tempDir, filepath.Base(uri)) - - client := &getter.Client{ - Ctx: context.Background(), - // Define the destination where the files will be stored. This will create the directory if it doesn't exist - Dst: tempDir, - // Source - Src: uri, - Mode: getter.ClientModeAny, - } - - if err = client.Get(); err != nil { - return err - } + defer removeTempDir(cliConfig, tempDir) + return processOciImage(cliConfig, uri, tempDir) } - - // Copy from the temp directory to the destination folder and skip the excluded files - copyOptions := cp.Options{ - // Skip specifies which files should be skipped - Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { - if strings.HasSuffix(src, ".git") { - return true, nil - } - - trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) - - // Exclude the files that match the 'excluded_paths' patterns - // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) - // https://en.wikipedia.org/wiki/Glob_(programming) - // https://github.com/bmatcuk/doublestar#patterns - for _, excludePath := range s.ExcludedPaths { - excludeMatch, err := u.PathMatch(excludePath, src) - if err != nil { - return true, err - } else if excludeMatch { - // If the file matches ANY of the 'excluded_paths' patterns, exclude the file - u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", - trimmedSrc, - excludePath, - )) - return true, nil - } - } - - // Only include the files that match the 'included_paths' patterns (if any pattern is specified) - if len(s.IncludedPaths) > 0 { - anyMatches := false - for _, includePath := range s.IncludedPaths { - includeMatch, err := u.PathMatch(includePath, src) - if err != nil { - return true, err - } else if includeMatch { - // If the file matches ANY of the 'included_paths' patterns, include the file - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", - trimmedSrc, - includePath, - )) - anyMatches = true - break - } - } - - if anyMatches { - return false, nil - } else { - u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) - return true, nil - } - } - - // If 'included_paths' is not provided, include all files that were not excluded - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) - return false, nil - }, - - // Preserve the atime and the mtime of the entries - // On linux we can preserve only up to 1 millisecond accuracy - PreserveTimes: false, - - // Preserve the uid and the gid of all entries - PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - if sourceIsLocalFile { - if filepath.Ext(targetPath) == "" { - targetPath = path.Join(targetPath, filepath.Base(uri)) - } + if useLocalFileSystem { + logPullingAction(cliConfig, s.Component, uri, targetPath) + return localFileSystem(cliConfig, uri, sourceIsLocalFile) } - - if err = cp.Copy(tempDir, targetPath, copyOptions); err != nil { + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { return err } + packages = append( + packages, + pkg{ + uri: uri, tempDir: tempDir, name: s.Component, targetPath: targetPath, + cliConfig: cliConfig, + s: s, + sourceIsLocalFile: sourceIsLocalFile, + }, + ) + + } + + } + if len(packages) > 0 && !dryRun { + model, err := newModel(packages) + if err != nil { + return fmt.Errorf("error initializing model error %v", err) + } + if _, err := tea.NewProgram(model).Run(); err != nil { + return fmt.Errorf("running download error %w", err) } + } return nil } @@ -540,3 +541,119 @@ func processVendorImports( return append(mergedSources, sources...), allImports, nil } +func logInitialMessage(cliConfig schema.CliConfiguration, vendorConfigFileName string, tags []string) { + logMessage := fmt.Sprintf("Processing vendor config file '%s'", vendorConfigFileName) + if len(tags) > 0 { + logMessage = fmt.Sprintf("%s for tags {%s}", logMessage, strings.Join(tags, ", ")) + } + u.LogInfo(cliConfig, logMessage) + +} +func validateSourceFields(s schema.AtmosVendorSource, vendorConfigFileName string) error { + // Ensure necessary fields are present + if s.File == "" { + s.File = vendorConfigFileName + } + if s.Source == "" { + return fmt.Errorf("'source' must be specified in 'sources' in the vendor config file '%s'", s.File) + } + if len(s.Targets) == 0 { + return fmt.Errorf("'targets' must be specified for the source '%s' in the vendor config file '%s'", s.Source, s.File) + } + return nil +} +func shouldSkipSource(s schema.AtmosVendorSource, component string, tags []string) bool { + // Skip if component or tags do not match + return (component != "" && s.Component != component) || (len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0) +} +func generateSourceURI(s schema.AtmosVendorSource, tmplData interface{}, indexSource int) (string, error) { + return ProcessTmpl(fmt.Sprintf("source-%d", indexSource), s.Source, tmplData, false) +} + +func determineSourceType(uri, vendorConfigFilePath string) (bool, bool, bool) { + // Determine if the URI is an OCI scheme, a local file, or remote + useOciScheme := strings.HasPrefix(uri, "oci://") + if useOciScheme { + uri = strings.TrimPrefix(uri, "oci://") + } + + useLocalFileSystem := false + sourceIsLocalFile := false + if !useOciScheme { + if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, uri); err == nil { + uri = absPath + useLocalFileSystem = true + sourceIsLocalFile = u.FileExists(uri) + } + } + return useOciScheme, useLocalFileSystem, sourceIsLocalFile +} + +func logPullingAction(cliConfig schema.CliConfiguration, component, uri, targetPath string) { + if component != "" { + u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources for the component '%s' from '%s' into '%s'", component, uri, targetPath)) + } else { + u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources from '%s' into '%s'", uri, targetPath)) + } +} + +func copyToTarget(cliConfig schema.CliConfiguration, tempDir, targetPath string, s schema.AtmosVendorSource, sourceIsLocalFile bool, uri string) error { + copyOptions := cp.Options{ + Skip: generateSkipFunction(cliConfig, tempDir, s), + PreserveTimes: false, + PreserveOwner: false, + OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, + } + + // Adjust the target path if it's a local file with no extension + if sourceIsLocalFile && filepath.Ext(targetPath) == "" { + targetPath = path.Join(targetPath, filepath.Base(uri)) + } + + return cp.Copy(tempDir, targetPath, copyOptions) +} + +func generateSkipFunction(cliConfig schema.CliConfiguration, tempDir string, s schema.AtmosVendorSource) func(os.FileInfo, string, string) (bool, error) { + return func(srcInfo os.FileInfo, src, dest string) (bool, error) { + if strings.HasSuffix(src, ".git") { + return true, nil + } + trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) + + for _, excludePath := range s.ExcludedPaths { + if match, _ := u.PathMatch(excludePath, src); match { + u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' matching '%s'", trimmedSrc, excludePath)) + return true, nil + } + } + + if len(s.IncludedPaths) > 0 { + for _, includePath := range s.IncludedPaths { + if match, _ := u.PathMatch(includePath, src); match { + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' matching '%s'", trimmedSrc, includePath)) + return false, nil + } + } + return true, nil + } + + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'", trimmedSrc)) + return false, nil + } +} +func localFileSystem(cliConfig schema.CliConfiguration, uri string, sourceIsLocalFile bool) error { + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return err + } + defer removeTempDir(cliConfig, tempDir) + copyOptions := cp.Options{ + PreserveTimes: false, + PreserveOwner: false, + OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, + } + if sourceIsLocalFile { + tempDir = path.Join(tempDir, filepath.Base(uri)) + } + return cp.Copy(uri, tempDir, copyOptions) +} From 8a1c415e2b22a78bfd7478c60c46dfc3ad880072 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 3 Nov 2024 18:22:54 +0200 Subject: [PATCH 03/45] Refactor package handling to support dry run functionality and introduce package types --- internal/exec/vendor_utils.go | 185 ++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 62 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index de096ee27..5e340ab4e 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -3,7 +3,6 @@ package exec import ( "context" "fmt" - "log" "os" "path" "path/filepath" @@ -25,6 +24,14 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) +type pkgType int + +const ( + pkgTypeRemote pkgType = iota + pkgTypeOci + pkgTypeLocal +) + type pkg struct { uri string tempDir string @@ -33,6 +40,7 @@ type pkg struct { cliConfig schema.CliConfiguration s schema.AtmosVendorSource sourceIsLocalFile bool + pkgType pkgType } type model struct { packages []pkg @@ -42,6 +50,7 @@ type model struct { spinner spinner.Model progress progress.Model done bool + dryRun bool } var ( @@ -51,7 +60,7 @@ var ( xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") ) -func newModel(pkg []pkg) (model, error) { +func newModel(pkg []pkg, dryRun bool) (model, error) { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(40), @@ -64,12 +73,13 @@ func newModel(pkg []pkg) (model, error) { packages: pkg, spinner: s, progress: p, + dryRun: dryRun, }, nil } func (m model) Init() tea.Cmd { // Start downloading with the `uri`, package name, and `tempDir` directly from the model - return tea.Batch(downloadAndInstall(m.packages[0]), m.spinner.Tick) + return tea.Batch(downloadAndInstall(m.packages[0], m.dryRun), m.spinner.Tick) } func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { @@ -92,12 +102,22 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { ) } + successMsg := "" + if pkg.name != "" { + successMsg = fmt.Sprintf("pull component %s into %s", pkg.name, pkg.targetPath) + } else { + successMsg = fmt.Sprintf("pull sources from %s into %s", pkg.uri, pkg.targetPath) + } + if m.dryRun { + successMsg = fmt.Sprintf("Dry run: %s", successMsg) + } if m.index >= len(m.packages)-1 { + // Everything's been installed. We're done! m.done = true return m, tea.Sequence( - tea.Printf("%s %s", checkMark, pkg.name), // print the last success message - tea.Quit, // exit the program + tea.Printf("%s %s", checkMark, successMsg), // print the last success message + tea.Quit, // exit the program ) } @@ -106,8 +126,8 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) return m, tea.Batch( progressCmd, - tea.Printf("%s Successfully pull component %s into %s", checkMark, pkg.name, pkg.targetPath), // print success message above our program - downloadAndInstall(m.packages[m.index]), // download the next package + tea.Printf("%s %s", checkMark, successMsg), // print success message above our program + downloadAndInstall(m.packages[m.index], m.dryRun), // download the next package ) case spinner.TickMsg: var cmd tea.Cmd @@ -128,6 +148,9 @@ func (m model) View() string { w := lipgloss.Width(fmt.Sprintf("%d", n)) if m.done { + if m.dryRun { + return doneStyle.Render("Done! Dry run completed. No components installed.\n") + } return doneStyle.Render(fmt.Sprintf("Done! Installed %d components.\n", n)) } @@ -152,29 +175,72 @@ type installedPkgMsg struct { name string } -func downloadAndInstall(p pkg) tea.Cmd { +func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { return func() tea.Msg { - // Set up the getter.Client with `tempDir` as destination defer removeTempDir(p.cliConfig, p.tempDir) - client := &getter.Client{ - Ctx: context.Background(), - Dst: p.tempDir, - Src: p.uri, - Mode: getter.ClientModeAny, - } - // Download the package - if err := client.Get(); err != nil { + // Log the action + //logPullingAction(p.cliConfig, p.name, p.uri, p.targetPath) + + if dryRun { + // Simulate the action + time.Sleep(1 * time.Second) return installedPkgMsg{ - err: err, + err: nil, name: p.name, } } - err := copyToTarget(p.cliConfig, p.tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri) - if err != nil { - //u.LogError(p.cliConfig, err) - log.Println(err) + + switch p.pkgType { + case pkgTypeRemote: + // Use go-getter to download remote packages + client := &getter.Client{ + Ctx: context.Background(), + Dst: p.tempDir, + Src: p.uri, + Mode: getter.ClientModeAny, + } + if err := client.Get(); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + // Copy to target + if err := copyToTarget(p.cliConfig, p.tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + + case pkgTypeOci: + // Process OCI images + if err := processOciImage(p.cliConfig, p.uri, p.tempDir); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + + case pkgTypeLocal: + // Copy from local file system + copyOptions := cp.Options{ + PreserveTimes: false, + PreserveOwner: false, + OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, + } + if p.sourceIsLocalFile { + p.tempDir = path.Join(p.tempDir, filepath.Base(p.uri)) + } + if err := cp.Copy(p.uri, p.tempDir, copyOptions); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } } + return installedPkgMsg{ err: nil, name: p.name, @@ -440,6 +506,17 @@ func ExecuteAtmosVendorInternal( } useOciScheme, useLocalFileSystem, sourceIsLocalFile := determineSourceType(uri, vendorConfigFilePath) + + // Determine package type + var pType pkgType + if useOciScheme { + pType = pkgTypeOci + } else if useLocalFileSystem { + pType = pkgTypeLocal + } else { + pType = pkgTypeRemote + } + // Process each target within the source for indexTarget, tgt := range s.Targets { target, err := ProcessTmpl(fmt.Sprintf("target-%d-%d", indexSource, indexTarget), tgt, tmplData, false) @@ -447,50 +524,42 @@ func ExecuteAtmosVendorInternal( return err } targetPath := path.Join(vendorConfigFilePath, target) - if dryRun { - logPullingAction(cliConfig, s.Component, uri, targetPath) - return nil - } - if useOciScheme { - logPullingAction(cliConfig, s.Component, uri, targetPath) - tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - defer removeTempDir(cliConfig, tempDir) - return processOciImage(cliConfig, uri, tempDir) - } - if useLocalFileSystem { - logPullingAction(cliConfig, s.Component, uri, targetPath) - return localFileSystem(cliConfig, uri, sourceIsLocalFile) - } + + // Create temp directory tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { return err } - packages = append( - packages, - pkg{ - uri: uri, tempDir: tempDir, name: s.Component, targetPath: targetPath, - cliConfig: cliConfig, - s: s, - sourceIsLocalFile: sourceIsLocalFile, - }, - ) - } + // Create package struct + p := pkg{ + uri: uri, + tempDir: tempDir, + name: s.Component, + targetPath: targetPath, + cliConfig: cliConfig, + s: s, + sourceIsLocalFile: sourceIsLocalFile, + pkgType: pType, + } + + packages = append(packages, p) + // Log the action (handled in downloadAndInstall) + } } - if len(packages) > 0 && !dryRun { - model, err := newModel(packages) + + // Run TUI to process packages + if len(packages) > 0 { + model, err := newModel(packages, dryRun) if err != nil { - return fmt.Errorf("error initializing model error %v", err) + return fmt.Errorf("error initializing model: %v", err) } if _, err := tea.NewProgram(model).Run(); err != nil { - return fmt.Errorf("running download error %w", err) + return fmt.Errorf("running download error: %w", err) } - } + return nil } @@ -589,14 +658,6 @@ func determineSourceType(uri, vendorConfigFilePath string) (bool, bool, bool) { return useOciScheme, useLocalFileSystem, sourceIsLocalFile } -func logPullingAction(cliConfig schema.CliConfiguration, component, uri, targetPath string) { - if component != "" { - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources for the component '%s' from '%s' into '%s'", component, uri, targetPath)) - } else { - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources from '%s' into '%s'", uri, targetPath)) - } -} - func copyToTarget(cliConfig schema.CliConfiguration, tempDir, targetPath string, s schema.AtmosVendorSource, sourceIsLocalFile bool, uri string) error { copyOptions := cp.Options{ Skip: generateSkipFunction(cliConfig, tempDir, s), From 086312af700bf51ac320ca88362cc76016199234 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 3 Nov 2024 18:43:42 +0200 Subject: [PATCH 04/45] Refactor local file handling in downloadAndInstall function --- internal/exec/vendor_utils.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 5e340ab4e..c8a1b6368 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -224,16 +224,8 @@ func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { } case pkgTypeLocal: - // Copy from local file system - copyOptions := cp.Options{ - PreserveTimes: false, - PreserveOwner: false, - OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, - } - if p.sourceIsLocalFile { - p.tempDir = path.Join(p.tempDir, filepath.Base(p.uri)) - } - if err := cp.Copy(p.uri, p.tempDir, copyOptions); err != nil { + // Process local files + if err := localFileSystem(p.cliConfig, p.uri, p.sourceIsLocalFile); err != nil { return installedPkgMsg{ err: err, name: p.name, From b6fe0d1be3d4fc7c6d10640bb848dbd6edca9529 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 3 Nov 2024 19:27:00 +0200 Subject: [PATCH 05/45] Refactor downloadAndInstall function to use a temporary directory for package handling --- internal/exec/vendor_utils.go | 56 ++++++++++++++++------------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index c8a1b6368..bb911ae62 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -34,7 +34,6 @@ const ( type pkg struct { uri string - tempDir string name string targetPath string cliConfig schema.CliConfiguration @@ -177,7 +176,6 @@ type installedPkgMsg struct { func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { return func() tea.Msg { - defer removeTempDir(p.cliConfig, p.tempDir) // Log the action //logPullingAction(p.cliConfig, p.name, p.uri, p.targetPath) @@ -190,13 +188,19 @@ func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { name: p.name, } } + // Create temp directory + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return err + } + defer removeTempDir(p.cliConfig, tempDir) switch p.pkgType { case pkgTypeRemote: // Use go-getter to download remote packages client := &getter.Client{ Ctx: context.Background(), - Dst: p.tempDir, + Dst: tempDir, Src: p.uri, Mode: getter.ClientModeAny, } @@ -207,7 +211,7 @@ func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { } } // Copy to target - if err := copyToTarget(p.cliConfig, p.tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTarget(p.cliConfig, tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: err, name: p.name, @@ -216,7 +220,7 @@ func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { case pkgTypeOci: // Process OCI images - if err := processOciImage(p.cliConfig, p.uri, p.tempDir); err != nil { + if err := processOciImage(p.cliConfig, p.uri, tempDir); err != nil { return installedPkgMsg{ err: err, name: p.name, @@ -224,15 +228,28 @@ func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { } case pkgTypeLocal: - // Process local files - if err := localFileSystem(p.cliConfig, p.uri, p.sourceIsLocalFile); err != nil { + // Copy from local file system + copyOptions := cp.Options{ + PreserveTimes: false, + PreserveOwner: false, + OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, + } + if p.sourceIsLocalFile { + tempDir = path.Join(tempDir, filepath.Base(p.uri)) + } + if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { return installedPkgMsg{ err: err, name: p.name, } } } - + if err := copyToTarget(p.cliConfig, tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } return installedPkgMsg{ err: nil, name: p.name, @@ -517,16 +534,9 @@ func ExecuteAtmosVendorInternal( } targetPath := path.Join(vendorConfigFilePath, target) - // Create temp directory - tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - // Create package struct p := pkg{ uri: uri, - tempDir: tempDir, name: s.Component, targetPath: targetPath, cliConfig: cliConfig, @@ -694,19 +704,3 @@ func generateSkipFunction(cliConfig schema.CliConfiguration, tempDir string, s s return false, nil } } -func localFileSystem(cliConfig schema.CliConfiguration, uri string, sourceIsLocalFile bool) error { - tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - defer removeTempDir(cliConfig, tempDir) - copyOptions := cp.Options{ - PreserveTimes: false, - PreserveOwner: false, - OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, - } - if sourceIsLocalFile { - tempDir = path.Join(tempDir, filepath.Base(uri)) - } - return cp.Copy(uri, tempDir, copyOptions) -} From dd6c509e745521c0903447c0ec423ad006ea73de Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 4 Nov 2024 09:47:38 +0200 Subject: [PATCH 06/45] Enhance model to track failed packages during installation process --- internal/exec/vendor_utils.go | 54 +++++++++++++---------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index bb911ae62..73cb0db1a 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -42,14 +42,15 @@ type pkg struct { pkgType pkgType } type model struct { - packages []pkg - index int - width int - height int - spinner spinner.Model - progress progress.Model - done bool - dryRun bool + packages []pkg + index int + width int + height int + spinner spinner.Model + progress progress.Model + done bool + dryRun bool + failedPkg int } var ( @@ -92,31 +93,19 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case installedPkgMsg: pkg := m.packages[m.index] - if msg.err != nil { - u.LogError(pkg.cliConfig, msg.err) - m.done = false - return m, tea.Sequence( - tea.Printf("%s %s", xMark, pkg.name), - tea.Quit, - ) + mark := checkMark - } - successMsg := "" - if pkg.name != "" { - successMsg = fmt.Sprintf("pull component %s into %s", pkg.name, pkg.targetPath) - } else { - successMsg = fmt.Sprintf("pull sources from %s into %s", pkg.uri, pkg.targetPath) - } - if m.dryRun { - successMsg = fmt.Sprintf("Dry run: %s", successMsg) + if msg.err != nil { + u.LogDebug(pkg.cliConfig, fmt.Sprintf("Failed to pull component %s error %s", pkg.name, msg.err)) + mark = xMark + m.failedPkg++ } if m.index >= len(m.packages)-1 { - // Everything's been installed. We're done! m.done = true return m, tea.Sequence( - tea.Printf("%s %s", checkMark, successMsg), // print the last success message - tea.Quit, // exit the program + tea.Printf("%s %s", mark, pkg.name), + tea.Quit, ) } @@ -125,7 +114,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) return m, tea.Batch( progressCmd, - tea.Printf("%s %s", checkMark, successMsg), // print success message above our program + tea.Printf("%s %s", mark, pkg.name), // print success message above our program downloadAndInstall(m.packages[m.index], m.dryRun), // download the next package ) case spinner.TickMsg: @@ -145,16 +134,17 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m model) View() string { n := len(m.packages) w := lipgloss.Width(fmt.Sprintf("%d", n)) - if m.done { if m.dryRun { return doneStyle.Render("Done! Dry run completed. No components installed.\n") } + if m.failedPkg > 0 { + return doneStyle.Render(fmt.Sprintf("Done! Installed %d components. %d components failed to install.\n", n-m.failedPkg, m.failedPkg)) + } return doneStyle.Render(fmt.Sprintf("Done! Installed %d components.\n", n)) } pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) - spin := m.spinner.View() + " " prog := m.progress.View() cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) @@ -176,10 +166,6 @@ type installedPkgMsg struct { func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { return func() tea.Msg { - - // Log the action - //logPullingAction(p.cliConfig, p.name, p.uri, p.targetPath) - if dryRun { // Simulate the action time.Sleep(1 * time.Second) From 06f6d01e34dae32704dcb006a45ab9d1b3faa51f Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 4 Nov 2024 09:59:40 +0200 Subject: [PATCH 07/45] Add version field to pkg update success message --- internal/exec/vendor_utils.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 73cb0db1a..12dd75888 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -40,6 +40,7 @@ type pkg struct { s schema.AtmosVendorSource sourceIsLocalFile bool pkgType pkgType + version string } type model struct { packages []pkg @@ -104,7 +105,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // Everything's been installed. We're done! m.done = true return m, tea.Sequence( - tea.Printf("%s %s", mark, pkg.name), + tea.Printf("%s %s %s", mark, pkg.name, pkg.version), tea.Quit, ) } @@ -114,8 +115,8 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) return m, tea.Batch( progressCmd, - tea.Printf("%s %s", mark, pkg.name), // print success message above our program - downloadAndInstall(m.packages[m.index], m.dryRun), // download the next package + tea.Printf("%s %s %s", mark, pkg.name, pkg.version), // print success message above our program + downloadAndInstall(m.packages[m.index], m.dryRun), // download the next package ) case spinner.TickMsg: var cmd tea.Cmd @@ -529,6 +530,7 @@ func ExecuteAtmosVendorInternal( s: s, sourceIsLocalFile: sourceIsLocalFile, pkgType: pType, + version: s.Version, } packages = append(packages, p) From 2dc0c2a1744782fe58672372491e093717b2a8d1 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 4 Nov 2024 10:59:00 +0200 Subject: [PATCH 08/45] Enhance file inclusion/exclusion logic to support patterns and improve logging --- internal/exec/vendor_utils.go | 43 +++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 12dd75888..6306ced0e 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -669,26 +669,55 @@ func generateSkipFunction(cliConfig schema.CliConfiguration, tempDir string, s s if strings.HasSuffix(src, ".git") { return true, nil } + trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) + // Exclude the files that match the 'excluded_paths' patterns + // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) + // https://en.wikipedia.org/wiki/Glob_(programming) + // https://github.com/bmatcuk/doublestar#patterns for _, excludePath := range s.ExcludedPaths { - if match, _ := u.PathMatch(excludePath, src); match { - u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' matching '%s'", trimmedSrc, excludePath)) + excludeMatch, err := u.PathMatch(excludePath, src) + if err != nil { + return true, err + } else if excludeMatch { + // If the file matches ANY of the 'excluded_paths' patterns, exclude the file + u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", + trimmedSrc, + excludePath, + )) return true, nil } } + // Only include the files that match the 'included_paths' patterns (if any pattern is specified) if len(s.IncludedPaths) > 0 { + anyMatches := false for _, includePath := range s.IncludedPaths { - if match, _ := u.PathMatch(includePath, src); match { - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' matching '%s'", trimmedSrc, includePath)) - return false, nil + includeMatch, err := u.PathMatch(includePath, src) + if err != nil { + return true, err + } else if includeMatch { + // If the file matches ANY of the 'included_paths' patterns, include the file + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", + trimmedSrc, + includePath, + )) + anyMatches = true + break } } - return true, nil + + if anyMatches { + return false, nil + } else { + u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) + return true, nil + } } - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'", trimmedSrc)) + // If 'included_paths' is not provided, include all files that were not excluded + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) return false, nil } } From 1ff92d3679d7c146b04139520ecda43c8d4f05b1 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 4 Nov 2024 11:15:03 +0200 Subject: [PATCH 09/45] improve log --- internal/exec/vendor_utils.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 6306ced0e..100629d70 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -496,7 +496,8 @@ func ExecuteAtmosVendorInternal( Version string }{s.Component, s.Version} - uri, err := generateSourceURI(s, tmplData, indexSource) + // Parse 'source' template + uri, err := ProcessTmpl(fmt.Sprintf("source-%d", indexSource), s.Source, tmplData, false) if err != nil { return err } @@ -520,11 +521,14 @@ func ExecuteAtmosVendorInternal( return err } targetPath := path.Join(vendorConfigFilePath, target) - + pkgName := s.Component + if pkgName == "" { + pkgName = uri + } // Create package struct p := pkg{ uri: uri, - name: s.Component, + name: pkgName, targetPath: targetPath, cliConfig: cliConfig, s: s, @@ -623,11 +627,10 @@ func validateSourceFields(s schema.AtmosVendorSource, vendorConfigFileName strin } func shouldSkipSource(s schema.AtmosVendorSource, component string, tags []string) bool { // Skip if component or tags do not match + // If `--component` is specified, and it's not equal to this component, skip this component + // If `--tags` list is specified, and it does not contain any tags defined in this component, skip this component return (component != "" && s.Component != component) || (len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0) } -func generateSourceURI(s schema.AtmosVendorSource, tmplData interface{}, indexSource int) (string, error) { - return ProcessTmpl(fmt.Sprintf("source-%d", indexSource), s.Source, tmplData, false) -} func determineSourceType(uri, vendorConfigFilePath string) (bool, bool, bool) { // Determine if the URI is an OCI scheme, a local file, or remote From 0cd82f6b058d2dc2568f77fbb4d4e82f7264de5c Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 4 Nov 2024 11:19:33 +0200 Subject: [PATCH 10/45] remove repeated call of copyToTarget --- internal/exec/vendor_utils.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 100629d70..3a1f5c29b 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -197,13 +197,6 @@ func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { name: p.name, } } - // Copy to target - if err := copyToTarget(p.cliConfig, tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { - return installedPkgMsg{ - err: err, - name: p.name, - } - } case pkgTypeOci: // Process OCI images From 3bc0ed0b02ef30a79a8d6893c321ff4b1ba926df Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 4 Nov 2024 11:24:36 +0200 Subject: [PATCH 11/45] Add error handling for unknown package types in downloadAndInstall --- internal/exec/vendor_utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 3a1f5c29b..a054d2c87 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -223,6 +223,12 @@ func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { name: p.name, } } + default: + return installedPkgMsg{ + err: fmt.Errorf("unknown package type"), + name: p.name, + } + } if err := copyToTarget(p.cliConfig, tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ From 47188c52593ea29cdeb049f1bd19d7bd6e9c060f Mon Sep 17 00:00:00 2001 From: haitham911 Date: Thu, 7 Nov 2024 11:03:55 +0200 Subject: [PATCH 12/45] Implement vendor model with package management and installation logic --- internal/exec/vendor_component_utils.go | 169 ++++++++-------- internal/exec/vendor_model.go | 255 ++++++++++++++++++++++++ internal/exec/vendor_utils.go | 238 +--------------------- 3 files changed, 345 insertions(+), 317 deletions(-) create mode 100644 internal/exec/vendor_model.go diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index ac31f3d9d..f73fc8ff9 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -223,87 +223,8 @@ func ExecuteComponentVendorInternal( } } - // Copy from the temp folder to the destination folder and skip the excluded files - copyOptions := cp.Options{ - // Skip specifies which files should be skipped - Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { - if strings.HasSuffix(src, ".git") { - return true, nil - } - - trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) - - // Exclude the files that match the 'excluded_paths' patterns - // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) - // https://en.wikipedia.org/wiki/Glob_(programming) - // https://github.com/bmatcuk/doublestar#patterns - for _, excludePath := range vendorComponentSpec.Source.ExcludedPaths { - excludeMatch, err := u.PathMatch(excludePath, src) - if err != nil { - return true, err - } else if excludeMatch { - // If the file matches ANY of the 'excluded_paths' patterns, exclude the file - u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", - trimmedSrc, - excludePath, - )) - return true, nil - } - } - - // Only include the files that match the 'included_paths' patterns (if any pattern is specified) - if len(vendorComponentSpec.Source.IncludedPaths) > 0 { - anyMatches := false - for _, includePath := range vendorComponentSpec.Source.IncludedPaths { - includeMatch, err := u.PathMatch(includePath, src) - if err != nil { - return true, err - } else if includeMatch { - // If the file matches ANY of the 'included_paths' patterns, include the file - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", - trimmedSrc, - includePath, - )) - anyMatches = true - break - } - } - - if anyMatches { - return false, nil - } else { - u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) - return true, nil - } - } - - // If 'included_paths' is not provided, include all files that were not excluded - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) - return false, nil - }, - - // Preserve the atime and the mtime of the entries - // On linux we can preserve only up to 1 millisecond accuracy - PreserveTimes: false, - - // Preserve the uid and the gid of all entries - PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - componentPath2 := componentPath - if sourceIsLocalFile { - if filepath.Ext(componentPath) == "" { - componentPath2 = path.Join(componentPath, filepath.Base(uri)) - } - } - - if err = cp.Copy(tempDir, componentPath2, copyOptions); err != nil { + // Copy from the temp folder to the destination folder + if err = copyComponentToDestination(cliConfig, tempDir, componentPath, vendorComponentSpec, sourceIsLocalFile, uri); err != nil { return err } } @@ -421,3 +342,89 @@ func ExecuteStackVendorInternal( ) error { return fmt.Errorf("command 'atmos vendor pull --stack ' is not supported yet") } +func copyComponentToDestination(cliConfig schema.CliConfiguration, tempDir, componentPath string, vendorComponentSpec schema.VendorComponentSpec, sourceIsLocalFile bool, uri string) error { + // Copy from the temp folder to the destination folder and skip the excluded files + copyOptions := cp.Options{ + // Skip specifies which files should be skipped + Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { + if strings.HasSuffix(src, ".git") { + return true, nil + } + + trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) + + // Exclude the files that match the 'excluded_paths' patterns + // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) + // https://en.wikipedia.org/wiki/Glob_(programming) + // https://github.com/bmatcuk/doublestar#patterns + for _, excludePath := range vendorComponentSpec.Source.ExcludedPaths { + excludeMatch, err := u.PathMatch(excludePath, src) + if err != nil { + return true, err + } else if excludeMatch { + // If the file matches ANY of the 'excluded_paths' patterns, exclude the file + u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", + trimmedSrc, + excludePath, + )) + return true, nil + } + } + + // Only include the files that match the 'included_paths' patterns (if any pattern is specified) + if len(vendorComponentSpec.Source.IncludedPaths) > 0 { + anyMatches := false + for _, includePath := range vendorComponentSpec.Source.IncludedPaths { + includeMatch, err := u.PathMatch(includePath, src) + if err != nil { + return true, err + } else if includeMatch { + // If the file matches ANY of the 'included_paths' patterns, include the file + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", + trimmedSrc, + includePath, + )) + anyMatches = true + break + } + } + + if anyMatches { + return false, nil + } else { + u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) + return true, nil + } + } + + // If 'included_paths' is not provided, include all files that were not excluded + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) + return false, nil + }, + + // Preserve the atime and the mtime of the entries + // On linux we can preserve only up to 1 millisecond accuracy + PreserveTimes: false, + + // Preserve the uid and the gid of all entries + PreserveOwner: false, + + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, + } + + componentPath2 := componentPath + if sourceIsLocalFile { + if filepath.Ext(componentPath) == "" { + componentPath2 = path.Join(componentPath, filepath.Base(uri)) + } + } + + if err := cp.Copy(tempDir, componentPath2, copyOptions); err != nil { + return err + } + return nil +} diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go new file mode 100644 index 000000000..9c425a2f6 --- /dev/null +++ b/internal/exec/vendor_model.go @@ -0,0 +1,255 @@ +package exec + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/charmbracelet/bubbles/progress" + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + "github.com/hashicorp/go-getter" + cp "github.com/otiai10/copy" +) + +type pkgType int + +const ( + pkgTypeRemote pkgType = iota + pkgTypeOci + pkgTypeLocal +) + +type pkg struct { + uri string + name string + targetPath string + sourceIsLocalFile bool + pkgType pkgType + version string + s schema.AtmosVendorSource +} +type model struct { + packages []pkg + index int + width int + height int + spinner spinner.Model + progress progress.Model + done bool + dryRun bool + failedPkg int + cliConfig schema.CliConfiguration +} + +var ( + currentPkgNameStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("211")) + doneStyle = lipgloss.NewStyle().Margin(1, 2) + checkMark = lipgloss.NewStyle().Foreground(lipgloss.Color("42")).SetString("✓") + xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") +) + +func newModel(pkg []pkg, dryRun bool, cliConfig schema.CliConfiguration) (model, error) { + p := progress.New( + progress.WithDefaultGradient(), + progress.WithWidth(30), + progress.WithoutPercentage(), + ) + s := spinner.New() + s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) + + return model{ + packages: pkg, + spinner: s, + progress: p, + dryRun: dryRun, + cliConfig: cliConfig, + }, nil +} + +func (m model) Init() tea.Cmd { + // Start downloading with the `uri`, package name, and `tempDir` directly from the model + return tea.Batch(downloadAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) +} +func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + if m.width > 120 { // Example: max width set to 80 characters + m.width = 120 + } + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c", "esc", "q": + return m, tea.Quit + } + + case installedPkgMsg: + + pkg := m.packages[m.index] + mark := checkMark + + if msg.err != nil { + u.LogDebug(m.cliConfig, fmt.Sprintf("Failed to vendor component %s error %s", pkg.name, msg.err)) + mark = xMark + m.failedPkg++ + } + version := "" + if pkg.version != "" { + version = fmt.Sprintf("(%s)", pkg.version) + } + if m.index >= len(m.packages)-1 { + // Everything's been installed. We're done! + m.done = true + return m, tea.Sequence( + tea.Printf("%s %s %s", mark, pkg.name, version), + tea.Quit, + ) + } + + // Update progress bar + m.index++ + progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) + return m, tea.Batch( + progressCmd, + tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program + downloadAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package + ) + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + case progress.FrameMsg: + newModel, cmd := m.progress.Update(msg) + if newModel, ok := newModel.(progress.Model); ok { + m.progress = newModel + } + return m, cmd + } + return m, nil +} + +func (m model) View() string { + n := len(m.packages) + w := lipgloss.Width(fmt.Sprintf("%d", n)) + if m.done { + if m.dryRun { + return doneStyle.Render("Done! Dry run completed. No components vendored.\n") + } + if m.failedPkg > 0 { + return doneStyle.Render(fmt.Sprintf("Vendored %d components.Failed to vendor %d components.\n", n-m.failedPkg, m.failedPkg)) + } + return doneStyle.Render(fmt.Sprintf("Vendored %d components.\n", n)) + } + + pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) + spin := m.spinner.View() + " " + prog := m.progress.View() + cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) + + pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) + + info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) + + cellsRemaining := max(0, m.width-lipgloss.Width(spin+info+prog+pkgCount)) + gap := strings.Repeat(" ", cellsRemaining) + + return spin + info + gap + prog + pkgCount +} + +type installedPkgMsg struct { + err error + name string +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} +func downloadAndInstall(p pkg, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { + return func() tea.Msg { + if dryRun { + // Simulate the action + time.Sleep(100 * time.Millisecond) + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + // Create temp directory + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return err + } + defer removeTempDir(cliConfig, tempDir) + + switch p.pkgType { + case pkgTypeRemote: + // Use go-getter to download remote packages + client := &getter.Client{ + Ctx: context.Background(), + Dst: tempDir, + Src: p.uri, + Mode: getter.ClientModeAny, + } + if err := client.Get(); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + + case pkgTypeOci: + // Process OCI images + if err := processOciImage(cliConfig, p.uri, tempDir); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + + case pkgTypeLocal: + // Copy from local file system + copyOptions := cp.Options{ + PreserveTimes: false, + PreserveOwner: false, + OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, + } + if p.sourceIsLocalFile { + tempDir = path.Join(tempDir, filepath.Base(p.uri)) + } + if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + default: + return installedPkgMsg{ + err: fmt.Errorf("unknown package type"), + name: p.name, + } + + } + if err := copyToTarget(cliConfig, tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + } + return installedPkgMsg{ + err: nil, + name: p.name, + } + } +} diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index a054d2c87..852b05428 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -1,20 +1,13 @@ package exec import ( - "context" "fmt" "os" "path" "path/filepath" - "strconv" "strings" - "time" - "github.com/charmbracelet/bubbles/progress" - "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" - "github.com/charmbracelet/lipgloss" - "github.com/hashicorp/go-getter" cp "github.com/otiai10/copy" "github.com/samber/lo" "github.com/spf13/cobra" @@ -24,232 +17,6 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) -type pkgType int - -const ( - pkgTypeRemote pkgType = iota - pkgTypeOci - pkgTypeLocal -) - -type pkg struct { - uri string - name string - targetPath string - cliConfig schema.CliConfiguration - s schema.AtmosVendorSource - sourceIsLocalFile bool - pkgType pkgType - version string -} -type model struct { - packages []pkg - index int - width int - height int - spinner spinner.Model - progress progress.Model - done bool - dryRun bool - failedPkg int -} - -var ( - currentPkgNameStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("211")) - doneStyle = lipgloss.NewStyle().Margin(1, 2) - checkMark = lipgloss.NewStyle().Foreground(lipgloss.Color("42")).SetString("✓") - xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") -) - -func newModel(pkg []pkg, dryRun bool) (model, error) { - p := progress.New( - progress.WithDefaultGradient(), - progress.WithWidth(40), - progress.WithoutPercentage(), - ) - s := spinner.New() - s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) - - return model{ - packages: pkg, - spinner: s, - progress: p, - dryRun: dryRun, - }, nil -} - -func (m model) Init() tea.Cmd { - // Start downloading with the `uri`, package name, and `tempDir` directly from the model - return tea.Batch(downloadAndInstall(m.packages[0], m.dryRun), m.spinner.Tick) -} -func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { - switch msg := msg.(type) { - case tea.WindowSizeMsg: - m.width, m.height = msg.Width, msg.Height - case tea.KeyMsg: - switch msg.String() { - case "ctrl+c", "esc", "q": - return m, tea.Quit - } - case installedPkgMsg: - - pkg := m.packages[m.index] - mark := checkMark - - if msg.err != nil { - u.LogDebug(pkg.cliConfig, fmt.Sprintf("Failed to pull component %s error %s", pkg.name, msg.err)) - mark = xMark - m.failedPkg++ - } - if m.index >= len(m.packages)-1 { - // Everything's been installed. We're done! - m.done = true - return m, tea.Sequence( - tea.Printf("%s %s %s", mark, pkg.name, pkg.version), - tea.Quit, - ) - } - - // Update progress bar - m.index++ - progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) - return m, tea.Batch( - progressCmd, - tea.Printf("%s %s %s", mark, pkg.name, pkg.version), // print success message above our program - downloadAndInstall(m.packages[m.index], m.dryRun), // download the next package - ) - case spinner.TickMsg: - var cmd tea.Cmd - m.spinner, cmd = m.spinner.Update(msg) - return m, cmd - case progress.FrameMsg: - newModel, cmd := m.progress.Update(msg) - if newModel, ok := newModel.(progress.Model); ok { - m.progress = newModel - } - return m, cmd - } - return m, nil -} - -func (m model) View() string { - n := len(m.packages) - w := lipgloss.Width(fmt.Sprintf("%d", n)) - if m.done { - if m.dryRun { - return doneStyle.Render("Done! Dry run completed. No components installed.\n") - } - if m.failedPkg > 0 { - return doneStyle.Render(fmt.Sprintf("Done! Installed %d components. %d components failed to install.\n", n-m.failedPkg, m.failedPkg)) - } - return doneStyle.Render(fmt.Sprintf("Done! Installed %d components.\n", n)) - } - - pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) - spin := m.spinner.View() + " " - prog := m.progress.View() - cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) - - pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) - - info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) - - cellsRemaining := max(0, m.width-lipgloss.Width(spin+info+prog+pkgCount)) - gap := strings.Repeat(" ", cellsRemaining) - - return spin + info + gap + prog + pkgCount -} - -type installedPkgMsg struct { - err error - name string -} - -func downloadAndInstall(p pkg, dryRun bool) tea.Cmd { - return func() tea.Msg { - if dryRun { - // Simulate the action - time.Sleep(1 * time.Second) - return installedPkgMsg{ - err: nil, - name: p.name, - } - } - // Create temp directory - tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - defer removeTempDir(p.cliConfig, tempDir) - - switch p.pkgType { - case pkgTypeRemote: - // Use go-getter to download remote packages - client := &getter.Client{ - Ctx: context.Background(), - Dst: tempDir, - Src: p.uri, - Mode: getter.ClientModeAny, - } - if err := client.Get(); err != nil { - return installedPkgMsg{ - err: err, - name: p.name, - } - } - - case pkgTypeOci: - // Process OCI images - if err := processOciImage(p.cliConfig, p.uri, tempDir); err != nil { - return installedPkgMsg{ - err: err, - name: p.name, - } - } - - case pkgTypeLocal: - // Copy from local file system - copyOptions := cp.Options{ - PreserveTimes: false, - PreserveOwner: false, - OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, - } - if p.sourceIsLocalFile { - tempDir = path.Join(tempDir, filepath.Base(p.uri)) - } - if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { - return installedPkgMsg{ - err: err, - name: p.name, - } - } - default: - return installedPkgMsg{ - err: fmt.Errorf("unknown package type"), - name: p.name, - } - - } - if err := copyToTarget(p.cliConfig, tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { - return installedPkgMsg{ - err: err, - name: p.name, - } - } - return installedPkgMsg{ - err: nil, - name: p.name, - } - } -} - -func max(a, b int) int { - if a > b { - return a - } - return b -} - // ExecuteVendorPullCommand executes `atmos vendor` commands func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { info, err := processCommandLineArgs("terraform", cmd, args, nil) @@ -529,11 +296,10 @@ func ExecuteAtmosVendorInternal( uri: uri, name: pkgName, targetPath: targetPath, - cliConfig: cliConfig, - s: s, sourceIsLocalFile: sourceIsLocalFile, pkgType: pType, version: s.Version, + s: s, } packages = append(packages, p) @@ -544,7 +310,7 @@ func ExecuteAtmosVendorInternal( // Run TUI to process packages if len(packages) > 0 { - model, err := newModel(packages, dryRun) + model, err := newModel(packages, dryRun, cliConfig) if err != nil { return fmt.Errorf("error initializing model: %v", err) } From 5248650b724c2bcc9103baccb7127aa91aa24a7a Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sat, 9 Nov 2024 14:59:11 +0200 Subject: [PATCH 13/45] Refactor package handling to use pkgAtmosVendor struct and update related functions --- internal/exec/vendor_component_utils.go | 363 +++++++++--------------- internal/exec/vendor_model.go | 24 +- internal/exec/vendor_model_component.go | 322 +++++++++++++++++++++ internal/exec/vendor_utils.go | 8 +- 4 files changed, 479 insertions(+), 238 deletions(-) create mode 100644 internal/exec/vendor_model_component.go diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index f73fc8ff9..6820ce2ed 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -8,17 +8,15 @@ import ( "os" "path" "path/filepath" - "strconv" "strings" "text/template" - "time" "github.com/Masterminds/sprig/v3" + tea "github.com/charmbracelet/bubbletea" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" "github.com/hairyhenderson/gomplate/v3" - "github.com/hashicorp/go-getter" cp "github.com/otiai10/copy" ) @@ -95,6 +93,101 @@ func ReadAndProcessComponentVendorConfigFile( // https://opencontainers.org/ // https://github.com/google/go-containerregistry // https://docs.aws.amazon.com/AmazonECR/latest/public/public-registries.html + +// ExecuteStackVendorInternal executes the command to vendor an Atmos stack +// TODO: implement this +func ExecuteStackVendorInternal( + stack string, + dryRun bool, +) error { + return fmt.Errorf("command 'atmos vendor pull --stack ' is not supported yet") +} +func copyComponentToDestination(cliConfig schema.CliConfiguration, tempDir, componentPath string, vendorComponentSpec schema.VendorComponentSpec, sourceIsLocalFile bool, uri string) error { + // Copy from the temp folder to the destination folder and skip the excluded files + copyOptions := cp.Options{ + // Skip specifies which files should be skipped + Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { + if strings.HasSuffix(src, ".git") { + return true, nil + } + + trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) + + // Exclude the files that match the 'excluded_paths' patterns + // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) + // https://en.wikipedia.org/wiki/Glob_(programming) + // https://github.com/bmatcuk/doublestar#patterns + for _, excludePath := range vendorComponentSpec.Source.ExcludedPaths { + excludeMatch, err := u.PathMatch(excludePath, src) + if err != nil { + return true, err + } else if excludeMatch { + // If the file matches ANY of the 'excluded_paths' patterns, exclude the file + u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", + trimmedSrc, + excludePath, + )) + return true, nil + } + } + + // Only include the files that match the 'included_paths' patterns (if any pattern is specified) + if len(vendorComponentSpec.Source.IncludedPaths) > 0 { + anyMatches := false + for _, includePath := range vendorComponentSpec.Source.IncludedPaths { + includeMatch, err := u.PathMatch(includePath, src) + if err != nil { + return true, err + } else if includeMatch { + // If the file matches ANY of the 'included_paths' patterns, include the file + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", + trimmedSrc, + includePath, + )) + anyMatches = true + break + } + } + + if anyMatches { + return false, nil + } else { + u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) + return true, nil + } + } + + // If 'included_paths' is not provided, include all files that were not excluded + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) + return false, nil + }, + + // Preserve the atime and the mtime of the entries + // On linux we can preserve only up to 1 millisecond accuracy + PreserveTimes: false, + + // Preserve the uid and the gid of all entries + PreserveOwner: false, + + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, + } + + componentPath2 := componentPath + if sourceIsLocalFile { + if filepath.Ext(componentPath) == "" { + componentPath2 = path.Join(componentPath, filepath.Base(uri)) + } + } + + if err := cp.Copy(tempDir, componentPath2, copyOptions); err != nil { + return err + } + return nil +} func ExecuteComponentVendorInternal( cliConfig schema.CliConfiguration, vendorComponentSpec schema.VendorComponentSpec, @@ -102,7 +195,6 @@ func ExecuteComponentVendorInternal( componentPath string, dryRun bool, ) error { - var tempDir string var err error var t *template.Template var uri string @@ -128,7 +220,6 @@ func ExecuteComponentVendorInternal( } else { uri = vendorComponentSpec.Source.Uri } - useOciScheme := false useLocalFileSystem := false sourceIsLocalFile := false @@ -152,83 +243,31 @@ func ExecuteComponentVendorInternal( } } } - + var pType pkgType + if useOciScheme { + pType = pkgTypeOci + } else if useLocalFileSystem { + pType = pkgTypeLocal + } else { + pType = pkgTypeRemote + } u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources for the component '%s' from '%s' into '%s'", component, uri, componentPath, )) - - if !dryRun { - // Create temp folder - // We are using a temp folder for the following reasons: - // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) - // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder - tempDir, err = os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - - defer removeTempDir(cliConfig, tempDir) - - // Download the source into the temp directory - if useOciScheme { - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(cliConfig, uri, tempDir) - if err != nil { - return err - } - } else if useLocalFileSystem { - copyOptions := cp.Options{ - PreserveTimes: false, - PreserveOwner: false, - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - tempDir2 := tempDir - if sourceIsLocalFile { - tempDir2 = path.Join(tempDir, filepath.Base(uri)) - } - - if err = cp.Copy(uri, tempDir2, copyOptions); err != nil { - return err - } - } else { - // Use `go-getter` to download the sources into the temp directory - // When cloning from the root of a repo w/o using modules (sub-paths), `go-getter` does the following: - // - If the destination directory does not exist, it creates it and runs `git init` - // - If the destination directory exists, it should be an already initialized Git repository (otherwise an error will be thrown) - // For more details, refer to - // - https://github.com/hashicorp/go-getter/issues/114 - // - https://github.com/hashicorp/go-getter?tab=readme-ov-file#subdirectories - // We add the `uri` to the already created `tempDir` so it does not exist to allow `go-getter` to create - // and correctly initialize it - tempDir = path.Join(tempDir, filepath.Base(uri)) - - client := &getter.Client{ - Ctx: context.Background(), - // Define the destination where the files will be stored. This will create the directory if it doesn't exist - Dst: tempDir, - // Source - Src: uri, - Mode: getter.ClientModeAny, - } - - if err = client.Get(); err != nil { - return err - } - } - - // Copy from the temp folder to the destination folder - if err = copyComponentToDestination(cliConfig, tempDir, componentPath, vendorComponentSpec, sourceIsLocalFile, uri); err != nil { - return err - } + componentPkg := pkgComponentVendor{ + uri: uri, + name: component, + componentPath: componentPath, + sourceIsLocalFile: sourceIsLocalFile, + pkgType: pType, + version: vendorComponentSpec.Source.Version, + vendorComponentSpec: vendorComponentSpec, + IsComponent: true, } - + var packages []pkgComponentVendor + packages = append(packages, componentPkg) // Process mixins if len(vendorComponentSpec.Mixins) > 0 { for _, mixin := range vendorComponentSpec.Mixins { @@ -273,158 +312,38 @@ func ExecuteComponentVendorInternal( uri = absPath } } - - u.LogInfo(cliConfig, fmt.Sprintf( - "Pulling the mixin '%s' for the component '%s' into '%s'\n", - uri, - component, - path.Join(componentPath, mixin.Filename), - )) - - if !dryRun { - err = os.RemoveAll(tempDir) - if err != nil { - return err - } - - // Download the mixin into the temp file - if !useOciScheme { - client := &getter.Client{ - Ctx: context.Background(), - Dst: path.Join(tempDir, mixin.Filename), - Src: uri, - Mode: getter.ClientModeFile, - } - - if err = client.Get(); err != nil { - return err - } - } else { - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(cliConfig, uri, tempDir) - if err != nil { - return err - } - } - - // Copy from the temp folder to the destination folder - copyOptions := cp.Options{ - // Preserve the atime and the mtime of the entries - PreserveTimes: false, - - // Preserve the uid and the gid of all entries - PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - // Prevent the error: - // symlink components/terraform/mixins/context.tf components/terraform/infra/vpc-flow-logs-bucket/context.tf: file exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - if err = cp.Copy(tempDir, componentPath, copyOptions); err != nil { - return err - } - } - } - } - - return nil -} - -// ExecuteStackVendorInternal executes the command to vendor an Atmos stack -// TODO: implement this -func ExecuteStackVendorInternal( - stack string, - dryRun bool, -) error { - return fmt.Errorf("command 'atmos vendor pull --stack ' is not supported yet") -} -func copyComponentToDestination(cliConfig schema.CliConfiguration, tempDir, componentPath string, vendorComponentSpec schema.VendorComponentSpec, sourceIsLocalFile bool, uri string) error { - // Copy from the temp folder to the destination folder and skip the excluded files - copyOptions := cp.Options{ - // Skip specifies which files should be skipped - Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { - if strings.HasSuffix(src, ".git") { - return true, nil - } - - trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) - - // Exclude the files that match the 'excluded_paths' patterns - // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) - // https://en.wikipedia.org/wiki/Glob_(programming) - // https://github.com/bmatcuk/doublestar#patterns - for _, excludePath := range vendorComponentSpec.Source.ExcludedPaths { - excludeMatch, err := u.PathMatch(excludePath, src) - if err != nil { - return true, err - } else if excludeMatch { - // If the file matches ANY of the 'excluded_paths' patterns, exclude the file - u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", - trimmedSrc, - excludePath, - )) - return true, nil - } + if useOciScheme { + pType = pkgTypeOci + } else if useLocalFileSystem { + pType = pkgTypeLocal + } else { + pType = pkgTypeRemote } - // Only include the files that match the 'included_paths' patterns (if any pattern is specified) - if len(vendorComponentSpec.Source.IncludedPaths) > 0 { - anyMatches := false - for _, includePath := range vendorComponentSpec.Source.IncludedPaths { - includeMatch, err := u.PathMatch(includePath, src) - if err != nil { - return true, err - } else if includeMatch { - // If the file matches ANY of the 'included_paths' patterns, include the file - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", - trimmedSrc, - includePath, - )) - anyMatches = true - break - } - } - - if anyMatches { - return false, nil - } else { - u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) - return true, nil - } + pkg := pkgComponentVendor{ + uri: uri, + pkgType: pType, + name: "mixin " + uri, + sourceIsLocalFile: sourceIsLocalFile, + IsMixins: true, + vendorComponentSpec: vendorComponentSpec, + version: mixin.Version, + componentPath: componentPath, + mixinFilename: mixin.Filename, } - // If 'included_paths' is not provided, include all files that were not excluded - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) - return false, nil - }, - - // Preserve the atime and the mtime of the entries - // On linux we can preserve only up to 1 millisecond accuracy - PreserveTimes: false, - - // Preserve the uid and the gid of all entries - PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - componentPath2 := componentPath - if sourceIsLocalFile { - if filepath.Ext(componentPath) == "" { - componentPath2 = path.Join(componentPath, filepath.Base(uri)) + packages = append(packages, pkg) } } - - if err := cp.Copy(tempDir, componentPath2, copyOptions); err != nil { - return err + // Run TUI to process packages + if len(packages) > 0 { + model, err := newModelComponentVendorInternal(packages, dryRun, cliConfig) + if err != nil { + return fmt.Errorf("error initializing model: %v", err) + } + if _, err := tea.NewProgram(model).Run(); err != nil { + return fmt.Errorf("running download error: %w", err) + } } return nil } diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 9c425a2f6..59e5a66ee 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -28,17 +28,17 @@ const ( pkgTypeLocal ) -type pkg struct { +type pkgAtmosVendor struct { uri string name string targetPath string sourceIsLocalFile bool pkgType pkgType version string - s schema.AtmosVendorSource + atmosVendorSource schema.AtmosVendorSource } -type model struct { - packages []pkg +type modelAtmosVendorInternal struct { + packages []pkgAtmosVendor index int width int height int @@ -57,7 +57,7 @@ var ( xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") ) -func newModel(pkg []pkg, dryRun bool, cliConfig schema.CliConfiguration) (model, error) { +func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelAtmosVendorInternal, error) { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(30), @@ -66,7 +66,7 @@ func newModel(pkg []pkg, dryRun bool, cliConfig schema.CliConfiguration) (model, s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) - return model{ + return modelAtmosVendorInternal{ packages: pkg, spinner: s, progress: p, @@ -75,15 +75,15 @@ func newModel(pkg []pkg, dryRun bool, cliConfig schema.CliConfiguration) (model, }, nil } -func (m model) Init() tea.Cmd { +func (m modelAtmosVendorInternal) Init() tea.Cmd { // Start downloading with the `uri`, package name, and `tempDir` directly from the model return tea.Batch(downloadAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) } -func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { +func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: m.width, m.height = msg.Width, msg.Height - if m.width > 120 { // Example: max width set to 80 characters + if m.width > 120 { m.width = 120 } case tea.KeyMsg: @@ -137,7 +137,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } -func (m model) View() string { +func (m modelAtmosVendorInternal) View() string { n := len(m.packages) w := lipgloss.Width(fmt.Sprintf("%d", n)) if m.done { @@ -176,7 +176,7 @@ func max(a, b int) int { } return b } -func downloadAndInstall(p pkg, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { +func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { // Simulate the action @@ -241,7 +241,7 @@ func downloadAndInstall(p pkg, dryRun bool, cliConfig schema.CliConfiguration) t } } - if err := copyToTarget(cliConfig, tempDir, p.targetPath, p.s, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTarget(cliConfig, tempDir, p.targetPath, p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: err, name: p.name, diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go new file mode 100644 index 000000000..62aa6cc5d --- /dev/null +++ b/internal/exec/vendor_model_component.go @@ -0,0 +1,322 @@ +package exec + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/charmbracelet/bubbles/progress" + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + "github.com/hashicorp/go-getter" + cp "github.com/otiai10/copy" +) + +type pkgComponentVendor struct { + uri string + name string + sourceIsLocalFile bool + pkgType pkgType + version string + vendorComponentSpec schema.VendorComponentSpec + componentPath string + IsComponent bool + IsMixins bool + mixinFilename string +} +type modelComponentVendorInternal struct { + packages []pkgComponentVendor + index int + width int + height int + spinner spinner.Model + progress progress.Model + done bool + dryRun bool + failedPkg int + cliConfig schema.CliConfiguration +} + +func newModelComponentVendorInternal(pkg []pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelComponentVendorInternal, error) { + p := progress.New( + progress.WithDefaultGradient(), + progress.WithWidth(30), + progress.WithoutPercentage(), + ) + s := spinner.New() + s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) + + return modelComponentVendorInternal{ + packages: pkg, + spinner: s, + progress: p, + dryRun: dryRun, + cliConfig: cliConfig, + }, nil +} + +func (m modelComponentVendorInternal) Init() tea.Cmd { + // Start downloading with the `uri`, package name, and `tempDir` directly from the model + return tea.Batch(downloadComponentAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) +} +func (m modelComponentVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + if m.width > 120 { + m.width = 120 + } + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c", "esc", "q": + return m, tea.Quit + } + + case installedPkgMsg: + + pkg := m.packages[m.index] + mark := checkMark + + if msg.err != nil { + u.LogDebug(m.cliConfig, fmt.Sprintf("Failed to vendor component %s error %s", pkg.name, msg.err)) + mark = xMark + m.failedPkg++ + } + version := "" + if pkg.version != "" { + version = fmt.Sprintf("(%s)", pkg.version) + } + if m.index >= len(m.packages)-1 { + // Everything's been installed. We're done! + m.done = true + return m, tea.Sequence( + tea.Printf("%s %s %s", mark, pkg.name, version), + tea.Quit, + ) + } + + // Update progress bar + m.index++ + progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) + return m, tea.Batch( + progressCmd, + tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program + downloadComponentAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package + ) + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + case progress.FrameMsg: + newModel, cmd := m.progress.Update(msg) + if newModel, ok := newModel.(progress.Model); ok { + m.progress = newModel + } + return m, cmd + } + return m, nil +} + +func (m modelComponentVendorInternal) View() string { + n := len(m.packages) + w := lipgloss.Width(fmt.Sprintf("%d", n)) + if m.done { + if m.dryRun { + return doneStyle.Render("Done! Dry run completed. No components vendored.\n") + } + if m.failedPkg > 0 { + return doneStyle.Render(fmt.Sprintf("Vendored %d components.Failed to vendor %d components.\n", n-m.failedPkg, m.failedPkg)) + } + return doneStyle.Render(fmt.Sprintf("Vendored %d components.\n", n)) + } + + pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) + spin := m.spinner.View() + " " + prog := m.progress.View() + cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) + + pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) + + info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) + + cellsRemaining := max(0, m.width-lipgloss.Width(spin+info+prog+pkgCount)) + gap := strings.Repeat(" ", cellsRemaining) + + return spin + info + gap + prog + pkgCount +} + +func downloadComponentAndInstall(p pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { + return func() tea.Msg { + if dryRun { + // Simulate the action + time.Sleep(100 * time.Millisecond) + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + if p.IsComponent { + err := installComponent(p, cliConfig) + if err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + + } + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + if p.IsMixins { + err := installMixin(p, cliConfig) + if err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + + } + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + + return installedPkgMsg{ + err: fmt.Errorf("unknown install operation"), + name: p.name, + } + } +} +func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) error { + + // Create temp folder + // We are using a temp folder for the following reasons: + // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) + // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return err + } + + defer removeTempDir(cliConfig, tempDir) + + switch p.pkgType { + case pkgTypeRemote: + tempDir = path.Join(tempDir, filepath.Base(p.uri)) + + client := &getter.Client{ + Ctx: context.Background(), + // Define the destination where the files will be stored. This will create the directory if it doesn't exist + Dst: tempDir, + // Source + Src: p.uri, + Mode: getter.ClientModeAny, + } + + if err = client.Get(); err != nil { + return err + } + + case pkgTypeOci: + // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory + err = processOciImage(cliConfig, p.uri, tempDir) + if err != nil { + return err + } + + case pkgTypeLocal: + copyOptions := cp.Options{ + PreserveTimes: false, + PreserveOwner: false, + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, + } + + tempDir2 := tempDir + if p.sourceIsLocalFile { + tempDir2 = path.Join(tempDir, filepath.Base(p.uri)) + } + + if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { + return err + } + default: + return fmt.Errorf("unknown package type") + + } + if err = copyComponentToDestination(cliConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { + return err + } + + return nil + +} +func installMixin(p pkgComponentVendor, cliConfig schema.CliConfiguration) error { + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return err + } + + defer removeTempDir(cliConfig, tempDir) + switch p.pkgType { + case pkgTypeRemote: + client := &getter.Client{ + Ctx: context.Background(), + Dst: path.Join(tempDir, p.mixinFilename), + Src: p.uri, + Mode: getter.ClientModeFile, + } + + if err = client.Get(); err != nil { + return err + } + case pkgTypeOci: + // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory + err = processOciImage(cliConfig, p.uri, tempDir) + if err != nil { + return err + } + case pkgTypeLocal: + return nil + + default: + return fmt.Errorf("unknown package type") + + } + // Copy from the temp folder to the destination folder + copyOptions := cp.Options{ + // Preserve the atime and the mtime of the entries + PreserveTimes: false, + + // Preserve the uid and the gid of all entries + PreserveOwner: false, + + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + // Prevent the error: + // symlink components/terraform/mixins/context.tf components/terraform/infra/vpc-flow-logs-bucket/context.tf: file exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, + } + + if err = cp.Copy(tempDir, p.componentPath, copyOptions); err != nil { + return err + } + + return nil +} diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 852b05428..d72a9e929 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -247,7 +247,7 @@ func ExecuteAtmosVendorInternal( //} // Process sources - var packages []pkg + var packages []pkgAtmosVendor for indexSource, s := range sources { if shouldSkipSource(s, component, tags) { continue @@ -292,14 +292,14 @@ func ExecuteAtmosVendorInternal( pkgName = uri } // Create package struct - p := pkg{ + p := pkgAtmosVendor{ uri: uri, name: pkgName, targetPath: targetPath, sourceIsLocalFile: sourceIsLocalFile, pkgType: pType, version: s.Version, - s: s, + atmosVendorSource: s, } packages = append(packages, p) @@ -310,7 +310,7 @@ func ExecuteAtmosVendorInternal( // Run TUI to process packages if len(packages) > 0 { - model, err := newModel(packages, dryRun, cliConfig) + model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) if err != nil { return fmt.Errorf("error initializing model: %v", err) } From b7ae2f135be6f7995d8e2908048c17821c54340b Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 10:54:32 +0200 Subject: [PATCH 14/45] Add support for daemon mode in TUI by conditionally disabling renderer --- internal/exec/vendor_component_utils.go | 15 +++++++++------ internal/exec/vendor_model.go | 5 ++--- internal/exec/vendor_utils.go | 10 +++++++++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 6820ce2ed..ffffd233b 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -17,6 +17,7 @@ import ( "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" "github.com/hairyhenderson/gomplate/v3" + "github.com/mattn/go-isatty" cp "github.com/otiai10/copy" ) @@ -251,11 +252,7 @@ func ExecuteComponentVendorInternal( } else { pType = pkgTypeRemote } - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources for the component '%s' from '%s' into '%s'", - component, - uri, - componentPath, - )) + componentPkg := pkgComponentVendor{ uri: uri, name: component, @@ -341,7 +338,13 @@ func ExecuteComponentVendorInternal( if err != nil { return fmt.Errorf("error initializing model: %v", err) } - if _, err := tea.NewProgram(model).Run(); err != nil { + var opts []tea.ProgramOption + if !isatty.IsTerminal(os.Stdout.Fd()) { + // If we're in daemon mode don't render the TUI + opts = []tea.ProgramOption{tea.WithoutRenderer()} + } + + if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } } diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 59e5a66ee..2dbaadb74 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -76,7 +76,6 @@ func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig sc } func (m modelAtmosVendorInternal) Init() tea.Cmd { - // Start downloading with the `uri`, package name, and `tempDir` directly from the model return tea.Batch(downloadAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) } func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { @@ -145,7 +144,7 @@ func (m modelAtmosVendorInternal) View() string { return doneStyle.Render("Done! Dry run completed. No components vendored.\n") } if m.failedPkg > 0 { - return doneStyle.Render(fmt.Sprintf("Vendored %d components.Failed to vendor %d components.\n", n-m.failedPkg, m.failedPkg)) + return doneStyle.Render(fmt.Sprintf("Vendored %d components. Failed to vendor %d components.\n", n-m.failedPkg, m.failedPkg)) } return doneStyle.Render(fmt.Sprintf("Vendored %d components.\n", n)) } @@ -180,7 +179,7 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi return func() tea.Msg { if dryRun { // Simulate the action - time.Sleep(100 * time.Millisecond) + time.Sleep(500 * time.Millisecond) return installedPkgMsg{ err: nil, name: p.name, diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index d72a9e929..ebacc4805 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -8,6 +8,7 @@ import ( "strings" tea "github.com/charmbracelet/bubbletea" + "github.com/mattn/go-isatty" cp "github.com/otiai10/copy" "github.com/samber/lo" "github.com/spf13/cobra" @@ -86,6 +87,7 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { return ExecuteAtmosVendorInternal(cliConfig, foundVendorConfigFile, vendorConfig.Spec, component, tags, dryRun) } else { // Check and process `component.yaml` + fmt.Println("No vendor config file found. Checking for component vendoring...") if component != "" { // Process component vendoring componentType, err := flags.GetString("type") @@ -314,7 +316,13 @@ func ExecuteAtmosVendorInternal( if err != nil { return fmt.Errorf("error initializing model: %v", err) } - if _, err := tea.NewProgram(model).Run(); err != nil { + var opts []tea.ProgramOption + if !isatty.IsTerminal(os.Stdout.Fd()) { + // If we're in daemon mode don't render the TUI + opts = []tea.ProgramOption{tea.WithoutRenderer()} + } + + if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } } From 08609e0b293ff2d821fa688d56c396477e731860 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 11:14:36 +0200 Subject: [PATCH 15/45] Add detailed logging for package installation errors in vendor model --- internal/exec/vendor_model.go | 6 ++++++ internal/exec/vendor_model_component.go | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 2dbaadb74..282a00762 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -188,6 +188,7 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi // Create temp directory tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to create temp directory %s", err)) return err } defer removeTempDir(cliConfig, tempDir) @@ -202,6 +203,7 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi Mode: getter.ClientModeAny, } if err := client.Get(); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to download package %s error %s", p.name, err)) return installedPkgMsg{ err: err, name: p.name, @@ -211,6 +213,7 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi case pkgTypeOci: // Process OCI images if err := processOciImage(cliConfig, p.uri, tempDir); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to process OCI image %s error %s", p.name, err)) return installedPkgMsg{ err: err, name: p.name, @@ -228,12 +231,14 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi tempDir = path.Join(tempDir, filepath.Base(p.uri)) } if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to copy package %s error %s", p.name, err)) return installedPkgMsg{ err: err, name: p.name, } } default: + u.LogTrace(cliConfig, fmt.Sprintf("Unknown package type %s", p.name)) return installedPkgMsg{ err: fmt.Errorf("unknown package type"), name: p.name, @@ -241,6 +246,7 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi } if err := copyToTarget(cliConfig, tempDir, p.targetPath, p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to copy package %s error %s", p.name, err)) return installedPkgMsg{ err: err, name: p.name, diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index 62aa6cc5d..8579bd594 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -191,7 +191,7 @@ func downloadComponentAndInstall(p pkgComponentVendor, dryRun bool, cliConfig sc name: p.name, } } - + u.LogTrace(cliConfig, fmt.Sprintf("Unknown install operation for %s", p.name)) return installedPkgMsg{ err: fmt.Errorf("unknown install operation"), name: p.name, @@ -206,6 +206,7 @@ func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) e // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to create temp directory %s", err)) return err } @@ -225,6 +226,7 @@ func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) e } if err = client.Get(); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to download package %s error %s", p.name, err)) return err } @@ -232,6 +234,7 @@ func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) e // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory err = processOciImage(cliConfig, p.uri, tempDir) if err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to process OCI image %s error %s", p.name, err)) return err } @@ -252,13 +255,16 @@ func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) e } if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to copy package %s error %s", p.name, err)) return err } default: + u.LogTrace(cliConfig, fmt.Sprintf("Unknown package type %s", p.name)) return fmt.Errorf("unknown package type") } if err = copyComponentToDestination(cliConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to copy package %s error %s", p.name, err)) return err } @@ -268,6 +274,7 @@ func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) e func installMixin(p pkgComponentVendor, cliConfig schema.CliConfiguration) error { tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to create temp directory %s", err)) return err } @@ -282,18 +289,21 @@ func installMixin(p pkgComponentVendor, cliConfig schema.CliConfiguration) error } if err = client.Get(); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to download package %s error %s", p.name, err)) return err } case pkgTypeOci: // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory err = processOciImage(cliConfig, p.uri, tempDir) if err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to process OCI image %s error %s", p.name, err)) return err } case pkgTypeLocal: return nil default: + u.LogTrace(cliConfig, fmt.Sprintf("Unknown package type %s", p.name)) return fmt.Errorf("unknown package type") } @@ -315,6 +325,7 @@ func installMixin(p pkgComponentVendor, cliConfig schema.CliConfiguration) error } if err = cp.Copy(tempDir, p.componentPath, copyOptions); err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("Failed to copy package %s error %s", p.name, err)) return err } From d17d4b0a774f3b2e8692871d71e6269366ff0c68 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 11:24:26 +0200 Subject: [PATCH 16/45] Fix index out of bounds errors in vendor model by adding checks before accessing packages --- go.mod | 2 +- internal/exec/vendor_model.go | 9 +++++++-- internal/exec/vendor_model_component.go | 8 ++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index bd0842dcf..cdf2fc27b 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/jwalton/go-supportscolor v1.2.0 github.com/kubescape/go-git-url v0.0.30 github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e + github.com/mattn/go-isatty v0.0.20 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/open-policy-agent/opa v0.70.0 @@ -169,7 +170,6 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 282a00762..cf889a63e 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -92,7 +92,10 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case installedPkgMsg: - + // ensure index is within bounds + if m.index >= len(m.packages) { + return m, nil + } pkg := m.packages[m.index] mark := checkMark @@ -153,7 +156,9 @@ func (m modelAtmosVendorInternal) View() string { spin := m.spinner.View() + " " prog := m.progress.View() cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) - + if m.index >= len(m.packages) { + return "" + } pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index 8579bd594..54d7fb5bf 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -81,7 +81,9 @@ func (m modelComponentVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case installedPkgMsg: - + if m.index >= len(m.packages) { + return m, nil + } pkg := m.packages[m.index] mark := checkMark @@ -142,7 +144,9 @@ func (m modelComponentVendorInternal) View() string { spin := m.spinner.View() + " " prog := m.progress.View() cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) - + if m.index >= len(m.packages) { + return "" + } pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) From 2355a7735dfb009e6b5c955c66f0c30d131077c5 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 11:27:34 +0200 Subject: [PATCH 17/45] Add nil checks for empty packages in vendor model initialization --- internal/exec/vendor_model.go | 8 +++++++- internal/exec/vendor_model_component.go | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index cf889a63e..426460833 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -65,7 +65,9 @@ func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig sc ) s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) - + if len(pkg) == 0 { + return modelAtmosVendorInternal{}, nil + } return modelAtmosVendorInternal{ packages: pkg, spinner: s, @@ -76,6 +78,10 @@ func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig sc } func (m modelAtmosVendorInternal) Init() tea.Cmd { + if len(m.packages) == 0 { + m.done = true + return nil + } return tea.Batch(downloadAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) } func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index 54d7fb5bf..d394dd166 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -53,7 +53,9 @@ func newModelComponentVendorInternal(pkg []pkgComponentVendor, dryRun bool, cliC ) s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) - + if len(pkg) == 0 { + return modelComponentVendorInternal{}, nil + } return modelComponentVendorInternal{ packages: pkg, spinner: s, @@ -65,6 +67,10 @@ func newModelComponentVendorInternal(pkg []pkgComponentVendor, dryRun bool, cliC func (m modelComponentVendorInternal) Init() tea.Cmd { // Start downloading with the `uri`, package name, and `tempDir` directly from the model + if len(m.packages) == 0 { + m.done = true + return nil + } return tea.Batch(downloadComponentAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) } func (m modelComponentVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { From c77eba3931fcac000861c0e5a13c68993ae4f90f Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 11:40:38 +0200 Subject: [PATCH 18/45] Add nil check for os.Stdin in daemon mode TUI rendering logic --- internal/exec/vendor_component_utils.go | 2 +- internal/exec/vendor_utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index ffffd233b..8e9cec908 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -339,7 +339,7 @@ func ExecuteComponentVendorInternal( return fmt.Errorf("error initializing model: %v", err) } var opts []tea.ProgramOption - if !isatty.IsTerminal(os.Stdout.Fd()) { + if os.Stdin == nil || !isatty.IsTerminal(os.Stdout.Fd()) { // If we're in daemon mode don't render the TUI opts = []tea.ProgramOption{tea.WithoutRenderer()} } diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index ebacc4805..cf801f2fc 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -317,7 +317,7 @@ func ExecuteAtmosVendorInternal( return fmt.Errorf("error initializing model: %v", err) } var opts []tea.ProgramOption - if !isatty.IsTerminal(os.Stdout.Fd()) { + if os.Stdin == nil || !isatty.IsTerminal(os.Stdout.Fd()) { // If we're in daemon mode don't render the TUI opts = []tea.ProgramOption{tea.WithoutRenderer()} } From b21c24e0ccebab359e6889d7929542f54ed4f198 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 12:06:58 +0200 Subject: [PATCH 19/45] Refactor TUI rendering logic to use CheckTTYSupport for better terminal compatibility --- internal/exec/vendor_component_utils.go | 17 +++++++++++++++-- internal/exec/vendor_utils.go | 4 +--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 8e9cec908..f1835eb32 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -339,8 +339,7 @@ func ExecuteComponentVendorInternal( return fmt.Errorf("error initializing model: %v", err) } var opts []tea.ProgramOption - if os.Stdin == nil || !isatty.IsTerminal(os.Stdout.Fd()) { - // If we're in daemon mode don't render the TUI + if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } @@ -350,3 +349,17 @@ func ExecuteComponentVendorInternal( } return nil } + +// CheckTTYSupport checks if both stdin and stdout support TTY. +func CheckTTYSupport() bool { + // Check for standard TTY support on stdin and stdout + stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) + stdoutTTY := isatty.IsTerminal(os.Stdout.Fd()) + + // Optionally, also check for Cygwin/MSYS compatibility if on Windows + stdinCygwinTTY := isatty.IsCygwinTerminal(os.Stdin.Fd()) + stdoutCygwinTTY := isatty.IsCygwinTerminal(os.Stdout.Fd()) + + // Return true if either standard TTY or Cygwin/MSYS TTY is available for both stdin and stdout + return (stdinTTY || stdinCygwinTTY) && (stdoutTTY || stdoutCygwinTTY) +} diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index cf801f2fc..c533572f1 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -8,7 +8,6 @@ import ( "strings" tea "github.com/charmbracelet/bubbletea" - "github.com/mattn/go-isatty" cp "github.com/otiai10/copy" "github.com/samber/lo" "github.com/spf13/cobra" @@ -317,8 +316,7 @@ func ExecuteAtmosVendorInternal( return fmt.Errorf("error initializing model: %v", err) } var opts []tea.ProgramOption - if os.Stdin == nil || !isatty.IsTerminal(os.Stdout.Fd()) { - // If we're in daemon mode don't render the TUI + if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } From 33c11f887a3381a003c72579974c4e3766b57e0b Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 12:24:58 +0200 Subject: [PATCH 20/45] Add TTY support check to disable TUI rendering when not available --- internal/exec/vendor_component_utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index f1835eb32..2a810f69a 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -19,6 +19,7 @@ import ( "github.com/hairyhenderson/gomplate/v3" "github.com/mattn/go-isatty" cp "github.com/otiai10/copy" + "golang.org/x/crypto/ssh/terminal" ) // findComponentConfigFile identifies the component vendoring config file (`component.yaml` or `component.yml`) @@ -339,6 +340,7 @@ func ExecuteComponentVendorInternal( return fmt.Errorf("error initializing model: %v", err) } var opts []tea.ProgramOption + // Disable TUI if no TTY support is available if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } @@ -352,6 +354,10 @@ func ExecuteComponentVendorInternal( // CheckTTYSupport checks if both stdin and stdout support TTY. func CheckTTYSupport() bool { + t := terminal.IsTerminal(int(os.Stdout.Fd())) + if !t { + return false + } // Check for standard TTY support on stdin and stdout stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) stdoutTTY := isatty.IsTerminal(os.Stdout.Fd()) From 9acf6fe4b88551c67e605fc56d3954ac80fcd84e Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 12:37:22 +0200 Subject: [PATCH 21/45] Refactor CheckTTYSupport to improve TTY detection on Linux and macOS --- internal/exec/vendor_component_utils.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 2a810f69a..acbea3796 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "text/template" @@ -19,7 +20,7 @@ import ( "github.com/hairyhenderson/gomplate/v3" "github.com/mattn/go-isatty" cp "github.com/otiai10/copy" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/sys/unix" ) // findComponentConfigFile identifies the component vendoring config file (`component.yaml` or `component.yml`) @@ -354,18 +355,16 @@ func ExecuteComponentVendorInternal( // CheckTTYSupport checks if both stdin and stdout support TTY. func CheckTTYSupport() bool { - t := terminal.IsTerminal(int(os.Stdout.Fd())) - if !t { - return false + nTTY := true + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { + _, err := unix.IoctlGetWinsize(int(os.Stdout.Fd()), unix.TIOCGWINSZ) + if err != nil { + nTTY = false + } } // Check for standard TTY support on stdin and stdout stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) - stdoutTTY := isatty.IsTerminal(os.Stdout.Fd()) - - // Optionally, also check for Cygwin/MSYS compatibility if on Windows - stdinCygwinTTY := isatty.IsCygwinTerminal(os.Stdin.Fd()) - stdoutCygwinTTY := isatty.IsCygwinTerminal(os.Stdout.Fd()) // Return true if either standard TTY or Cygwin/MSYS TTY is available for both stdin and stdout - return (stdinTTY || stdinCygwinTTY) && (stdoutTTY || stdoutCygwinTTY) + return stdinTTY && nTTY } From 9e7f6db4a9da75471d746bfae1c997fb81a756bb Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 12:51:17 +0200 Subject: [PATCH 22/45] debug stdout support TTY --- internal/exec/vendor_component_utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index acbea3796..8744ca6b7 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -355,6 +355,7 @@ func ExecuteComponentVendorInternal( // CheckTTYSupport checks if both stdin and stdout support TTY. func CheckTTYSupport() bool { + return false nTTY := true if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { _, err := unix.IoctlGetWinsize(int(os.Stdout.Fd()), unix.TIOCGWINSZ) From 068cff0c31b8acf139aa26ce93908c79c58d2a3c Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 13:09:56 +0200 Subject: [PATCH 23/45] Refactor CheckTTYSupport to only check stdin TTY support and simplify rendering options --- internal/exec/vendor_component_utils.go | 20 +++++--------------- internal/exec/vendor_utils.go | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 8744ca6b7..f80f83b68 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -8,7 +8,6 @@ import ( "os" "path" "path/filepath" - "runtime" "strings" "text/template" @@ -20,7 +19,6 @@ import ( "github.com/hairyhenderson/gomplate/v3" "github.com/mattn/go-isatty" cp "github.com/otiai10/copy" - "golang.org/x/sys/unix" ) // findComponentConfigFile identifies the component vendoring config file (`component.yaml` or `component.yml`) @@ -345,7 +343,7 @@ func ExecuteComponentVendorInternal( if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } - + opts = []tea.ProgramOption{tea.WithoutRenderer()} if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } @@ -353,19 +351,11 @@ func ExecuteComponentVendorInternal( return nil } -// CheckTTYSupport checks if both stdin and stdout support TTY. +// CheckTTYSupport checks if both stdin support TTY. func CheckTTYSupport() bool { - return false - nTTY := true - if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { - _, err := unix.IoctlGetWinsize(int(os.Stdout.Fd()), unix.TIOCGWINSZ) - if err != nil { - nTTY = false - } - } - // Check for standard TTY support on stdin and stdout + // Check for standard TTY support on stdin stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) - // Return true if either standard TTY or Cygwin/MSYS TTY is available for both stdin and stdout - return stdinTTY && nTTY + // Return true if either standard TTY + return stdinTTY } diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index c533572f1..7144c3c0f 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -319,7 +319,7 @@ func ExecuteAtmosVendorInternal( if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } - + opts = []tea.ProgramOption{tea.WithoutRenderer()} if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } From 5adcfa2aa7152255bfe1899b793fc2b9ef9b9863 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 13:14:58 +0200 Subject: [PATCH 24/45] Suppress log output when TTY support is not available in vendor execution --- internal/exec/vendor_component_utils.go | 5 ++++- internal/exec/vendor_utils.go | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index f80f83b68..3150e4186 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -5,6 +5,8 @@ import ( "context" "errors" "fmt" + "io" + "log" "os" "path" "path/filepath" @@ -343,7 +345,7 @@ func ExecuteComponentVendorInternal( if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } - opts = []tea.ProgramOption{tea.WithoutRenderer()} + log.SetOutput(io.Discard) if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } @@ -353,6 +355,7 @@ func ExecuteComponentVendorInternal( // CheckTTYSupport checks if both stdin support TTY. func CheckTTYSupport() bool { + // Check for standard TTY support on stdin stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 7144c3c0f..0b9be63ef 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -2,6 +2,8 @@ package exec import ( "fmt" + "io" + "log" "os" "path" "path/filepath" @@ -319,7 +321,7 @@ func ExecuteAtmosVendorInternal( if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } - opts = []tea.ProgramOption{tea.WithoutRenderer()} + log.SetOutput(io.Discard) if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } From 6d8512ce48443d4b0de1facf84b43fc601dce753 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 13:50:03 +0200 Subject: [PATCH 25/45] Remove log output suppression and add TTY support messages for non-interactive mode --- internal/exec/vendor_component_utils.go | 3 --- internal/exec/vendor_utils.go | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 3150e4186..3f437b2a8 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -5,8 +5,6 @@ import ( "context" "errors" "fmt" - "io" - "log" "os" "path" "path/filepath" @@ -345,7 +343,6 @@ func ExecuteComponentVendorInternal( if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} } - log.SetOutput(io.Discard) if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 0b9be63ef..826081201 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -2,8 +2,6 @@ package exec import ( "fmt" - "io" - "log" "os" "path" "path/filepath" @@ -320,8 +318,10 @@ func ExecuteAtmosVendorInternal( var opts []tea.ProgramOption if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} + fmt.Println("TTY is not supported. Running in non-interactive mode.") + } else { + fmt.Println("TTY is supported. Running in interactive mode.") } - log.SetOutput(io.Discard) if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) } From 256d08f1a08eeec8dddd6825c249674214a03a5c Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 14:33:13 +0200 Subject: [PATCH 26/45] Add TTYSupport field to modelAtmosVendorInternal and update logic for non-interactive mode --- internal/exec/vendor_model.go | 48 +++++++++++++++++++++++------------ internal/exec/vendor_utils.go | 11 ++++---- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 426460833..073027851 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -38,16 +38,17 @@ type pkgAtmosVendor struct { atmosVendorSource schema.AtmosVendorSource } type modelAtmosVendorInternal struct { - packages []pkgAtmosVendor - index int - width int - height int - spinner spinner.Model - progress progress.Model - done bool - dryRun bool - failedPkg int - cliConfig schema.CliConfiguration + packages []pkgAtmosVendor + index int + width int + height int + spinner spinner.Model + progress progress.Model + done bool + dryRun bool + failedPkg int + cliConfig schema.CliConfiguration + TTYSupport bool } var ( @@ -68,12 +69,14 @@ func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig sc if len(pkg) == 0 { return modelAtmosVendorInternal{}, nil } + tty := CheckTTYSupport() return modelAtmosVendorInternal{ - packages: pkg, - spinner: s, - progress: p, - dryRun: dryRun, - cliConfig: cliConfig, + packages: pkg, + spinner: s, + progress: p, + dryRun: dryRun, + cliConfig: cliConfig, + TTYSupport: tty, }, nil } @@ -125,6 +128,12 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // Update progress bar m.index++ + if !m.TTYSupport { + return m, tea.Batch( + tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program + downloadAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package + ) + } progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) return m, tea.Batch( progressCmd, @@ -136,6 +145,9 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.spinner, cmd = m.spinner.Update(msg) return m, cmd case progress.FrameMsg: + if !m.TTYSupport { + return m, nil + } newModel, cmd := m.progress.Update(msg) if newModel, ok := newModel.(progress.Model); ok { m.progress = newModel @@ -160,7 +172,11 @@ func (m modelAtmosVendorInternal) View() string { pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) spin := m.spinner.View() + " " - prog := m.progress.View() + prog := "" + if m.TTYSupport { + prog = m.progress.View() + } + cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) if m.index >= len(m.packages) { return "" diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 826081201..0eb2957f5 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -311,16 +311,15 @@ func ExecuteAtmosVendorInternal( // Run TUI to process packages if len(packages) > 0 { - model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) - if err != nil { - return fmt.Errorf("error initializing model: %v", err) - } + var opts []tea.ProgramOption if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer()} fmt.Println("TTY is not supported. Running in non-interactive mode.") - } else { - fmt.Println("TTY is supported. Running in interactive mode.") + } + model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) + if err != nil { + return fmt.Errorf("error initializing model: %v", err) } if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) From 70a63a22f31a7dd82b39f1951b799182eff64eb4 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 14:44:41 +0200 Subject: [PATCH 27/45] Refactor modelAtmosVendorInternal to remove TTYSupport field and related logic --- internal/exec/vendor_model.go | 60 ++++++++++------------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 073027851..fc8a670f3 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -38,17 +38,15 @@ type pkgAtmosVendor struct { atmosVendorSource schema.AtmosVendorSource } type modelAtmosVendorInternal struct { - packages []pkgAtmosVendor - index int - width int - height int - spinner spinner.Model - progress progress.Model - done bool - dryRun bool - failedPkg int - cliConfig schema.CliConfiguration - TTYSupport bool + packages []pkgAtmosVendor + index int + width int + height int + spinner spinner.Model + done bool + dryRun bool + failedPkg int + cliConfig schema.CliConfiguration } var ( @@ -59,24 +57,17 @@ var ( ) func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelAtmosVendorInternal, error) { - p := progress.New( - progress.WithDefaultGradient(), - progress.WithWidth(30), - progress.WithoutPercentage(), - ) + s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) if len(pkg) == 0 { return modelAtmosVendorInternal{}, nil } - tty := CheckTTYSupport() return modelAtmosVendorInternal{ - packages: pkg, - spinner: s, - progress: p, - dryRun: dryRun, - cliConfig: cliConfig, - TTYSupport: tty, + packages: pkg, + spinner: s, + dryRun: dryRun, + cliConfig: cliConfig, }, nil } @@ -128,15 +119,8 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // Update progress bar m.index++ - if !m.TTYSupport { - return m, tea.Batch( - tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program - downloadAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package - ) - } - progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) + //progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) return m, tea.Batch( - progressCmd, tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program downloadAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package ) @@ -145,14 +129,7 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.spinner, cmd = m.spinner.Update(msg) return m, cmd case progress.FrameMsg: - if !m.TTYSupport { - return m, nil - } - newModel, cmd := m.progress.Update(msg) - if newModel, ok := newModel.(progress.Model); ok { - m.progress = newModel - } - return m, cmd + return m, nil } return m, nil } @@ -172,11 +149,8 @@ func (m modelAtmosVendorInternal) View() string { pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) spin := m.spinner.View() + " " + //prog := m.progress.View() prog := "" - if m.TTYSupport { - prog = m.progress.View() - } - cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) if m.index >= len(m.packages) { return "" From 063bab078f603c600dca9a2c2d742e8d595c435f Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 14:53:33 +0200 Subject: [PATCH 28/45] Add progress bar to modelAtmosVendorInternal for package installation updates --- internal/exec/vendor_model.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index fc8a670f3..426460833 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -43,6 +43,7 @@ type modelAtmosVendorInternal struct { width int height int spinner spinner.Model + progress progress.Model done bool dryRun bool failedPkg int @@ -57,7 +58,11 @@ var ( ) func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelAtmosVendorInternal, error) { - + p := progress.New( + progress.WithDefaultGradient(), + progress.WithWidth(30), + progress.WithoutPercentage(), + ) s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) if len(pkg) == 0 { @@ -66,6 +71,7 @@ func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig sc return modelAtmosVendorInternal{ packages: pkg, spinner: s, + progress: p, dryRun: dryRun, cliConfig: cliConfig, }, nil @@ -119,8 +125,9 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // Update progress bar m.index++ - //progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) + progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) return m, tea.Batch( + progressCmd, tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program downloadAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package ) @@ -129,7 +136,11 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.spinner, cmd = m.spinner.Update(msg) return m, cmd case progress.FrameMsg: - return m, nil + newModel, cmd := m.progress.Update(msg) + if newModel, ok := newModel.(progress.Model); ok { + m.progress = newModel + } + return m, cmd } return m, nil } @@ -149,8 +160,7 @@ func (m modelAtmosVendorInternal) View() string { pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) spin := m.spinner.View() + " " - //prog := m.progress.View() - prog := "" + prog := m.progress.View() cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) if m.index >= len(m.packages) { return "" From 8f51cabc6e613c8c9e95d36a20d721fc12f9b54a Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 15:01:33 +0200 Subject: [PATCH 29/45] package pull status when TTY support is unavailable --- internal/exec/vendor_model.go | 3 +++ internal/exec/vendor_utils.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 426460833..9de17df3d 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -146,6 +146,9 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } func (m modelAtmosVendorInternal) View() string { + if !CheckTTYSupport() { + return fmt.Sprintf("Pulling %s", m.packages[m.index].name) + } n := len(m.packages) w := lipgloss.Width(fmt.Sprintf("%d", n)) if m.done { diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 0eb2957f5..fa2ab4fdd 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -322,7 +322,7 @@ func ExecuteAtmosVendorInternal( return fmt.Errorf("error initializing model: %v", err) } if _, err := tea.NewProgram(model, opts...).Run(); err != nil { - return fmt.Errorf("running download error: %w", err) + return fmt.Errorf("running atmos vendor internal download error: %w", err) } } From 7e28e179aa527aab7cdf277f34b4f9e90f6eeb9d Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 15:08:11 +0200 Subject: [PATCH 30/45] Enhance TTY support check to include stdout terminal status --- internal/exec/vendor_component_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 3f437b2a8..62a658f04 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -354,7 +354,7 @@ func ExecuteComponentVendorInternal( func CheckTTYSupport() bool { // Check for standard TTY support on stdin - stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) + stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) && isatty.IsTerminal(os.Stdout.Fd()) // Return true if either standard TTY return stdinTTY From 772c681524f80fa35c6a9ace73574d8b91fce29a Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 15:15:49 +0200 Subject: [PATCH 31/45] Implement non-interactive mode with spinner feedback and remove TTY dependency --- internal/exec/vendor_model.go | 4 +- internal/exec/vendor_tty.go | 104 ++++++++++++++++++++++++++++++++++ internal/exec/vendor_utils.go | 17 ++++-- 3 files changed, 116 insertions(+), 9 deletions(-) create mode 100644 internal/exec/vendor_tty.go diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 9de17df3d..3bcfc83ef 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -146,9 +146,7 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } func (m modelAtmosVendorInternal) View() string { - if !CheckTTYSupport() { - return fmt.Sprintf("Pulling %s", m.packages[m.index].name) - } + n := len(m.packages) w := lipgloss.Width(fmt.Sprintf("%d", n)) if m.done { diff --git a/internal/exec/vendor_tty.go b/internal/exec/vendor_tty.go new file mode 100644 index 000000000..f4b318219 --- /dev/null +++ b/internal/exec/vendor_tty.go @@ -0,0 +1,104 @@ +package exec + +import ( + "fmt" + "log" + "math/rand" + "time" + + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +var ( + helpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("241")).Render + mainStyle = lipgloss.NewStyle().MarginLeft(1) +) + +type result struct { + duration time.Duration + emoji string +} + +type model struct { + spinner spinner.Model + results []result + quitting bool +} + +func newModel() model { + const showLastResults = 5 + + sp := spinner.New() + sp.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("206")) + + return model{ + spinner: sp, + results: make([]result, showLastResults), + } +} + +func (m model) Init() tea.Cmd { + log.Println("Starting work...") + return tea.Batch( + m.spinner.Tick, + runPretendProcess, + ) +} + +func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + m.quitting = true + return m, tea.Quit + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + case processFinishedMsg: + d := time.Duration(msg) + res := result{emoji: randomEmoji(), duration: d} + log.Printf("%s Job finished in %s", res.emoji, res.duration) + m.results = append(m.results[1:], res) + return m, runPretendProcess + default: + return m, nil + } +} + +func (m model) View() string { + s := "\n" + + m.spinner.View() + " Doing some work...\n\n" + + for _, res := range m.results { + if res.duration == 0 { + s += "........................\n" + } else { + s += fmt.Sprintf("%s Job finished in %s\n", res.emoji, res.duration) + } + } + + s += helpStyle("\nPress any key to exit\n") + + if m.quitting { + s += "\n" + } + + return mainStyle.Render(s) +} + +// processFinishedMsg is sent when a pretend process completes. +type processFinishedMsg time.Duration + +// pretendProcess simulates a long-running process. +func runPretendProcess() tea.Msg { + pause := time.Duration(rand.Int63n(899)+100) * time.Millisecond // nolint:gosec + time.Sleep(pause) + return processFinishedMsg(pause) +} + +func randomEmoji() string { + emojis := []rune("🍦🧋🍡🤠👾😭🦊🐯🦆🥨🎏🍔🍒🍥🎮📦🦁🐶🐸🍕🥐🧲🚒🥇🏆🌽") + return string(emojis[rand.Intn(len(emojis))]) // nolint:gosec +} diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index fa2ab4fdd..221ed955e 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -317,12 +317,17 @@ func ExecuteAtmosVendorInternal( opts = []tea.ProgramOption{tea.WithoutRenderer()} fmt.Println("TTY is not supported. Running in non-interactive mode.") } - model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) - if err != nil { - return fmt.Errorf("error initializing model: %v", err) - } - if _, err := tea.NewProgram(model, opts...).Run(); err != nil { - return fmt.Errorf("running atmos vendor internal download error: %w", err) + // model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) + // if err != nil { + // return fmt.Errorf("error initializing model: %v", err) + // } + // if _, err := tea.NewProgram(model, opts...).Run(); err != nil { + // return fmt.Errorf("running atmos vendor internal download error: %w", err) + // } + p := tea.NewProgram(newModel(), opts...) + if _, err := p.Run(); err != nil { + fmt.Println("Error starting Bubble Tea program:", err) + os.Exit(1) } } From 0214dc1188bee30103f3db05a59fabca37c54361 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 15:38:17 +0200 Subject: [PATCH 32/45] Update bubbletea and ansi dependencies to latest versions --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index cdf2fc27b..ae5fe1cee 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/arsham/figurine v1.3.0 github.com/bmatcuk/doublestar/v4 v4.7.1 github.com/charmbracelet/bubbles v0.20.0 - github.com/charmbracelet/bubbletea v1.1.2 + github.com/charmbracelet/bubbletea v1.2.1 github.com/charmbracelet/glamour v0.8.0 github.com/charmbracelet/lipgloss v1.0.0 github.com/elewis787/boa v0.1.2 @@ -95,7 +95,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chainguard-dev/git-urls v1.0.2 // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect - github.com/charmbracelet/x/ansi v0.4.2 // indirect + github.com/charmbracelet/x/ansi v0.4.5 // indirect github.com/charmbracelet/x/term v0.2.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect diff --git a/go.sum b/go.sum index 5a1060077..f52347495 100644 --- a/go.sum +++ b/go.sum @@ -397,6 +397,8 @@ github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQW github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= github.com/charmbracelet/bubbletea v1.1.2 h1:naQXF2laRxyLyil/i7fxdpiz1/k06IKquhm4vBfHsIc= github.com/charmbracelet/bubbletea v1.1.2/go.mod h1:9HIU/hBV24qKjlehyj8z1r/tR9TYTQEag+cWZnuXo8E= +github.com/charmbracelet/bubbletea v1.2.1 h1:J041h57zculJKEKf/O2pS4edXGIz+V0YvojvfGXePIk= +github.com/charmbracelet/bubbletea v1.2.1/go.mod h1:viLoDL7hG4njLJSKU2gw7kB3LSEmWsrM80rO1dBJWBI= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= @@ -405,6 +407,8 @@ github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo= github.com/charmbracelet/x/ansi v0.4.2 h1:0JM6Aj/g/KC154/gOP4vfxun0ff6itogDYk41kof+qk= github.com/charmbracelet/x/ansi v0.4.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/ansi v0.4.5 h1:LqK4vwBNaXw2AyGIICa5/29Sbdq58GbGdFngSexTdRM= +github.com/charmbracelet/x/ansi v0.4.5/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q= github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/term v0.2.0 h1:cNB9Ot9q8I711MyZ7myUR5HFWL/lc3OpU8jZ4hwm0x0= From ba96f1610975a76ebd4023d36f2db95620e32031 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 16:18:51 +0200 Subject: [PATCH 33/45] Remove vendor_tty.go and update non-interactive mode handling in vendor_utils.go --- internal/exec/vendor_tty.go | 104 ---------------------------------- internal/exec/vendor_utils.go | 20 +++---- 2 files changed, 8 insertions(+), 116 deletions(-) delete mode 100644 internal/exec/vendor_tty.go diff --git a/internal/exec/vendor_tty.go b/internal/exec/vendor_tty.go deleted file mode 100644 index f4b318219..000000000 --- a/internal/exec/vendor_tty.go +++ /dev/null @@ -1,104 +0,0 @@ -package exec - -import ( - "fmt" - "log" - "math/rand" - "time" - - "github.com/charmbracelet/bubbles/spinner" - tea "github.com/charmbracelet/bubbletea" - "github.com/charmbracelet/lipgloss" -) - -var ( - helpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("241")).Render - mainStyle = lipgloss.NewStyle().MarginLeft(1) -) - -type result struct { - duration time.Duration - emoji string -} - -type model struct { - spinner spinner.Model - results []result - quitting bool -} - -func newModel() model { - const showLastResults = 5 - - sp := spinner.New() - sp.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("206")) - - return model{ - spinner: sp, - results: make([]result, showLastResults), - } -} - -func (m model) Init() tea.Cmd { - log.Println("Starting work...") - return tea.Batch( - m.spinner.Tick, - runPretendProcess, - ) -} - -func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { - switch msg := msg.(type) { - case tea.KeyMsg: - m.quitting = true - return m, tea.Quit - case spinner.TickMsg: - var cmd tea.Cmd - m.spinner, cmd = m.spinner.Update(msg) - return m, cmd - case processFinishedMsg: - d := time.Duration(msg) - res := result{emoji: randomEmoji(), duration: d} - log.Printf("%s Job finished in %s", res.emoji, res.duration) - m.results = append(m.results[1:], res) - return m, runPretendProcess - default: - return m, nil - } -} - -func (m model) View() string { - s := "\n" + - m.spinner.View() + " Doing some work...\n\n" - - for _, res := range m.results { - if res.duration == 0 { - s += "........................\n" - } else { - s += fmt.Sprintf("%s Job finished in %s\n", res.emoji, res.duration) - } - } - - s += helpStyle("\nPress any key to exit\n") - - if m.quitting { - s += "\n" - } - - return mainStyle.Render(s) -} - -// processFinishedMsg is sent when a pretend process completes. -type processFinishedMsg time.Duration - -// pretendProcess simulates a long-running process. -func runPretendProcess() tea.Msg { - pause := time.Duration(rand.Int63n(899)+100) * time.Millisecond // nolint:gosec - time.Sleep(pause) - return processFinishedMsg(pause) -} - -func randomEmoji() string { - emojis := []rune("🍦🧋🍡🤠👾😭🦊🐯🦆🥨🎏🍔🍒🍥🎮📦🦁🐶🐸🍕🥐🧲🚒🥇🏆🌽") - return string(emojis[rand.Intn(len(emojis))]) // nolint:gosec -} diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 221ed955e..e65f47b36 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -314,21 +314,17 @@ func ExecuteAtmosVendorInternal( var opts []tea.ProgramOption if !CheckTTYSupport() { - opts = []tea.ProgramOption{tea.WithoutRenderer()} + opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} fmt.Println("TTY is not supported. Running in non-interactive mode.") } - // model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) - // if err != nil { - // return fmt.Errorf("error initializing model: %v", err) - // } - // if _, err := tea.NewProgram(model, opts...).Run(); err != nil { - // return fmt.Errorf("running atmos vendor internal download error: %w", err) - // } - p := tea.NewProgram(newModel(), opts...) - if _, err := p.Run(); err != nil { - fmt.Println("Error starting Bubble Tea program:", err) - os.Exit(1) + model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) + if err != nil { + return fmt.Errorf("error initializing model: %v", err) + } + if _, err := tea.NewProgram(model, opts...).Run(); err != nil { + return fmt.Errorf("running atmos vendor internal download error: %w", err) } + } return nil From 84455d36751231275ebd56788561620b2938adbe Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 16:31:55 +0200 Subject: [PATCH 34/45] Fix non-interactive mode handling by adding input option for TTY support check --- internal/exec/vendor_component_utils.go | 2 +- internal/exec/vendor_utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 62a658f04..56a556d94 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -341,7 +341,7 @@ func ExecuteComponentVendorInternal( var opts []tea.ProgramOption // Disable TUI if no TTY support is available if !CheckTTYSupport() { - opts = []tea.ProgramOption{tea.WithoutRenderer()} + opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} } if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index e65f47b36..ee2855a1b 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -314,8 +314,8 @@ func ExecuteAtmosVendorInternal( var opts []tea.ProgramOption if !CheckTTYSupport() { + // set tea.WithInput(nil) workaround the issue on non TTY mode https://github.com/charmbracelet/bubbletea/issues/761 opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} - fmt.Println("TTY is not supported. Running in non-interactive mode.") } model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) if err != nil { From 68d6e613618164e8022d4db760531eeb8d3f89bb Mon Sep 17 00:00:00 2001 From: haitham911 Date: Sun, 10 Nov 2024 16:33:56 +0200 Subject: [PATCH 35/45] Refactor TTY support check to only evaluate stdin and update comments for clarity --- internal/exec/vendor_component_utils.go | 2 +- internal/exec/vendor_utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 56a556d94..a43ce8388 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -354,7 +354,7 @@ func ExecuteComponentVendorInternal( func CheckTTYSupport() bool { // Check for standard TTY support on stdin - stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) && isatty.IsTerminal(os.Stdout.Fd()) + stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) // Return true if either standard TTY return stdinTTY diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index ee2855a1b..5258db7ad 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -314,7 +314,7 @@ func ExecuteAtmosVendorInternal( var opts []tea.ProgramOption if !CheckTTYSupport() { - // set tea.WithInput(nil) workaround the issue on non TTY mode https://github.com/charmbracelet/bubbletea/issues/761 + // set tea.WithInput(nil) workaround tea program not run on not TTY mod issue on non TTY mode https://github.com/charmbracelet/bubbletea/issues/761 opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} } model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) From 0ace6987423174364b294f80778226b6fdb54ed5 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 11 Nov 2024 14:43:54 +0200 Subject: [PATCH 36/45] Add '--everything' flag to vendor pull command and update handling logic --- cmd/vendor_pull.go | 2 +- internal/exec/vendor_model.go | 77 ++++++++++--- internal/exec/vendor_model_component.go | 139 ++++-------------------- internal/exec/vendor_utils.go | 17 ++- 4 files changed, 97 insertions(+), 138 deletions(-) diff --git a/cmd/vendor_pull.go b/cmd/vendor_pull.go index 50fe6cec1..b616d365d 100644 --- a/cmd/vendor_pull.go +++ b/cmd/vendor_pull.go @@ -32,6 +32,6 @@ func init() { vendorPullCmd.PersistentFlags().StringP("type", "t", "terraform", "atmos vendor pull --component --type=terraform|helmfile") vendorPullCmd.PersistentFlags().Bool("dry-run", false, "atmos vendor pull --component --dry-run") vendorPullCmd.PersistentFlags().String("tags", "", "Only vendor the components that have the specified tags: atmos vendor pull --tags=dev,test") - + vendorPullCmd.PersistentFlags().Bool("everything", false, "Vendor all components: atmos vendor pull --everything") vendorCmd.AddCommand(vendorPullCmd) } diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 3bcfc83ef..aba78cc09 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -28,6 +28,12 @@ const ( pkgTypeLocal ) +type pkgVendor struct { + name string + version string + atmosPackage *pkgAtmosVendor + componentPackage *pkgComponentVendor +} type pkgAtmosVendor struct { uri string name string @@ -37,8 +43,8 @@ type pkgAtmosVendor struct { version string atmosVendorSource schema.AtmosVendorSource } -type modelAtmosVendorInternal struct { - packages []pkgAtmosVendor +type modelVendor struct { + packages []pkgVendor index int width int height int @@ -48,6 +54,7 @@ type modelAtmosVendorInternal struct { dryRun bool failedPkg int cliConfig schema.CliConfiguration + isTTY bool } var ( @@ -57,7 +64,7 @@ var ( xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") ) -func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelAtmosVendorInternal, error) { +func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelVendor, error) { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(30), @@ -65,26 +72,37 @@ func newModelAtmosVendorInternal(pkg []pkgAtmosVendor, dryRun bool, cliConfig sc ) s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) - if len(pkg) == 0 { - return modelAtmosVendorInternal{}, nil + if len(pkgs) == 0 { + return modelVendor{}, nil + } + tty := CheckTTYSupport() + var vendorPks []pkgVendor + for _, pkg := range pkgs { + p := pkgVendor{ + name: pkg.name, + version: pkg.version, + atmosPackage: &pkg, + } + vendorPks = append(vendorPks, p) } - return modelAtmosVendorInternal{ - packages: pkg, + return modelVendor{ + packages: vendorPks, spinner: s, progress: p, dryRun: dryRun, cliConfig: cliConfig, + isTTY: tty, }, nil } -func (m modelAtmosVendorInternal) Init() tea.Cmd { +func (m modelVendor) Init() tea.Cmd { if len(m.packages) == 0 { m.done = true return nil } - return tea.Batch(downloadAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) + return tea.Batch(ExecuteInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) } -func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { +func (m modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: m.width, m.height = msg.Width, msg.Height @@ -103,6 +121,7 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } pkg := m.packages[m.index] + mark := checkMark if msg.err != nil { @@ -117,19 +136,35 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if m.index >= len(m.packages)-1 { // Everything's been installed. We're done! m.done = true + if !m.isTTY { + u.LogInfo(m.cliConfig, fmt.Sprintf("%s %s %s", mark, pkg.name, version)) + + if m.dryRun { + u.LogInfo(m.cliConfig, "Done! Dry run completed. No components vendored.\n") + } + if m.failedPkg > 0 { + u.LogInfo(m.cliConfig, fmt.Sprintf("Vendored %d components. Failed to vendor %d components.\n", len(m.packages)-m.failedPkg, m.failedPkg)) + } + u.LogInfo(m.cliConfig, fmt.Sprintf("Vendored %d components.\n", len(m.packages))) + } return m, tea.Sequence( tea.Printf("%s %s %s", mark, pkg.name, version), tea.Quit, ) } - - // Update progress bar + if !m.isTTY { + u.LogInfo(m.cliConfig, fmt.Sprintf("%s %s %s", mark, pkg.name, version)) + } m.index++ + // Update progress bar progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) + if !m.isTTY { + u.LogInfo(m.cliConfig, fmt.Sprintf("Pulling %s", m.packages[m.index].name)) + } return m, tea.Batch( progressCmd, - tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program - downloadAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package + tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program + ExecuteInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package ) case spinner.TickMsg: var cmd tea.Cmd @@ -145,7 +180,7 @@ func (m modelAtmosVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } -func (m modelAtmosVendorInternal) View() string { +func (m modelVendor) View() string { n := len(m.packages) w := lipgloss.Width(fmt.Sprintf("%d", n)) @@ -187,7 +222,7 @@ func max(a, b int) int { } return b } -func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { +func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { // Simulate the action @@ -197,6 +232,7 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi name: p.name, } } + // Create temp directory tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { @@ -270,3 +306,12 @@ func downloadAndInstall(p pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfi } } } +func ExecuteInstall(installer pkgVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { + if installer.atmosPackage != nil { + return downloadAndInstall(installer.atmosPackage, dryRun, cliConfig) + } + if installer.componentPackage != nil { + return downloadComponentAndInstall(installer.componentPackage, dryRun, cliConfig) + } + return nil +} diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index d394dd166..cd610e24b 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -7,7 +7,6 @@ import ( "path" "path/filepath" "strconv" - "strings" "time" "github.com/charmbracelet/bubbles/progress" @@ -32,20 +31,8 @@ type pkgComponentVendor struct { IsMixins bool mixinFilename string } -type modelComponentVendorInternal struct { - packages []pkgComponentVendor - index int - width int - height int - spinner spinner.Model - progress progress.Model - done bool - dryRun bool - failedPkg int - cliConfig schema.CliConfiguration -} -func newModelComponentVendorInternal(pkg []pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelComponentVendorInternal, error) { +func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelVendor, error) { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(30), @@ -53,117 +40,31 @@ func newModelComponentVendorInternal(pkg []pkgComponentVendor, dryRun bool, cliC ) s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) - if len(pkg) == 0 { - return modelComponentVendorInternal{}, nil + if len(pkgs) == 0 { + return modelVendor{}, nil + } + vendorPks := []pkgVendor{} + for _, pkg := range pkgs { + vendorPkg := pkgVendor{ + name: pkg.name, + version: pkg.version, + componentPackage: &pkg, + } + vendorPks = append(vendorPks, vendorPkg) + } - return modelComponentVendorInternal{ - packages: pkg, + tty := CheckTTYSupport() + return modelVendor{ + packages: vendorPks, spinner: s, progress: p, dryRun: dryRun, cliConfig: cliConfig, + isTTY: tty, }, nil } -func (m modelComponentVendorInternal) Init() tea.Cmd { - // Start downloading with the `uri`, package name, and `tempDir` directly from the model - if len(m.packages) == 0 { - m.done = true - return nil - } - return tea.Batch(downloadComponentAndInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) -} -func (m modelComponentVendorInternal) Update(msg tea.Msg) (tea.Model, tea.Cmd) { - switch msg := msg.(type) { - case tea.WindowSizeMsg: - m.width, m.height = msg.Width, msg.Height - if m.width > 120 { - m.width = 120 - } - case tea.KeyMsg: - switch msg.String() { - case "ctrl+c", "esc", "q": - return m, tea.Quit - } - - case installedPkgMsg: - if m.index >= len(m.packages) { - return m, nil - } - pkg := m.packages[m.index] - mark := checkMark - - if msg.err != nil { - u.LogDebug(m.cliConfig, fmt.Sprintf("Failed to vendor component %s error %s", pkg.name, msg.err)) - mark = xMark - m.failedPkg++ - } - version := "" - if pkg.version != "" { - version = fmt.Sprintf("(%s)", pkg.version) - } - if m.index >= len(m.packages)-1 { - // Everything's been installed. We're done! - m.done = true - return m, tea.Sequence( - tea.Printf("%s %s %s", mark, pkg.name, version), - tea.Quit, - ) - } - - // Update progress bar - m.index++ - progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) - return m, tea.Batch( - progressCmd, - tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program - downloadComponentAndInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package - ) - case spinner.TickMsg: - var cmd tea.Cmd - m.spinner, cmd = m.spinner.Update(msg) - return m, cmd - case progress.FrameMsg: - newModel, cmd := m.progress.Update(msg) - if newModel, ok := newModel.(progress.Model); ok { - m.progress = newModel - } - return m, cmd - } - return m, nil -} - -func (m modelComponentVendorInternal) View() string { - n := len(m.packages) - w := lipgloss.Width(fmt.Sprintf("%d", n)) - if m.done { - if m.dryRun { - return doneStyle.Render("Done! Dry run completed. No components vendored.\n") - } - if m.failedPkg > 0 { - return doneStyle.Render(fmt.Sprintf("Vendored %d components.Failed to vendor %d components.\n", n-m.failedPkg, m.failedPkg)) - } - return doneStyle.Render(fmt.Sprintf("Vendored %d components.\n", n)) - } - - pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) - spin := m.spinner.View() + " " - prog := m.progress.View() - cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) - if m.index >= len(m.packages) { - return "" - } - pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) - - info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) - - cellsRemaining := max(0, m.width-lipgloss.Width(spin+info+prog+pkgCount)) - gap := strings.Repeat(" ", cellsRemaining) - - return spin + info + gap + prog + pkgCount -} - -func downloadComponentAndInstall(p pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { +func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { // Simulate the action @@ -208,7 +109,7 @@ func downloadComponentAndInstall(p pkgComponentVendor, dryRun bool, cliConfig sc } } } -func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) error { +func installComponent(p *pkgComponentVendor, cliConfig schema.CliConfiguration) error { // Create temp folder // We are using a temp folder for the following reasons: @@ -281,7 +182,7 @@ func installComponent(p pkgComponentVendor, cliConfig schema.CliConfiguration) e return nil } -func installMixin(p pkgComponentVendor, cliConfig schema.CliConfiguration) error { +func installMixin(p *pkgComponentVendor, cliConfig schema.CliConfiguration) error { tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { u.LogTrace(cliConfig, fmt.Sprintf("Failed to create temp directory %s", err)) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 5258db7ad..506239c20 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -19,6 +19,7 @@ import ( // ExecuteVendorPullCommand executes `atmos vendor` commands func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { + info, err := processCommandLineArgs("terraform", cmd, args, nil) if err != nil { return err @@ -69,6 +70,17 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { if component != "" && len(tags) > 0 { return fmt.Errorf("either '--component' or '--tags' flag can to be provided, but not both") } + everything, err := flags.GetBool("everything") + if err != nil { + return err + } + if !everything && !flags.Changed("everything") && (component == "" && stack == "" && len(tags) == 0) { + // Display help and return an error to prevent execution + return fmt.Errorf("either '--component', '--stack', '--tags', or '--everything' flag must be provided") + } + if everything && (component != "" || stack != "" || len(tags) > 0) { + return fmt.Errorf("'--everything' flag cannot be combined with '--component', '--stack', or '--tags' flags") + } if stack != "" { // Process stack vendoring @@ -80,13 +92,14 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { if vendorConfigExists && err != nil { return err } - + if !vendorConfigExists && everything { + return fmt.Errorf("the '--everything' flag is set, but the vendor config file '%s' does not exist", cfg.AtmosVendorConfigFileName) + } if vendorConfigExists { // Process `vendor.yaml` return ExecuteAtmosVendorInternal(cliConfig, foundVendorConfigFile, vendorConfig.Spec, component, tags, dryRun) } else { // Check and process `component.yaml` - fmt.Println("No vendor config file found. Checking for component vendoring...") if component != "" { // Process component vendoring componentType, err := flags.GetString("type") From 3e973f3be0cdf676e280805dfc8d781113973f27 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 11 Nov 2024 14:59:58 +0200 Subject: [PATCH 37/45] Update atmos.yaml to include '--everything' flag in vendor pull command --- examples/demo-vendoring/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/demo-vendoring/atmos.yaml b/examples/demo-vendoring/atmos.yaml index 1c7d3ad24..149947338 100644 --- a/examples/demo-vendoring/atmos.yaml +++ b/examples/demo-vendoring/atmos.yaml @@ -27,4 +27,4 @@ commands: - name: "test" description: "Run all tests" steps: - - atmos vendor pull + - atmos vendor pull --everything From 201af7c8afd69a12d8522d578a588d43ab800090 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Mon, 11 Nov 2024 15:03:54 +0200 Subject: [PATCH 38/45] Add '--everything' flag to vendor pull command in atmos.yaml --- examples/demo-component-versions/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/demo-component-versions/atmos.yaml b/examples/demo-component-versions/atmos.yaml index e24b091f4..35ad30e31 100644 --- a/examples/demo-component-versions/atmos.yaml +++ b/examples/demo-component-versions/atmos.yaml @@ -25,4 +25,4 @@ commands: - name: "test" description: "Run all tests" steps: - - atmos vendor pull + - atmos vendor pull --everything From 82a150759978d7eb1edef4cdd627057638828410 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Tue, 12 Nov 2024 09:50:29 +0200 Subject: [PATCH 39/45] refactor: update vendor component handling and improve error logging --- internal/exec/vendor_component_utils.go | 5 ++--- internal/exec/vendor_model.go | 15 +++++++++++++-- internal/exec/vendor_model_component.go | 6 +++++- internal/exec/vendor_utils.go | 13 +++++++------ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index a43ce8388..605768742 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -311,8 +311,6 @@ func ExecuteComponentVendorInternal( } if useOciScheme { pType = pkgTypeOci - } else if useLocalFileSystem { - pType = pkgTypeLocal } else { pType = pkgTypeRemote } @@ -321,7 +319,7 @@ func ExecuteComponentVendorInternal( uri: uri, pkgType: pType, name: "mixin " + uri, - sourceIsLocalFile: sourceIsLocalFile, + sourceIsLocalFile: false, IsMixins: true, vendorComponentSpec: vendorComponentSpec, version: mixin.Version, @@ -342,6 +340,7 @@ func ExecuteComponentVendorInternal( // Disable TUI if no TTY support is available if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} + u.LogWarning(cliConfig, "No TTY detected. Falling back to basic output. This can happen when no terminal is attached or when commands are pipelined.") } if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index aba78cc09..07550dbbe 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -62,6 +62,7 @@ var ( doneStyle = lipgloss.NewStyle().Margin(1, 2) checkMark = lipgloss.NewStyle().Foreground(lipgloss.Color("42")).SetString("✓") xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") + grayColor = lipgloss.NewStyle().Foreground(lipgloss.Color("241")) ) func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelVendor, error) { @@ -147,6 +148,7 @@ func (m modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } u.LogInfo(m.cliConfig, fmt.Sprintf("Vendored %d components.\n", len(m.packages))) } + version := grayColor.Render(version) return m, tea.Sequence( tea.Printf("%s %s %s", mark, pkg.name, version), tea.Quit, @@ -161,6 +163,7 @@ func (m modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if !m.isTTY { u.LogInfo(m.cliConfig, fmt.Sprintf("Pulling %s", m.packages[m.index].name)) } + version = grayColor.Render(version) return m, tea.Batch( progressCmd, tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program @@ -293,7 +296,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, cliConfig schema.CliConf } } - if err := copyToTarget(cliConfig, tempDir, p.targetPath, p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTarget(cliConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { u.LogTrace(cliConfig, fmt.Sprintf("Failed to copy package %s error %s", p.name, err)) return installedPkgMsg{ err: err, @@ -313,5 +316,13 @@ func ExecuteInstall(installer pkgVendor, dryRun bool, cliConfig schema.CliConfig if installer.componentPackage != nil { return downloadComponentAndInstall(installer.componentPackage, dryRun, cliConfig) } - return nil + // No valid package provided + return func() tea.Msg { + err := fmt.Errorf("no valid installer package provided for %s", installer.name) + u.LogError(cliConfig, err) + return installedPkgMsg{ + err: err, + name: installer.name, + } + } } diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index cd610e24b..ceb2ed0b4 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -211,7 +211,11 @@ func installMixin(p *pkgComponentVendor, cliConfig schema.CliConfiguration) erro return err } case pkgTypeLocal: - return nil + if p.uri == "" { + return fmt.Errorf("local mixin URI cannot be empty") + } + // Implement local mixin installation logic + return fmt.Errorf("local mixin installation not implemented") default: u.LogTrace(cliConfig, fmt.Sprintf("Unknown package type %s", p.name)) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 506239c20..1684ef2c9 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -263,11 +263,11 @@ func ExecuteAtmosVendorInternal( // Process sources var packages []pkgAtmosVendor for indexSource, s := range sources { - if shouldSkipSource(s, component, tags) { + if shouldSkipSource(&s, component, tags) { continue } - if err := validateSourceFields(s, vendorConfigFileName); err != nil { + if err := validateSourceFields(&s, vendorConfigFileName); err != nil { return err } @@ -329,6 +329,7 @@ func ExecuteAtmosVendorInternal( if !CheckTTYSupport() { // set tea.WithInput(nil) workaround tea program not run on not TTY mod issue on non TTY mode https://github.com/charmbracelet/bubbletea/issues/761 opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} + u.LogWarning(cliConfig, "No TTY detected. Falling back to basic output. This can happen when no terminal is attached or when commands are pipelined.") } model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) if err != nil { @@ -398,7 +399,7 @@ func logInitialMessage(cliConfig schema.CliConfiguration, vendorConfigFileName s u.LogInfo(cliConfig, logMessage) } -func validateSourceFields(s schema.AtmosVendorSource, vendorConfigFileName string) error { +func validateSourceFields(s *schema.AtmosVendorSource, vendorConfigFileName string) error { // Ensure necessary fields are present if s.File == "" { s.File = vendorConfigFileName @@ -411,7 +412,7 @@ func validateSourceFields(s schema.AtmosVendorSource, vendorConfigFileName strin } return nil } -func shouldSkipSource(s schema.AtmosVendorSource, component string, tags []string) bool { +func shouldSkipSource(s *schema.AtmosVendorSource, component string, tags []string) bool { // Skip if component or tags do not match // If `--component` is specified, and it's not equal to this component, skip this component // If `--tags` list is specified, and it does not contain any tags defined in this component, skip this component @@ -437,7 +438,7 @@ func determineSourceType(uri, vendorConfigFilePath string) (bool, bool, bool) { return useOciScheme, useLocalFileSystem, sourceIsLocalFile } -func copyToTarget(cliConfig schema.CliConfiguration, tempDir, targetPath string, s schema.AtmosVendorSource, sourceIsLocalFile bool, uri string) error { +func copyToTarget(cliConfig schema.CliConfiguration, tempDir, targetPath string, s *schema.AtmosVendorSource, sourceIsLocalFile bool, uri string) error { copyOptions := cp.Options{ Skip: generateSkipFunction(cliConfig, tempDir, s), PreserveTimes: false, @@ -453,7 +454,7 @@ func copyToTarget(cliConfig schema.CliConfiguration, tempDir, targetPath string, return cp.Copy(tempDir, targetPath, copyOptions) } -func generateSkipFunction(cliConfig schema.CliConfiguration, tempDir string, s schema.AtmosVendorSource) func(os.FileInfo, string, string) (bool, error) { +func generateSkipFunction(cliConfig schema.CliConfiguration, tempDir string, s *schema.AtmosVendorSource) func(os.FileInfo, string, string) (bool, error) { return func(srcInfo os.FileInfo, src, dest string) (bool, error) { if strings.HasSuffix(src, ".git") { return true, nil From e60ecda8369b31b5948735bb99eb1e9af2d0b2b3 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Tue, 12 Nov 2024 10:07:56 +0200 Subject: [PATCH 40/45] feat: add '--everything' flag to 'atmos vendor pull' command in documentation --- website/docs/cheatsheets/commands.mdx | 2 +- website/docs/cheatsheets/vendoring.mdx | 2 +- website/docs/cli/commands/vendor/vendor-pull.mdx | 6 +++--- website/docs/core-concepts/vendor/vendor-manifest.mdx | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/website/docs/cheatsheets/commands.mdx b/website/docs/cheatsheets/commands.mdx index 0128c5567..ad79dd11e 100644 --- a/website/docs/cheatsheets/commands.mdx +++ b/website/docs/cheatsheets/commands.mdx @@ -190,7 +190,7 @@ import CardGroup from '@site/src/components/CardGroup' ``` - atmos vendor pull + atmos vendor pull --everything ```

Pull sources and mixins from remote repositories for Terraform and Helmfile components and other artifacts

 diff --git a/website/docs/cheatsheets/vendoring.mdx b/website/docs/cheatsheets/vendoring.mdx index fc8d845b3..90d7b193e 100644 --- a/website/docs/cheatsheets/vendoring.mdx +++ b/website/docs/cheatsheets/vendoring.mdx @@ -67,7 +67,7 @@ import CardGroup from '@site/src/components/CardGroup' ```shell - atmos vendor pull + atmos vendor pull --everything atmos vendor pull --component vpc-mixin-1 atmos vendor pull -c vpc-mixin-2 atmos vendor pull -c vpc-mixin-3 diff --git a/website/docs/cli/commands/vendor/vendor-pull.mdx b/website/docs/cli/commands/vendor/vendor-pull.mdx index 944b1579b..a3f84a534 100644 --- a/website/docs/cli/commands/vendor/vendor-pull.mdx +++ b/website/docs/cli/commands/vendor/vendor-pull.mdx @@ -29,7 +29,7 @@ With Atmos vendoring, you can copy components and other artifacts from the follo Execute the `vendor pull` command like this: ```shell -atmos vendor pull +atmos vendor pull --everything atmos vendor pull --component [options] atmos vendor pull -c [options] atmos vendor pull --tags , [options] @@ -114,7 +114,7 @@ Run `atmos vendor pull --help` to see all the available options ## Examples ```shell -atmos vendor pull +atmos vendor pull --everything atmos vendor pull --component vpc atmos vendor pull -c vpc-flow-logs-bucket atmos vendor pull -c echo-server --type helmfile @@ -124,7 +124,7 @@ atmos vendor pull --tags networking --dry-run :::note -When executing the `atmos vendor pull` command, Atmos performs the following steps to decide which vendoring manifest to use: +When executing the `atmos vendor pull --everything` command, Atmos performs the following steps to decide which vendoring manifest to use: - If `vendor.yaml` manifest is found (in the directory from which the command is executed), Atmos will parse the file and execute the command against it. If the flag `--component` is not specified, Atmos will vendor all the artifacts defined in the `vendor.yaml` manifest. diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index 55f6e319c..52b0d080f 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -27,7 +27,7 @@ Atmos searches for the vendoring manifest in the following locations and uses th After defining the `vendor.yaml` manifest, all the remote artifacts can be downloaded by running the following command: ```shell -atmos vendor pull +atmos vendor pull --everything ``` To vendor a particular component or other artifact, execute the following command: @@ -154,7 +154,7 @@ To vendor remote artifacts, create a `vendor.yaml` file similar to the example b With this configuration, it would be possible to run the following commands: ```shell -# atmos vendor pull +# atmos vendor pull --everything # atmos vendor pull --component vpc-mixin-1 # atmos vendor pull -c vpc-mixin-2 # atmos vendor pull -c vpc-mixin-3 @@ -411,7 +411,7 @@ are defined: ```shell -> atmos vendor pull +> atmos vendor pull --everything Processing vendor config file 'vendor.yaml' Pulling sources for the component 'my-vpc6' from 'github.com/cloudposse/terraform-aws-components.git//modules/vpc?ref=1.315.0' into 'components/terraform/infra/my-vpc6' From 643018933323754ddd6e060a6d30928b54f7efa1 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Wed, 13 Nov 2024 14:07:45 +0200 Subject: [PATCH 41/45] refactor: improve TTY support checks and enhance logging messages; add URI validation --- internal/exec/vendor_component_utils.go | 13 +++++-------- internal/exec/vendor_utils.go | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 605768742..cc8aa97eb 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -340,7 +340,7 @@ func ExecuteComponentVendorInternal( // Disable TUI if no TTY support is available if !CheckTTYSupport() { opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} - u.LogWarning(cliConfig, "No TTY detected. Falling back to basic output. This can happen when no terminal is attached or when commands are pipelined.") + u.LogWarning(cliConfig, "TTY is not supported. Running in non-interactive mode") } if _, err := tea.NewProgram(model, opts...).Run(); err != nil { return fmt.Errorf("running download error: %w", err) @@ -349,12 +349,9 @@ func ExecuteComponentVendorInternal( return nil } -// CheckTTYSupport checks if both stdin support TTY. +// CheckTTYSupport checks stdin support TTY. func CheckTTYSupport() bool { - - // Check for standard TTY support on stdin - stdinTTY := isatty.IsTerminal(os.Stdin.Fd()) - - // Return true if either standard TTY - return stdinTTY + stdinTTYStdin := isatty.IsTerminal(os.Stdin.Fd()) + stdoutTTYStdout := isatty.IsTerminal(os.Stdout.Fd()) + return stdinTTYStdin && stdoutTTYStdout } diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 1684ef2c9..4d4266957 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -281,7 +281,10 @@ func ExecuteAtmosVendorInternal( if err != nil { return err } - + err = validateURI(uri) + if err != nil { + return err + } useOciScheme, useLocalFileSystem, sourceIsLocalFile := determineSourceType(uri, vendorConfigFilePath) // Determine package type @@ -329,7 +332,7 @@ func ExecuteAtmosVendorInternal( if !CheckTTYSupport() { // set tea.WithInput(nil) workaround tea program not run on not TTY mod issue on non TTY mode https://github.com/charmbracelet/bubbletea/issues/761 opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} - u.LogWarning(cliConfig, "No TTY detected. Falling back to basic output. This can happen when no terminal is attached or when commands are pipelined.") + u.LogWarning(cliConfig, "TTY is not supported. Running in non-interactive mode") } model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) if err != nil { @@ -392,7 +395,7 @@ func processVendorImports( return append(mergedSources, sources...), allImports, nil } func logInitialMessage(cliConfig schema.CliConfiguration, vendorConfigFileName string, tags []string) { - logMessage := fmt.Sprintf("Processing vendor config file '%s'", vendorConfigFileName) + logMessage := fmt.Sprintf("Vendoring from '%s'", vendorConfigFileName) if len(tags) > 0 { logMessage = fmt.Sprintf("%s for tags {%s}", logMessage, strings.Join(tags, ", ")) } @@ -511,3 +514,10 @@ func generateSkipFunction(cliConfig schema.CliConfiguration, tempDir string, s * return false, nil } } +func validateURI(uri string) error { + if uri == "" { + return fmt.Errorf("URI cannot be empty") + } + // Add more validation as needed + return nil +} From aa16b2b25e40fc6aef470622d62d6ed6862d8466 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Wed, 13 Nov 2024 14:33:34 +0200 Subject: [PATCH 42/45] refactor: implement context timeout for package download process --- internal/exec/vendor_model.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 07550dbbe..e67c25c64 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -243,12 +243,13 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, cliConfig schema.CliConf return err } defer removeTempDir(cliConfig, tempDir) - + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() switch p.pkgType { case pkgTypeRemote: // Use go-getter to download remote packages client := &getter.Client{ - Ctx: context.Background(), + Ctx: ctx, Dst: tempDir, Src: p.uri, Mode: getter.ClientModeAny, From 7ac6601f05a3ef82e7986082363a9774f82d7877 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Wed, 13 Nov 2024 14:38:12 +0200 Subject: [PATCH 43/45] chore: remove outdated dependencies from go.sum --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index 88b523249..822b319f9 100644 --- a/go.sum +++ b/go.sum @@ -395,8 +395,6 @@ github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiw github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= -github.com/charmbracelet/bubbletea v1.1.2 h1:naQXF2laRxyLyil/i7fxdpiz1/k06IKquhm4vBfHsIc= -github.com/charmbracelet/bubbletea v1.1.2/go.mod h1:9HIU/hBV24qKjlehyj8z1r/tR9TYTQEag+cWZnuXo8E= github.com/charmbracelet/bubbletea v1.2.1 h1:J041h57zculJKEKf/O2pS4edXGIz+V0YvojvfGXePIk= github.com/charmbracelet/bubbletea v1.2.1/go.mod h1:viLoDL7hG4njLJSKU2gw7kB3LSEmWsrM80rO1dBJWBI= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= @@ -405,8 +403,6 @@ github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo= -github.com/charmbracelet/x/ansi v0.4.2 h1:0JM6Aj/g/KC154/gOP4vfxun0ff6itogDYk41kof+qk= -github.com/charmbracelet/x/ansi v0.4.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/ansi v0.4.5 h1:LqK4vwBNaXw2AyGIICa5/29Sbdq58GbGdFngSexTdRM= github.com/charmbracelet/x/ansi v0.4.5/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q= From 7757b4423ca8dc89115dc8fd32107bbd0162c74d Mon Sep 17 00:00:00 2001 From: haitham911 Date: Wed, 13 Nov 2024 18:07:24 +0200 Subject: [PATCH 44/45] refactor: improve error messages and flag validation in ExecuteVendorPullCommand --- internal/exec/vendor_utils.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 4d4266957..6ef2cfb85 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -64,20 +64,25 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { } if component != "" && stack != "" { - return fmt.Errorf("either '--component' or '--stack' flag can to be provided, but not both") + return fmt.Errorf("either '--component' or '--stack' flag can be provided, but not both") } if component != "" && len(tags) > 0 { - return fmt.Errorf("either '--component' or '--tags' flag can to be provided, but not both") + return fmt.Errorf("either '--component' or '--tags' flag can be provided, but not both") } + + // Retrieve the 'everything' flag and set default behavior if no other flags are set everything, err := flags.GetBool("everything") if err != nil { return err } - if !everything && !flags.Changed("everything") && (component == "" && stack == "" && len(tags) == 0) { - // Display help and return an error to prevent execution - return fmt.Errorf("either '--component', '--stack', '--tags', or '--everything' flag must be provided") + + // If neither `everything`, `component`, `stack`, nor `tags` flags are set, default to `everything = true` + if !everything && !flags.Changed("everything") && component == "" && stack == "" && len(tags) == 0 { + everything = true } + + // Validate that only one of `--everything`, `--component`, `--stack`, or `--tags` is provided if everything && (component != "" || stack != "" || len(tags) > 0) { return fmt.Errorf("'--everything' flag cannot be combined with '--component', '--stack', or '--tags' flags") } From e1f1fff3d28c76277d4b901962f471a306f88129 Mon Sep 17 00:00:00 2001 From: haitham911 Date: Wed, 13 Nov 2024 18:35:50 +0200 Subject: [PATCH 45/45] refactor: update modelVendor return value to indicate completion when no packages are provided --- internal/exec/vendor_model.go | 2 +- internal/exec/vendor_model_component.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index e67c25c64..c9dc66a65 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -74,7 +74,7 @@ func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, cliConfig s s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) if len(pkgs) == 0 { - return modelVendor{}, nil + return modelVendor{done: true}, nil } tty := CheckTTYSupport() var vendorPks []pkgVendor diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index ceb2ed0b4..c3e9eaa73 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -41,7 +41,7 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, cli s := spinner.New() s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) if len(pkgs) == 0 { - return modelVendor{}, nil + return modelVendor{done: true}, nil } vendorPks := []pkgVendor{} for _, pkg := range pkgs {