builder: introduce optimize subcommand for prefetch

We can statically analyze the image entrypoint dependency, or use runtime dynamic
analysis technologies such as ebpf, fanotify, metric, etc. to obtain the container
file access pattern, and then build this part of data into an independent image layer:

* preferentially fetch blob during the image startup phase to reduce network and disk IO.
* avoid frequent image builds, allows for better local cache utilization.

Implement optimize subcommand to optimize image bootstrap
from a prefetch file list, generate a new blob.

```
nydus-image optimize --prefetch-files /path/to/prefetch-files.txt \
  --bootstrap /path/to/bootstrap \
  --blob-dir /path/to/blobs
```
This will generate a new bootstrap and new blob in `blob-dir`.

Signed-off-by: daiyongxuan <[email protected]>

builder/src/chunkdict_generator.rs

@@ -18,18 +18,20 @@ use super::core::node::{ChunkSource, NodeInfo};
 use super::{BlobManager, Bootstrap, BootstrapManager, BuildContext, BuildOutput, Tree};
 use crate::core::node::Node;
 use crate::NodeChunk;
-use anyhow::Result;
+use crate::OsString;
+use anyhow::{Ok, Result};
 use nydus_rafs::metadata::chunk::ChunkWrapper;
 use nydus_rafs::metadata::inode::InodeWrapper;
 use nydus_rafs::metadata::layout::RafsXAttrs;
 use nydus_storage::meta::BlobChunkInfoV1Ondisk;
 use nydus_utils::compress::Algorithm;
 use nydus_utils::digest::RafsDigest;
-use std::ffi::OsString;
+
 use std::mem::size_of;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::u32;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct ChunkdictChunkInfo {

builder/src/core/blob.rs

@@ -94,7 +94,7 @@ impl Blob {
         Ok(())
     }
 
-    fn finalize_blob_data(
+    pub fn finalize_blob_data(
         ctx: &BuildContext,
         blob_mgr: &mut BlobManager,
         blob_writer: &mut dyn Artifact,

builder/src/core/context.rs

@@ -13,6 +13,7 @@ use std::io::{BufWriter, Cursor, Read, Seek, Write};
 use std::mem::size_of;
 use std::os::unix::fs::FileTypeExt;
 use std::path::{Display, Path, PathBuf};
+use std::result::Result::Ok;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 use std::{fmt, fs};
@@ -459,6 +460,7 @@ impl BlobCacheGenerator {
     }
 }
 
+#[derive(Clone)]
 /// BlobContext is used to hold the blob information of a layer during build.
 pub struct BlobContext {
     /// Blob id (user specified or sha256(blob)).
@@ -898,6 +900,11 @@ impl BlobManager {
         }
     }
 
+    /// Set current blob index
+    pub fn set_current_blob_index(&mut self, index: usize) {
+        self.current_blob_index = Some(index as u32)
+    }
+
     fn new_blob_ctx(ctx: &BuildContext) -> Result<BlobContext> {
         let (cipher_object, cipher_ctx) = match ctx.cipher {
             crypt::Algorithm::None => (Default::default(), None),

builder/src/core/overlay.rs

@@ -71,6 +71,16 @@ pub enum WhiteoutSpec {
     None,
 }
 
+impl fmt::Display for WhiteoutSpec {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        match self {
+            WhiteoutSpec::Oci => write!(f, "oci"),
+            WhiteoutSpec::Overlayfs => write!(f, "overlayfs"),
+            WhiteoutSpec::None => write!(f, "none"),
+        }
+    }
+}
+
 impl Default for WhiteoutSpec {
     fn default() -> Self {
         Self::Oci

builder/src/core/tree.rs

@@ -174,6 +174,30 @@ impl Tree {
         Some(tree)
     }
 
+    /// Get the mutable tree node corresponding to the path.
+    pub fn get_node_mut(&mut self, path: &Path) -> Option<&mut Tree> {
+        let target_vec = Node::generate_target_vec(path);
+        assert!(!target_vec.is_empty());
+        let mut tree = self;
+
+        let last_idx = target_vec.len() - 1;
+        for name in &target_vec[1..last_idx] {
+            match tree.get_child_idx(name.as_bytes()) {
+                Some(idx) => tree = &mut tree.children[idx],
+                None => return None,
+            }
+        }
+
+        if let Some(last_name) = target_vec.last() {
+            match tree.get_child_idx(last_name.as_bytes()) {
+                Some(idx) => Some(&mut tree.children[idx]),
+                None => None,
+            }
+        } else {
+            Some(tree)
+        }
+    }
+
     /// Merge the upper layer tree into the lower layer tree, applying whiteout rules.
     pub fn merge_overaly(&mut self, ctx: &BuildContext, upper: Tree) -> Result<()> {
         assert_eq!(self.name, "/".as_bytes());

builder/src/lib.rs

@@ -41,6 +41,8 @@ pub use self::core::prefetch::{Prefetch, PrefetchPolicy};
 pub use self::core::tree::{MetadataTreeBuilder, Tree, TreeNode};
 pub use self::directory::DirectoryBuilder;
 pub use self::merge::Merger;
+pub use self::optimize_prefetch::update_ctx_from_bootstrap;
+pub use self::optimize_prefetch::OptimizePrefetch;
 pub use self::stargz::StargzBuilder;
 pub use self::tarball::TarballBuilder;
 
@@ -49,6 +51,7 @@ mod compact;
 mod core;
 mod directory;
 mod merge;
+mod optimize_prefetch;
 mod stargz;
 mod tarball;
 
@@ -249,7 +252,6 @@ fn finalize_blob(
             blob_cache.finalize(&blob_ctx.blob_id)?;
         }
     }
-
     Ok(())
 }