Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 5ad8112

Browse files
imeoerimeoer
committedFeb 8, 2024
smoke: add test for external blob
Signed-off-by: Yan Song <[email protected]>
1 parent 9a4d0ea commit 5ad8112

File tree

26 files changed

+248
-240
lines changed

26 files changed

+248
-240
lines changed
 

‎builder/src/attributes.rs

+81-48
Original file line numberDiff line numberDiff line change
@@ -10,79 +10,112 @@ use anyhow::Result;
1010
use gix_attributes::parse;
1111
use gix_attributes::parse::Kind;
1212

13-
pub struct Attribute {}
13+
const KEY_TYPE: &str = "type";
14+
const VAL_EXTERNAL: &str = "external";
1415

15-
impl Attribute {
16+
pub struct Parser {}
17+
18+
#[derive(Clone, Debug, Eq, PartialEq, Default)]
19+
pub struct Item {
20+
pub pattern: PathBuf,
21+
pub attributes: HashMap<String, String>,
22+
}
23+
24+
#[derive(Clone, Debug, Eq, PartialEq, Default)]
25+
pub struct Attributes {
26+
pub items: Vec<Item>,
27+
}
28+
29+
impl Attributes {
1630
/// Parse nydus attributes from a file.
17-
pub fn parse<P: AsRef<Path>>(path: P) -> Result<HashMap<PathBuf, u32>> {
31+
pub fn from<P: AsRef<Path>>(path: P) -> Result<Attributes> {
1832
let content = fs::read(path)?;
19-
let attributes = parse(&content);
20-
let mut result = HashMap::new();
21-
for attribute in attributes {
22-
let attribute = attribute?;
23-
if let Kind::Pattern(pattern) = attribute.0 {
24-
let mut path: Option<PathBuf> = None;
25-
let mut backend_index: Option<u32> = None;
26-
for line in attribute.1 {
27-
let line = line?;
28-
if line.name.as_str() == "type"
29-
&& line.state.as_bstr().unwrap_or_default() == "external"
30-
{
31-
let _path = PathBuf::from(pattern.text.to_string());
32-
if !_path.is_absolute() {
33-
path = Some(path::Path::new("/").join(_path));
34-
}
35-
}
36-
if line.name.as_str() == "backend_index" {
37-
backend_index = Some(
38-
line.state
39-
.as_bstr()
40-
.unwrap_or_default()
41-
.to_string()
42-
.parse()?,
43-
);
44-
}
33+
let _items = parse(&content);
34+
35+
let mut items = Vec::new();
36+
for _item in _items {
37+
let _item = _item?;
38+
if let Kind::Pattern(pattern) = _item.0 {
39+
let mut path = PathBuf::from(pattern.text.to_string());
40+
if !path.is_absolute() {
41+
path = path::Path::new("/").join(path);
4542
}
46-
match (path, backend_index) {
47-
(Some(path), Some(backend_index)) => {
48-
result.insert(path, backend_index);
49-
}
50-
_ => {}
43+
let mut attributes = HashMap::new();
44+
for line in _item.1 {
45+
let line = line?;
46+
let name = line.name.as_str();
47+
let state = line.state.as_bstr().unwrap_or_default();
48+
attributes.insert(name.to_string(), state.to_string());
5149
}
50+
items.push(Item {
51+
pattern: path,
52+
attributes,
53+
});
5254
}
5355
}
54-
Ok(result)
56+
57+
Ok(Attributes { items })
58+
}
59+
60+
fn check_external(&self, item: &Item) -> bool {
61+
item.attributes.get(KEY_TYPE) == Some(&VAL_EXTERNAL.to_string())
62+
}
63+
64+
pub fn is_external<P: AsRef<Path>>(&self, path: P) -> bool {
65+
self.items
66+
.iter()
67+
.any(|item| item.pattern == path.as_ref() && self.check_external(item))
68+
}
69+
70+
pub fn is_prefix_external<P: AsRef<Path>>(&self, target: P) -> bool {
71+
self.items
72+
.iter()
73+
.any(|item| item.pattern.starts_with(&target) && self.check_external(item))
5574
}
5675
}
5776

5877
#[cfg(test)]
5978
mod tests {
6079
use std::{collections::HashMap, fs, path::PathBuf};
6180

62-
use super::Attribute;
81+
use super::{Attributes, Item};
6382
use vmm_sys_util::tempfile::TempFile;
6483

6584
#[test]
6685
fn test_attribute_parse() {
6786
let file = TempFile::new().unwrap();
6887
fs::write(
6988
file.as_path(),
70-
"/foo type=external backend_index=0
71-
/bar type=external backend_index=0
72-
/models/foo type=external backend_index=1",
89+
"/foo type=external
90+
/bar type=external
91+
/models/foo type=external",
7392
)
7493
.unwrap();
75-
let paths = Attribute::parse(file.as_path()).unwrap();
76-
assert_eq!(
77-
paths,
78-
[
79-
(PathBuf::from("/foo"), 0),
80-
(PathBuf::from("/bar"), 0),
81-
(PathBuf::from("/models/foo"), 1)
82-
]
94+
95+
let attributes = Attributes::from(file.as_path()).unwrap();
96+
let _attributes: HashMap<String, String> = [("type".to_string(), "external".to_string())]
8397
.iter()
8498
.cloned()
85-
.collect::<HashMap<PathBuf, u32>>()
99+
.collect();
100+
101+
assert_eq!(
102+
attributes,
103+
Attributes {
104+
items: vec![
105+
Item {
106+
pattern: PathBuf::from("/foo"),
107+
attributes: _attributes.clone()
108+
},
109+
Item {
110+
pattern: PathBuf::from("/bar"),
111+
attributes: _attributes.clone()
112+
},
113+
Item {
114+
pattern: PathBuf::from("/models/foo"),
115+
attributes: _attributes.clone()
116+
}
117+
]
118+
},
86119
);
87120
}
88121
}

‎builder/src/compact.rs

+4-3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use nydus_utils::{digest, try_round_up_4k};
2121
use serde::{Deserialize, Serialize};
2222
use sha2::Digest;
2323

24+
use crate::attributes::Attributes;
2425
use crate::core::context::Artifact;
2526

2627
use super::core::blob::Blob;
@@ -615,7 +616,7 @@ impl BlobCompactor {
615616
false,
616617
Features::new(),
617618
false,
618-
HashMap::new(),
619+
Attributes::default(),
619620
);
620621
let mut bootstrap_mgr =
621622
BootstrapManager::new(Some(ArtifactStorage::SingleFile(d_bootstrap)), None);
@@ -1142,7 +1143,7 @@ mod tests {
11421143
false,
11431144
Features::new(),
11441145
false,
1145-
HashMap::new(),
1146+
Attributes::default(),
11461147
);
11471148

11481149
let mut compactor = blob_compactor_load_and_dedup_chunks().unwrap();
@@ -1246,7 +1247,7 @@ mod tests {
12461247
false,
12471248
Features::new(),
12481249
false,
1249-
HashMap::new(),
1250+
Attributes::default(),
12501251
);
12511252
let mut blob_ctx1 = BlobContext::new(
12521253
"blob_id1".to_owned(),

‎builder/src/core/context.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ use nydus_utils::digest::DigestData;
4444
use nydus_utils::{compress, digest, div_round_up, round_down, try_round_up_4k, BufReaderInfo};
4545

4646
use super::node::ChunkSource;
47+
use crate::attributes::Attributes;
4748
use crate::core::tree::TreeNode;
4849
use crate::{ChunkDict, Feature, Features, HashChunkDict, Prefetch, PrefetchPolicy, WhiteoutSpec};
4950

@@ -613,7 +614,6 @@ impl BlobContext {
613614
blob_ctx
614615
.blob_meta_header
615616
.set_encrypted(features.contains(BlobFeatures::ENCRYPTED));
616-
println!("EXTERNAL {}", features.contains(BlobFeatures::EXTERNAL));
617617
blob_ctx
618618
.blob_meta_header
619619
.set_external(features.contains(BlobFeatures::EXTERNAL));
@@ -938,7 +938,7 @@ impl BlobManager {
938938
)))
939939
}
940940
};
941-
let mut blob_features = ctx.blob_features.clone();
941+
let mut blob_features = ctx.blob_features;
942942
let mut compressor = ctx.compressor;
943943
if self.external {
944944
blob_features.insert(BlobFeatures::EXTERNAL);
@@ -1328,7 +1328,7 @@ pub struct BuildContext {
13281328
pub blob_cache_generator: Option<BlobCacheGenerator>,
13291329

13301330
/// Nydus attributes for different build behavior.
1331-
pub attributes: HashMap<PathBuf, u32>,
1331+
pub attributes: Attributes,
13321332
}
13331333

13341334
impl BuildContext {
@@ -1349,7 +1349,7 @@ impl BuildContext {
13491349
blob_inline_meta: bool,
13501350
features: Features,
13511351
encrypt: bool,
1352-
attributes: HashMap<PathBuf, u32>,
1352+
attributes: Attributes,
13531353
) -> Self {
13541354
// It's a flag for images built with new nydus-image 2.2 and newer.
13551355
let mut blob_features = BlobFeatures::CAP_TAR_TOC;
@@ -1454,7 +1454,7 @@ impl Default for BuildContext {
14541454
configuration: Arc::new(ConfigV2::default()),
14551455
blob_cache_generator: None,
14561456

1457-
attributes: HashMap::new(),
1457+
attributes: Attributes::default(),
14581458
}
14591459
}
14601460
}

‎builder/src/core/node.rs

+4-7
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,6 @@ impl Node {
334334
.layered_chunk_dict
335335
.add_chunk(chunk.clone(), ctx.digester);
336336
}
337-
println!("CHUNK {:?} {}", self.target(), chunk);
338337
self.chunks.push(NodeChunk {
339338
source: ChunkSource::Build,
340339
inner: chunk,
@@ -382,12 +381,10 @@ impl Node {
382381
.read_exact(buf)
383382
.with_context(|| format!("failed to read node file {:?}", self.path()))?;
384383
}
385-
} else {
386-
if !external {
387-
reader
388-
.read_exact(buf)
389-
.with_context(|| format!("failed to read node file {:?}", self.path()))?;
390-
}
384+
} else if !external {
385+
reader
386+
.read_exact(buf)
387+
.with_context(|| format!("failed to read node file {:?}", self.path()))?;
391388
}
392389

393390
// For tar-tarfs case, no need to compute chunk id.

‎builder/src/core/v6.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ impl Node {
4141
orig_meta_addr: u64,
4242
meta_addr: u64,
4343
chunk_cache: &mut BTreeMap<DigestWithBlobIndex, Arc<ChunkWrapper>>,
44-
blobs: &Vec<Arc<BlobInfo>>,
44+
blobs: &[Arc<BlobInfo>],
4545
) -> Result<()> {
4646
let xattr_inline_count = self.info.xattrs.count_v6();
4747
ensure!(
@@ -454,7 +454,7 @@ impl Node {
454454
f_bootstrap: &mut dyn RafsIoWrite,
455455
chunk_cache: &mut BTreeMap<DigestWithBlobIndex, Arc<ChunkWrapper>>,
456456
inode: &mut Box<dyn RafsV6OndiskInode>,
457-
blobs: &Vec<Arc<BlobInfo>>,
457+
blobs: &[Arc<BlobInfo>],
458458
) -> Result<()> {
459459
let mut is_continuous = true;
460460
let mut prev = None;

‎builder/src/directory.rs

+7-10
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ impl FilesystemTreeBuilder {
7171

7272
let (mut child, mut external_child) = (Tree::new(child.clone()), Tree::new(child));
7373

74-
let external = ctx.attributes.get(&target).is_some();
74+
let external = ctx.attributes.is_external(&target);
7575
if external {
7676
info!("ignore external file data: {:?}", path);
7777
}
@@ -92,11 +92,8 @@ impl FilesystemTreeBuilder {
9292
external_trees.push(external_child);
9393
} else {
9494
trees.push(child.clone());
95-
for (path, _) in &ctx.attributes {
96-
if path.starts_with(&target) {
97-
external_trees.push(external_child);
98-
break;
99-
}
95+
if ctx.attributes.is_prefix_external(target) {
96+
external_trees.push(external_child);
10097
}
10198
};
10299
}
@@ -236,10 +233,10 @@ impl Builder for DirectoryBuilder {
236233
ctx.prefetch = prefetch::Prefetch::new(prefetch::PrefetchPolicy::None)?;
237234
let mut external_blob_mgr = BlobManager::new(ctx.digester, true);
238235
let mut external_bootstrap_mgr = bootstrap_mgr.clone();
239-
external_bootstrap_mgr
240-
.bootstrap_storage
241-
.as_mut()
242-
.map(|stor| stor.add_suffix("external"));
236+
if let Some(stor) = external_bootstrap_mgr.bootstrap_storage.as_mut() {
237+
stor.add_suffix("external")
238+
}
239+
243240
let mut external_blob_writer: Box<dyn Artifact> =
244241
if let Some(blob_stor) = ctx.external_blob_storage.clone() {
245242
Box::new(ArtifactWriter::new(blob_stor)?)

‎builder/src/stargz.rs

+4-2
Original file line numberDiff line numberDiff line change
@@ -911,7 +911,9 @@ impl Builder for StargzBuilder {
911911
#[cfg(test)]
912912
mod tests {
913913
use super::*;
914-
use crate::{ArtifactStorage, ConversionType, Features, Prefetch, WhiteoutSpec};
914+
use crate::{
915+
attributes::Attributes, ArtifactStorage, ConversionType, Features, Prefetch, WhiteoutSpec,
916+
};
915917

916918
#[test]
917919
fn test_build_stargz_toc() {
@@ -937,7 +939,7 @@ mod tests {
937939
false,
938940
Features::new(),
939941
false,
940-
HashMap::new(),
942+
Attributes::default(),
941943
);
942944
ctx.fs_version = RafsVersion::V6;
943945
ctx.conversion_type = ConversionType::EStargzToRafs;

‎builder/src/tarball.rs

+3-4
Original file line numberDiff line numberDiff line change
@@ -665,9 +665,8 @@ impl Builder for TarballBuilder {
665665

666666
#[cfg(test)]
667667
mod tests {
668-
use std::collections::HashMap;
669-
670668
use super::*;
669+
use crate::attributes::Attributes;
671670
use crate::{ArtifactStorage, Features, Prefetch, WhiteoutSpec};
672671
use nydus_utils::{compress, digest};
673672

@@ -694,7 +693,7 @@ mod tests {
694693
false,
695694
Features::new(),
696695
false,
697-
HashMap::new(),
696+
Attributes::default(),
698697
);
699698
let mut bootstrap_mgr = BootstrapManager::new(
700699
Some(ArtifactStorage::FileDir((tmp_dir, String::new()))),
@@ -730,7 +729,7 @@ mod tests {
730729
false,
731730
Features::new(),
732731
true,
733-
HashMap::new(),
732+
Attributes::default(),
734733
);
735734
let mut bootstrap_mgr = BootstrapManager::new(
736735
Some(ArtifactStorage::FileDir((tmp_dir, String::new()))),

‎smoke/.nydusattributes

-3
This file was deleted.

‎smoke/go.mod

+3-1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ require (
3333
github.com/pmezard/go-difflib v1.0.0 // indirect
3434
github.com/rogpeppe/go-internal v1.12.0 // indirect
3535
github.com/sirupsen/logrus v1.9.3 // indirect
36+
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
37+
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
3638
go.opencensus.io v0.24.0 // indirect
3739
golang.org/x/mod v0.14.0 // indirect
3840
golang.org/x/sync v0.5.0 // indirect
@@ -45,4 +47,4 @@ require (
4547
gopkg.in/yaml.v3 v3.0.1 // indirect
4648
)
4749

48-
replace github.com/containerd/nydus-snapshotter => github.com/imeoer/nydus-snapshotter v0.13.10
50+
replace github.com/containerd/nydus-snapshotter => github.com/imeoer/nydus-snapshotter v0.13.13

0 commit comments

Comments
 (0)
Please sign in to comment.