diff --git a/Cargo.lock b/Cargo.lock index 687c4dcd..8cdb092f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,43 +156,69 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "aws-credential-types" -version = "0.56.1" +name = "aws-config" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a66ac8ef5fa9cf01c2d999f39d16812e90ec1467bd382cbbb74ba23ea86201" +checksum = "004dc45f6b869e6a70725df448004a720b7f52f6607d55d8815cbd5448f86def" dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", + "aws-types", + "bytes", "fastrand", + "hex", + "http", + "hyper", + "ring", + "time", "tokio", "tracing", "zeroize", ] +[[package]] +name = "aws-credential-types" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfa51c87f10211f37cd78e6d01d6f18b3f96a086906ed361d11e04ac53e29508" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-http" -version = "0.56.1" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e626370f9ba806ae4c439e49675fd871f5767b093075cdf4fef16cac42ba900" +checksum = "361c4310fdce94328cc2d1ca0c8a48c13f43009c61d3367585685a50ca8c66b6" dependencies = [ - "aws-credential-types", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", "http", "http-body", - "lazy_static", - "percent-encoding 2.3.0", "pin-project-lite", "tracing", ] [[package]] name = "aws-runtime" -version = "0.56.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ac5cf0ff19c1bca0cea7932e11b239d1025a45696a4f44f72ea86e2b8bdd07" +checksum = "ce0953f7fc1c4428511345e28ea3e98c8b59c9e91eafae30bf76d71d70642693" dependencies = [ "aws-credential-types", "aws-http", @@ -212,9 +238,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.34.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f7a233b27af6e70094eafd43d9ee11da6e78eb2c1a31e5a7de737b782c627d" +checksum = "c1c12ff4dee4cbcfb0f7b7c19b1f72d5603ff8cd1183db7678a10ecf378ba534" dependencies = [ "aws-credential-types", "aws-http", @@ -222,7 +248,6 @@ dependencies = [ "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", - "aws-smithy-client", "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-json", @@ -237,49 +262,122 @@ dependencies = [ "once_cell", "percent-encoding 2.3.0", "regex", - "tokio-stream", "tracing", "url 2.4.1", ] +[[package]] +name = "aws-sdk-sso" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86575c7604dcdb583aba3390200e5333d8e4fe597bad54f57b190aaf4fac9771" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef0d7c1d0730adb5e85407174483a579e39576e0f4350ecd0fac69ec1217b1b" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f45778089751d5aa8645a02dd60865fa0eea39f00be5db2c7779bc50b83db19a" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http", + "regex", + "tracing", +] + [[package]] name = "aws-sigv4" -version = "0.56.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b28f4910bb956b7ab320b62e98096402354eca976c587d1eeccd523d9bac03" +checksum = "b6bcbad6e0f130232b22e4b4e28834348ce5b79c23b5059b387c08fd0dc8f876" dependencies = [ + "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac", "http", "once_cell", + "p256", "percent-encoding 2.3.0", "regex", + "ring", "sha2", + "subtle", "time", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-async" -version = "0.56.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cdb73f85528b9d19c23a496034ac53703955a59323d581c06aa27b4e4e247af" +checksum = "1e9f65000917e3aa94c259d67fe01fa9e4cd456187d026067d642436e6311a81" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "aws-smithy-checksums" -version = "0.56.1" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb15946af1b8d3beeff53ad991d9bff68ac22426b6d40372b958a75fa61eaed" +checksum = "4c2a63681f82fb85ca58d566534b7dc619c782fee0c61c1aa51e2b560c21cb4f" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -296,35 +394,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-client" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27b2756264c82f830a91cb4d2d485b2d19ad5bea476d9a966e03d27f27ba59a" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-eventstream" -version = "0.56.1" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "850233feab37b591b7377fd52063aa37af615687f5896807abe7f49bd4e1d25b" +checksum = "a85e16fa903c70c49ab3785e5f4ac2ad2171b36e0616f321011fa57962404bb6" dependencies = [ "aws-smithy-types", "bytes", @@ -333,122 +407,126 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.56.1" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cdcf365d8eee60686885f750a34c190e513677db58bbc466c44c588abf4199" +checksum = "e4e816425a6b9caea4929ac97d0cb33674849bd5f0086418abc0d02c63f7a1bf" dependencies = [ "aws-smithy-eventstream", + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", "http", "http-body", - "hyper", "once_cell", "percent-encoding 2.3.0", "pin-project-lite", "pin-utils", - "tokio", - "tokio-util", "tracing", ] [[package]] -name = "aws-smithy-http-tower" -version = "0.56.1" +name = "aws-smithy-json" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822de399d0ce62829a69dfa8c5cd08efdbe61a7426b953e2268f8b8b52a607bd" +checksum = "8ab3f6d49e08df2f8d05e1bb5b68998e1e67b76054d3c43e7b954becb9a5e9ac" dependencies = [ - "aws-smithy-http", "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", ] [[package]] -name = "aws-smithy-json" -version = "0.56.1" +name = "aws-smithy-query" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1e7ab8fa7ad10c193af7ae56d2420989e9f4758bf03601a342573333ea34f" +checksum = "0f94a7a3aa509ff9e8b8d80749851d04e5eee0954c43f2e7d6396c4740028737" dependencies = [ "aws-smithy-types", + "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "0.56.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745e096b3553e7e0f40622aa04971ce52765af82bebdeeac53aa6fc82fe801e6" +checksum = "8da5b0a3617390e769576321816112f711c13d7e1114685e022505cf51fe5e48" dependencies = [ "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "fastrand", + "h2", "http", "http-body", + "hyper", + "hyper-rustls", "once_cell", "pin-project-lite", "pin-utils", + "rustls", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "0.56.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d0ae0c9cfd57944e9711ea610b48a963fb174a53aabacc08c5794a594b1d02" +checksum = "2404c9eb08bfe9af255945254d9afc69a367b7ee008b8db75c05e3bca485fc65" dependencies = [ "aws-smithy-async", - "aws-smithy-http", "aws-smithy-types", "bytes", "http", + "pin-project-lite", "tokio", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "0.56.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d90dbc8da2f6be461fa3c1906b20af8f79d14968fe47f2b7d29d086f62a51728" +checksum = "2aba8136605d14ac88f57dc3a693a9f8a4eab4a3f52bc03ff13746f0cd704e97" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", "serde", "time", + "tokio", + "tokio-util", ] [[package]] name = "aws-smithy-xml" -version = "0.56.1" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01d2dedcdd8023043716cfeeb3c6c59f2d447fce365d8e194838891794b23b6" +checksum = "2e8f03926587fc881b12b102048bb04305bf7fb8c83e776f0ccc51eaa2378263" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.56.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85aa0451bf8af1bf22a4f028d5d28054507a14be43cb8ac0597a8471fba9edfe" +checksum = "faa59f6f26a3472ca2ce7e7802d037a0a9a7ac23de5761eadd9b68f31ac4fd21" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "http", "rustc_version", @@ -470,6 +548,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -492,6 +576,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -697,6 +787,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "const-oid" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" + [[package]] name = "core-foundation" version = "0.9.3" @@ -727,10 +823,8 @@ name = "crater" version = "0.1.0" dependencies = [ "assert_cmd", - "aws-credential-types", + "aws-config", "aws-sdk-s3", - "aws-smithy-async", - "aws-types", "base64 0.21.5", "bytes", "cargo_metadata", @@ -754,6 +848,7 @@ dependencies = [ "log", "mime", "minifier", + "nix 0.27.1", "percent-encoding 2.3.0", "predicates", "prometheus", @@ -888,6 +983,28 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -965,6 +1082,16 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.9" @@ -1040,12 +1167,44 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.33" @@ -1143,6 +1302,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "filetime" version = "0.2.22" @@ -2045,6 +2214,17 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + [[package]] name = "h2" version = "0.3.21" @@ -2763,6 +2943,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2875,6 +3066,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.27" @@ -3139,6 +3340,17 @@ dependencies = [ "winreg", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.5" @@ -3342,6 +3554,20 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -3494,6 +3720,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "slab" version = "0.4.9" @@ -3553,6 +3789,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "strsim" version = "0.10.0" @@ -3877,28 +4123,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -4103,6 +4327,12 @@ dependencies = [ "percent-encoding 2.3.0", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" diff --git a/Cargo.toml b/Cargo.toml index d5eb8dc1..259fc153 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,11 +56,10 @@ prometheus = "0.13.3" cargo_metadata = "0.18.1" indexmap = { version = "2.0.2", features = ["serde"] } tokio = "1.24" -aws-types = "0.56.1" -aws-credential-types = "0.56.1" -aws-smithy-async = "0.56.1" -aws-sdk-s3 = "0.34" +aws-sdk-s3 = "1.7" +aws-config = { version = "1", features = ["behavior-version-latest"] } thiserror = "1.0.38" +nix = { version = "0.27.1", features = ["mman"] } [dev-dependencies] assert_cmd = "2.0.4" diff --git a/src/report/archives.rs b/src/report/archives.rs index 32b4410d..06fb2553 100644 --- a/src/report/archives.rs +++ b/src/report/archives.rs @@ -1,3 +1,7 @@ +use std::fs::File; +use std::num::NonZeroUsize; +use std::ptr::NonNull; + use crate::config::Config; use crate::crates::Crate; use crate::experiments::Experiment; @@ -7,6 +11,52 @@ use crate::results::{EncodedLog, EncodingType, ReadResults}; use flate2::{write::GzEncoder, Compression}; use indexmap::IndexMap; use tar::{Builder as TarBuilder, Header as TarHeader}; +use tempfile::tempfile; + +#[cfg(unix)] +struct TempfileBackedBuffer { + _file: File, + mmap: NonNull<[u8]>, +} + +#[cfg(unix)] +impl TempfileBackedBuffer { + fn new(file: File) -> Fallible { + let len = file.metadata()?.len().try_into().unwrap(); + unsafe { + let base = nix::sys::mman::mmap( + None, + NonZeroUsize::new(len).unwrap(), + nix::sys::mman::ProtFlags::PROT_READ, + nix::sys::mman::MapFlags::MAP_PRIVATE, + Some(&file), + 0, + )?; + let Some(base) = NonNull::new(base as *mut u8) else { + panic!("Failed to map file"); + }; + Ok(TempfileBackedBuffer { + _file: file, + mmap: NonNull::slice_from_raw_parts(base, len), + }) + } + } + + fn buffer(&self) -> &[u8] { + unsafe { self.mmap.as_ref() } + } +} + +#[cfg(unix)] +impl Drop for TempfileBackedBuffer { + fn drop(&mut self) { + unsafe { + if let Err(e) = nix::sys::mman::munmap(self.mmap.as_ptr() as *mut _, self.mmap.len()) { + eprintln!("Failed to unmap temporary file: {:?}", e); + } + } + } +} #[derive(Serialize)] pub struct Archive { @@ -92,6 +142,7 @@ fn iterate<'a, DB: ReadResults + 'a>( }) } +#[allow(unused_mut)] fn write_all_archive( db: &DB, ex: &Experiment, @@ -100,18 +151,37 @@ fn write_all_archive( config: &Config, ) -> Fallible { for i in 1..=RETRIES { - let mut all = TarBuilder::new(GzEncoder::new(Vec::new(), Compression::default())); + // We write this large-ish tarball into a tempfile, which moves the I/O to disk operations + // rather than keeping it in memory. This avoids complicating the code by doing incremental + // writes to S3 (requiring buffer management etc) while avoiding keeping the blob entirely + // in memory. + let backing = tempfile()?; + let mut all = TarBuilder::new(GzEncoder::new(backing, Compression::default())); for entry in iterate(db, ex, crates, config) { let entry = entry?; let mut header = entry.header(); all.append_data(&mut header, &entry.path, &entry.log_bytes[..])?; } - let data = all.into_inner()?.finish()?; - let len = data.len(); + let mut data = all.into_inner()?.finish()?; + let mut buffer; + let view; + #[cfg(unix)] + { + buffer = TempfileBackedBuffer::new(data)?; + view = buffer.buffer(); + } + #[cfg(not(unix))] + { + use std::io::{Read, Seek}; + data.rewind()?; + buffer = Vec::new(); + data.read_to_end(&mut buffer)?; + view = &buffer[..]; + } match dest.write_bytes( "logs-archives/all.tar.gz", - data, + view, &"application/gzip".parse().unwrap(), EncodingType::Plain, ) { @@ -123,7 +193,10 @@ fn write_all_archive( std::thread::sleep(std::time::Duration::from_secs(2)); warn!( "retry ({}/{}) writing logs-archives/all.tar.gz ({} bytes) (error: {:?})", - i, RETRIES, len, e, + i, + RETRIES, + view.len(), + e, ); continue; } @@ -164,7 +237,7 @@ pub fn write_logs_archives( let data = archive.into_inner()?.finish()?; dest.write_bytes( format!("logs-archives/{comparison}.tar.gz"), - data, + &data, &"application/gzip".parse().unwrap(), EncodingType::Plain, )?; diff --git a/src/report/html.rs b/src/report/html.rs index c8d7d7f2..812f0c07 100644 --- a/src/report/html.rs +++ b/src/report/html.rs @@ -298,13 +298,13 @@ pub fn write_html_report( info!("copying static assets"); dest.write_bytes( "report.js", - js_in.content()?.into_owned(), + &js_in.content()?, js_in.mime(), EncodingType::Plain, )?; dest.write_bytes( "report.css", - css_in.content()?.into_owned(), + &css_in.content()?, css_in.mime(), EncodingType::Plain, )?; diff --git a/src/report/mod.rs b/src/report/mod.rs index 46016c2e..b1648c04 100644 --- a/src/report/mod.rs +++ b/src/report/mod.rs @@ -293,7 +293,7 @@ fn write_logs( s.spawn(move || { while let Ok((log_path, data, encoding)) = rx.recv() { if let Err(e) = - dest.write_bytes(log_path, data, &mime::TEXT_PLAIN_UTF_8, encoding) + dest.write_bytes(log_path, &data, &mime::TEXT_PLAIN_UTF_8, encoding) { errors.lock().unwrap().push(e); } @@ -548,7 +548,7 @@ pub trait ReportWriter: Send + Sync { fn write_bytes>( &self, path: P, - b: Vec, + b: &[u8], mime: &Mime, encoding_type: EncodingType, ) -> Fallible<()>; @@ -574,7 +574,7 @@ impl ReportWriter for FileWriter { fn write_bytes>( &self, path: P, - b: Vec, + b: &[u8], _: &Mime, _: EncodingType, ) -> Fallible<()> { @@ -619,14 +619,14 @@ impl ReportWriter for DummyWriter { fn write_bytes>( &self, path: P, - b: Vec, + b: &[u8], mime: &Mime, _: EncodingType, ) -> Fallible<()> { self.results .lock() .unwrap() - .insert((path.as_ref().to_path_buf(), mime.clone()), b); + .insert((path.as_ref().to_path_buf(), mime.clone()), b.to_vec()); Ok(()) } diff --git a/src/report/s3.rs b/src/report/s3.rs index 6e3a2e2c..240fc1f2 100644 --- a/src/report/s3.rs +++ b/src/report/s3.rs @@ -77,12 +77,12 @@ impl ReportWriter for S3Writer { fn write_bytes>( &self, path: P, - s: Vec, + body: &[u8], mime: &Mime, encoding_type: EncodingType, ) -> Fallible<()> { // At least 50 MB, then use a multipart upload... - if s.len() >= 50 * 1024 * 1024 { + if body.len() >= 50 * 1024 * 1024 { let mut request = self .client .create_multipart_upload() @@ -108,12 +108,12 @@ impl ReportWriter for S3Writer { }; let chunk_size = 20 * 1024 * 1024; - let bytes = bytes::Bytes::from(s); let mut part = 1; let mut start = 0; let mut parts = aws_sdk_s3::types::CompletedMultipartUpload::builder(); - while start < bytes.len() { - let chunk = bytes.slice(start..std::cmp::min(start + chunk_size, bytes.len())); + while start < body.len() { + let chunk = &body[start..std::cmp::min(start + chunk_size, body.len())]; + let chunk = bytes::Bytes::copy_from_slice(chunk); let request = self .client @@ -160,7 +160,9 @@ impl ReportWriter for S3Writer { let mut request = self .client .put_object() - .body(aws_sdk_s3::primitives::ByteStream::from(s)) + .body(aws_sdk_s3::primitives::ByteStream::from( + bytes::Bytes::copy_from_slice(body), + )) .acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead) .key(format!( "{}/{}", @@ -185,7 +187,7 @@ impl ReportWriter for S3Writer { } fn write_string>(&self, path: P, s: Cow, mime: &Mime) -> Fallible<()> { - self.write_bytes(path, s.into_owned().into_bytes(), mime, EncodingType::Plain) + self.write_bytes(path, s.as_bytes(), mime, EncodingType::Plain) } } diff --git a/src/results/dummy.rs b/src/results/dummy.rs index a7f5a1b4..90129bad 100644 --- a/src/results/dummy.rs +++ b/src/results/dummy.rs @@ -27,7 +27,7 @@ impl DummyDB { pub fn add_dummy_log(&mut self, ex: &Experiment, krate: Crate, tc: Toolchain, log: EncodedLog) { self.experiments .entry(ex.name.to_string()) - .or_insert_with(DummyData::default) + .or_default() .logs .insert((krate, tc), log); } @@ -41,7 +41,7 @@ impl DummyDB { ) { self.experiments .entry(ex.name.to_string()) - .or_insert_with(DummyData::default) + .or_default() .results .insert((krate, tc), res); } diff --git a/src/server/reports.rs b/src/server/reports.rs index b4218bb9..37231552 100644 --- a/src/server/reports.rs +++ b/src/server/reports.rs @@ -1,5 +1,3 @@ -use aws_sdk_s3::config::retry::RetryConfig; - use crate::experiments::{Experiment, Status}; use crate::prelude::*; use crate::report::{self, Comparison, TestResults}; @@ -17,24 +15,21 @@ use super::tokens::BucketRegion; const AUTOMATIC_THREAD_WAKEUP: u64 = 600; fn generate_report(data: &Data, ex: &Experiment, results: &DatabaseDB) -> Fallible { - let mut config = aws_types::SdkConfig::builder(); + let mut config = aws_config::from_env(); match &data.tokens.reports_bucket.region { BucketRegion::S3 { region } => { - config.set_region(Some(aws_types::region::Region::new(region.to_owned()))); + config = config.region(aws_sdk_s3::config::Region::new(region.to_owned())); } BucketRegion::Custom { url } => { - config.set_region(Some(aws_types::region::Region::from_static("us-east-1"))); - config.set_endpoint_url(Some(url.clone())); + config = config.region(aws_sdk_s3::config::Region::from_static("us-east-1")); + config = config.endpoint_url(url.clone()); } } - config.set_credentials_provider(Some(data.tokens.reports_bucket.to_aws_credentials())); - // https://github.com/awslabs/aws-sdk-rust/issues/586 -- without this, the - // SDK will just completely not retry requests. - config.set_sleep_impl(Some(aws_sdk_s3::config::SharedAsyncSleep::new( - aws_smithy_async::rt::sleep::TokioSleep::new(), - ))); - config.set_retry_config(Some(RetryConfig::standard())); - let config = config.build(); + config = config.credentials_provider(data.tokens.reports_bucket.to_aws_credentials()); + let config = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(config.load()); let client = aws_sdk_s3::Client::new(&config); let writer = report::S3Writer::create( client, diff --git a/src/server/tokens.rs b/src/server/tokens.rs index 25d7eef9..7cddf440 100644 --- a/src/server/tokens.rs +++ b/src/server/tokens.rs @@ -30,17 +30,13 @@ pub struct ReportsBucket { } impl ReportsBucket { - pub(crate) fn to_aws_credentials( - &self, - ) -> aws_credential_types::provider::SharedCredentialsProvider { - aws_credential_types::provider::SharedCredentialsProvider::new( - aws_sdk_s3::config::Credentials::new( - self.access_key.clone(), - self.secret_key.clone(), - None, - None, - "crater-credentials", - ), + pub(crate) fn to_aws_credentials(&self) -> aws_sdk_s3::config::Credentials { + aws_sdk_s3::config::Credentials::new( + self.access_key.clone(), + self.secret_key.clone(), + None, + None, + "crater-credentials", ) } }