diff --git a/Cargo.lock b/Cargo.lock index 2b360a5e52..8a12cc81be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "addr2line" version = "0.24.2" @@ -481,7 +487,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -512,13 +518,14 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -545,7 +552,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -558,6 +565,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-s3" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7ce6d85596c4bcb3aba8ad5bb134b08e204c8a475c9999c1af9290f80aa8ad" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json 0.61.2", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand 2.2.0", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sso" version = "1.49.0" @@ -568,7 +609,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -590,7 +631,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -612,7 +653,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -627,11 +668,12 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.5" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" +checksum = "9bfe75fad52793ce6dec0dc3d4b1f388f038b5eb866c8d4d7f3a8e21b5ea5051" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -650,21 +692,55 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" dependencies = [ "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f45a1c384d7a393026bc5f5c177105aa9fa68e4749653b985707ac27d77295" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "461e5e02f9864cba17cff30f007c2e37ade94d01e87cdb5204e44a84e6d38c17" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -688,6 +764,15 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" +dependencies = [ + "aws-smithy-types", +] + [[package]] name = "aws-smithy-query" version = "0.60.7" @@ -700,9 +785,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.3" +version = "1.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -744,9 +829,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" dependencies = [ "base64-simd", "bytes", @@ -779,9 +864,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -838,32 +923,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "azure_core" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "dyn-clone", - "futures", - "getrandom 0.2.15", - "http-types", - "once_cell", - "paste", - "pin-project", - "rand 0.8.5", - "reqwest 0.12.9", - "rustc_version", - "serde", - "serde_json", - "time", - "tracing", - "url", - "uuid", -] - [[package]] name = "azure_core" version = "0.21.0" @@ -881,6 +940,7 @@ dependencies = [ "once_cell", "paste", "pin-project", + "quick-xml 0.31.0", "rand 0.8.5", "reqwest 0.12.9", "rustc_version", @@ -900,7 +960,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa5603f2de38c21165a1b5dfed94d64b1ab265526b0686e8557c907a53a0ee2" dependencies = [ "async-trait", - "azure_core 0.21.0", + "azure_core", "bytes", "futures", "serde", @@ -914,13 +974,14 @@ dependencies = [ [[package]] name = "azure_identity" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" dependencies = [ "async-lock", "async-process", "async-trait", - "azure_core 0.20.0", + "azure_core", "futures", "oauth2", "pin-project", @@ -933,34 +994,70 @@ dependencies = [ ] [[package]] -name = "azure_identity" +name = "azure_security_keyvault" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +checksum = "bd94f507b75349a0e381c0a23bd77cc654fb509f0e6797ce4f99dd959d9e2d68" +dependencies = [ + "async-trait", + "azure_core", + "futures", + "serde", + "serde_json", + "time", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" dependencies = [ + "RustyXML", "async-lock", - "async-process", "async-trait", - "azure_core 0.21.0", + "azure_core", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core", + "azure_storage", + "azure_svc_blobstorage", + "bytes", "futures", - "oauth2", - "pin-project", "serde", + "serde_derive", + "serde_json", "time", "tracing", - "tz-rs", "url", "uuid", ] [[package]] -name = "azure_security_keyvault" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" dependencies = [ - "async-trait", - "azure_core 0.20.0", + "azure_core", + "bytes", "futures", + "log", + "once_cell", "serde", "serde_json", "time", @@ -1944,6 +2041,30 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1953,6 +2074,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -3843,6 +3973,7 @@ dependencies = [ "hyper 1.5.0", "hyper-util", "rustls 0.23.18", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -4463,9 +4594,9 @@ checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" [[package]] name = "libc" -version = "0.2.162" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libdbus-sys" @@ -4503,7 +4634,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -4954,6 +5085,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + [[package]] name = "monostate" version = "0.1.13" @@ -5144,7 +5286,7 @@ dependencies = [ "inotify", "kqueue", "libc", - "mio", + "mio 0.8.11", "walkdir", "windows-sys 0.45.0", ] @@ -5336,6 +5478,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "humantime", + "hyper 1.5.0", + "itertools 0.13.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.37.4", + "rand 0.8.5", + "reqwest 0.12.9", + "ring", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "oci-client" version = "0.14.0" @@ -6293,6 +6465,26 @@ dependencies = [ "reborrow", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.37.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.5" @@ -6726,6 +6918,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.18", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -7019,6 +7212,18 @@ dependencies = [ "security-framework 2.11.1", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -7536,6 +7741,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "snapbox" version = "0.4.17" @@ -7599,6 +7825,66 @@ dependencies = [ "toml", ] +[[package]] +name = "spin-blobstore-azure" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "azure_core", + "azure_storage", + "azure_storage_blobs", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-fs" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "walkdir", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-s3" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "async-once-cell", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-async", + "bytes", + "futures", + "http-body 1.0.1", + "http-body-util", + "object_store", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + [[package]] name = "spin-build" version = "3.3.0-pre0" @@ -7793,6 +8079,29 @@ dependencies = [ "toml", ] +[[package]] +name = "spin-factor-blobstore" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "bytes", + "futures", + "lru", + "serde", + "spin-core", + "spin-factor-wasi", + "spin-factors", + "spin-factors-test", + "spin-locked-app", + "spin-resource-table", + "spin-world", + "tempfile", + "tokio", + "toml", + "tracing", + "wasmtime-wasi", +] + [[package]] name = "spin-factor-key-value" version = "3.3.0-pre0" @@ -8089,9 +8398,9 @@ version = "3.3.0-pre0" dependencies = [ "anyhow", "async-trait", - "azure_core 0.21.0", + "azure_core", "azure_data_cosmos", - "azure_identity 0.21.0", + "azure_identity", "futures", "reqwest 0.12.9", "serde", @@ -8281,7 +8590,11 @@ name = "spin-runtime-config" version = "3.3.0-pre0" dependencies = [ "anyhow", + "spin-blobstore-azure", + "spin-blobstore-fs", + "spin-blobstore-s3", "spin-common", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -8315,6 +8628,7 @@ dependencies = [ "anyhow", "clap 3.2.25", "spin-common", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -8513,8 +8827,8 @@ dependencies = [ name = "spin-variables" version = "3.3.0-pre0" dependencies = [ - "azure_core 0.20.0", - "azure_identity 0.20.0", + "azure_core", + "azure_identity", "azure_security_keyvault", "dotenvy", "serde", @@ -8533,6 +8847,7 @@ version = "3.3.0-pre0" dependencies = [ "async-trait", "wasmtime", + "wasmtime-wasi", ] [[package]] @@ -9075,28 +9390,27 @@ dependencies = [ [[package]] name = "tokio" -version = "1.38.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", "libc", - "mio", - "num_cpus", + "mio 1.0.3", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", @@ -9631,6 +9945,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom 0.2.15", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6ee3206fb6..01ca508e01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,12 @@ members = [ [workspace.dependencies] anyhow = "1" async-trait = "0.1" +azure_core = "0.21.0" +azure_data_cosmos = "0.21.0" +azure_identity = "0.21.0" +azure_security_keyvault = "0.21.0" +azure_storage = "0.21.0" +azure_storage_blobs = "0.21.0" bytes = "1" conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" } dirs = "5.0" @@ -142,7 +148,7 @@ sha2 = "0.10" tempfile = "3" test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" } thiserror = "1" -tokio = "1" +tokio = "1.40" toml = "0.8" tracing = { version = "0.1", features = ["log"] } tracing-opentelemetry = { version = "0.29", default-features = false, features = ["metrics"] } diff --git a/crates/blobstore-azure/Cargo.toml b/crates/blobstore-azure/Cargo.toml new file mode 100644 index 0000000000..53fcf86834 --- /dev/null +++ b/crates/blobstore-azure/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "spin-blobstore-azure" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +azure_core = { workspace = true } +azure_storage = { workspace = true } +azure_storage_blobs = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-blobstore = { path = "../factor-blobstore" } +tokio = { workspace = true } +tokio-stream = "0.1.16" +tokio-util = { version = "0.7.12", features = ["compat"] } +uuid = { version = "1.0", features = ["v4"] } +wasmtime-wasi = { workspace = true } + +[lints] +workspace = true diff --git a/crates/blobstore-azure/src/lib.rs b/crates/blobstore-azure/src/lib.rs new file mode 100644 index 0000000000..f5169718a0 --- /dev/null +++ b/crates/blobstore-azure/src/lib.rs @@ -0,0 +1,54 @@ +mod store; + +use serde::Deserialize; +use spin_factor_blobstore::runtime_config::spin::MakeBlobStore; +use store::{ + auth::{AzureBlobAuthOptions, AzureKeyAuth}, + AzureContainerManager, +}; + +/// A key-value store that uses Azure Cosmos as the backend. +#[derive(Default)] +pub struct AzureBlobStoreBuilder { + _priv: (), +} + +impl AzureBlobStoreBuilder { + /// Creates a new `AzureBlobStoreBuilder`. + pub fn new() -> Self { + Self::default() + } +} + +/// Runtime configuration for the Azure Cosmos key-value store. +#[derive(Deserialize)] +pub struct AzureBlobStoreRuntimeConfig { + /// The authorization token for the Azure blob storage account. + key: Option, + /// The Azure blob storage account name. + account: String, +} + +impl MakeBlobStore for AzureBlobStoreBuilder { + const RUNTIME_CONFIG_TYPE: &'static str = "azure_blob"; + + type RuntimeConfig = AzureBlobStoreRuntimeConfig; + + type ContainerManager = AzureContainerManager; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + let auth = match &runtime_config.key { + Some(key) => AzureBlobAuthOptions::AccountKey(AzureKeyAuth::new( + runtime_config.account.clone(), + key.clone(), + )), + None => AzureBlobAuthOptions::Environmental, + }; + + let blob_store = AzureContainerManager::new(auth)?; + Ok(blob_store) + } +} diff --git a/crates/blobstore-azure/src/store.rs b/crates/blobstore-azure/src/store.rs new file mode 100644 index 0000000000..2bb59519b7 --- /dev/null +++ b/crates/blobstore-azure/src/store.rs @@ -0,0 +1,212 @@ +use std::sync::Arc; + +use anyhow::Result; +use azure_storage_blobs::prelude::{BlobServiceClient, ContainerClient}; +use spin_core::async_trait; +use spin_factor_blobstore::{Container, ContainerManager, Error}; + +pub mod auth; +mod incoming_data; +mod object_names; + +use auth::AzureBlobAuthOptions; +use incoming_data::AzureIncomingData; +use object_names::AzureObjectNames; + +pub struct AzureContainerManager { + client: BlobServiceClient, +} + +impl AzureContainerManager { + pub fn new(auth_options: AzureBlobAuthOptions) -> Result { + let (account, credentials) = match auth_options { + AzureBlobAuthOptions::AccountKey(config) => ( + config.account.clone(), + azure_storage::StorageCredentials::access_key(&config.account, config.key.clone()), + ), + AzureBlobAuthOptions::Environmental => { + let account = std::env::var("STORAGE_ACCOUNT").expect("missing STORAGE_ACCOUNT"); + let access_key = + std::env::var("STORAGE_ACCESS_KEY").expect("missing STORAGE_ACCOUNT_KEY"); + ( + account.clone(), + azure_storage::StorageCredentials::access_key(account, access_key), + ) + } + }; + + let client = azure_storage_blobs::prelude::ClientBuilder::new(account, credentials) + .blob_service_client(); + Ok(Self { client }) + } +} + +#[async_trait] +impl ContainerManager for AzureContainerManager { + async fn get(&self, name: &str) -> Result, Error> { + Ok(Arc::new(AzureContainer { + _label: name.to_owned(), + client: self.client.container_client(name), + })) + } + + fn is_defined(&self, _store_name: &str) -> bool { + true + } +} + +struct AzureContainer { + _label: String, + client: ContainerClient, +} + +/// Azure doesn't provide us with a container creation time +const DUMMY_CREATED_AT: u64 = 0; + +#[async_trait] +impl Container for AzureContainer { + async fn exists(&self) -> anyhow::Result { + Ok(self.client.exists().await?) + } + + async fn name(&self) -> String { + self.client.container_name().to_owned() + } + + async fn info(&self) -> anyhow::Result { + let properties = self.client.get_properties().await?; + Ok(spin_factor_blobstore::ContainerMetadata { + name: properties.container.name, + created_at: DUMMY_CREATED_AT, + }) + } + + async fn clear(&self) -> anyhow::Result<()> { + anyhow::bail!("Azure blob storage does not support clearing containers") + } + + async fn delete_object(&self, name: &str) -> anyhow::Result<()> { + self.client.blob_client(name).delete().await?; + Ok(()) + } + + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()> { + // TODO: are atomic semantics required? or efficiency guarantees? + let futures = names.iter().map(|name| self.delete_object(name)); + futures::future::try_join_all(futures).await?; + Ok(()) + } + + async fn has_object(&self, name: &str) -> anyhow::Result { + Ok(self.client.blob_client(name).exists().await?) + } + + async fn object_info( + &self, + name: &str, + ) -> anyhow::Result { + let response = self.client.blob_client(name).get_properties().await?; + Ok(spin_factor_blobstore::ObjectMetadata { + name: name.to_string(), + container: self.client.container_name().to_string(), + created_at: response + .blob + .properties + .creation_time + .unix_timestamp() + .try_into() + .unwrap(), + size: response.blob.properties.content_length, + }) + } + + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result> { + // We can't use a Rust range because the Azure type does not accept inclusive ranges, + // and we don't want to add 1 to `end` if it's already at MAX! + let range = if end == u64::MAX { + azure_core::request_options::Range::RangeFrom(start..) + } else { + azure_core::request_options::Range::Range(start..(end + 1)) + }; + let client = self.client.blob_client(name); + Ok(Box::new(AzureIncomingData::new(client, range))) + } + + async fn write_data( + &self, + name: &str, + data: tokio::io::ReadHalf, + finished_tx: tokio::sync::mpsc::Sender>, + ) -> anyhow::Result<()> { + let client = self.client.blob_client(name); + + tokio::spawn(async move { + let write_result = Self::write_data_core(data, client).await; + finished_tx + .send(write_result) + .await + .expect("should sent finish tx"); + }); + + Ok(()) + } + + async fn list_objects(&self) -> anyhow::Result> { + let stm = self.client.list_blobs().into_stream(); + Ok(Box::new(AzureObjectNames::new(stm))) + } +} + +impl AzureContainer { + async fn write_data_core( + mut data: tokio::io::ReadHalf, + client: azure_storage_blobs::prelude::BlobClient, + ) -> anyhow::Result<()> { + use tokio::io::AsyncReadExt; + + // Azure limits us to 50k blocks per blob. At 2MB/block that allows 100GB, which will be + // enough for most use cases. If users need flexibility for larger blobs, we could make + // the block size configurable via the runtime config ("size hint" or something). + const BLOCK_SIZE: usize = 2 * 1024 * 1024; + + let mut blocks = vec![]; + + 'put_blocks: loop { + let mut bytes = Vec::with_capacity(BLOCK_SIZE); + loop { + let read = data.read_buf(&mut bytes).await?; + let len = bytes.len(); + + if read == 0 { + // end of stream - send the last block and go + let id_bytes = uuid::Uuid::new_v4().as_bytes().to_vec(); + let block_id = azure_storage_blobs::prelude::BlockId::new(id_bytes); + client.put_block(block_id.clone(), bytes).await?; + blocks.push(azure_storage_blobs::blob::BlobBlockType::Uncommitted( + block_id, + )); + break 'put_blocks; + } + if len >= BLOCK_SIZE { + let id_bytes = uuid::Uuid::new_v4().as_bytes().to_vec(); + let block_id = azure_storage_blobs::prelude::BlockId::new(id_bytes); + client.put_block(block_id.clone(), bytes).await?; + blocks.push(azure_storage_blobs::blob::BlobBlockType::Uncommitted( + block_id, + )); + break; + } + } + } + + let block_list = azure_storage_blobs::blob::BlockList { blocks }; + client.put_block_list(block_list).await?; + + Ok(()) + } +} diff --git a/crates/blobstore-azure/src/store/auth.rs b/crates/blobstore-azure/src/store/auth.rs new file mode 100644 index 0000000000..4d818c0627 --- /dev/null +++ b/crates/blobstore-azure/src/store/auth.rs @@ -0,0 +1,27 @@ +/// Azure blob storage runtime config literal options for authentication +#[derive(Clone, Debug)] +pub struct AzureKeyAuth { + pub account: String, + pub key: String, +} + +impl AzureKeyAuth { + pub fn new(account: String, key: String) -> Self { + Self { account, key } + } +} + +/// Azure blob storage enumeration for the possible authentication options +#[derive(Clone, Debug)] +pub enum AzureBlobAuthOptions { + /// The account and key have been specified directly + AccountKey(AzureKeyAuth), + /// Spin should use the environment variables of the process to + /// create the StorageCredentials for the storage client. For now this uses old school credentials: + /// + /// STORAGE_ACCOUNT + /// STORAGE_ACCESS_KEY + /// + /// TODO: Thorsten pls make this proper with *hand waving* managed identity and stuff! + Environmental, +} diff --git a/crates/blobstore-azure/src/store/incoming_data.rs b/crates/blobstore-azure/src/store/incoming_data.rs new file mode 100644 index 0000000000..a1302b8893 --- /dev/null +++ b/crates/blobstore-azure/src/store/incoming_data.rs @@ -0,0 +1,86 @@ +use anyhow::Result; +use azure_core::Pageable; +use azure_storage_blobs::blob::operations::GetBlobResponse; +use azure_storage_blobs::prelude::BlobClient; +use futures::StreamExt; +use spin_core::async_trait; +use tokio::sync::Mutex; + +pub struct AzureIncomingData { + // The Mutex is used to make it Send + stm: Mutex>>, + client: BlobClient, +} + +impl AzureIncomingData { + pub fn new(client: BlobClient, range: azure_core::request_options::Range) -> Self { + let stm = client.get().range(range).into_stream(); + Self { + stm: Mutex::new(Some(stm)), + client, + } + } + + fn consume_async_impl(&mut self) -> wasmtime_wasi::pipe::AsyncReadStream { + use futures::TryStreamExt; + use tokio_util::compat::FuturesAsyncReadCompatExt; + let stm = self.consume_as_stream(); + let ar = stm.into_async_read(); + let arr = ar.compat(); + wasmtime_wasi::pipe::AsyncReadStream::new(arr) + } + + fn consume_as_stream( + &mut self, + ) -> impl futures::stream::Stream, std::io::Error>> { + let opt_stm = self.stm.get_mut(); + let stm = opt_stm.take().unwrap(); + stm.flat_map(|chunk| streamify_chunk(chunk.unwrap().data)) + } +} + +fn streamify_chunk( + chunk: azure_core::ResponseBody, +) -> impl futures::stream::Stream, std::io::Error>> { + chunk.map(|c| Ok(c.unwrap().to_vec())) +} + +#[async_trait] +impl spin_factor_blobstore::IncomingData for AzureIncomingData { + async fn consume_sync(&mut self) -> anyhow::Result> { + let mut data = vec![]; + let Some(pageable) = self.stm.get_mut() else { + anyhow::bail!("oh no"); + }; + + loop { + let Some(chunk) = pageable.next().await else { + break; + }; + let chunk = chunk.unwrap(); + let by = chunk.data.collect().await.unwrap(); + data.extend(by.to_vec()); + } + + Ok(data) + } + + fn consume_async(&mut self) -> wasmtime_wasi::pipe::AsyncReadStream { + self.consume_async_impl() + } + + async fn size(&mut self) -> anyhow::Result { + // TODO: in theory this should be infallible once we have the IncomingData + // object. But in practice if we use the Pageable for that we don't get it until + // we do the first read. So that would force us to either pre-fetch the + // first chunk or to issue a properties request *just in case* size() was + // called. So I'm making it fallible for now. + Ok(self + .client + .get_properties() + .await? + .blob + .properties + .content_length) + } +} diff --git a/crates/blobstore-azure/src/store/object_names.rs b/crates/blobstore-azure/src/store/object_names.rs new file mode 100644 index 0000000000..64681f10bf --- /dev/null +++ b/crates/blobstore-azure/src/store/object_names.rs @@ -0,0 +1,83 @@ +use azure_core::Pageable; +use azure_storage_blobs::container::operations::ListBlobsResponse; +use tokio::sync::Mutex; + +use spin_core::async_trait; + +pub struct AzureObjectNames { + // The Mutex is used to make it Send + stm: Mutex>, + read_but_not_yet_returned: Vec, + end_stm_after_read_but_not_yet_returned: bool, +} + +impl AzureObjectNames { + pub fn new(stm: Pageable) -> Self { + Self { + stm: Mutex::new(stm), + read_but_not_yet_returned: Default::default(), + end_stm_after_read_but_not_yet_returned: false, + } + } + + async fn read_impl(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + use futures::StreamExt; + + let len: usize = len.try_into().unwrap(); + + // If we have names outstanding, send that first. (We are allowed to send less than len, + // and so sending all pending stuff before paging, rather than trying to manage a mix of + // pending stuff with newly retrieved chunks, simplifies the code.) + if !self.read_but_not_yet_returned.is_empty() { + if self.read_but_not_yet_returned.len() <= len { + // We are allowed to send all pending names + let to_return = self.read_but_not_yet_returned.drain(..).collect(); + return Ok((to_return, self.end_stm_after_read_but_not_yet_returned)); + } else { + // Send as much as we can. The rest remains in the pending buffer to send, + // so this does not represent end of stream. + let to_return = self.read_but_not_yet_returned.drain(0..len).collect(); + return Ok((to_return, false)); + } + } + + // Get one chunk and send as much as we can of it. Aagin, we don't need to try to + // pack the full length here - we can send chunk by chunk. + + let Some(chunk) = self.stm.get_mut().next().await else { + return Ok((vec![], false)); + }; + let chunk = chunk.unwrap(); + + // TODO: do we need to prefix these with a prefix from somewhere or do they include it? + let mut names: Vec<_> = chunk.blobs.blobs().map(|blob| blob.name.clone()).collect(); + let at_end = chunk.next_marker.is_none(); + + if names.len() <= len { + // We can send them all! + Ok((names, at_end)) + } else { + // We have more names than we can send in this response. Send what we can and + // stash the rest. + let to_return: Vec<_> = names.drain(0..len).collect(); + self.read_but_not_yet_returned = names; + self.end_stm_after_read_but_not_yet_returned = at_end; + Ok((to_return, false)) + } + } +} + +#[async_trait] +impl spin_factor_blobstore::ObjectNames for AzureObjectNames { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + self.read_impl(len).await // Separate function because rust-analyser gives better intellisense when async_trait isn't in the picture! + } + + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)> { + // TODO: there is a question (raised as an issue on the repo) about the required behaviour + // here. For now I assume that skipping fewer than `num` is allowed as long as we are + // honest about it. Because it is easier that is why. + let (skipped, at_end) = self.read_impl(num).await?; + Ok((skipped.len().try_into().unwrap(), at_end)) + } +} diff --git a/crates/blobstore-fs/Cargo.toml b/crates/blobstore-fs/Cargo.toml new file mode 100644 index 0000000000..d393829b83 --- /dev/null +++ b/crates/blobstore-fs/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "spin-blobstore-fs" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-blobstore = { path = "../factor-blobstore" } +tokio = { workspace = true } +tokio-stream = "0.1.16" +tokio-util = { version = "0.7.12", features = ["codec", "compat"] } +walkdir = "2.5" +wasmtime-wasi = { workspace = true } + +[lints] +workspace = true diff --git a/crates/blobstore-fs/src/lib.rs b/crates/blobstore-fs/src/lib.rs new file mode 100644 index 0000000000..3ee49a5f6b --- /dev/null +++ b/crates/blobstore-fs/src/lib.rs @@ -0,0 +1,372 @@ +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; + +use spin_core::async_trait; +use spin_factor_blobstore::runtime_config::spin::MakeBlobStore; + +/// A blob store that uses a persistent file system volume +/// as a back end. +#[derive(Default)] +pub struct FileSystemBlobStore { + _priv: (), +} + +impl FileSystemBlobStore { + /// Creates a new `FileSystemBlobStore`. + pub fn new() -> Self { + Self::default() + } +} + +impl MakeBlobStore for FileSystemBlobStore { + const RUNTIME_CONFIG_TYPE: &'static str = "file_system"; + + type RuntimeConfig = FileSystemBlobStoreRuntimeConfig; + + type ContainerManager = BlobStoreFileSystem; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + Ok(BlobStoreFileSystem::new(runtime_config.path)) + } +} + +pub struct BlobStoreFileSystem { + path: PathBuf, +} + +impl BlobStoreFileSystem { + fn new(path: PathBuf) -> Self { + Self { path } + } +} + +/// The serialized runtime configuration for the in memory blob store. +#[derive(Deserialize, Serialize)] +pub struct FileSystemBlobStoreRuntimeConfig { + path: PathBuf, +} + +#[async_trait] +impl spin_factor_blobstore::ContainerManager for BlobStoreFileSystem { + async fn get(&self, name: &str) -> Result, String> { + let container = FileSystemContainer::new(name, &self.path); + Ok(Arc::new(container)) + } + + fn is_defined(&self, _container_name: &str) -> bool { + true + } +} + +struct FileSystemContainer { + name: String, + path: PathBuf, +} + +impl FileSystemContainer { + fn new(name: &str, path: &Path) -> Self { + Self { + name: name.to_string(), + path: path.to_owned(), + } + } + + fn object_path(&self, name: &str) -> anyhow::Result { + validate_no_escape(name)?; + Ok(self.path.join(name)) + } +} + +fn validate_no_escape(name: &str) -> anyhow::Result<()> { + // TODO: this is hopelessly naive but will do for testing + if name.contains("..") { + anyhow::bail!("path tries to escape from base directory"); + } + Ok(()) +} + +#[async_trait] +impl spin_factor_blobstore::Container for FileSystemContainer { + async fn exists(&self) -> anyhow::Result { + Ok(true) + } + async fn name(&self) -> String { + self.name.clone() + } + async fn info(&self) -> anyhow::Result { + let meta = self.path.metadata()?; + let created_at = created_at_nanos(&meta)?; + + Ok(spin_factor_blobstore::ContainerMetadata { + name: self.name.to_owned(), + created_at, + }) + } + async fn clear(&self) -> anyhow::Result<()> { + let entries = std::fs::read_dir(&self.path)?.collect::>(); + + for entry in entries { + let entry = entry?; + if entry.metadata()?.is_dir() { + std::fs::remove_dir_all(entry.path())?; + } else { + std::fs::remove_file(entry.path())?; + } + } + + Ok(()) + } + async fn delete_object(&self, name: &str) -> anyhow::Result<()> { + tokio::fs::remove_file(self.object_path(name)?).await?; + Ok(()) + } + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()> { + let futs = names.iter().map(|name| self.delete_object(name)); + let results = futures::future::join_all(futs).await; + + if let Some(err_result) = results.into_iter().find(|r| r.is_err()) { + err_result + } else { + Ok(()) + } + } + async fn has_object(&self, name: &str) -> anyhow::Result { + Ok(self.object_path(name)?.exists()) + } + async fn object_info( + &self, + name: &str, + ) -> anyhow::Result { + let meta = tokio::fs::metadata(self.object_path(name)?).await?; + let created_at = created_at_nanos(&meta)?; + Ok(spin_factor_blobstore::ObjectMetadata { + name: name.to_string(), + container: self.name.to_string(), + created_at, + size: meta.len(), + }) + } + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result> { + let path = self.object_path(name)?; + let file = tokio::fs::File::open(&path).await?; + + Ok(Box::new(BlobContent { + file: Some(file), + start, + end, + })) + } + + async fn write_data( + &self, + name: &str, + data: tokio::io::ReadHalf, + finished_tx: tokio::sync::mpsc::Sender>, + ) -> anyhow::Result<()> { + let path = self.object_path(name)?; + if let Some(dir) = path.parent() { + tokio::fs::create_dir_all(dir).await?; + } + let file = tokio::fs::File::create(&path).await?; + + tokio::spawn(async move { + let write_result = Self::write_data_core(data, file).await; + finished_tx + .send(write_result) + .await + .expect("shoulda sent finished_tx"); + }); + + Ok(()) + } + + async fn list_objects(&self) -> anyhow::Result> { + if !self.path.is_dir() { + anyhow::bail!( + "Backing store for {} does not exist or is not a directory", + self.name + ); + } + Ok(Box::new(BlobNames::new(&self.path))) + } +} + +impl FileSystemContainer { + async fn write_data_core( + data: tokio::io::ReadHalf, + file: tokio::fs::File, + ) -> anyhow::Result<()> { + use futures::SinkExt; + use tokio_util::codec::{BytesCodec, FramedWrite}; + + // Ceremonies to turn `file` and `data` into Sink and Stream + let mut file_sink = FramedWrite::new(file, BytesCodec::new()); + let mut data_stm = tokio_util::io::ReaderStream::new(data); + + file_sink.send_all(&mut data_stm).await?; + + Ok(()) + } +} + +struct BlobContent { + file: Option, + start: u64, + end: u64, +} + +#[async_trait] +impl spin_factor_blobstore::IncomingData for BlobContent { + async fn consume_sync(&mut self) -> anyhow::Result> { + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + + let mut file = self.file.take().context("already consumed")?; + + let mut buf = Vec::with_capacity(1000); + + file.seek(std::io::SeekFrom::Start(self.start)).await?; + file.take(self.end - self.start) + .read_to_end(&mut buf) + .await?; + + Ok(buf) + } + + fn consume_async(&mut self) -> wasmtime_wasi::pipe::AsyncReadStream { + use futures::StreamExt; + use futures::TryStreamExt; + use tokio_util::compat::FuturesAsyncReadCompatExt; + + let file = self.file.take().unwrap(); + let stm = tokio_util::io::ReaderStream::new(file) + .skip(self.start.try_into().unwrap()) + .take((self.end - self.start).try_into().unwrap()); + + let ar = stm.into_async_read().compat(); + wasmtime_wasi::pipe::AsyncReadStream::new(ar) + } + + async fn size(&mut self) -> anyhow::Result { + let file = self.file.as_ref().context("already consumed")?; + let meta = file.metadata().await?; + Ok(meta.len()) + } +} + +struct BlobNames { + // This isn't async like tokio ReadDir, but it saves us having + // to manage state ourselves as we traverse into subdirectories. + walk_dir: Box> + Send + Sync>, + + base_path: PathBuf, +} + +impl BlobNames { + fn new(path: &Path) -> Self { + let walk_dir = walkdir::WalkDir::new(path) + .into_iter() + .filter_map(as_file_path); + Self { + walk_dir: Box::new(walk_dir), + base_path: path.to_owned(), + } + } + + fn object_name(&self, path: &Path) -> anyhow::Result { + Ok(path + .strip_prefix(&self.base_path) + .map(|p| format!("{}", p.display()))?) + } +} + +fn as_file_path( + entry: Result, +) -> Option> { + match entry { + Err(err) => Some(Err(err)), + Ok(entry) => { + if entry.file_type().is_file() { + Some(Ok(entry.into_path())) + } else { + None + } + } + } +} + +#[async_trait] +impl spin_factor_blobstore::ObjectNames for BlobNames { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + let mut names = Vec::with_capacity(len.try_into().unwrap_or_default()); + let mut at_end = false; + + for _ in 0..len { + match self.walk_dir.next() { + None => { + at_end = true; + break; + } + Some(Err(e)) => { + anyhow::bail!(e); + } + Some(Ok(path)) => { + names.push(self.object_name(&path)?); + } + } + } + + // We could report "at end" when we actually just returned the last file. + // It's not worth messing around with peeking ahead because the cost to the + // guest of making a call that returns nothing is (hopefully) small. + Ok((names, at_end)) + } + + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)> { + // TODO: we could save semi-duplicate code by delegating to `read`? + // The cost would be a bunch of allocation but that seems minor when + // you're dealing with the filesystem. + + let mut count = 0; + let mut at_end = false; + + for _ in 0..num { + match self.walk_dir.next() { + None => { + at_end = true; + break; + } + Some(Err(e)) => { + anyhow::bail!(e); + } + Some(Ok(_)) => { + count += 1; + } + } + } + + Ok((count, at_end)) + } +} + +fn created_at_nanos(meta: &std::fs::Metadata) -> anyhow::Result { + let time_nanos = meta + .created()? + .duration_since(std::time::SystemTime::UNIX_EPOCH)? + .as_nanos() + .try_into() + .unwrap_or_default(); + Ok(time_nanos) +} diff --git a/crates/blobstore-s3/Cargo.toml b/crates/blobstore-s3/Cargo.toml new file mode 100644 index 0000000000..b570b64296 --- /dev/null +++ b/crates/blobstore-s3/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "spin-blobstore-s3" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +async-once-cell = "0.5.4" +# Turn off default features to avoid pulling in "aws-smithy-runtime/default-https-client" which messes up tls provider selection +aws-config = { version = "1.1.7", default-features = false, features = ["rt-tokio", "credentials-process", "sso"] } +aws-credential-types = "1.1.7" +aws-sdk-s3 = { version = "1.68", default-features = false, features = ["rustls", "rt-tokio"] } +aws-smithy-async = "1.2.5" +bytes = { workspace = true } +futures = { workspace = true } +http-body = "1.0" +http-body-util = "0.1" +object_store = { version = "0.11", features = ["aws"] } +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-blobstore = { path = "../factor-blobstore" } +tokio = { workspace = true } +tokio-stream = "0.1.16" +tokio-util = { version = "0.7.12", features = ["compat"] } +uuid = { version = "1.0", features = ["v4"] } +wasmtime-wasi = { workspace = true } + +[lints] +workspace = true diff --git a/crates/blobstore-s3/src/lib.rs b/crates/blobstore-s3/src/lib.rs new file mode 100644 index 0000000000..db53706b59 --- /dev/null +++ b/crates/blobstore-s3/src/lib.rs @@ -0,0 +1,67 @@ +mod store; + +use serde::Deserialize; +use spin_factor_blobstore::runtime_config::spin::MakeBlobStore; +use store::S3ContainerManager; + +/// A blob store that uses a S3-compatible service as the backend. +/// This currently supports only AWS S3 +#[derive(Default)] +pub struct S3BlobStore { + _priv: (), +} + +impl S3BlobStore { + /// Creates a new `S3BlobStore`. + pub fn new() -> Self { + Self::default() + } +} + +// TODO: allow URL configuration for compatible non-AWS services + +/// Runtime configuration for the S3 blob store. +#[derive(Deserialize)] +pub struct S3BlobStoreRuntimeConfig { + /// The access key for the AWS S3 account role. + access_key: Option, + /// The secret key for authorization on the AWS S3 account. + secret_key: Option, + /// The token for authorization on the AWS S3 account. + token: Option, + /// The AWS region where the S3 account is located + region: String, + /// The name of the bucket backing the store. The default is the store label. + bucket: Option, +} + +impl MakeBlobStore for S3BlobStore { + const RUNTIME_CONFIG_TYPE: &'static str = "s3"; + + type RuntimeConfig = S3BlobStoreRuntimeConfig; + + type ContainerManager = S3ContainerManager; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + let auth = match (&runtime_config.access_key, &runtime_config.secret_key) { + (Some(access_key), Some(secret_key)) => { + store::S3AuthOptions::AccessKey(store::S3KeyAuth::new( + access_key.clone(), + secret_key.clone(), + runtime_config.token.clone(), + )) + } + (None, None) => store::S3AuthOptions::Environmental, + _ => anyhow::bail!( + "either both of access_key and secret_key must be provided, or neither" + ), + }; + + let blob_store = + S3ContainerManager::new(runtime_config.region, auth, runtime_config.bucket)?; + Ok(blob_store) + } +} diff --git a/crates/blobstore-s3/src/store.rs b/crates/blobstore-s3/src/store.rs new file mode 100644 index 0000000000..50d60cb29f --- /dev/null +++ b/crates/blobstore-s3/src/store.rs @@ -0,0 +1,262 @@ +use std::sync::Arc; + +use anyhow::Result; +use spin_core::async_trait; +use spin_factor_blobstore::{Container, ContainerManager, Error}; + +mod auth; +mod incoming_data; +mod object_names; + +pub use auth::{S3AuthOptions, S3KeyAuth}; +use incoming_data::S3IncomingData; +use object_names::S3ObjectNames; + +pub struct S3ContainerManager { + builder: object_store::aws::AmazonS3Builder, + client: async_once_cell::Lazy< + aws_sdk_s3::Client, + std::pin::Pin + Send>>, + >, + bucket: Option, +} + +impl S3ContainerManager { + pub fn new( + region: String, + auth_options: S3AuthOptions, + bucket: Option, + ) -> Result { + let builder = match &auth_options { + S3AuthOptions::AccessKey(config) => object_store::aws::AmazonS3Builder::new() + .with_region(®ion) + .with_access_key_id(&config.access_key) + .with_secret_access_key(&config.secret_key) + .with_token(config.token.clone().unwrap_or_default()), + S3AuthOptions::Environmental => object_store::aws::AmazonS3Builder::from_env(), + }; + + let region_clone = region.clone(); + let client_fut = Box::pin(async move { + let sdk_config = match auth_options { + S3AuthOptions::AccessKey(config) => aws_config::SdkConfig::builder() + .credentials_provider(aws_sdk_s3::config::SharedCredentialsProvider::new( + config, + )) + .region(aws_config::Region::new(region_clone)) + .behavior_version(aws_config::BehaviorVersion::latest()) + .build(), + S3AuthOptions::Environmental => { + aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await + } + }; + aws_sdk_s3::Client::new(&sdk_config) + }); + + Ok(Self { + builder, + client: async_once_cell::Lazy::from_future(client_fut), + bucket, + }) + } +} + +#[async_trait] +impl ContainerManager for S3ContainerManager { + async fn get(&self, name: &str) -> Result, Error> { + let name = self.bucket.clone().unwrap_or_else(|| name.to_owned()); + + let store = self + .builder + .clone() + .with_bucket_name(&name) + .build() + .map_err(|e| e.to_string())?; + + Ok(Arc::new(S3Container { + name, + store, + client: self.client.get_unpin().await.clone(), + })) + } + + fn is_defined(&self, _store_name: &str) -> bool { + true + } +} + +struct S3Container { + name: String, + store: object_store::aws::AmazonS3, + client: aws_sdk_s3::Client, +} + +/// S3 doesn't provide us with a container creation time +const DUMMY_CREATED_AT: u64 = 0; + +#[async_trait] +impl Container for S3Container { + async fn exists(&self) -> anyhow::Result { + match self.client.head_bucket().bucket(&self.name).send().await { + Ok(_) => Ok(true), + Err(e) => match e.as_service_error() { + Some(se) => Ok(!se.is_not_found()), + None => anyhow::bail!(e), + }, + } + } + + async fn name(&self) -> String { + self.name.clone() + } + + async fn info(&self) -> anyhow::Result { + Ok(spin_factor_blobstore::ContainerMetadata { + name: self.name.clone(), + created_at: DUMMY_CREATED_AT, + }) + } + + async fn clear(&self) -> anyhow::Result<()> { + anyhow::bail!("AWS S3 blob storage does not support clearing containers") + } + + async fn delete_object(&self, name: &str) -> anyhow::Result<()> { + self.client + .delete_object() + .bucket(&self.name) + .key(name) + .send() + .await?; + Ok(()) + } + + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()> { + // TODO: are atomic semantics required? or efficiency guarantees? + let futures = names.iter().map(|name| self.delete_object(name)); + futures::future::try_join_all(futures).await?; + Ok(()) + } + + async fn has_object(&self, name: &str) -> anyhow::Result { + match self + .client + .head_object() + .bucket(&self.name) + .key(name) + .send() + .await + { + Ok(_) => Ok(true), + Err(e) => match e.as_service_error() { + Some(se) => Ok(!se.is_not_found()), + None => anyhow::bail!(e), + }, + } + } + + async fn object_info( + &self, + name: &str, + ) -> anyhow::Result { + let response = self + .client + .head_object() + .bucket(&self.name) + .key(name) + .send() + .await?; + Ok(spin_factor_blobstore::ObjectMetadata { + name: name.to_string(), + container: self.name.clone(), + created_at: response + .last_modified() + .and_then(|t| t.secs().try_into().ok()) + .unwrap_or(DUMMY_CREATED_AT), + size: response + .content_length + .and_then(|l| l.try_into().ok()) + .unwrap_or_default(), + }) + } + + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result> { + let range = if end == u64::MAX { + format!("bytes={start}-") + } else { + format!("bytes={start}-{end}") + }; + let resp = self + .client + .get_object() + .bucket(&self.name) + .key(name) + .range(range) + .send() + .await?; + Ok(Box::new(S3IncomingData::new(resp))) + } + + async fn write_data( + &self, + name: &str, + data: tokio::io::ReadHalf, + finished_tx: tokio::sync::mpsc::Sender>, + ) -> anyhow::Result<()> { + let store = self.store.clone(); + let path = object_store::path::Path::from(name); + + tokio::spawn(async move { + let write_result = Self::write_data_core(data, store, path).await; + finished_tx + .send(write_result) + .await + .expect("should sent finish tx"); + }); + + Ok(()) + } + + async fn list_objects(&self) -> anyhow::Result> { + let stm = self + .client + .list_objects_v2() + .bucket(&self.name) + .into_paginator() + .send(); + Ok(Box::new(S3ObjectNames::new(stm))) + } +} + +impl S3Container { + async fn write_data_core( + mut data: tokio::io::ReadHalf, + store: object_store::aws::AmazonS3, + path: object_store::path::Path, + ) -> anyhow::Result<()> { + use object_store::ObjectStore; + + const BUF_SIZE: usize = 5 * 1024 * 1024; + + let mupload = store.put_multipart(&path).await?; + let mut writer = object_store::WriteMultipart::new(mupload); + loop { + use tokio::io::AsyncReadExt; + let mut buf = vec![0; BUF_SIZE]; + let read_amount = data.read(&mut buf).await?; + if read_amount == 0 { + break; + } + buf.truncate(read_amount); + writer.put(buf.into()); + } + writer.finish().await?; + + Ok(()) + } +} diff --git a/crates/blobstore-s3/src/store/auth.rs b/crates/blobstore-s3/src/store/auth.rs new file mode 100644 index 0000000000..855732bd03 --- /dev/null +++ b/crates/blobstore-s3/src/store/auth.rs @@ -0,0 +1,48 @@ +/// AWS S3 runtime config literal options for authentication +#[derive(Clone, Debug)] +pub struct S3KeyAuth { + /// The access key for the AWS S3 account role. + pub access_key: String, + /// The secret key for authorization on the AWS S3 account. + pub secret_key: String, + /// The token for authorization on the AWS S3 account. + pub token: Option, +} + +impl S3KeyAuth { + pub fn new(access_key: String, secret_key: String, token: Option) -> Self { + Self { + access_key, + secret_key, + token, + } + } +} + +impl aws_credential_types::provider::ProvideCredentials for S3KeyAuth { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::ready(Ok( + aws_credential_types::Credentials::new( + self.access_key.clone(), + self.secret_key.clone(), + self.token.clone(), + None, // Optional expiration time + "spin_custom_s3_provider", + ), + )) + } +} + +/// AWS S3 authentication options +#[derive(Clone, Debug)] +pub enum S3AuthOptions { + /// The account and key have been specified directly + AccessKey(S3KeyAuth), + /// Use environment variables + Environmental, +} diff --git a/crates/blobstore-s3/src/store/incoming_data.rs b/crates/blobstore-s3/src/store/incoming_data.rs new file mode 100644 index 0000000000..7a3709033e --- /dev/null +++ b/crates/blobstore-s3/src/store/incoming_data.rs @@ -0,0 +1,67 @@ +use aws_sdk_s3::operation::get_object; + +use anyhow::Result; +use spin_core::async_trait; + +pub struct S3IncomingData { + get_obj_output: Option, +} + +impl S3IncomingData { + pub fn new(get_obj_output: get_object::GetObjectOutput) -> Self { + Self { + get_obj_output: Some(get_obj_output), + } + } + + /// Destructively takes the GetObjectOutput from self. + /// After this self will be unusable; but this cannot + /// consume self for resource lifetime reasons. + fn take_output(&mut self) -> get_object::GetObjectOutput { + self.get_obj_output + .take() + .expect("GetObject response was already consumed") + } + + fn consume_async_impl(&mut self) -> wasmtime_wasi::pipe::AsyncReadStream { + use futures::TryStreamExt; + use tokio_util::compat::FuturesAsyncReadCompatExt; + let stream = self.consume_as_stream(); + let reader = stream.into_async_read().compat(); + wasmtime_wasi::pipe::AsyncReadStream::new(reader) + } + + fn consume_as_stream( + &mut self, + ) -> impl futures::stream::Stream, std::io::Error>> { + use futures::StreamExt; + let get_obj_output = self.take_output(); + let reader = get_obj_output.body.into_async_read(); + let stream = tokio_util::io::ReaderStream::new(reader); + stream.map(|chunk| chunk.map(|b| b.to_vec())) + } +} + +#[async_trait] +impl spin_factor_blobstore::IncomingData for S3IncomingData { + async fn consume_sync(&mut self) -> anyhow::Result> { + let get_obj_output = self.take_output(); + Ok(get_obj_output.body.collect().await?.to_vec()) + } + + fn consume_async(&mut self) -> wasmtime_wasi::pipe::AsyncReadStream { + self.consume_async_impl() + } + + async fn size(&mut self) -> anyhow::Result { + use anyhow::Context; + let goo = self + .get_obj_output + .as_ref() + .context("object was already consumed")?; + Ok(goo + .content_length() + .context("content-length not returned")? + .try_into()?) + } +} diff --git a/crates/blobstore-s3/src/store/object_names.rs b/crates/blobstore-s3/src/store/object_names.rs new file mode 100644 index 0000000000..7539c4219f --- /dev/null +++ b/crates/blobstore-s3/src/store/object_names.rs @@ -0,0 +1,101 @@ +use aws_sdk_s3::config::http::HttpResponse as AwsHttpResponse; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::operation::list_objects_v2; +use aws_smithy_async::future::pagination_stream::PaginationStream; +use tokio::sync::Mutex; + +use anyhow::Result; +use spin_core::async_trait; + +pub struct S3ObjectNames { + stm: Mutex< + PaginationStream< + Result< + list_objects_v2::ListObjectsV2Output, + SdkError, + >, + >, + >, + read_but_not_yet_returned: Vec, + end_stm_after_read_but_not_yet_returned: bool, +} + +impl S3ObjectNames { + pub fn new( + stm: PaginationStream< + Result< + list_objects_v2::ListObjectsV2Output, + SdkError, + >, + >, + ) -> Self { + Self { + stm: Mutex::new(stm), + read_but_not_yet_returned: Default::default(), + end_stm_after_read_but_not_yet_returned: false, + } + } + + async fn read_impl(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + let len: usize = len.try_into().unwrap(); + + // If we have names outstanding, send that first. (We are allowed to send less than len, + // and so sending all pending stuff before paging, rather than trying to manage a mix of + // pending stuff with newly retrieved chunks, simplifies the code.) + if !self.read_but_not_yet_returned.is_empty() { + if self.read_but_not_yet_returned.len() <= len { + // We are allowed to send all pending names + let to_return = self.read_but_not_yet_returned.drain(..).collect(); + return Ok((to_return, self.end_stm_after_read_but_not_yet_returned)); + } else { + // Send as much as we can. The rest remains in the pending buffer to send, + // so this does not represent end of stream. + let to_return = self.read_but_not_yet_returned.drain(0..len).collect(); + return Ok((to_return, false)); + } + } + + // Get one chunk and send as much as we can of it. Aagin, we don't need to try to + // pack the full length here - we can send chunk by chunk. + + let Some(chunk) = self.stm.get_mut().next().await else { + return Ok((vec![], false)); + }; + let chunk = chunk.unwrap(); + + let at_end = chunk.continuation_token().is_none(); + let mut names: Vec<_> = chunk + .contents + .unwrap_or_default() + .into_iter() + .flat_map(|blob| blob.key) + .collect(); + + if names.len() <= len { + // We can send them all! + Ok((names, at_end)) + } else { + // We have more names than we can send in this response. Send what we can and + // stash the rest. + let to_return: Vec<_> = names.drain(0..len).collect(); + self.read_but_not_yet_returned = names; + self.end_stm_after_read_but_not_yet_returned = at_end; + Ok((to_return, false)) + } + } +} + +#[async_trait] +impl spin_factor_blobstore::ObjectNames for S3ObjectNames { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + self.read_impl(len).await // Separate function because rust-analyser gives better intellisense when async_trait isn't in the picture! + } + + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)> { + // TODO: there is a question (raised as an issue on the repo) about the required behaviour + // here. For now I assume that skipping fewer than `num` is allowed as long as we are + // honest about it. Because it is easier that is why. + let (skipped, at_end) = self.read_impl(num).await?; + Ok((skipped.len().try_into().unwrap(), at_end)) + } +} diff --git a/crates/factor-blobstore/Cargo.toml b/crates/factor-blobstore/Cargo.toml new file mode 100644 index 0000000000..ae329bb539 --- /dev/null +++ b/crates/factor-blobstore/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "spin-factor-blobstore" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +lru = "0.12" +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors = { path = "../factors" } +spin-locked-app = { path = "../locked-app" } +spin-resource-table = { path = "../table" } +spin-world = { path = "../world" } +tokio = { workspace = true, features = ["macros", "sync", "rt", "io-util"] } +toml = { workspace = true } +tracing = { workspace = true } +wasmtime-wasi = { workspace = true } + +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/factor-blobstore/src/host.rs b/crates/factor-blobstore/src/host.rs new file mode 100644 index 0000000000..6b4f3d044f --- /dev/null +++ b/crates/factor-blobstore/src/host.rs @@ -0,0 +1,195 @@ +use anyhow::Result; +use spin_core::wasmtime::component::ResourceTable; +use spin_core::{async_trait, wasmtime::component::Resource}; +use spin_resource_table::Table; +use spin_world::wasi::blobstore::{self as bs}; +use std::{collections::HashSet, sync::Arc}; +use tokio::io::{ReadHalf, SimplexStream}; +use tokio::sync::mpsc; +use tokio::sync::RwLock; + +pub use bs::types::Error; + +mod container; +mod incoming_value; +mod object_names; +mod outgoing_value; + +pub(crate) use outgoing_value::OutgoingValue; + +use crate::DelegatingContainerManager; + +// TODO: I feel like the notions of "container" and "container manager" are muddled. +// This was kinda modelled on the KV StoreManager but I am not sure it has worked. +// A "container manager" actually manages only one container, making the `get` and +// `is_defined` functions seemingly redundant. More clarity and better definition +// is needed here, although the existing code does work! +// +// Part of the trouble is, I think, that the WIT has operations for "create container" +// etc. which implies a level above "container" but whose semantics are very poorly +// defined (the implication in the WIT is that a `blobstore` implementation backs +// onto exactly one provider, and if you need to deal with multiple providers then +// you need to do some double-import trickery, which does not seem right). Clarification +// sought via https://github.com/WebAssembly/wasi-blobstore/issues/27, so we may need +// to do some rework once the authors define it more fully. + +/// Allows obtaining a container. The only interesting implementation is +/// [DelegatingContainerManager] (which is what [BlobStoreDispatch] uses); +/// other implementations currently manage only one container. (See comments.) +#[async_trait] +pub trait ContainerManager: Sync + Send { + async fn get(&self, name: &str) -> Result, Error>; + fn is_defined(&self, container_name: &str) -> bool; +} + +/// A container. This represents the system or network resource defined by +/// a label mapping in the runtime config, e.g. a file system directory, +/// Azure blob storage account, or S3 bucket. This trait is implemented +/// by providers; it is the interface through which the [BlobStoreDispatch] +/// WASI host talks to the different implementations. +#[async_trait] +pub trait Container: Sync + Send { + async fn exists(&self) -> anyhow::Result; + async fn name(&self) -> String; + async fn info(&self) -> anyhow::Result; + async fn clear(&self) -> anyhow::Result<()>; + async fn delete_object(&self, name: &str) -> anyhow::Result<()>; + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()>; + async fn has_object(&self, name: &str) -> anyhow::Result; + async fn object_info(&self, name: &str) -> anyhow::Result; + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result>; + async fn write_data( + &self, + name: &str, + data: ReadHalf, + finished_tx: mpsc::Sender>, + ) -> anyhow::Result<()>; + async fn list_objects(&self) -> anyhow::Result>; +} + +/// An interface implemented by providers when listing objects. +#[async_trait] +pub trait ObjectNames: Send + Sync { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)>; + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)>; +} + +/// The content of a blob being read from a container. Called by the host to +/// handle WIT incoming-value methods, and implemented by providers. +/// providers +#[async_trait] +pub trait IncomingData: Send + Sync { + async fn consume_sync(&mut self) -> anyhow::Result>; + fn consume_async(&mut self) -> wasmtime_wasi::pipe::AsyncReadStream; + async fn size(&mut self) -> anyhow::Result; +} + +/// Implements all the WIT host interfaces for wasi-blobstore. +pub struct BlobStoreDispatch<'a> { + allowed_containers: &'a HashSet, + manager: &'a DelegatingContainerManager, + wasi_resources: &'a mut ResourceTable, + containers: &'a RwLock>>, + incoming_values: &'a RwLock>>, + outgoing_values: &'a RwLock>, + object_names: &'a RwLock>>, +} + +impl<'a> BlobStoreDispatch<'a> { + pub(crate) fn new( + allowed_containers: &'a HashSet, + manager: &'a DelegatingContainerManager, + wasi_resources: &'a mut ResourceTable, + containers: &'a RwLock>>, + incoming_values: &'a RwLock>>, + outgoing_values: &'a RwLock>, + object_names: &'a RwLock>>, + ) -> Self { + Self { + allowed_containers, + manager, + wasi_resources, + containers, + incoming_values, + outgoing_values, + object_names, + } + } + + pub fn allowed_containers(&self) -> &HashSet { + self.allowed_containers + } + + async fn take_incoming_value( + &mut self, + resource: Resource, + ) -> Result, String> { + self.incoming_values + .write() + .await + .remove(resource.rep()) + .ok_or_else(|| "invalid incoming-value resource".to_string()) + } +} + +impl bs::blobstore::Host for BlobStoreDispatch<'_> { + async fn create_container( + &mut self, + _name: String, + ) -> Result, String> { + Err("This version of Spin does not support creating containers".to_owned()) + } + + async fn get_container( + &mut self, + name: String, + ) -> Result, String> { + if self.allowed_containers.contains(&name) { + let container = self.manager.get(&name).await?; + let rep = self.containers.write().await.push(container).unwrap(); + Ok(Resource::new_own(rep)) + } else { + Err(format!("Container {name:?} not defined or access denied")) + } + } + + async fn delete_container(&mut self, _name: String) -> Result<(), String> { + Err("This version of Spin does not support deleting containers".to_owned()) + } + + async fn container_exists(&mut self, name: String) -> Result { + if self.allowed_containers.contains(&name) { + let container = self.manager.get(&name).await?; + container.exists().await.map_err(|e| e.to_string()) + } else { + Ok(false) + } + } + + async fn copy_object( + &mut self, + _src: bs::blobstore::ObjectId, + _dest: bs::blobstore::ObjectId, + ) -> Result<(), String> { + Err("This version of Spin does not support copying objects".to_owned()) + } + + async fn move_object( + &mut self, + _src: bs::blobstore::ObjectId, + _dest: bs::blobstore::ObjectId, + ) -> Result<(), String> { + Err("This version of Spin does not support moving objects".to_owned()) + } +} + +impl bs::types::Host for BlobStoreDispatch<'_> { + fn convert_error(&mut self, error: String) -> anyhow::Result { + Ok(error) + } +} diff --git a/crates/factor-blobstore/src/host/container.rs b/crates/factor-blobstore/src/host/container.rs new file mode 100644 index 0000000000..ba39a430b8 --- /dev/null +++ b/crates/factor-blobstore/src/host/container.rs @@ -0,0 +1,154 @@ +use anyhow::Result; +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::{self as bs}; + +use super::BlobStoreDispatch; + +impl bs::container::Host for BlobStoreDispatch<'_> {} + +impl bs::container::HostContainer for BlobStoreDispatch<'_> { + async fn name(&mut self, self_: Resource) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + Ok(container.name().await) + } + + async fn info( + &mut self, + self_: Resource, + ) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container.info().await.map_err(|e| e.to_string()) + } + + async fn get_data( + &mut self, + self_: Resource, + name: bs::container::ObjectName, + start: u64, + end: u64, + ) -> Result, String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + let incoming = container + .get_data(&name, start, end) + .await + .map_err(|e| e.to_string())?; + let rep = self.incoming_values.write().await.push(incoming).unwrap(); + Ok(Resource::new_own(rep)) + } + + async fn write_data( + &mut self, + self_: Resource, + name: bs::container::ObjectName, + data: Resource, + ) -> Result<(), String> { + let lock_c = self.containers.read().await; + let container = lock_c + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + let mut lock_ov = self.outgoing_values.write().await; + let outgoing = lock_ov + .get_mut(data.rep()) + .ok_or_else(|| "invalid outgoing-value resource".to_string())?; + + let (stm, finished_tx) = outgoing.take_read_stream().map_err(|e| e.to_string())?; + container + .write_data(&name, stm, finished_tx) + .await + .map_err(|e| e.to_string())?; + + Ok(()) + } + + async fn list_objects( + &mut self, + self_: Resource, + ) -> Result, String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + let names = container.list_objects().await.map_err(|e| e.to_string())?; + let rep = self.object_names.write().await.push(names).unwrap(); + Ok(Resource::new_own(rep)) + } + + async fn delete_object( + &mut self, + self_: Resource, + name: String, + ) -> Result<(), String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container + .delete_object(&name) + .await + .map_err(|e| e.to_string()) + } + + async fn delete_objects( + &mut self, + self_: Resource, + names: Vec, + ) -> Result<(), String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container + .delete_objects(&names) + .await + .map_err(|e| e.to_string()) + } + + async fn has_object( + &mut self, + self_: Resource, + name: String, + ) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container.has_object(&name).await.map_err(|e| e.to_string()) + } + + async fn object_info( + &mut self, + self_: Resource, + name: String, + ) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container + .object_info(&name) + .await + .map_err(|e| e.to_string()) + } + + async fn clear(&mut self, self_: Resource) -> Result<(), String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container.clear().await.map_err(|e| e.to_string()) + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.containers.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/host/incoming_value.rs b/crates/factor-blobstore/src/host/incoming_value.rs new file mode 100644 index 0000000000..a21e456b20 --- /dev/null +++ b/crates/factor-blobstore/src/host/incoming_value.rs @@ -0,0 +1,43 @@ +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::{self as bs}; +use wasmtime_wasi::{HostInputStream, InputStream}; + +use super::BlobStoreDispatch; + +impl bs::types::HostIncomingValue for BlobStoreDispatch<'_> { + async fn incoming_value_consume_sync( + &mut self, + self_: Resource, + ) -> Result, String> { + let mut incoming = self.take_incoming_value(self_).await?; + incoming + .as_mut() + .consume_sync() + .await + .map_err(|e| e.to_string()) + } + + async fn incoming_value_consume_async( + &mut self, + self_: Resource, + ) -> Result, String> { + let mut incoming = self.take_incoming_value(self_).await?; + let async_body = incoming.as_mut().consume_async(); + let input_stream: Box = Box::new(async_body); + let resource = self.wasi_resources.push(input_stream).unwrap(); + Ok(resource) + } + + async fn size(&mut self, self_: Resource) -> anyhow::Result { + let mut lock = self.incoming_values.write().await; + let incoming = lock + .get_mut(self_.rep()) + .ok_or_else(|| anyhow::anyhow!("invalid incoming-value resource"))?; + incoming.size().await + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.incoming_values.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/host/object_names.rs b/crates/factor-blobstore/src/host/object_names.rs new file mode 100644 index 0000000000..c9a49a33bd --- /dev/null +++ b/crates/factor-blobstore/src/host/object_names.rs @@ -0,0 +1,35 @@ +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::container::{HostStreamObjectNames, StreamObjectNames}; + +use super::BlobStoreDispatch; + +impl HostStreamObjectNames for BlobStoreDispatch<'_> { + async fn read_stream_object_names( + &mut self, + self_: Resource, + len: u64, + ) -> Result<(Vec, bool), String> { + let mut lock = self.object_names.write().await; + let object_names = lock + .get_mut(self_.rep()) + .ok_or_else(|| "invalid stream-object-names resource".to_string())?; + object_names.read(len).await.map_err(|e| e.to_string()) + } + + async fn skip_stream_object_names( + &mut self, + self_: Resource, + num: u64, + ) -> Result<(u64, bool), String> { + let mut lock = self.object_names.write().await; + let object_names = lock + .get_mut(self_.rep()) + .ok_or_else(|| "invalid stream-object-names resource".to_string())?; + object_names.skip(num).await.map_err(|e| e.to_string()) + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.object_names.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/host/outgoing_value.rs b/crates/factor-blobstore/src/host/outgoing_value.rs new file mode 100644 index 0000000000..6283d71c1f --- /dev/null +++ b/crates/factor-blobstore/src/host/outgoing_value.rs @@ -0,0 +1,116 @@ +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::types::HostOutgoingValue; +use spin_world::wasi::blobstore::{self as bs}; +use tokio::io::{ReadHalf, SimplexStream, WriteHalf}; +use tokio::sync::mpsc; + +use super::BlobStoreDispatch; + +pub struct OutgoingValue { + read: Option>, + write: Option>, + stop_tx: Option>, + finished_rx: Option>>, +} + +const OUTGOING_VALUE_BUF_SIZE: usize = 16 * 1024; + +impl OutgoingValue { + fn new() -> Self { + let (read, write) = tokio::io::simplex(OUTGOING_VALUE_BUF_SIZE); + Self { + read: Some(read), + write: Some(write), + stop_tx: None, + finished_rx: None, + } + } + + fn write_stream(&mut self) -> anyhow::Result { + let Some(write) = self.write.take() else { + anyhow::bail!("OutgoingValue has already returned its write stream"); + }; + + let (stop_tx, stop_rx) = mpsc::channel(1); + + self.stop_tx = Some(stop_tx); + + let stm = crate::AsyncWriteStream::new_closeable(OUTGOING_VALUE_BUF_SIZE, write, stop_rx); + Ok(stm) + } + + fn syncers( + &mut self, + ) -> ( + Option<&mpsc::Sender<()>>, + Option<&mut mpsc::Receiver>>, + ) { + (self.stop_tx.as_ref(), self.finished_rx.as_mut()) + } + + pub(crate) fn take_read_stream( + &mut self, + ) -> anyhow::Result<(ReadHalf, mpsc::Sender>)> { + let Some(read) = self.read.take() else { + anyhow::bail!("OutgoingValue has already been connected to a blob"); + }; + + let (finished_tx, finished_rx) = mpsc::channel(1); + self.finished_rx = Some(finished_rx); + + Ok((read, finished_tx)) + } +} + +impl HostOutgoingValue for BlobStoreDispatch<'_> { + async fn new_outgoing_value(&mut self) -> anyhow::Result> { + let outgoing_value = OutgoingValue::new(); + let rep = self + .outgoing_values + .write() + .await + .push(outgoing_value) + .unwrap(); + Ok(Resource::new_own(rep)) + } + + async fn outgoing_value_write_body( + &mut self, + self_: Resource, + ) -> anyhow::Result, ()>> { + let mut lock = self.outgoing_values.write().await; + let outgoing = lock + .get_mut(self_.rep()) + .ok_or_else(|| anyhow::anyhow!("invalid outgoing-value resource"))?; + let stm = outgoing.write_stream()?; + + let host_stm: Box = Box::new(stm); + let resource = self.wasi_resources.push(host_stm).unwrap(); + + Ok(Ok(resource)) + } + + async fn finish(&mut self, self_: Resource) -> Result<(), String> { + let mut lock = self.outgoing_values.write().await; + let outgoing = lock + .get_mut(self_.rep()) + .ok_or_else(|| "invalid outgoing-value resource".to_string())?; + // Separate methods cause "mutable borrow while immutably borrowed" so get it all in one go + let (stop_tx, finished_rx) = outgoing.syncers(); + let stop_tx = stop_tx.expect("shoulda had a stop_tx"); + let finished_rx = finished_rx.expect("shoulda had a finished_rx"); + + stop_tx.send(()).await.expect("shoulda sent a stop"); + let result = finished_rx.recv().await; + + match result { + None | Some(Ok(())) => Ok(()), + Some(Err(e)) => Err(format!("{e}")), + } + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.outgoing_values.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/lib.rs b/crates/factor-blobstore/src/lib.rs new file mode 100644 index 0000000000..2253337fee --- /dev/null +++ b/crates/factor-blobstore/src/lib.rs @@ -0,0 +1,181 @@ +//! Example usage: +//! +//! -------------------- +//! +//! spin.toml: +//! +//! [component.foo] +//! blob_containers = ["default"] +//! +//! -------------------- +//! +//! runtime-config.toml +//! +//! [blob_store.default] +//! type = "file_system" | "s3" | "azure_blob" +//! # further config settings per type +//! +//! -------------------- +//! +//! TODO: the naming here is not very consistent and we should make a more conscious +//! decision about whether these things are "blob stores" or "containers" or what + +mod host; +pub mod runtime_config; +mod stream; +mod util; + +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use anyhow::ensure; +use spin_factors::{ConfigureAppContext, Factor, InitContext, PrepareContext, RuntimeFactors}; +use spin_locked_app::MetadataKey; +use spin_resource_table::Table; + +pub use host::{BlobStoreDispatch, Container, ContainerManager, Error, IncomingData, ObjectNames}; +pub use runtime_config::RuntimeConfig; +pub use spin_world::wasi::blobstore::types::{ContainerMetadata, ObjectMetadata}; +pub use stream::AsyncWriteStream; +use tokio::sync::RwLock; +pub use util::DelegatingContainerManager; + +/// Lockfile metadata key for blob stores. +pub const BLOB_CONTAINERS_KEY: MetadataKey> = MetadataKey::new("blob_containers"); + +/// A factor that provides blob storage. +#[derive(Default)] +pub struct BlobStoreFactor { + _priv: (), +} + +impl BlobStoreFactor { + /// Create a new BlobStoreFactor. + pub fn new() -> Self { + Self { _priv: () } + } +} + +impl Factor for BlobStoreFactor { + type RuntimeConfig = RuntimeConfig; + type AppState = AppState; + type InstanceBuilder = InstanceBuilder; + + fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { + fn type_annotate(f: F) -> F + where + F: Fn(&mut T) -> BlobStoreDispatch, + { + f + } + + let get_data_with_table = ctx.get_data_with_table_fn(); + let closure = type_annotate(move |data| { + let (state, table) = get_data_with_table(data); + BlobStoreDispatch::new( + &state.allowed_containers, + state.container_manager.as_ref(), + table, + &state.containers, + &state.incoming_values, + &state.outgoing_values, + &state.object_names, + ) + }); + let linker = ctx.linker(); + + spin_world::wasi::blobstore::blobstore::add_to_linker_get_host(linker, closure)?; + spin_world::wasi::blobstore::container::add_to_linker_get_host(linker, closure)?; + spin_world::wasi::blobstore::types::add_to_linker_get_host(linker, closure)?; + + Ok(()) + } + + fn configure_app( + &self, + mut ctx: ConfigureAppContext, + ) -> anyhow::Result { + let runtime_config = ctx.take_runtime_config().unwrap_or_default(); + + let delegating_manager = DelegatingContainerManager::new(runtime_config); + let container_manager = Arc::new(delegating_manager); + + // Build component -> allowed containers map + let mut component_allowed_containers = HashMap::new(); + for component in ctx.app().components() { + let component_id = component.id().to_string(); + let containers = component + .get_metadata(BLOB_CONTAINERS_KEY)? + .unwrap_or_default() + .into_iter() + .collect::>(); + for label in &containers { + ensure!( + container_manager.is_defined(label), + "unknown {} label {label:?} for component {component_id:?}", + BLOB_CONTAINERS_KEY.as_ref(), + ); + } + component_allowed_containers.insert(component_id, containers); + } + + Ok(AppState { + container_manager, + component_allowed_containers, + }) + } + + fn prepare( + &self, + ctx: PrepareContext, + ) -> anyhow::Result { + let app_state = ctx.app_state(); + let allowed_containers = app_state + .component_allowed_containers + .get(ctx.app_component().id()) + .expect("component should be in component_allowed_containers") + .clone(); + let capacity = u32::MAX; + Ok(InstanceBuilder { + container_manager: app_state.container_manager.clone(), + allowed_containers, + containers: RwLock::new(Table::new(capacity)), + incoming_values: RwLock::new(Table::new(capacity)), + object_names: RwLock::new(Table::new(capacity)), + outgoing_values: RwLock::new(Table::new(capacity)), + }) + } +} + +pub struct AppState { + /// The container manager for the app. + container_manager: Arc, + /// The allowed containers for each component. + /// + /// This is a map from component ID to the set of container labels that the + /// component is allowed to use. + component_allowed_containers: HashMap>, +} + +pub struct InstanceBuilder { + /// The container manager for the app. This contains *all* container mappings. + container_manager: Arc, + /// The allowed containers for this component instance. + allowed_containers: HashSet, + /// There are multiple WASI interfaces in play here. The factor adds each of them + /// to the linker, passing a closure that derives the interface implementation + /// from the InstanceBuilder. + /// + /// For the different interfaces to agree on their resource tables, each closure + /// needs to derive the same resource table from the InstanceBuilder. + /// So the InstanceBuilder (which is also the instance state) sets up all the resource + /// tables and RwLocks them, then the dispatch object borrows them. + containers: RwLock>>, + incoming_values: RwLock>>, + outgoing_values: RwLock>, + object_names: RwLock>>, +} + +impl spin_factors::SelfInstanceBuilder for InstanceBuilder {} diff --git a/crates/factor-blobstore/src/runtime_config.rs b/crates/factor-blobstore/src/runtime_config.rs new file mode 100644 index 0000000000..9abbc938bf --- /dev/null +++ b/crates/factor-blobstore/src/runtime_config.rs @@ -0,0 +1,44 @@ +pub mod spin; + +use std::{collections::HashMap, sync::Arc}; + +use crate::ContainerManager; + +/// Runtime configuration for all blob containers. +#[derive(Default, Clone)] +pub struct RuntimeConfig { + /// Map of container names to container managers. + container_managers: HashMap>, +} + +impl RuntimeConfig { + /// Adds a container manager for the container with the given label to the runtime configuration. + /// + /// If a container manager already exists for the given label, it will be replaced. + pub fn add_container_manager( + &mut self, + label: String, + container_manager: Arc, + ) { + self.container_managers.insert(label, container_manager); + } + + /// Returns whether a container manager exists for the given label. + pub fn has_container_manager(&self, label: &str) -> bool { + self.container_managers.contains_key(label) + } + + /// Returns the container manager for the container with the given label. + pub fn get_container_manager(&self, label: &str) -> Option> { + self.container_managers.get(label).cloned() + } +} + +impl IntoIterator for RuntimeConfig { + type Item = (String, Arc); + type IntoIter = std::collections::hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.container_managers.into_iter() + } +} diff --git a/crates/factor-blobstore/src/runtime_config/spin.rs b/crates/factor-blobstore/src/runtime_config/spin.rs new file mode 100644 index 0000000000..1f709e0e40 --- /dev/null +++ b/crates/factor-blobstore/src/runtime_config/spin.rs @@ -0,0 +1,135 @@ +//! Runtime configuration implementation used by Spin CLI. + +use crate::{ContainerManager, RuntimeConfig}; +use anyhow::Context as _; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use spin_factors::runtime_config::toml::GetTomlValue; +use std::{collections::HashMap, sync::Arc}; + +/// Defines the construction of a blob store from a serialized runtime config. +pub trait MakeBlobStore: 'static + Send + Sync { + /// Unique type identifier for the store. + const RUNTIME_CONFIG_TYPE: &'static str; + /// Runtime configuration for the store. + type RuntimeConfig: DeserializeOwned; + /// The store manager for the store. + type ContainerManager: ContainerManager; + + /// Creates a new store manager from the runtime configuration. + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result; +} + +/// A function that creates a container manager from a TOML table. +type StoreFromToml = + Arc anyhow::Result> + Send + Sync>; + +/// Creates a `StoreFromToml` function from a `MakeBlobStore` implementation. +fn store_from_toml_fn(provider_type: T) -> StoreFromToml { + Arc::new(move |table| { + let runtime_config: T::RuntimeConfig = table + .try_into() + .context("could not parse blobstore runtime config")?; + let provider = provider_type + .make_store(runtime_config) + .context("could not make blobstore from runtime config")?; + Ok(Arc::new(provider)) + }) +} + +/// Converts from toml based runtime configuration into a [`RuntimeConfig`]. +/// +/// The various container types (i.e., the "type" field in the toml field) are registered with the +/// resolver using `add_store_type`. The default store for a label is registered using `add_default_store`. +#[derive(Default, Clone)] +pub struct RuntimeConfigResolver { + /// A map of store types to a function that returns the appropriate store + /// manager from runtime config TOML. + store_types: HashMap<&'static str, StoreFromToml>, +} + +impl RuntimeConfigResolver { + /// Create a new RuntimeConfigResolver. + pub fn new() -> Self { + ::default() + } + + /// Registers a store type to the resolver. + pub fn register_store_type(&mut self, store_type: T) -> anyhow::Result<()> { + if self + .store_types + .insert(T::RUNTIME_CONFIG_TYPE, store_from_toml_fn(store_type)) + .is_some() + { + anyhow::bail!( + "duplicate key value store type {:?}", + T::RUNTIME_CONFIG_TYPE + ); + } + Ok(()) + } + + /// Resolves a toml table into a runtime config. + pub fn resolve(&self, table: Option<&impl GetTomlValue>) -> anyhow::Result { + let runtime_config = self.resolve_from_toml(table)?.unwrap_or_default(); + Ok(runtime_config) + } + + fn resolve_from_toml( + &self, + table: Option<&impl GetTomlValue>, + ) -> anyhow::Result> { + let Some(table) = table.and_then(|t| t.get("blob_store")) else { + return Ok(None); + }; + let table: HashMap = table.clone().try_into()?; + + let mut runtime_config = RuntimeConfig::default(); + for (label, config) in table { + let store_manager = self + .container_manager_from_config(config) + .with_context(|| format!("could not configure blob store with label '{label}'"))?; + runtime_config.add_container_manager(label.clone(), store_manager); + } + + Ok(Some(runtime_config)) + } + + /// Given a [`ContainerConfig`], returns a container manager. + /// + /// Errors if there is no [`MakeBlobStore`] registered for the container config's type + /// or if the container manager cannot be created from the config. + fn container_manager_from_config( + &self, + config: ContainerConfig, + ) -> anyhow::Result> { + let config_type = config.type_.as_str(); + let maker = self.store_types.get(config_type).with_context(|| { + format!("the store type '{config_type}' was not registered with the config resolver") + })?; + maker(config.config) + } +} + +#[derive(Deserialize, Clone)] +pub struct ContainerConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} + +impl ContainerConfig { + pub fn new(type_: String, config: T) -> anyhow::Result + where + T: Serialize, + { + Ok(Self { + type_, + config: toml::value::Table::try_from(config)?, + }) + } +} diff --git a/crates/factor-blobstore/src/stream/mod.rs b/crates/factor-blobstore/src/stream/mod.rs new file mode 100644 index 0000000000..9231368b84 --- /dev/null +++ b/crates/factor-blobstore/src/stream/mod.rs @@ -0,0 +1,15 @@ +//! Adapts the WASI streams to allow closing without resource mapping. +//! +//! The solution that the Wasmtime/WASI folks advice is to map your child +//! resources to a custom type, and have the parent "close child" function +//! get the child resource and call a suitable function to termimate it. +//! Unfortunately, that requires (as far as I know) the binding expression +//! to know about the custom type. And since we do all our binding in +//! `spin-world`, which cannot depend on factor crates because it would make +//! things circular, we need to work around it by implementing a close +//! side channel on our own HostOutputStream implementation. +//! And that is what this module does. + +mod write_stream; + +pub use write_stream::AsyncWriteStream; diff --git a/crates/factor-blobstore/src/stream/write_stream.rs b/crates/factor-blobstore/src/stream/write_stream.rs new file mode 100644 index 0000000000..3ae189c9e8 --- /dev/null +++ b/crates/factor-blobstore/src/stream/write_stream.rs @@ -0,0 +1,278 @@ +use anyhow::anyhow; +use bytes::Bytes; +use std::sync::{Arc, Mutex}; +use wasmtime_wasi::{HostOutputStream, StreamError, Subscribe}; + +#[derive(Debug)] +struct WorkerState { + alive: bool, + items: std::collections::VecDeque, + write_budget: usize, + flush_pending: bool, + shutdown_pending: bool, + error: Option, +} + +impl WorkerState { + fn check_error(&mut self) -> Result<(), StreamError> { + if let Some(e) = self.error.take() { + return Err(StreamError::LastOperationFailed(e)); + } + if !self.alive { + return Err(StreamError::Closed); + } + Ok(()) + } +} + +struct Worker { + state: Mutex, + new_work: tokio::sync::Notify, + write_ready_changed: tokio::sync::Notify, +} + +enum Job { + Shutdown, + Flush, + Write(Bytes), +} + +impl Worker { + fn new(write_budget: usize) -> Self { + Self { + state: Mutex::new(WorkerState { + alive: true, + items: std::collections::VecDeque::new(), + write_budget, + flush_pending: false, + shutdown_pending: false, + error: None, + }), + new_work: tokio::sync::Notify::new(), + write_ready_changed: tokio::sync::Notify::new(), + } + } + async fn ready(&self) { + loop { + { + let state = self.state(); + if state.error.is_some() + || !state.alive + || (!state.flush_pending && !state.shutdown_pending && state.write_budget > 0) + { + return; + } + } + self.write_ready_changed.notified().await; + } + } + fn check_write(&self) -> Result { + let mut state = self.state(); + state.check_error()?; + + if state.flush_pending || state.shutdown_pending || state.write_budget == 0 { + return Ok(0); + } + + Ok(state.write_budget) + } + fn state(&self) -> std::sync::MutexGuard { + self.state.lock().unwrap() + } + fn pop(&self) -> Option { + let mut state = self.state(); + if state.items.is_empty() { + if state.flush_pending { + return Some(Job::Flush); + } + if state.shutdown_pending { + return Some(Job::Shutdown); + } + } else if let Some(bytes) = state.items.pop_front() { + return Some(Job::Write(bytes)); + } + + None + } + fn report_error(&self, e: std::io::Error) { + { + let mut state = self.state(); + state.alive = false; + state.error = Some(e.into()); + state.flush_pending = false; + state.shutdown_pending = false; + } + self.write_ready_changed.notify_one(); + } + async fn work(&self, mut writer: T) { + use tokio::io::AsyncWriteExt; + loop { + while let Some(job) = self.pop() { + match job { + Job::Flush => { + if let Err(e) = writer.flush().await { + self.report_error(e); + return; + } + + tracing::debug!("worker marking flush complete"); + self.state().flush_pending = false; + } + + Job::Shutdown => { + if let Err(e) = writer.shutdown().await { + self.report_error(e); + return; + } + self.state().shutdown_pending = false; + } + + Job::Write(mut bytes) => { + tracing::debug!("worker writing: {bytes:?}"); + let len = bytes.len(); + match writer.write_all_buf(&mut bytes).await { + Err(e) => { + self.report_error(e); + return; + } + Ok(_) => { + self.state().write_budget += len; + } + } + } + } + + self.write_ready_changed.notify_one(); + } + self.new_work.notified().await; + } + } +} + +/// Provides a [`HostOutputStream`] impl from a [`tokio::io::AsyncWrite`] impl +pub struct AsyncWriteStream { + worker: Arc, + join_handle: Option>, + shutdown_join_handle: Option, +} + +impl AsyncWriteStream { + /// Create a [`AsyncWriteStream`]. In order to use the [`HostOutputStream`] impl + /// provided by this struct, the argument must impl [`tokio::io::AsyncWrite`]. + pub fn new( + write_budget: usize, + writer: T, + ) -> Self { + let worker = Arc::new(Worker::new(write_budget)); + + let w = Arc::clone(&worker); + let join_handle = wasmtime_wasi::runtime::spawn(async move { w.work(writer).await }); + + AsyncWriteStream { + worker, + join_handle: Some(join_handle), + shutdown_join_handle: None, + } + } + + /// Create a [`AsyncWriteStream`]. In order to use the [`HostOutputStream`] impl + /// provided by this struct, the argument must impl [`tokio::io::AsyncWrite`]. + /// + /// The [`AsyncWriteStream`] created by this constructor can be shut down (that is, + /// graceful EOF) by sending a message through the sender side of the `shutdown_rx` + /// sync channel. + pub fn new_closeable( + write_budget: usize, + writer: T, + mut shutdown_rx: tokio::sync::mpsc::Receiver<()>, + ) -> Self { + let worker = Arc::new(Worker::new(write_budget)); + + let w = Arc::clone(&worker); + let join_handle = wasmtime_wasi::runtime::spawn(async move { w.work(writer).await }); + + let w_clone = worker.clone(); + let shutdown_join_handle = tokio::spawn(async move { + let shutdown_msg = shutdown_rx.recv().await; + if shutdown_msg.is_some() { + let mut state = w_clone.state(); + if state.check_error().is_err() { + // The stream is already failing - no point shutting it down. + return; + } + + state.shutdown_pending = true; + w_clone.new_work.notify_one(); + } + }) + .abort_handle(); + + AsyncWriteStream { + worker, + join_handle: Some(join_handle), + shutdown_join_handle: Some(shutdown_join_handle), + } + } +} + +#[spin_core::async_trait] +impl HostOutputStream for AsyncWriteStream { + fn write(&mut self, bytes: Bytes) -> Result<(), StreamError> { + let mut state = self.worker.state(); + state.check_error()?; + if state.flush_pending { + return Err(StreamError::Trap(anyhow!( + "write not permitted while flush pending" + ))); + } + match state.write_budget.checked_sub(bytes.len()) { + Some(remaining_budget) => { + state.write_budget = remaining_budget; + state.items.push_back(bytes); + } + None => return Err(StreamError::Trap(anyhow!("write exceeded budget"))), + } + drop(state); + self.worker.new_work.notify_one(); + Ok(()) + } + fn flush(&mut self) -> Result<(), StreamError> { + let mut state = self.worker.state(); + state.check_error()?; + + state.flush_pending = true; + self.worker.new_work.notify_one(); + + Ok(()) + } + + fn check_write(&mut self) -> Result { + self.worker.check_write() + } + + async fn cancel(&mut self) { + if let Some(handle) = self.shutdown_join_handle.take() { + handle.abort(); + }; + if let Some(task) = self.join_handle.take() { + _ = cancel(task).await; + }; + } +} +#[spin_core::async_trait] +impl Subscribe for AsyncWriteStream { + async fn ready(&mut self) { + self.worker.ready().await; + } +} + +async fn cancel(mut handle: wasmtime_wasi::runtime::AbortOnDropJoinHandle<()>) -> Option<()> { + use std::ops::DerefMut; + + handle.deref_mut().abort(); + match handle.deref_mut().await { + Ok(value) => Some(value), + Err(err) if err.is_cancelled() => None, + Err(err) => std::panic::resume_unwind(err.into_panic()), + } +} diff --git a/crates/factor-blobstore/src/util.rs b/crates/factor-blobstore/src/util.rs new file mode 100644 index 0000000000..df8d4266d0 --- /dev/null +++ b/crates/factor-blobstore/src/util.rs @@ -0,0 +1,29 @@ +use crate::{Container, ContainerManager, Error}; +use spin_core::async_trait; +use std::{collections::HashMap, sync::Arc}; + +/// A [`ContainerManager`] which delegates to other `ContainerManager`s based on the label. +pub struct DelegatingContainerManager { + delegates: HashMap>, +} + +impl DelegatingContainerManager { + pub fn new(delegates: impl IntoIterator)>) -> Self { + let delegates = delegates.into_iter().collect(); + Self { delegates } + } +} + +#[async_trait] +impl ContainerManager for DelegatingContainerManager { + async fn get(&self, name: &str) -> Result, Error> { + match self.delegates.get(name) { + Some(cm) => cm.get(name).await, + None => Err("no such container".to_string()), + } + } + + fn is_defined(&self, label: &str) -> bool { + self.delegates.contains_key(label) + } +} diff --git a/crates/factor-blobstore/tests/factor_test.rs b/crates/factor-blobstore/tests/factor_test.rs new file mode 100644 index 0000000000..d116c82028 --- /dev/null +++ b/crates/factor-blobstore/tests/factor_test.rs @@ -0,0 +1,146 @@ +// use anyhow::bail; +// use spin_core::async_trait; +// use spin_factor_blobstore::{BlobStoreFactor, RuntimeConfig, Store, StoreManager}; +// use spin_factors::RuntimeFactors; +// use spin_factors_test::{toml, TestEnvironment}; +// use spin_world::wasi::blobstore::types::Error; +// use std::{collections::HashSet, sync::Arc}; + +// #[derive(RuntimeFactors)] +// struct TestFactors { +// blobstore: BlobStoreFactor, +// } + +// impl From for TestFactorsRuntimeConfig { +// fn from(value: RuntimeConfig) -> Self { +// Self { +// blobstore: Some(value), +// } +// } +// } + +// #[tokio::test] +// async fn works_when_allowed_store_is_defined() -> anyhow::Result<()> { +// todo!("this test") +// // let mut runtime_config = RuntimeConfig::default(); +// // runtime_config.add_store_manager("default".into(), mock_store_manager()); +// // let factors = TestFactors { +// // key_value: KeyValueFactor::new(), +// // }; +// // let env = TestEnvironment::new(factors).extend_manifest(toml! { +// // [component.test-component] +// // source = "does-not-exist.wasm" +// // key_value_stores = ["default"] +// // }); +// // let mut state = env +// // .runtime_config(runtime_config)? +// // .build_instance_state() +// // .await?; + +// // assert_eq!( +// // state.key_value.allowed_stores(), +// // &["default".into()].into_iter().collect::>() +// // ); + +// // assert!(state.key_value.open("default".to_owned()).await?.is_ok()); +// // Ok(()) +// } + +// #[tokio::test] +// async fn errors_when_store_is_not_defined() -> anyhow::Result<()> { +// todo!("this test") +// // let runtime_config = RuntimeConfig::default(); +// // let factors = TestFactors { +// // key_value: KeyValueFactor::new(), +// // }; +// // let env = TestEnvironment::new(factors).extend_manifest(toml! { +// // [component.test-component] +// // source = "does-not-exist.wasm" +// // key_value_stores = ["default"] +// // }); +// // let Err(err) = env +// // .runtime_config(runtime_config)? +// // .build_instance_state() +// // .await +// // else { +// // bail!("expected instance build to fail but it didn't"); +// // }; + +// // assert!(err +// // .to_string() +// // .contains(r#"unknown key_value_stores label "default""#)); + +// // Ok(()) +// } + +// #[tokio::test] +// async fn errors_when_store_is_not_allowed() -> anyhow::Result<()> { +// todo!("this test") +// // let mut runtime_config = RuntimeConfig::default(); +// // runtime_config.add_store_manager("default".into(), mock_store_manager()); +// // let factors = TestFactors { +// // key_value: KeyValueFactor::new(), +// // }; +// // let env = TestEnvironment::new(factors).extend_manifest(toml! { +// // [component.test-component] +// // source = "does-not-exist.wasm" +// // key_value_stores = [] +// // }); +// // let mut state = env +// // .runtime_config(runtime_config)? +// // .build_instance_state() +// // .await?; + +// // assert_eq!(state.key_value.allowed_stores(), &HashSet::new()); + +// // assert!(matches!( +// // state.key_value.open("default".to_owned()).await?, +// // Err(Error::AccessDenied) +// // )); + +// // Ok(()) +// } + +// fn mock_store_manager() -> Arc { +// Arc::new(MockStoreManager) +// } + +// struct MockStoreManager; + +// #[async_trait] +// impl StoreManager for MockStoreManager { +// async fn get(&self, name: &str) -> Result, Error> { +// let _ = name; +// Ok(Arc::new(MockStore)) +// } + +// fn is_defined(&self, store_name: &str) -> bool { +// let _ = store_name; +// todo!() +// } +// } + +// struct MockStore; + +// #[async_trait] +// impl Store for MockStore { +// async fn get(&self, key: &str) -> Result>, Error> { +// let _ = key; +// todo!() +// } +// async fn set(&self, key: &str, value: &[u8]) -> Result<(), Error> { +// let _ = (key, value); +// todo!() +// } +// async fn delete(&self, key: &str) -> Result<(), Error> { +// let _ = key; +// todo!() +// } +// async fn exists(&self, key: &str) -> Result { +// let _ = key; +// todo!() +// } +// async fn get_keys(&self) -> Result, Error> { +// todo!() +// } +// } diff --git a/crates/key-value-azure/Cargo.toml b/crates/key-value-azure/Cargo.toml index feaba85109..53845f77d7 100644 --- a/crates/key-value-azure/Cargo.toml +++ b/crates/key-value-azure/Cargo.toml @@ -10,9 +10,9 @@ rust-version.workspace = true [dependencies] anyhow = { workspace = true } -azure_data_cosmos = "0.21.0" -azure_identity = "0.21.0" -azure_core = "0.21.0" +azure_data_cosmos = { workspace = true } +azure_identity = { workspace = true } +azure_core = { workspace = true } futures = { workspace = true } serde = { workspace = true } async-trait = { workspace = true } diff --git a/crates/loader/src/local.rs b/crates/loader/src/local.rs index 1168330be7..ab9dc4c410 100644 --- a/crates/loader/src/local.rs +++ b/crates/loader/src/local.rs @@ -155,6 +155,7 @@ impl LocalLoader { .string_array("allowed_outbound_hosts", allowed_outbound_hosts) .string_array("key_value_stores", component.key_value_stores) .string_array("databases", component.sqlite_databases) + .string_array("blob_containers", component.blob_containers) .string_array("ai_models", component.ai_models) .serializable("build", component.build)? .take(); diff --git a/crates/manifest/src/compat.rs b/crates/manifest/src/compat.rs index 638fbf3d01..96cdebffd7 100644 --- a/crates/manifest/src/compat.rs +++ b/crates/manifest/src/compat.rs @@ -69,6 +69,7 @@ pub fn v1_to_v2_app(manifest: v1::AppManifestV1) -> Result, + /// `blob_containers = ["default", "my-container"]` + #[serde( + default, + with = "kebab_or_snake_case", + skip_serializing_if = "Vec::is_empty" + )] + pub blob_containers: Vec, /// `ai_models = ["llama2-chat"]` #[serde(default, skip_serializing_if = "Vec::is_empty")] pub ai_models: Vec, @@ -541,7 +548,8 @@ mod tests { allowed_http_hosts: vec![], allowed_outbound_hosts: vec![], key_value_stores: labels.clone(), - sqlite_databases: labels, + sqlite_databases: labels.clone(), + blob_containers: labels, ai_models: vec![], build: None, tool: Map::new(), diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index 75839f47a6..20f7766461 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -10,7 +10,11 @@ rust-version.workspace = true [dependencies] anyhow = { workspace = true } +spin-blobstore-s3 = { path = "../blobstore-s3" } +spin-blobstore-azure = { path = "../blobstore-azure" } +spin-blobstore-fs = { path = "../blobstore-fs" } spin-common = { path = "../common" } +spin-factor-blobstore = { path = "../factor-blobstore" } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-llm = { path = "../factor-llm" } spin-factor-outbound-http = { path = "../factor-outbound-http" } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 3e7f22ada7..e5f36f86c7 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; use spin_common::ui::quoted_path; +use spin_factor_blobstore::BlobStoreFactor; use spin_factor_key_value::runtime_config::spin::{self as key_value}; use spin_factor_key_value::KeyValueFactor; use spin_factor_llm::{spin as llm, LlmFactor}; @@ -134,6 +135,7 @@ where let key_value_resolver = key_value_config_resolver(runtime_config_dir, state_dir.clone()); let sqlite_resolver = sqlite_config_resolver(state_dir.clone()) .context("failed to resolve sqlite runtime config")?; + let blobstore_config_resolver = blobstore_config_resolver(toml_resolver.state_dir()?); let toml = toml_resolver.toml(); let log_dir = toml_resolver.log_dir()?; @@ -142,6 +144,7 @@ where &key_value_resolver, tls_resolver.as_ref(), &sqlite_resolver, + &blobstore_config_resolver, ); // Note: all valid fields in the runtime config must have been referenced at // this point or the finalizer will fail due to `validate_all_keys_used` @@ -277,6 +280,7 @@ pub struct TomlRuntimeConfigSource<'a, 'b> { key_value: &'a key_value::RuntimeConfigResolver, tls: Option<&'a SpinTlsRuntimeConfig>, sqlite: &'a sqlite::RuntimeConfigResolver, + blob_store: &'a spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver, } impl<'a, 'b> TomlRuntimeConfigSource<'a, 'b> { @@ -285,12 +289,14 @@ impl<'a, 'b> TomlRuntimeConfigSource<'a, 'b> { key_value: &'a key_value::RuntimeConfigResolver, tls: Option<&'a SpinTlsRuntimeConfig>, sqlite: &'a sqlite::RuntimeConfigResolver, + blob_store: &'a spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver, ) -> Self { Self { toml: toml_resolver, key_value, tls, sqlite, + blob_store, } } } @@ -373,6 +379,15 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result> { + // TODO: actually + Ok(Some(self.blob_store.resolve(Some(&self.toml.table))?)) + } +} + impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_, '_> { fn finalize(&mut self) -> anyhow::Result<()> { Ok(self.toml.validate_all_keys_used()?) @@ -440,6 +455,29 @@ fn sqlite_config_resolver( )) } +/// The blob store runtime configuration resolver. +pub fn blobstore_config_resolver( + // local_store_base_path: Option, + _default_store_base_path: Option, // TODO: used? +) -> spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver { + let mut blobstore_resolver = + spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver::new(); + + // Register the supported store types. + // Unwraps are safe because the store types are known to not overlap. + blobstore_resolver + .register_store_type(spin_blobstore_fs::FileSystemBlobStore::new()) + .unwrap(); + blobstore_resolver + .register_store_type(spin_blobstore_azure::AzureBlobStoreBuilder::new()) + .unwrap(); + blobstore_resolver + .register_store_type(spin_blobstore_s3::S3BlobStore::new()) + .unwrap(); + + blobstore_resolver +} + #[cfg(test)] mod tests { use std::{collections::HashMap, sync::Arc}; diff --git a/crates/runtime-factors/Cargo.toml b/crates/runtime-factors/Cargo.toml index fd982d8bee..9b804a956d 100644 --- a/crates/runtime-factors/Cargo.toml +++ b/crates/runtime-factors/Cargo.toml @@ -17,6 +17,7 @@ llm-cublas = ["spin-factor-llm/llm-cublas"] anyhow = { workspace = true } clap = { version = "3.1.18", features = ["derive", "env"] } spin-common = { path = "../common" } +spin-factor-blobstore = { path = "../factor-blobstore" } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-llm = { path = "../factor-llm" } spin-factor-outbound-http = { path = "../factor-outbound-http" } diff --git a/crates/runtime-factors/src/lib.rs b/crates/runtime-factors/src/lib.rs index c091e58162..5727178995 100644 --- a/crates/runtime-factors/src/lib.rs +++ b/crates/runtime-factors/src/lib.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use anyhow::Context as _; use spin_common::arg_parser::parse_kv; + +use spin_factor_blobstore::BlobStoreFactor; use spin_factor_key_value::KeyValueFactor; use spin_factor_llm::LlmFactor; use spin_factor_outbound_http::OutboundHttpFactor; @@ -33,6 +35,7 @@ pub struct TriggerFactors { pub pg: OutboundPgFactor, pub mysql: OutboundMysqlFactor, pub llm: LlmFactor, + pub blobstore: BlobStoreFactor, } impl TriggerFactors { @@ -56,6 +59,7 @@ impl TriggerFactors { spin_factor_llm::spin::default_engine_creator(state_dir) .context("failed to configure LLM factor")?, ), + blobstore: BlobStoreFactor::new(), }) } } diff --git a/crates/templates/Cargo.toml b/crates/templates/Cargo.toml index c0b8a07aa1..552e2a1b01 100644 --- a/crates/templates/Cargo.toml +++ b/crates/templates/Cargo.toml @@ -14,9 +14,9 @@ flate2 = "1" indexmap = { version = "2", features = ["serde"] } itertools = { workspace = true } lazy_static = "1" -liquid = "0.26" -liquid-core = "0.26" -liquid-derive = "0.26" +liquid = "=0.26.9" +liquid-core = "=0.26.9" +liquid-derive = "=0.26.8" path-absolutize = "3" pathdiff = "0.2" regex = { workspace = true } diff --git a/crates/variables/Cargo.toml b/crates/variables/Cargo.toml index f97cbdcda1..e27b1571c4 100644 --- a/crates/variables/Cargo.toml +++ b/crates/variables/Cargo.toml @@ -9,9 +9,9 @@ repository.workspace = true rust-version.workspace = true [dependencies] -azure_core = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } -azure_identity = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } -azure_security_keyvault = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } +azure_core = { workspace = true } +azure_identity = { workspace = true } +azure_security_keyvault = { workspace = true } dotenvy = "0.15" serde = { workspace = true } spin-expressions = { path = "../expressions" } diff --git a/crates/world/Cargo.toml b/crates/world/Cargo.toml index fd4763100d..3fbffddf52 100644 --- a/crates/world/Cargo.toml +++ b/crates/world/Cargo.toml @@ -7,3 +7,4 @@ edition = { workspace = true } [dependencies] async-trait = { workspace = true } wasmtime = { workspace = true } +wasmtime-wasi = { workspace = true } diff --git a/crates/world/src/lib.rs b/crates/world/src/lib.rs index 501ca9c568..8a0f58303b 100644 --- a/crates/world/src/lib.rs +++ b/crates/world/src/lib.rs @@ -34,8 +34,12 @@ wasmtime::component::bindgen!({ "wasi:config/store@0.2.0-draft-2024-09-27/error" => wasi::config::store::Error, "wasi:keyvalue/store/error" => wasi::keyvalue::store::Error, "wasi:keyvalue/atomics/cas-error" => wasi::keyvalue::atomics::CasError, + "wasi:blobstore/types@0.2.0-draft-2024-09-01/error" => wasi::blobstore::types::Error, }, trappable_imports: true, + with: { + "wasi:io": wasmtime_wasi::bindings::io, + }, }); pub use fermyon::spin as v1; diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index e7d3078fde..f560e94530 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "addr2line" version = "0.24.2" @@ -308,6 +314,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-lc-rs" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +dependencies = [ + "bindgen 0.69.5", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-runtime" version = "1.5.6" @@ -317,6 +346,7 @@ dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -356,6 +386,41 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-s3" +version = "1.82.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand 2.3.0", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sso" version = "1.64.0" @@ -433,20 +498,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac", "http 0.2.12", "http 1.3.1", "once_cell", + "p256", "percent-encoding", + "ring", "sha2", + "subtle", "time", "tracing", + "zeroize", ] [[package]] @@ -460,12 +531,46 @@ dependencies = [ "tokio", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -492,12 +597,20 @@ dependencies = [ "aws-smithy-types", "h2 0.4.8", "http 0.2.12", + "http 1.3.1", "http-body 0.4.6", "hyper 0.14.32", + "hyper 1.6.0", "hyper-rustls 0.24.2", + "hyper-rustls 0.27.5", + "hyper-util", "pin-project-lite", "rustls 0.21.12", + "rustls 0.23.25", + "rustls-native-certs 0.8.1", + "rustls-pki-types", "tokio", + "tower 0.5.2", "tracing", ] @@ -668,32 +781,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "azure_core" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "dyn-clone", - "futures", - "getrandom 0.2.15", - "http-types", - "once_cell", - "paste", - "pin-project", - "rand 0.8.5", - "reqwest", - "rustc_version", - "serde", - "serde_json", - "time", - "tracing", - "url", - "uuid", -] - [[package]] name = "azure_core" version = "0.21.0" @@ -711,6 +798,7 @@ dependencies = [ "once_cell", "paste", "pin-project", + "quick-xml 0.31.0", "rand 0.8.5", "reqwest", "rustc_version", @@ -730,7 +818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa5603f2de38c21165a1b5dfed94d64b1ab265526b0686e8557c907a53a0ee2" dependencies = [ "async-trait", - "azure_core 0.21.0", + "azure_core", "bytes", "futures", "serde", @@ -744,13 +832,14 @@ dependencies = [ [[package]] name = "azure_identity" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" dependencies = [ "async-lock", "async-process", "async-trait", - "azure_core 0.20.0", + "azure_core", "futures", "oauth2", "pin-project", @@ -763,34 +852,70 @@ dependencies = [ ] [[package]] -name = "azure_identity" +name = "azure_security_keyvault" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +checksum = "bd94f507b75349a0e381c0a23bd77cc654fb509f0e6797ce4f99dd959d9e2d68" dependencies = [ + "async-trait", + "azure_core", + "futures", + "serde", + "serde_json", + "time", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +dependencies = [ + "RustyXML", "async-lock", - "async-process", "async-trait", - "azure_core 0.21.0", + "azure_core", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core", + "azure_storage", + "azure_svc_blobstorage", + "bytes", "futures", - "oauth2", - "pin-project", "serde", + "serde_derive", + "serde_json", "time", "tracing", - "tz-rs", "url", "uuid", ] [[package]] -name = "azure_security_keyvault" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" dependencies = [ - "async-trait", - "azure_core 0.20.0", + "azure_core", + "bytes", "futures", + "log", + "once_cell", "serde", "serde_json", "time", @@ -820,6 +945,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -848,6 +979,35 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.100", + "which", +] + [[package]] name = "bindgen" version = "0.71.1" @@ -861,7 +1021,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 2.1.1", "shlex", "syn 2.0.100", ] @@ -1155,6 +1315,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const_fn" version = "0.4.11" @@ -1171,6 +1337,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1232,7 +1408,7 @@ dependencies = [ "hashbrown 0.14.5", "log", "regalloc2", - "rustc-hash", + "rustc-hash 2.1.1", "serde", "smallvec", "target-lexicon", @@ -1302,6 +1478,30 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1311,6 +1511,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -1367,6 +1576,28 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1431,6 +1662,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.4.0" @@ -1562,18 +1803,56 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "embedded-io" version = "0.4.0" @@ -1693,6 +1972,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1767,6 +2056,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1971,6 +2266,17 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.26" @@ -2184,6 +2490,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" + [[package]] name = "hyper" version = "0.14.32" @@ -2274,6 +2586,7 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "rustls 0.23.25", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", @@ -2657,6 +2970,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "leb128" version = "0.2.5" @@ -2682,7 +3001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -2937,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc" dependencies = [ "base64 0.21.7", - "bindgen", + "bindgen 0.71.1", "bitflags 2.9.0", "btoi", "byteorder", @@ -2975,7 +3294,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -3086,6 +3405,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "humantime", + "hyper 1.6.0", + "itertools 0.13.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.37.4", + "rand 0.8.5", + "reqwest", + "ring", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3254,6 +3603,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -3397,6 +3757,16 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -3487,6 +3857,16 @@ dependencies = [ "zerocopy 0.8.24", ] +[[package]] +name = "prettyplease" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +dependencies = [ + "proc-macro2", + "syn 2.0.100", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3587,6 +3967,26 @@ dependencies = [ "wasmtime-math", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.37.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.7" @@ -3598,7 +3998,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.25", "socket2", "thiserror 2.0.12", @@ -3617,7 +4017,7 @@ dependencies = [ "getrandom 0.3.2", "rand 0.9.0", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.25", "rustls-pki-types", "slab", @@ -3856,7 +4256,7 @@ dependencies = [ "bumpalo", "hashbrown 0.15.2", "log", - "rustc-hash", + "rustc-hash 2.1.1", "smallvec", ] @@ -3941,6 +4341,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.25", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -3963,6 +4364,17 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.14" @@ -4016,6 +4428,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -4125,6 +4543,7 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -4143,7 +4562,7 @@ dependencies = [ "openssl-probe", "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] @@ -4156,7 +4575,19 @@ dependencies = [ "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", ] [[package]] @@ -4213,6 +4644,7 @@ version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4230,6 +4662,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "sanitize-filename" version = "0.5.0" @@ -4271,6 +4712,20 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -4278,7 +4733,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.9.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -4438,6 +4906,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "siphasher" version = "1.0.1" @@ -4462,6 +4940,27 @@ dependencies = [ "serde", ] +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "socket2" version = "0.5.9" @@ -4500,6 +4999,66 @@ dependencies = [ "spin-locked-app", ] +[[package]] +name = "spin-blobstore-azure" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "azure_core", + "azure_storage", + "azure_storage_blobs", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-fs" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "walkdir", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-s3" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "async-once-cell", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-async", + "bytes", + "futures", + "http-body 1.0.1", + "http-body-util", + "object_store", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + [[package]] name = "spin-common" version = "3.3.0-pre0" @@ -4563,6 +5122,27 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "spin-factor-blobstore" +version = "3.3.0-pre0" +dependencies = [ + "anyhow", + "bytes", + "futures", + "lru", + "serde", + "spin-core", + "spin-factor-wasi", + "spin-factors", + "spin-locked-app", + "spin-resource-table", + "spin-world", + "tokio", + "toml", + "tracing", + "wasmtime-wasi", +] + [[package]] name = "spin-factor-key-value" version = "3.3.0-pre0" @@ -4797,9 +5377,9 @@ version = "3.3.0-pre0" dependencies = [ "anyhow", "async-trait", - "azure_core 0.21.0", + "azure_core", "azure_data_cosmos", - "azure_identity 0.21.0", + "azure_identity", "futures", "reqwest", "serde", @@ -4882,7 +5462,11 @@ name = "spin-runtime-config" version = "3.3.0-pre0" dependencies = [ "anyhow", + "spin-blobstore-azure", + "spin-blobstore-fs", + "spin-blobstore-s3", "spin-common", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -4912,6 +5496,7 @@ dependencies = [ "anyhow", "clap", "spin-common", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -5024,8 +5609,8 @@ dependencies = [ name = "spin-variables" version = "3.3.0-pre0" dependencies = [ - "azure_core 0.20.0", - "azure_identity 0.20.0", + "azure_core", + "azure_identity", "azure_security_keyvault", "dotenvy", "serde", @@ -5044,6 +5629,17 @@ version = "3.3.0-pre0" dependencies = [ "async-trait", "wasmtime", + "wasmtime-wasi", +] + +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", ] [[package]] @@ -5158,7 +5754,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.9.0", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -5463,6 +6059,7 @@ checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -5815,6 +6412,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ "getrandom 0.3.2", + "serde", ] [[package]] @@ -5900,6 +6498,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -6541,6 +7149,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "1.6.0" @@ -6616,7 +7236,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/wit/deps/wasi-blobstore-2024-09-01/blobstore.wit b/wit/deps/wasi-blobstore-2024-09-01/blobstore.wit new file mode 100644 index 0000000000..5a50214f6b --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/blobstore.wit @@ -0,0 +1,27 @@ +// wasi-cloud Blobstore service definition +interface blobstore { + use container.{container}; + use types.{error, container-name, object-id}; + + // creates a new empty container + create-container: func(name: container-name) -> result; + + // retrieves a container by name + get-container: func(name: container-name) -> result; + + // deletes a container and all objects within it + delete-container: func(name: container-name) -> result<_, error>; + + // returns true if the container exists + container-exists: func(name: container-name) -> result; + + // copies (duplicates) an object, to the same or a different container. + // returns an error if the target container does not exist. + // overwrites destination object if it already existed. + copy-object: func(src: object-id, dest: object-id) -> result<_, error>; + + // moves or renames an object, to the same or a different container + // returns an error if the destination container does not exist. + // overwrites destination object if it already existed. + move-object: func(src:object-id, dest: object-id) -> result<_, error>; +} \ No newline at end of file diff --git a/wit/deps/wasi-blobstore-2024-09-01/container.wit b/wit/deps/wasi-blobstore-2024-09-01/container.wit new file mode 100644 index 0000000000..f4c577e5d7 --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/container.wit @@ -0,0 +1,66 @@ +// a Container is a collection of objects +interface container { + use wasi:io/streams@0.2.0.{ + input-stream, + output-stream, + }; + + use types.{ + container-metadata, + error, + incoming-value, + object-metadata, + object-name, + outgoing-value, + }; + + // this defines the `container` resource + resource container { + // returns container name + name: func() -> result; + + // returns container metadata + info: func() -> result; + + // retrieves an object or portion of an object, as a resource. + // Start and end offsets are inclusive. + // Once a data-blob resource has been created, the underlying bytes are held by the blobstore service for the lifetime + // of the data-blob resource, even if the object they came from is later deleted. + get-data: func(name: object-name, start: u64, end: u64) -> result; + + // creates or replaces an object with the data blob. + write-data: func(name: object-name, data: borrow) -> result<_, error>; + + // returns list of objects in the container. Order is undefined. + list-objects: func() -> result; + + // deletes object. + // does not return error if object did not exist. + delete-object: func(name: object-name) -> result<_, error>; + + // deletes multiple objects in the container + delete-objects: func(names: list) -> result<_, error>; + + // returns true if the object exists in this container + has-object: func(name: object-name) -> result; + + // returns metadata for the object + object-info: func(name: object-name) -> result; + + // removes all objects within the container, leaving the container empty. + clear: func() -> result<_, error>; + } + + // this defines the `stream-object-names` resource which is a representation of stream + resource stream-object-names { + // reads the next number of objects from the stream + // + // This function returns the list of objects read, and a boolean indicating if the end of the stream was reached. + read-stream-object-names: func(len: u64) -> result, bool>, error>; + + // skip the next number of objects in the stream + // + // This function returns the number of objects skipped, and a boolean indicating if the end of the stream was reached. + skip-stream-object-names: func(num: u64) -> result, error>; + } +} \ No newline at end of file diff --git a/wit/deps/wasi-blobstore-2024-09-01/types.wit b/wit/deps/wasi-blobstore-2024-09-01/types.wit new file mode 100644 index 0000000000..ca5972494b --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/types.wit @@ -0,0 +1,91 @@ +// Types used by blobstore +interface types { + use wasi:io/streams@0.2.0.{input-stream, output-stream}; + + // name of a container, a collection of objects. + // The container name may be any valid UTF-8 string. + type container-name = string; + + // name of an object within a container + // The object name may be any valid UTF-8 string. + type object-name = string; + + // TODO: define timestamp to include seconds since + // Unix epoch and nanoseconds + // https://github.com/WebAssembly/wasi-blob-store/issues/7 + type timestamp = u64; + + // size of an object, in bytes + type object-size = u64; + + type error = string; + + // information about a container + record container-metadata { + // the container's name + name: container-name, + // date and time container was created + created-at: timestamp, + } + + // information about an object + record object-metadata { + // the object's name + name: object-name, + // the object's parent container + container: container-name, + // date and time the object was created + created-at: timestamp, + // size of the object, in bytes + size: object-size, + } + + // identifier for an object that includes its container name + record object-id { + container: container-name, + object: object-name + } + + /// A data is the data stored in a data blob. The value can be of any type + /// that can be represented in a byte array. It provides a way to write the value + /// to the output-stream defined in the `wasi-io` interface. + // Soon: switch to `resource value { ... }` + resource outgoing-value { + new-outgoing-value: static func() -> outgoing-value; + + /// Returns a stream for writing the value contents. + /// + /// The returned `output-stream` is a child resource: it must be dropped + /// before the parent `outgoing-value` resource is dropped (or finished), + /// otherwise the `outgoing-value` drop or `finish` will trap. + /// + /// Returns success on the first call: the `output-stream` resource for + /// this `outgoing-value` may be retrieved at most once. Subsequent calls + /// will return error. + outgoing-value-write-body: func() -> result; + + /// Finalize an outgoing value. This must be + /// called to signal that the outgoing value is complete. If the `outgoing-value` + /// is dropped without calling `outgoing-value.finalize`, the implementation + /// should treat the value as corrupted. + finish: static func(this: outgoing-value) -> result<_, error>; + } + + /// A incoming-value is a wrapper around a value. It provides a way to read the value + /// from the input-stream defined in the `wasi-io` interface. + /// + /// The incoming-value provides two ways to consume the value: + /// 1. `incoming-value-consume-sync` consumes the value synchronously and returns the + /// value as a list of bytes. + /// 2. `incoming-value-consume-async` consumes the value asynchronously and returns the + /// value as an input-stream. + // Soon: switch to `resource incoming-value { ... }` + resource incoming-value { + incoming-value-consume-sync: static func(this: incoming-value) -> result; + incoming-value-consume-async: static func(this: incoming-value) -> result; + size: func() -> u64; + } + + type incoming-value-async-body = input-stream; + type incoming-value-sync-body = list; +} diff --git a/wit/deps/wasi-blobstore-2024-09-01/world.wit b/wit/deps/wasi-blobstore-2024-09-01/world.wit new file mode 100644 index 0000000000..61f4225282 --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/world.wit @@ -0,0 +1,5 @@ +package wasi:blobstore@0.2.0-draft-2024-09-01; + +world imports { + import blobstore; +} \ No newline at end of file diff --git a/wit/world.wit b/wit/world.wit index 07da9cd4bf..e92690507f 100644 --- a/wit/world.wit +++ b/wit/world.wit @@ -11,5 +11,6 @@ world platform { include fermyon:spin/platform@2.0.0; include wasi:keyvalue/imports@0.2.0-draft2; import spin:postgres/postgres@3.0.0; + import wasi:blobstore/blobstore@0.2.0-draft-2024-09-01; import wasi:config/store@0.2.0-draft-2024-09-27; }