From 68c27a310d0fa3bfbc272e70b5c68a062861df5b Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Fri, 5 May 2023 19:53:17 +0200 Subject: [PATCH 01/21] Create base Rust project --- .gitignore | 2 + Cargo.lock | 1842 ++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 17 + src/crd.rs | 49 ++ src/lib.rs | 1 + src/main.rs | 10 + yaml/minio-bucket.yaml | 2 +- 7 files changed, 1922 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 src/crd.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3a8cabc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +.idea diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..1f65ec3 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1842 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom", + "instant", + "rand", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "serde", + "winapi", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cxx" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.15", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dyn-clone" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-openssl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" +dependencies = [ + "http", + "hyper", + "linked_hash_set", + "once_cell", + "openssl", + "openssl-sys", + "parking_lot", + "tokio", + "tokio-openssl", + "tower-layer", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-patch" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f54898088ccb91df1b492cc80029a6fdf1c48ca0db7c6822a8babad69c94658" +dependencies = [ + "serde", + "serde_json", + "thiserror", + "treediff", +] + +[[package]] +name = "jsonpath_lib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "k8s-openapi" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd990069640f9db34b3b0f7a1afc62a05ffaa3be9b66aa3c313f58346df7f788" +dependencies = [ + "base64 0.21.0", + "bytes", + "chrono", + "http", + "percent-encoding", + "serde", + "serde-value", + "serde_json", + "url", +] + +[[package]] +name = "kube" +version = "0.82.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc7d3d52dd5c871991679102e80dfb192faaaa09fecdbccdd8c55af264ce7a8f" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "0.82.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "544339f1665488243f79080441cacb09c997746fd763342303e66eebb9d3ba13" +dependencies = [ + "base64 0.20.0", + "bytes", + "chrono", + "dirs-next", + "either", + "futures", + "http", + "http-body", + "hyper", + "hyper-openssl", + "hyper-timeout", + "jsonpath_lib", + "k8s-openapi", + "kube-core", + "openssl", + "pem", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.82.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25983d07f414dfffba08c5951fe110f649113416b1d8e22f7c89c750eb2555a7" +dependencies = [ + "chrono", + "form_urlencoded", + "http", + "json-patch", + "k8s-openapi", + "once_cell", + "schemars", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "kube-derive" +version = "0.82.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5af652b642aca19ef5194de3506aa39f89d788d5326a570da68b13a02d6c5ba2" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "kube-runtime" +version = "0.82.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125331201e3073707ac79c294c89021faa76c84da3a566a3749a2a93d295c98a" +dependencies = [ + "ahash", + "async-trait", + "backoff", + "derivative", + "futures", + "json-patch", + "k8s-openapi", + "kube-client", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "libc" +version = "0.2.142" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linked_hash_set" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "linux-raw-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minio-operator" +version = "0.1.0" +dependencies = [ + "anyhow", + "env_logger", + "k8s-openapi", + "kube", + "log", + "schemars", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "openssl" +version = "0.10.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "openssl-sys" +version = "0.9.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.45.0", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "schemars" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "serde", + "zeroize", +] + +[[package]] +name = "serde" +version = "1.0.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tokio-openssl" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +dependencies = [ + "futures-util", + "openssl", + "openssl-sys", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "slab", + "tokio", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +dependencies = [ + "base64 0.20.0", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "mime", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "treediff" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" +dependencies = [ + "serde_json", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..cbbf30e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "minio-operator" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +log = "0.4.17" +env_logger = "0.10.0" +anyhow = "1.0.71" +serde = { version = "1.0.162", features = ["derive"] } +serde_json = "1.0.96" +schemars = "0.8.12" +tokio = { version = "1.28.0", features = ["full"] } +kube = { version = "0.82.2", features = ["runtime", "derive"] } +k8s-openapi = { version = "0.18.0", features = ["v1_26"] } # TODO : switch to v1_27 diff --git a/src/crd.rs b/src/crd.rs new file mode 100644 index 0000000..27ce17f --- /dev/null +++ b/src/crd.rs @@ -0,0 +1,49 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] +#[kube( + group = "communiquons.org", + version = "v1", + kind = "MinioInstance", + namespaced +)] +pub struct MinioInstanceSpec { + pub endpoint: String, + pub credentials: String, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] +pub enum RetentionType { + #[default] + #[serde(rename_all = "lowercase")] + Compliance, + #[serde(rename_all = "lowercase")] + Governance, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] +pub struct BucketRetention { + pub validity: usize, + pub r#type: RetentionType, +} + +#[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] +#[kube( + group = "communiquons.org", + version = "v1", + kind = "MinioBucket", + namespaced +)] +pub struct MinioBucketSpec { + pub instance: String, + pub name: String, + pub secret: String, + #[serde(default)] + pub anonymous_read_access: bool, + #[serde(default)] + versioning: bool, + quota: Option, + retention: Option, +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..b89e978 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1 @@ +pub mod crd; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..baa238c --- /dev/null +++ b/src/main.rs @@ -0,0 +1,10 @@ +use kube::Client; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + env_logger::init_from_env(env_logger::Env::new().default_filter_or("info")); + + let client = Client::try_default().await?; + + Ok(()) +} diff --git a/yaml/minio-bucket.yaml b/yaml/minio-bucket.yaml index 809414d..c6efed3 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/minio-bucket.yaml @@ -60,7 +60,7 @@ spec: type: integer description: The number of days the data shall be kept example: 180 - mode: + type: type: string description: Retention type. In governance mode, some privileged user can bypass retention policy, while in governance policy, no one, including root user, can delete the data enum: -- 2.45.2 From 4b7748bfbf4926a6a129c331a61e4abc90beaa03 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Fri, 5 May 2023 20:05:22 +0200 Subject: [PATCH 02/21] Create first test bucket --- README.md | 8 +++++++- test/first-test.yaml | 25 +++++++++++++++++++++++++ yaml/minio-bucket.yaml | 6 ++---- yaml/minio-instance.yaml | 3 ++- yaml/service_account.yaml | 1 + 5 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 test/first-test.yaml diff --git a/README.md b/README.md index fb323ad..6c0fb2a 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,10 @@ Automatically create Minio buckets based on K8S CRD. -WIP, early project \ No newline at end of file +WIP, early project + +Apply all K8s config files manually: + +```bash +cat yaml/*.yaml | kubectl apply -f - +``` \ No newline at end of file diff --git a/test/first-test.yaml b/test/first-test.yaml new file mode 100644 index 0000000..da2f0ba --- /dev/null +++ b/test/first-test.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-root +type: Opaque +data: + accessKey: bWluaW8= + secretKey: bWluaW8= +--- +apiVersion: "communiquons.org/v1" +kind: MinioInstance +metadata: + name: my-minio-instance +spec: + endpoint: http://minio:9000/ + credentials: minio-root +--- +apiVersion: "communiquons.org/v1" +kind: MinioBucket +metadata: + name: first-bucket +spec: + instance: my-minio-instance + name: first-bucket + secret: bucket-secret \ No newline at end of file diff --git a/yaml/minio-bucket.yaml b/yaml/minio-bucket.yaml index c6efed3..7f74c7f 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/minio-bucket.yaml @@ -66,9 +66,6 @@ spec: enum: - compliance - governance - - - # either Namespaced or Cluster scope: Namespaced names: @@ -81,4 +78,5 @@ spec: # shortNames allow shorter string to match your resource on the CLI shortNames: - mbs - - buckets \ No newline at end of file + - buckets +--- diff --git a/yaml/minio-instance.yaml b/yaml/minio-instance.yaml index a4a48a5..c87f77b 100644 --- a/yaml/minio-instance.yaml +++ b/yaml/minio-instance.yaml @@ -45,4 +45,5 @@ spec: kind: MinioInstance # shortNames allow shorter string to match your resource on the CLI shortNames: - - mis \ No newline at end of file + - mis +--- diff --git a/yaml/service_account.yaml b/yaml/service_account.yaml index 7d375ae..3d89363 100644 --- a/yaml/service_account.yaml +++ b/yaml/service_account.yaml @@ -30,3 +30,4 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: minio-buckets +--- -- 2.45.2 From 547cc02800e7918e9817e9164da3268c3ff7e8a3 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Sat, 6 May 2023 09:35:18 +0200 Subject: [PATCH 03/21] Can listen to Minio buckets creation / update --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 25 ++++++++++++++++++++++++- test/first-test.yaml | 2 +- test/second-bucket.yaml | 9 +++++++++ 5 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 test/second-bucket.yaml diff --git a/Cargo.lock b/Cargo.lock index 1f65ec3..ce6bfeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -864,6 +864,7 @@ version = "0.1.0" dependencies = [ "anyhow", "env_logger", + "futures", "k8s-openapi", "kube", "log", diff --git a/Cargo.toml b/Cargo.toml index cbbf30e..926dc9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,3 +15,4 @@ schemars = "0.8.12" tokio = { version = "1.28.0", features = ["full"] } kube = { version = "0.82.2", features = ["runtime", "derive"] } k8s-openapi = { version = "0.18.0", features = ["v1_26"] } # TODO : switch to v1_27 +futures = "0.3.28" diff --git a/src/main.rs b/src/main.rs index baa238c..df7d984 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,7 @@ -use kube::Client; +use futures::TryStreamExt; +use kube::{Api, Client}; +use kube::runtime::{watcher, WatchStreamExt}; +use minio_operator::crd::MinioBucket; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -6,5 +9,25 @@ async fn main() -> anyhow::Result<()> { let client = Client::try_default().await?; + let buckets: Api = Api::default_namespaced(client.clone()); + + // Listen for events / buckets creation or update (deletion is not supported) + let wc = watcher::Config::default(); + let bw = watcher(buckets, wc).applied_objects(); + futures::pin_mut!(bw); + + while let Some(b) = bw.try_next().await? { + if let Err(e) = apply_bucket(&b, &client).await { + log::error!("Failed to apply desired configuration for applied bucket {} : {}", b.spec.name, e) + } + } + Ok(()) } + + +/// Make sure a bucket is compliant with a desired configuration +async fn apply_bucket(b: &MinioBucket, _client: &Client) -> anyhow::Result<()> { + log::info!("Apply configuration for bucket {}", b.spec.name); + Ok(()) +} \ No newline at end of file diff --git a/test/first-test.yaml b/test/first-test.yaml index da2f0ba..46f19ed 100644 --- a/test/first-test.yaml +++ b/test/first-test.yaml @@ -22,4 +22,4 @@ metadata: spec: instance: my-minio-instance name: first-bucket - secret: bucket-secret \ No newline at end of file + secret: first-bucket-secret \ No newline at end of file diff --git a/test/second-bucket.yaml b/test/second-bucket.yaml new file mode 100644 index 0000000..e14d20f --- /dev/null +++ b/test/second-bucket.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: "communiquons.org/v1" +kind: MinioBucket +metadata: + name: second-bucket +spec: + instance: my-minio-instance + name: second-bucket + secret: second-bucket-secret \ No newline at end of file -- 2.45.2 From 36aaf5fb4d8779dc1d7dc1eb6cc4f26b22489a87 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Sat, 6 May 2023 10:58:18 +0200 Subject: [PATCH 04/21] Read minio instance secret key --- Cargo.lock | 1 + Cargo.toml | 1 + src/constants.rs | 6 ++++++ src/crd.rs | 6 +++--- src/lib.rs | 3 +++ src/main.rs | 24 ++++++++++++++++++++++-- src/minio.rs | 6 ++++++ src/secrets.rs | 20 ++++++++++++++++++++ test/first-test.yaml | 4 ++-- yaml/minio-bucket.yaml | 2 +- 10 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 src/constants.rs create mode 100644 src/minio.rs create mode 100644 src/secrets.rs diff --git a/Cargo.lock b/Cargo.lock index ce6bfeb..b641fe1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -871,6 +871,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "thiserror", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 926dc9e..c9882b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,3 +16,4 @@ tokio = { version = "1.28.0", features = ["full"] } kube = { version = "0.82.2", features = ["runtime", "derive"] } k8s-openapi = { version = "0.18.0", features = ["v1_26"] } # TODO : switch to v1_27 futures = "0.3.28" +thiserror = "1.0.40" diff --git a/src/constants.rs b/src/constants.rs new file mode 100644 index 0000000..53f5800 --- /dev/null +++ b/src/constants.rs @@ -0,0 +1,6 @@ +//! # Application constants +pub const SECRET_MINIO_INSTANCE_ACCESS_KEY: &str = "accessKey"; +pub const SECRET_MINIO_INSTANCE_SECRET_KEY: &str = "secretKey"; + +pub const SECRET_MINIO_BUCKET_ACCESS_KEY: &str = "accessKey"; +pub const SECRET_MINIO_BUCKET_SECRET_KEY: &str = "secretKey"; \ No newline at end of file diff --git a/src/crd.rs b/src/crd.rs index 27ce17f..5f0d147 100644 --- a/src/crd.rs +++ b/src/crd.rs @@ -43,7 +43,7 @@ pub struct MinioBucketSpec { #[serde(default)] pub anonymous_read_access: bool, #[serde(default)] - versioning: bool, - quota: Option, - retention: Option, + pub versioning: bool, + pub quota: Option, + pub retention: Option, } diff --git a/src/lib.rs b/src/lib.rs index b89e978..2aab6f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1 +1,4 @@ +pub mod constants; pub mod crd; +pub mod secrets; +pub mod minio; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index df7d984..c8e9878 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,11 @@ use futures::TryStreamExt; +use k8s_openapi::api::core::v1::Secret; use kube::{Api, Client}; use kube::runtime::{watcher, WatchStreamExt}; -use minio_operator::crd::MinioBucket; +use minio_operator::constants::{SECRET_MINIO_INSTANCE_ACCESS_KEY, SECRET_MINIO_INSTANCE_SECRET_KEY}; +use minio_operator::crd::{MinioBucket, MinioInstance}; +use minio_operator::minio::MinioService; +use minio_operator::secrets::read_secret_str; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -27,7 +31,23 @@ async fn main() -> anyhow::Result<()> { /// Make sure a bucket is compliant with a desired configuration -async fn apply_bucket(b: &MinioBucket, _client: &Client) -> anyhow::Result<()> { +async fn apply_bucket(b: &MinioBucket, client: &Client) -> anyhow::Result<()> { log::info!("Apply configuration for bucket {}", b.spec.name); + + // Get instance information + let instances: Api = Api::default_namespaced(client.clone()); + let instance = instances.get(&b.spec.instance).await?; + + // Get instance configuration + let secrets: Api = Api::default_namespaced(client.clone()); + let instance_secret = secrets.get(&instance.spec.credentials).await?; + let service = MinioService { + hostname: instance.spec.endpoint, + access_key: read_secret_str(&instance_secret, SECRET_MINIO_INSTANCE_ACCESS_KEY)?, + secret_key: read_secret_str(&instance_secret, SECRET_MINIO_INSTANCE_SECRET_KEY)?, + }; + + println!("{:?}", service); + Ok(()) } \ No newline at end of file diff --git a/src/minio.rs b/src/minio.rs new file mode 100644 index 0000000..00a6cb3 --- /dev/null +++ b/src/minio.rs @@ -0,0 +1,6 @@ +#[derive(Debug, Clone)] +pub struct MinioService { + pub hostname: String, + pub access_key: String, + pub secret_key: String, +} \ No newline at end of file diff --git a/src/secrets.rs b/src/secrets.rs new file mode 100644 index 0000000..fac72e6 --- /dev/null +++ b/src/secrets.rs @@ -0,0 +1,20 @@ +use k8s_openapi::api::core::v1::Secret; + +#[derive(thiserror::Error, Debug)] +enum SecretError { + #[error("Secret has no data!")] + MissingData, + #[error("The key '{0}' is not present in the secret!")] + MissingKey(String), +} + +/// Attempt to read a value contained in a secret. Returns an error in case +/// of failure +pub fn read_secret_str(s: &Secret, key: &str) -> anyhow::Result { + let data = s.data.as_ref().ok_or(SecretError::MissingData)?; + + let value = data.get(key) + .ok_or(SecretError::MissingKey(key.to_string()))?; + + Ok(String::from_utf8(value.0.clone())?) +} \ No newline at end of file diff --git a/test/first-test.yaml b/test/first-test.yaml index 46f19ed..37cb409 100644 --- a/test/first-test.yaml +++ b/test/first-test.yaml @@ -4,8 +4,8 @@ metadata: name: minio-root type: Opaque data: - accessKey: bWluaW8= - secretKey: bWluaW8= + accessKey: bWluaW9hZG1pbg== + secretKey: bWluaW9hZG1pbg== --- apiVersion: "communiquons.org/v1" kind: MinioInstance diff --git a/yaml/minio-bucket.yaml b/yaml/minio-bucket.yaml index 7f74c7f..d2067f6 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/minio-bucket.yaml @@ -34,7 +34,7 @@ spec: type: string example: mybucket secret: - description: The name of the secret that will receive an access key & token with write access on the bucket + description: The name of the secret that will receive an access key & a secret key with write access on the bucket type: string example: secret-name anonymous_read_access: -- 2.45.2 From 76c22150c00b13918d7fa7436f7e7e61228c5a7a Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Sat, 6 May 2023 11:47:18 +0200 Subject: [PATCH 05/21] Automatically create secret for bucket if missing --- Cargo.lock | 1 + Cargo.toml | 1 + src/constants.rs | 5 ++++- src/lib.rs | 2 +- src/main.rs | 43 ++++++++++++++++++++++++++++------- src/minio.rs | 19 ++++++++++++++++ src/secrets.rs | 50 ++++++++++++++++++++++++++++++++++++++--- test/bucket-policy.yaml | 28 +++++++++++++++++++++++ 8 files changed, 136 insertions(+), 13 deletions(-) create mode 100644 test/bucket-policy.yaml diff --git a/Cargo.lock b/Cargo.lock index b641fe1..9c74a0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -868,6 +868,7 @@ dependencies = [ "k8s-openapi", "kube", "log", + "rand", "schemars", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index c9882b1..ac465dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,4 @@ kube = { version = "0.82.2", features = ["runtime", "derive"] } k8s-openapi = { version = "0.18.0", features = ["v1_26"] } # TODO : switch to v1_27 futures = "0.3.28" thiserror = "1.0.40" +rand = "0.8.5" diff --git a/src/constants.rs b/src/constants.rs index 53f5800..f9d9341 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -3,4 +3,7 @@ pub const SECRET_MINIO_INSTANCE_ACCESS_KEY: &str = "accessKey"; pub const SECRET_MINIO_INSTANCE_SECRET_KEY: &str = "secretKey"; pub const SECRET_MINIO_BUCKET_ACCESS_KEY: &str = "accessKey"; -pub const SECRET_MINIO_BUCKET_SECRET_KEY: &str = "secretKey"; \ No newline at end of file +pub const SECRET_MINIO_BUCKET_SECRET_KEY: &str = "secretKey"; + +pub const SECRET_MINIO_BUCKET_ACCESS_LEN: usize = 20; +pub const SECRET_MINIO_BUCKET_SECRET_LEN: usize = 35; \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 2aab6f6..2130aff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ pub mod constants; pub mod crd; +pub mod minio; pub mod secrets; -pub mod minio; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index c8e9878..8863459 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,15 @@ +use std::collections::BTreeMap; use futures::TryStreamExt; use k8s_openapi::api::core::v1::Secret; -use kube::{Api, Client}; use kube::runtime::{watcher, WatchStreamExt}; -use minio_operator::constants::{SECRET_MINIO_INSTANCE_ACCESS_KEY, SECRET_MINIO_INSTANCE_SECRET_KEY}; +use kube::{Api, Client}; +use minio_operator::constants::{ + SECRET_MINIO_BUCKET_ACCESS_KEY, SECRET_MINIO_BUCKET_SECRET_KEY, + SECRET_MINIO_INSTANCE_ACCESS_KEY, SECRET_MINIO_INSTANCE_SECRET_KEY, +}; use minio_operator::crd::{MinioBucket, MinioInstance}; -use minio_operator::minio::MinioService; -use minio_operator::secrets::read_secret_str; +use minio_operator::minio::{MinioService, MinioUser}; +use minio_operator::secrets::{create_secret, read_secret_str}; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -22,14 +26,17 @@ async fn main() -> anyhow::Result<()> { while let Some(b) = bw.try_next().await? { if let Err(e) = apply_bucket(&b, &client).await { - log::error!("Failed to apply desired configuration for applied bucket {} : {}", b.spec.name, e) + log::error!( + "Failed to apply desired configuration for applied bucket {} : {}", + b.spec.name, + e + ) } } Ok(()) } - /// Make sure a bucket is compliant with a desired configuration async fn apply_bucket(b: &MinioBucket, client: &Client) -> anyhow::Result<()> { log::info!("Apply configuration for bucket {}", b.spec.name); @@ -46,8 +53,28 @@ async fn apply_bucket(b: &MinioBucket, client: &Client) -> anyhow::Result<()> { access_key: read_secret_str(&instance_secret, SECRET_MINIO_INSTANCE_ACCESS_KEY)?, secret_key: read_secret_str(&instance_secret, SECRET_MINIO_INSTANCE_SECRET_KEY)?, }; - + + // Get user key & password + let user_secret = match secrets.get_opt(&b.spec.secret).await? { + Some(s) => s, + None => { + log::info!("Needs to create the secret {} for the bucket {}", b.spec.secret, b.spec.name); + + // The secret needs to be created + let new_user = MinioUser::gen_random(); + create_secret(&secrets, &b.spec.secret, BTreeMap::from([ + (SECRET_MINIO_BUCKET_ACCESS_KEY.to_string(), new_user.username), + (SECRET_MINIO_BUCKET_SECRET_KEY.to_string(), new_user.password) + ])).await? + } + }; + let user = MinioUser { + username: read_secret_str(&user_secret, SECRET_MINIO_BUCKET_ACCESS_KEY)?, + password: read_secret_str(&user_secret, SECRET_MINIO_BUCKET_SECRET_KEY)?, + }; + println!("{:?}", service); + println!("{:?}", user); Ok(()) -} \ No newline at end of file +} diff --git a/src/minio.rs b/src/minio.rs index 00a6cb3..408f906 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -1,6 +1,25 @@ +use rand::distributions::Alphanumeric; +use rand::Rng; +use crate::constants::{SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; + #[derive(Debug, Clone)] pub struct MinioService { pub hostname: String, pub access_key: String, pub secret_key: String, +} + +#[derive(Debug, Clone)] +pub struct MinioUser { + pub username: String, + pub password: String, +} + +impl MinioUser { + pub fn gen_random() -> Self { + Self { + username: rand::thread_rng().sample_iter(&Alphanumeric).take(SECRET_MINIO_BUCKET_ACCESS_LEN).map(char::from).collect(), + password: rand::thread_rng().sample_iter(&Alphanumeric).take(SECRET_MINIO_BUCKET_SECRET_LEN).map(char::from).collect(), + } + } } \ No newline at end of file diff --git a/src/secrets.rs b/src/secrets.rs index fac72e6..c03b608 100644 --- a/src/secrets.rs +++ b/src/secrets.rs @@ -1,4 +1,8 @@ use k8s_openapi::api::core::v1::Secret; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use kube::{Api}; +use std::collections::{BTreeMap}; +use kube::api::PostParams; #[derive(thiserror::Error, Debug)] enum SecretError { @@ -11,10 +15,50 @@ enum SecretError { /// Attempt to read a value contained in a secret. Returns an error in case /// of failure pub fn read_secret_str(s: &Secret, key: &str) -> anyhow::Result { - let data = s.data.as_ref().ok_or(SecretError::MissingData)?; + let data = s.data.as_ref().ok_or(SecretError::MissingData)?; - let value = data.get(key) + let value = data + .get(key) .ok_or(SecretError::MissingKey(key.to_string()))?; Ok(String::from_utf8(value.0.clone())?) -} \ No newline at end of file +} + +/// Create a secret consisting only of string key / value pairs +pub async fn create_secret( + secrets: &Api, + name: &str, + values: BTreeMap, +) -> anyhow::Result { + Ok(secrets + .create( + &PostParams::default(), + &Secret { + data: None, + immutable: None, + metadata: ObjectMeta { + annotations: None, + creation_timestamp: None, + deletion_grace_period_seconds: None, + deletion_timestamp: None, + finalizers: None, + generate_name: None, + generation: None, + labels: Some(BTreeMap::from([( + "created-by".to_string(), + "miniok8sbuckets".to_string(), + )])), + managed_fields: None, + name: Some(name.to_string()), + namespace: None, + owner_references: None, + resource_version: None, + self_link: None, + uid: None, + }, + string_data: Some(values), + type_: None, + }, + ) + .await?) +} diff --git a/test/bucket-policy.yaml b/test/bucket-policy.yaml new file mode 100644 index 0000000..951f4d8 --- /dev/null +++ b/test/bucket-policy.yaml @@ -0,0 +1,28 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListObjectsInBucket", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::bucket" + ] + }, + { + "Sid": "AllObjectActions", + "Effect": "Allow", + "Action": [ + "s3:DeleteObject", + "s3:Get*", + "s3:PutObject", + "s3:*Object" + ], + "Resource": [ + "arn:aws:s3:::bucket/*" + ] + } + ] +} \ No newline at end of file -- 2.45.2 From 11d2ba5312a0a33715be7326d2f0757257850e33 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Sat, 6 May 2023 13:15:17 +0200 Subject: [PATCH 06/21] Spwan test minio instance --- Cargo.lock | 280 +++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 4 + README.md | 4 +- src/constants.rs | 2 +- src/lib.rs | 3 + src/main.rs | 27 ++++- src/minio.rs | 31 ++++- src/minio_server.rs | 113 ++++++++++++++++++ src/secrets.rs | 4 +- src/utils.rs | 11 ++ 10 files changed, 462 insertions(+), 17 deletions(-) create mode 100644 src/minio_server.rs create mode 100644 src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 9c74a0e..3720972 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,6 +137,16 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.4" @@ -266,6 +276,15 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -300,6 +319,15 @@ dependencies = [ "libc", ] +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fnv" version = "1.0.7" @@ -430,6 +458,25 @@ dependencies = [ "wasi", ] +[[package]] +name = "h2" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -507,6 +554,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "httparse", @@ -550,6 +598,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.56" @@ -620,6 +681,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ipnet" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" + [[package]] name = "is-terminal" version = "0.4.7" @@ -791,6 +858,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.142" @@ -868,7 +941,9 @@ dependencies = [ "k8s-openapi", "kube", "log", + "mktemp", "rand", + "reqwest", "schemars", "serde", "serde_json", @@ -888,6 +963,33 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "mktemp" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bdc1f74dd7bb717d39f784f844e490d935b3aa7e383008006dbbf29c1f7820a" +dependencies = [ + "uuid", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -949,6 +1051,12 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "openssl-sys" version = "0.9.87" @@ -988,7 +1096,7 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "windows-sys 0.45.0", ] @@ -1109,6 +1217,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -1116,7 +1233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] @@ -1137,6 +1254,43 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +[[package]] +name = "reqwest" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rustix" version = "0.37.19" @@ -1157,6 +1311,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + [[package]] name = "schemars" version = "0.8.12" @@ -1203,6 +1366,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.162" @@ -1256,6 +1442,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.21" @@ -1331,6 +1529,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.45.0", +] + [[package]] name = "termcolor" version = "1.2.0" @@ -1415,6 +1626,16 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-openssl" version = "0.6.3" @@ -1584,6 +1805,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +dependencies = [ + "getrandom", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -1637,6 +1867,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.84" @@ -1666,6 +1908,16 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1706,6 +1958,21 @@ dependencies = [ "windows-targets 0.48.0", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -1838,6 +2105,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "zeroize" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index ac465dc..d8af726 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,7 @@ k8s-openapi = { version = "0.18.0", features = ["v1_26"] } # TODO : switch to v1 futures = "0.3.28" thiserror = "1.0.40" rand = "0.8.5" + +[dev-dependencies] +mktemp = "0.5.0" +reqwest = "0.11.17" \ No newline at end of file diff --git a/README.md b/README.md index 6c0fb2a..ad717f0 100644 --- a/README.md +++ b/README.md @@ -8,4 +8,6 @@ Apply all K8s config files manually: ```bash cat yaml/*.yaml | kubectl apply -f - -``` \ No newline at end of file +``` + +Note : [mc tool](https://min.io/download) is required \ No newline at end of file diff --git a/src/constants.rs b/src/constants.rs index f9d9341..9ba1ead 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -6,4 +6,4 @@ pub const SECRET_MINIO_BUCKET_ACCESS_KEY: &str = "accessKey"; pub const SECRET_MINIO_BUCKET_SECRET_KEY: &str = "secretKey"; pub const SECRET_MINIO_BUCKET_ACCESS_LEN: usize = 20; -pub const SECRET_MINIO_BUCKET_SECRET_LEN: usize = 35; \ No newline at end of file +pub const SECRET_MINIO_BUCKET_SECRET_LEN: usize = 35; diff --git a/src/lib.rs b/src/lib.rs index 2130aff..9a6443a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,7 @@ pub mod constants; pub mod crd; pub mod minio; +#[cfg(test)] +pub mod minio_server; pub mod secrets; +pub mod utils; diff --git a/src/main.rs b/src/main.rs index 8863459..3e1af02 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeMap; use futures::TryStreamExt; use k8s_openapi::api::core::v1::Secret; use kube::runtime::{watcher, WatchStreamExt}; @@ -10,6 +9,7 @@ use minio_operator::constants::{ use minio_operator::crd::{MinioBucket, MinioInstance}; use minio_operator::minio::{MinioService, MinioUser}; use minio_operator::secrets::{create_secret, read_secret_str}; +use std::collections::BTreeMap; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -58,14 +58,29 @@ async fn apply_bucket(b: &MinioBucket, client: &Client) -> anyhow::Result<()> { let user_secret = match secrets.get_opt(&b.spec.secret).await? { Some(s) => s, None => { - log::info!("Needs to create the secret {} for the bucket {}", b.spec.secret, b.spec.name); + log::info!( + "Needs to create the secret {} for the bucket {}", + b.spec.secret, + b.spec.name + ); // The secret needs to be created let new_user = MinioUser::gen_random(); - create_secret(&secrets, &b.spec.secret, BTreeMap::from([ - (SECRET_MINIO_BUCKET_ACCESS_KEY.to_string(), new_user.username), - (SECRET_MINIO_BUCKET_SECRET_KEY.to_string(), new_user.password) - ])).await? + create_secret( + &secrets, + &b.spec.secret, + BTreeMap::from([ + ( + SECRET_MINIO_BUCKET_ACCESS_KEY.to_string(), + new_user.username, + ), + ( + SECRET_MINIO_BUCKET_SECRET_KEY.to_string(), + new_user.password, + ), + ]), + ) + .await? } }; let user = MinioUser { diff --git a/src/minio.rs b/src/minio.rs index 408f906..f672b06 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -1,6 +1,5 @@ -use rand::distributions::Alphanumeric; -use rand::Rng; use crate::constants::{SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; +use crate::utils::rand_str; #[derive(Debug, Clone)] pub struct MinioService { @@ -18,8 +17,30 @@ pub struct MinioUser { impl MinioUser { pub fn gen_random() -> Self { Self { - username: rand::thread_rng().sample_iter(&Alphanumeric).take(SECRET_MINIO_BUCKET_ACCESS_LEN).map(char::from).collect(), - password: rand::thread_rng().sample_iter(&Alphanumeric).take(SECRET_MINIO_BUCKET_SECRET_LEN).map(char::from).collect(), + username: rand_str(SECRET_MINIO_BUCKET_ACCESS_LEN), + password: rand_str(SECRET_MINIO_BUCKET_SECRET_LEN), } } -} \ No newline at end of file +} + +impl MinioService { + /// Check if the Minio Service is ready to respond to our requests + pub async fn is_ready(&self) -> bool { + match reqwest::get(format!("{}/minio/health/live", self.hostname)).await { + Ok(r) => { + if r.status() == 200 { + log::info!("Minio is ready!"); + return true; + } + + log::info!( + "Minio not ready yet, check failed with status code {}", + r.status() + ); + } + Err(e) => log::info!("Minio not ready yet, check failed with error {e}"), + } + + false + } +} diff --git a/src/minio_server.rs b/src/minio_server.rs new file mode 100644 index 0000000..103add0 --- /dev/null +++ b/src/minio_server.rs @@ -0,0 +1,113 @@ +//! # Minio server controller +//! +//! Used for testing only + +use crate::minio::MinioService; +use crate::utils::rand_str; +use rand::RngCore; +use std::io::ErrorKind; +use std::process::{Child, Command}; + +pub struct MinioServer { + #[allow(dead_code)] + storage_base_dir: mktemp::Temp, + child: Child, + pub api_port: u16, + pub root_user: String, + pub root_password: String, +} + +impl MinioServer { + pub async fn start() -> anyhow::Result { + let storage_dir = mktemp::Temp::new_dir()?; + + let root_user = rand_str(30); + let root_password = rand_str(30); + let api_port = (2000 + rand::thread_rng().next_u64() % 5000) as u16; + log::info!( + "Spwan a new Minio server on port {} with root credentials {}:{}", + api_port, + root_user, + root_password + ); + + let child = Command::new("minio") + .current_dir(storage_dir.clone()) + .arg("server") + .arg("--address") + .arg(format!(":{api_port}")) + .arg(storage_dir.to_str().unwrap()) + .env("MINIO_ROOT_USER", &root_user) + .env("MINIO_ROOT_PASSWORD", &root_password) + .spawn()?; + + let instance = Self { + storage_base_dir: storage_dir, + child, + api_port, + root_user, + root_password, + }; + + // Wait for Minio to become ready + let mut check_count = 0; + loop { + if check_count >= 100 { + log::error!("Minio failed to respond properly in time!"); + return Err(std::io::Error::new( + ErrorKind::Other, + "Minio failed to respond in time!", + ) + .into()); + } + check_count += 1; + + if instance.as_service().is_ready().await { + break; + } + } + + Ok(instance) + } + + pub fn base_url(&self) -> String { + format!("http://127.0.0.1:{}", self.api_port) + } + + /// Get a MinioService instance of this temporary server + pub fn as_service(&self) -> MinioService { + MinioService { + hostname: self.base_url(), + access_key: self.root_user.clone(), + secret_key: self.root_password.clone(), + } + } +} + +impl Drop for MinioServer { + fn drop(&mut self) { + if let Err(e) = self.child.kill() { + log::error!("Failed to kill child server! {}", e); + } + } +} + +#[cfg(test)] +mod test { + use crate::minio_server::MinioServer; + + #[tokio::test] + async fn start_minio() { + let _ = env_logger::builder().is_test(true).try_init(); + + let server = MinioServer::start().await.unwrap(); + let service = server.as_service(); + println!("{:?}", service); + + assert!(service.is_ready().await); + + // Check if minio properly exit + drop(server); + assert!(!service.is_ready().await); + } +} diff --git a/src/secrets.rs b/src/secrets.rs index c03b608..107b275 100644 --- a/src/secrets.rs +++ b/src/secrets.rs @@ -1,8 +1,8 @@ use k8s_openapi::api::core::v1::Secret; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; -use kube::{Api}; -use std::collections::{BTreeMap}; use kube::api::PostParams; +use kube::Api; +use std::collections::BTreeMap; #[derive(thiserror::Error, Debug)] enum SecretError { diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..f2e11fd --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,11 @@ +use rand::distributions::Alphanumeric; +use rand::Rng; + +/// Generate a random string of a given size +pub fn rand_str(len: usize) -> String { + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() +} -- 2.45.2 From a3f523410fd73978eab81c9e50793bfa6beaf872 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Sat, 6 May 2023 16:41:36 +0200 Subject: [PATCH 07/21] Add buckets creation --- Cargo.toml | 2 - src/constants.rs | 2 + src/crd.rs | 2 + src/lib.rs | 2 +- src/minio.rs | 260 +++++++++++++++++- src/{minio_server.rs => minio_test_server.rs} | 10 +- yaml/minio-bucket.yaml | 4 + 7 files changed, 273 insertions(+), 9 deletions(-) rename src/{minio_server.rs => minio_test_server.rs} (93%) diff --git a/Cargo.toml b/Cargo.toml index d8af726..2971ef7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,5 @@ k8s-openapi = { version = "0.18.0", features = ["v1_26"] } # TODO : switch to v1 futures = "0.3.28" thiserror = "1.0.40" rand = "0.8.5" - -[dev-dependencies] mktemp = "0.5.0" reqwest = "0.11.17" \ No newline at end of file diff --git a/src/constants.rs b/src/constants.rs index 9ba1ead..f380b30 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -7,3 +7,5 @@ pub const SECRET_MINIO_BUCKET_SECRET_KEY: &str = "secretKey"; pub const SECRET_MINIO_BUCKET_ACCESS_LEN: usize = 20; pub const SECRET_MINIO_BUCKET_SECRET_LEN: usize = 35; + +pub const MC_EXE: &str = "mc"; diff --git a/src/crd.rs b/src/crd.rs index 5f0d147..e6374ec 100644 --- a/src/crd.rs +++ b/src/crd.rs @@ -45,5 +45,7 @@ pub struct MinioBucketSpec { #[serde(default)] pub versioning: bool, pub quota: Option, + #[serde(default)] + pub lock: bool, pub retention: Option, } diff --git a/src/lib.rs b/src/lib.rs index 9a6443a..86bbbb1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,6 @@ pub mod constants; pub mod crd; pub mod minio; #[cfg(test)] -pub mod minio_server; +pub mod minio_test_server; pub mod secrets; pub mod utils; diff --git a/src/minio.rs b/src/minio.rs index f672b06..9950a6d 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -1,5 +1,27 @@ -use crate::constants::{SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; +use crate::constants::{MC_EXE, SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; +use crate::crd::{MinioBucketSpec, RetentionType}; use crate::utils::rand_str; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use std::process::Command; + +const MC_ALIAS_NAME: &str = "managedminioinst"; + +#[derive(thiserror::Error, Debug)] +enum MinioError { + #[error("Failed to set 'mc' alias!")] + SetMcAlias, + #[error("Failed to execute 'mc' command!")] + ExecMc, + #[error("Failed to execute 'mc mb' command!")] + MakeBucketFailed, + #[error("Failed to set anonymous access!")] + SetAnonymousAcccessFailed, + #[error("Failed to set bucket quota!")] + SetQuotaFailed, + #[error("Failed to set bucket retention!")] + SetRetentionFailed, +} #[derive(Debug, Clone)] pub struct MinioService { @@ -23,6 +45,29 @@ impl MinioUser { } } +#[derive(Debug, Clone, Deserialize)] +pub struct BucketEntry { + pub status: String, + key: String, +} + +impl BucketEntry { + pub fn bucket_name(&self) -> &str { + &self.key[0..self.key.len() - 1] + } +} + +#[derive(Debug, Clone, Deserialize)] +struct BasicMinioResult { + pub status: String, +} + +impl BasicMinioResult { + pub fn success(&self) -> bool { + self.status == "success" + } +} + impl MinioService { /// Check if the Minio Service is ready to respond to our requests pub async fn is_ready(&self) -> bool { @@ -43,4 +88,217 @@ impl MinioService { false } + + /// Execute a minio mc command + async fn exec_mc_cmd(&self, args: &[&str]) -> anyhow::Result> + where + A: DeserializeOwned, + { + log::debug!("exec_mc_cmd with args {:?}", args); + + let conf_dir = mktemp::Temp::new_dir()?; + let global_flags = ["--config-dir", conf_dir.to_str().unwrap(), "--json"]; + + // First, set our alias to mc in a temporary directory + let res = Command::new(MC_EXE) + .args(global_flags) + .args([ + "alias", + "set", + MC_ALIAS_NAME, + self.hostname.as_str(), + self.access_key.as_str(), + self.secret_key.as_str(), + ]) + .output()?; + if res.status.code() != Some(0) { + log::error!( + "Failed to configure mc alias! (status code {:?}, stderr={}, stdout={})", + res.status, + String::from_utf8_lossy(&res.stderr), + String::from_utf8_lossy(&res.stdout) + ); + return Err(MinioError::SetMcAlias.into()); + } + + // Execute requested command + let res = Command::new(MC_EXE) + .args(global_flags) + .args(args) + .output()?; + + if res.status.code() != Some(0) { + log::error!( + "Failed execute command! (status code {:?}, stderr={}, stdout={})", + res.status, + String::from_utf8_lossy(&res.stderr), + String::from_utf8_lossy(&res.stdout) + ); + return Err(MinioError::ExecMc.into()); + } + + let stdout = String::from_utf8_lossy(&res.stdout); + log::debug!( + "stdout='{}' stderr='{}'", + stdout, + String::from_utf8_lossy(&res.stderr) + ); + + if stdout.is_empty() { + log::info!("Command returned no result!"); + return Ok(vec![]); + } + + let mut out = vec![]; + for l in stdout.split('\n') { + if !l.trim().is_empty() { + out.push(serde_json::from_str(l)?); + } + } + Ok(out) + } + + /// Get the list of buckets + pub async fn buckets_list(&self) -> anyhow::Result> { + self.exec_mc_cmd::(&["ls", MC_ALIAS_NAME]) + .await + } + + /// Check if a bucket exists or not + pub async fn bucket_exists(&self, name: &str) -> anyhow::Result { + Ok(self + .buckets_list() + .await? + .iter() + .any(|b| b.bucket_name().eq(name))) + } + + /// Create a bucket + pub async fn create_bucket(&self, b: &MinioBucketSpec) -> anyhow::Result<()> { + // Set base parameters + let bucket_name = format!("{}/{}", MC_ALIAS_NAME, b.name); + let mut args = ["mb", bucket_name.as_str()].to_vec(); + + if b.versioning { + args.push("--with-versioning"); + } + + if b.lock || b.retention.is_some() { + args.push("--with-lock"); + } + + let res = self.exec_mc_cmd::(&args).await?; + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::MakeBucketFailed.into()); + } + + // Enable anonymous access, eg. public hosting + if b.anonymous_read_access { + let target = format!("{}/*", bucket_name); + + let res = self + .exec_mc_cmd::(&["anonymous", "set", "download", target.as_str()]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetAnonymousAcccessFailed.into()); + } + } + + // Set quota, if requested + if let Some(quota) = &b.quota { + let quota = format!("{}MB", quota); + + let res = self + .exec_mc_cmd::(&[ + "quota", + "set", + bucket_name.as_str(), + "--size", + quota.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetQuotaFailed.into()); + } + } + + // Set retention, if requested + if let Some(retention) = &b.retention { + let days = format!("{}d", retention.validity); + + let res = self + .exec_mc_cmd::(&[ + "retention", + "set", + "--default", + match retention.r#type { + RetentionType::Compliance => "compliance", + RetentionType::Governance => "governance", + }, + days.as_str(), + bucket_name.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetRetentionFailed.into()); + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use crate::crd::MinioBucketSpec; + use crate::minio_test_server::MinioTestServer; + + #[tokio::test] + async fn list_buckets_empty_instance() { + let srv = MinioTestServer::start().await.unwrap(); + let buckets = srv.as_service().buckets_list().await.unwrap(); + assert!(buckets.is_empty()); + } + + #[tokio::test] + async fn bucket_exists_no_bucket() { + let srv = MinioTestServer::start().await.unwrap(); + assert!(!srv.as_service().bucket_exists("mybucket").await.unwrap()); + } + + #[tokio::test] + async fn bucket_basic_creation() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: "mybucket".to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + assert!(service.bucket_exists("mybucket").await.unwrap()); + } + + // TODO : with anonymous access + // TODO : without anonymous access + // TODO : with versioning + // TODO : without versioning + // TODO : with quota + // TODO : without quota + // TODO : with lock + // TODO : without lock + // TODO : with retention + // TODO : without retention } diff --git a/src/minio_server.rs b/src/minio_test_server.rs similarity index 93% rename from src/minio_server.rs rename to src/minio_test_server.rs index 103add0..5e8654a 100644 --- a/src/minio_server.rs +++ b/src/minio_test_server.rs @@ -8,7 +8,7 @@ use rand::RngCore; use std::io::ErrorKind; use std::process::{Child, Command}; -pub struct MinioServer { +pub struct MinioTestServer { #[allow(dead_code)] storage_base_dir: mktemp::Temp, child: Child, @@ -17,7 +17,7 @@ pub struct MinioServer { pub root_password: String, } -impl MinioServer { +impl MinioTestServer { pub async fn start() -> anyhow::Result { let storage_dir = mktemp::Temp::new_dir()?; @@ -84,7 +84,7 @@ impl MinioServer { } } -impl Drop for MinioServer { +impl Drop for MinioTestServer { fn drop(&mut self) { if let Err(e) = self.child.kill() { log::error!("Failed to kill child server! {}", e); @@ -94,13 +94,13 @@ impl Drop for MinioServer { #[cfg(test)] mod test { - use crate::minio_server::MinioServer; + use crate::minio_test_server::MinioTestServer; #[tokio::test] async fn start_minio() { let _ = env_logger::builder().is_test(true).try_init(); - let server = MinioServer::start().await.unwrap(); + let server = MinioTestServer::start().await.unwrap(); let service = server.as_service(); println!("{:?}", service); diff --git a/yaml/minio-bucket.yaml b/yaml/minio-bucket.yaml index d2067f6..8fa4a55 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/minio-bucket.yaml @@ -49,6 +49,10 @@ spec: type: integer description: Limits the amount of data in the bucket, in Megabytes. By default it is unlimited example: 100 + lock: + description: Object locking prevent objects from being deleted. Will be considered as set to true when retention is defined. + type: boolean + default: false retention: type: object description: Impose rules to prevent object deletion for a period of time. It requires versioning to be enabled/disabled -- 2.45.2 From f779715d650ed059ccd1fc43f6017bb07eb8ad03 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 14:10:15 +0200 Subject: [PATCH 08/21] Test properly versioning --- src/minio.rs | 210 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 197 insertions(+), 13 deletions(-) diff --git a/src/minio.rs b/src/minio.rs index 9950a6d..b9dfc55 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -1,9 +1,11 @@ +use std::process::Command; + +use serde::de::DeserializeOwned; +use serde::Deserialize; + use crate::constants::{MC_EXE, SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; use crate::crd::{MinioBucketSpec, RetentionType}; use crate::utils::rand_str; -use serde::de::DeserializeOwned; -use serde::Deserialize; -use std::process::Command; const MC_ALIAS_NAME: &str = "managedminioinst"; @@ -62,6 +64,16 @@ struct BasicMinioResult { pub status: String, } +#[derive(Debug, Clone, Deserialize)] +struct MinioGetVersioningResult { + pub versioning: Option, +} + +#[derive(Debug, Clone, Deserialize)] +struct MinioVersioning { + pub status: String, +} + impl BasicMinioResult { pub fn success(&self) -> bool { self.status == "success" @@ -89,6 +101,11 @@ impl MinioService { false } + /// Get bucket name prefixed by mc alias name + fn absolute_bucket_name(&self, name: &str) -> String { + format!("{}/{name}", MC_ALIAS_NAME) + } + /// Execute a minio mc command async fn exec_mc_cmd(&self, args: &[&str]) -> anyhow::Result> where @@ -179,10 +196,6 @@ impl MinioService { let bucket_name = format!("{}/{}", MC_ALIAS_NAME, b.name); let mut args = ["mb", bucket_name.as_str()].to_vec(); - if b.versioning { - args.push("--with-versioning"); - } - if b.lock || b.retention.is_some() { args.push("--with-lock"); } @@ -192,6 +205,8 @@ impl MinioService { return Err(MinioError::MakeBucketFailed.into()); } + self.bucket_set_versioning(&b.name, b.versioning).await?; + // Enable anonymous access, eg. public hosting if b.anonymous_read_access { let target = format!("{}/*", bucket_name); @@ -249,6 +264,38 @@ impl MinioService { Ok(()) } + + /// Set bucket versioning + pub async fn bucket_set_versioning(&self, bucket: &str, enable: bool) -> anyhow::Result<()> { + let bucket_name = self.absolute_bucket_name(bucket); + + let res = self + .exec_mc_cmd::(&[ + "version", + match enable { + true => "enable", + false => "suspend", + }, + bucket_name.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetQuotaFailed.into()); + } + Ok(()) + } + + /// Get current bucket versioning status + pub async fn bucket_get_versioning(&self, bucket_name: &str) -> anyhow::Result { + let bucket_name = self.absolute_bucket_name(bucket_name); + Ok(self.exec_mc_cmd::(&["version", "info", bucket_name.as_str()]) + .await? + .remove(0) + .versioning + .map(|v| v.status.to_lowercase().eq("enabled")) + .unwrap_or_default()) + } } #[cfg(test)] @@ -256,6 +303,8 @@ mod test { use crate::crd::MinioBucketSpec; use crate::minio_test_server::MinioTestServer; + const TEST_BUCKET_NAME: &str = "mybucket"; + #[tokio::test] async fn list_buckets_empty_instance() { let srv = MinioTestServer::start().await.unwrap(); @@ -278,7 +327,7 @@ mod test { service .create_bucket(&MinioBucketSpec { instance: "".to_string(), - name: "mybucket".to_string(), + name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), anonymous_read_access: false, versioning: false, @@ -288,13 +337,148 @@ mod test { }) .await .unwrap(); - assert!(service.bucket_exists("mybucket").await.unwrap()); + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + } + + #[tokio::test] + async fn bucket_creation_with_anonymous_access() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: true, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert_eq!( + reqwest::get(format!("{}/{}/test", service.hostname, TEST_BUCKET_NAME)) + .await + .unwrap() + .status() + .as_u16(), + 404 + ); + } + + #[tokio::test] + async fn bucket_creation_without_anonymous_access() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert_eq!( + reqwest::get(format!("{}/{}/test", service.hostname, TEST_BUCKET_NAME)) + .await + .unwrap() + .status() + .as_u16(), + 403 + ); + } + + // With versioning + #[tokio::test] + async fn bucket_with_versioning() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: true, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert!(service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); + } + + #[tokio::test] + async fn bucket_without_versioning() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert!(!service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); + } + + #[tokio::test] + async fn change_versioning() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert!(!service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); + service.bucket_set_versioning(TEST_BUCKET_NAME, true).await.unwrap(); + assert!(service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); + service.bucket_set_versioning(TEST_BUCKET_NAME, false).await.unwrap(); + assert!(!service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); } - // TODO : with anonymous access - // TODO : without anonymous access - // TODO : with versioning - // TODO : without versioning // TODO : with quota // TODO : without quota // TODO : with lock -- 2.45.2 From fb2a9f39fb4fcf60b614938f35b90841524a55ff Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 14:23:30 +0200 Subject: [PATCH 09/21] Test properly anonymous access --- src/minio.rs | 153 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 21 deletions(-) diff --git a/src/minio.rs b/src/minio.rs index b9dfc55..7bea047 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -74,6 +74,11 @@ struct MinioVersioning { pub status: String, } +#[derive(Debug, Clone, Deserialize)] +struct MinioAnonymousAccess { + pub permission: String, +} + impl BasicMinioResult { pub fn success(&self) -> bool { self.status == "success" @@ -206,19 +211,8 @@ impl MinioService { } self.bucket_set_versioning(&b.name, b.versioning).await?; - - // Enable anonymous access, eg. public hosting - if b.anonymous_read_access { - let target = format!("{}/*", bucket_name); - - let res = self - .exec_mc_cmd::(&["anonymous", "set", "download", target.as_str()]) - .await?; - - if res.get(0).map(|r| r.success()) != Some(true) { - return Err(MinioError::SetAnonymousAcccessFailed.into()); - } - } + self.bucket_set_anonymous_access(&b.name, b.anonymous_read_access) + .await?; // Set quota, if requested if let Some(quota) = &b.quota { @@ -289,13 +283,52 @@ impl MinioService { /// Get current bucket versioning status pub async fn bucket_get_versioning(&self, bucket_name: &str) -> anyhow::Result { let bucket_name = self.absolute_bucket_name(bucket_name); - Ok(self.exec_mc_cmd::(&["version", "info", bucket_name.as_str()]) + Ok(self + .exec_mc_cmd::(&["version", "info", bucket_name.as_str()]) .await? .remove(0) .versioning .map(|v| v.status.to_lowercase().eq("enabled")) .unwrap_or_default()) } + + /// Set bucket anonymous access + pub async fn bucket_set_anonymous_access( + &self, + bucket_name: &str, + access: bool, + ) -> anyhow::Result<()> { + let target = self.absolute_bucket_name(bucket_name); + + let res = self + .exec_mc_cmd::(&[ + "anonymous", + "set", + match access { + true => "download", + false => "private", + }, + target.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetAnonymousAcccessFailed.into()); + } + + Ok(()) + } + + /// Get current bucket anonymous access status + pub async fn bucket_get_anonymous_access(&self, bucket_name: &str) -> anyhow::Result { + let bucket_name = self.absolute_bucket_name(bucket_name); + Ok(self + .exec_mc_cmd::(&["anonymous", "get", bucket_name.as_str()]) + .await? + .remove(0) + .permission + == "download") + } } #[cfg(test)] @@ -361,6 +394,10 @@ mod test { .unwrap(); assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert!(service + .bucket_get_anonymous_access(TEST_BUCKET_NAME) + .await + .unwrap()); assert_eq!( reqwest::get(format!("{}/{}/test", service.hostname, TEST_BUCKET_NAME)) .await @@ -402,6 +439,59 @@ mod test { ); } + #[tokio::test] + async fn bucket_creation_without_anonymous_access_updating_status() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + let test_url = format!("{}/{}/test", service.hostname, TEST_BUCKET_NAME); + assert_eq!( + reqwest::get(&test_url).await.unwrap().status().as_u16(), + 403 + ); + service + .bucket_set_anonymous_access(TEST_BUCKET_NAME, true) + .await + .unwrap(); + assert!(service + .bucket_get_anonymous_access(TEST_BUCKET_NAME) + .await + .unwrap()); + assert_eq!( + reqwest::get(&test_url).await.unwrap().status().as_u16(), + 404 + ); + + service + .bucket_set_anonymous_access(TEST_BUCKET_NAME, false) + .await + .unwrap(); + assert!(!service + .bucket_get_anonymous_access(TEST_BUCKET_NAME) + .await + .unwrap()); + assert_eq!( + reqwest::get(&test_url).await.unwrap().status().as_u16(), + 403 + ); + } + // With versioning #[tokio::test] async fn bucket_with_versioning() { @@ -424,7 +514,10 @@ mod test { .unwrap(); assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); - assert!(service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); + assert!(service + .bucket_get_versioning(TEST_BUCKET_NAME) + .await + .unwrap()); } #[tokio::test] @@ -448,7 +541,10 @@ mod test { .unwrap(); assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); - assert!(!service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); + assert!(!service + .bucket_get_versioning(TEST_BUCKET_NAME) + .await + .unwrap()); } #[tokio::test] @@ -472,11 +568,26 @@ mod test { .unwrap(); assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); - assert!(!service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); - service.bucket_set_versioning(TEST_BUCKET_NAME, true).await.unwrap(); - assert!(service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); - service.bucket_set_versioning(TEST_BUCKET_NAME, false).await.unwrap(); - assert!(!service.bucket_get_versioning(TEST_BUCKET_NAME).await.unwrap()); + assert!(!service + .bucket_get_versioning(TEST_BUCKET_NAME) + .await + .unwrap()); + service + .bucket_set_versioning(TEST_BUCKET_NAME, true) + .await + .unwrap(); + assert!(service + .bucket_get_versioning(TEST_BUCKET_NAME) + .await + .unwrap()); + service + .bucket_set_versioning(TEST_BUCKET_NAME, false) + .await + .unwrap(); + assert!(!service + .bucket_get_versioning(TEST_BUCKET_NAME) + .await + .unwrap()); } // TODO : with quota -- 2.45.2 From e3860d9f934f2b494636fc8ddb04712069f7be87 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 14:38:54 +0200 Subject: [PATCH 10/21] Test properly quota definition --- src/minio.rs | 135 ++++++++++++++++++++++++++++++++++------- yaml/minio-bucket.yaml | 4 +- 2 files changed, 115 insertions(+), 24 deletions(-) diff --git a/src/minio.rs b/src/minio.rs index 7bea047..5516352 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -79,6 +79,11 @@ struct MinioAnonymousAccess { pub permission: String, } +#[derive(Debug, Clone, Deserialize)] +struct MinioQuota { + pub quota: Option, +} + impl BasicMinioResult { pub fn success(&self) -> bool { self.status == "success" @@ -215,23 +220,7 @@ impl MinioService { .await?; // Set quota, if requested - if let Some(quota) = &b.quota { - let quota = format!("{}MB", quota); - - let res = self - .exec_mc_cmd::(&[ - "quota", - "set", - bucket_name.as_str(), - "--size", - quota.as_str(), - ]) - .await?; - - if res.get(0).map(|r| r.success()) != Some(true) { - return Err(MinioError::SetQuotaFailed.into()); - } - } + self.bucket_set_quota(&b.name, b.quota).await?; // Set retention, if requested if let Some(retention) = &b.retention { @@ -329,6 +318,41 @@ impl MinioService { .permission == "download") } + + /// Set bucket quota, in bytes + pub async fn bucket_set_quota(&self, bucket: &str, quota: Option) -> anyhow::Result<()> { + let bucket_name = self.absolute_bucket_name(bucket); + + let res = if let Some(quota) = "a { + let quota = format!("{}B", quota); + self.exec_mc_cmd::(&[ + "quota", + "set", + bucket_name.as_str(), + "--size", + quota.as_str(), + ]) + .await? + } else { + self.exec_mc_cmd::(&["quota", "clear", bucket_name.as_str()]) + .await? + }; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetQuotaFailed.into()); + } + Ok(()) + } + + /// Get current bucket quota, in bytes + pub async fn bucket_get_quota(&self, bucket_name: &str) -> anyhow::Result> { + let bucket_name = self.absolute_bucket_name(bucket_name); + Ok(self + .exec_mc_cmd::(&["quota", "info", bucket_name.as_str()]) + .await? + .remove(0) + .quota) + } } #[cfg(test)] @@ -492,7 +516,6 @@ mod test { ); } - // With versioning #[tokio::test] async fn bucket_with_versioning() { let _ = env_logger::builder().is_test(true).try_init(); @@ -590,10 +613,78 @@ mod test { .unwrap()); } - // TODO : with quota - // TODO : without quota - // TODO : with lock - // TODO : without lock + #[tokio::test] + async fn bucket_without_qutoa() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert_eq!( + service.bucket_get_quota(TEST_BUCKET_NAME).await.unwrap(), + None + ); + + service + .bucket_set_quota(TEST_BUCKET_NAME, Some(5122600)) + .await + .unwrap(); + assert_eq!( + service.bucket_get_quota(TEST_BUCKET_NAME).await.unwrap(), + Some(5122600) + ); + + service + .bucket_set_quota(TEST_BUCKET_NAME, None) + .await + .unwrap(); + assert_eq!( + service.bucket_get_quota(TEST_BUCKET_NAME).await.unwrap(), + None + ); + } + + #[tokio::test] + async fn bucket_with_qutoa() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: Some(42300), + lock: false, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert_eq!( + service.bucket_get_quota(TEST_BUCKET_NAME).await.unwrap(), + Some(42300) + ); + } + // TODO : with retention // TODO : without retention } diff --git a/yaml/minio-bucket.yaml b/yaml/minio-bucket.yaml index 8fa4a55..7a4795d 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/minio-bucket.yaml @@ -47,8 +47,8 @@ spec: default: false quota: type: integer - description: Limits the amount of data in the bucket, in Megabytes. By default it is unlimited - example: 100 + description: Limits the amount of data in the bucket, in bytes. By default it is unlimited + example: 1000000000 lock: description: Object locking prevent objects from being deleted. Will be considered as set to true when retention is defined. type: boolean -- 2.45.2 From 239b58d8db3d9d147f481bbdc4988f07cda3dd71 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 15:23:19 +0200 Subject: [PATCH 11/21] Properly test retention --- src/crd.rs | 4 +- src/minio.rs | 240 +++++++++++++++++++++++++++++++++------ src/minio_test_server.rs | 3 + yaml/minio-bucket.yaml | 2 +- 4 files changed, 214 insertions(+), 35 deletions(-) diff --git a/src/crd.rs b/src/crd.rs index e6374ec..0921e7d 100644 --- a/src/crd.rs +++ b/src/crd.rs @@ -14,7 +14,7 @@ pub struct MinioInstanceSpec { pub credentials: String, } -#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, Default, Copy, Clone, JsonSchema, PartialEq, Eq)] pub enum RetentionType { #[default] #[serde(rename_all = "lowercase")] @@ -23,7 +23,7 @@ pub enum RetentionType { Governance, } -#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] pub struct BucketRetention { pub validity: usize, pub r#type: RetentionType, diff --git a/src/minio.rs b/src/minio.rs index 5516352..00b1d90 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -4,7 +4,7 @@ use serde::de::DeserializeOwned; use serde::Deserialize; use crate::constants::{MC_EXE, SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; -use crate::crd::{MinioBucketSpec, RetentionType}; +use crate::crd::{BucketRetention, MinioBucketSpec, RetentionType}; use crate::utils::rand_str; const MC_ALIAS_NAME: &str = "managedminioinst"; @@ -84,6 +84,13 @@ struct MinioQuota { pub quota: Option, } +#[derive(Debug, Clone, Deserialize)] +struct MinioRetentionResult { + pub enabled: Option, + pub mode: Option, + pub validity: Option, +} + impl BasicMinioResult { pub fn success(&self) -> bool { self.status == "success" @@ -206,7 +213,7 @@ impl MinioService { let bucket_name = format!("{}/{}", MC_ALIAS_NAME, b.name); let mut args = ["mb", bucket_name.as_str()].to_vec(); - if b.lock || b.retention.is_some() { + if b.lock { args.push("--with-lock"); } @@ -215,36 +222,15 @@ impl MinioService { return Err(MinioError::MakeBucketFailed.into()); } - self.bucket_set_versioning(&b.name, b.versioning).await?; + self.bucket_set_versioning(&b.name, b.versioning || b.lock) + .await?; self.bucket_set_anonymous_access(&b.name, b.anonymous_read_access) .await?; - - // Set quota, if requested self.bucket_set_quota(&b.name, b.quota).await?; - - // Set retention, if requested - if let Some(retention) = &b.retention { - let days = format!("{}d", retention.validity); - - let res = self - .exec_mc_cmd::(&[ - "retention", - "set", - "--default", - match retention.r#type { - RetentionType::Compliance => "compliance", - RetentionType::Governance => "governance", - }, - days.as_str(), - bucket_name.as_str(), - ]) + if b.lock { + self.bucket_set_default_retention(&b.name, b.retention) .await?; - - if res.get(0).map(|r| r.success()) != Some(true) { - return Err(MinioError::SetRetentionFailed.into()); - } } - Ok(()) } @@ -353,11 +339,84 @@ impl MinioService { .remove(0) .quota) } + + /// Set bucket default retention policy + pub async fn bucket_set_default_retention( + &self, + bucket_name: &str, + retention: Option, + ) -> anyhow::Result<()> { + let bucket_name = self.absolute_bucket_name(bucket_name); + let res = if let Some(retention) = &retention { + let days = format!("{}d", retention.validity); + + self.exec_mc_cmd::(&[ + "retention", + "set", + "--default", + match retention.r#type { + RetentionType::Compliance => "compliance", + RetentionType::Governance => "governance", + }, + days.as_str(), + bucket_name.as_str(), + ]) + .await? + } else { + self.exec_mc_cmd::(&[ + "retention", + "clear", + "--default", + bucket_name.as_str(), + ]) + .await? + }; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetRetentionFailed.into()); + } + + Ok(()) + } + + /// Get bucket default retention policy + pub async fn bucket_get_default_retention( + &self, + bucket: &str, + ) -> anyhow::Result> { + let bucket_name = self.absolute_bucket_name(bucket); + let res = self + .exec_mc_cmd::(&[ + "retention", + "info", + bucket_name.as_str(), + "--default", + ]) + .await? + .remove(0); + + if let (Some(mode), Some(validity), Some(enabled)) = (res.mode, res.validity, res.enabled) { + if enabled.to_lowercase().eq("enabled") { + return Ok(Some(BucketRetention { + validity: validity.to_lowercase().replace("days", "").parse()?, + r#type: match mode.to_lowercase().as_str() { + "governance" => RetentionType::Governance, + "compliance" => RetentionType::Compliance, + o => { + log::error!("Unknown retention type: {}", o); + return Ok(None); + } + }, + })); + } + } + Ok(None) + } } #[cfg(test)] mod test { - use crate::crd::MinioBucketSpec; + use crate::crd::{BucketRetention, MinioBucketSpec, RetentionType}; use crate::minio_test_server::MinioTestServer; const TEST_BUCKET_NAME: &str = "mybucket"; @@ -614,7 +673,7 @@ mod test { } #[tokio::test] - async fn bucket_without_qutoa() { + async fn bucket_without_quota() { let _ = env_logger::builder().is_test(true).try_init(); let srv = MinioTestServer::start().await.unwrap(); @@ -659,7 +718,7 @@ mod test { } #[tokio::test] - async fn bucket_with_qutoa() { + async fn bucket_with_quota() { let _ = env_logger::builder().is_test(true).try_init(); let srv = MinioTestServer::start().await.unwrap(); @@ -685,6 +744,123 @@ mod test { ); } - // TODO : with retention - // TODO : without retention + #[tokio::test] + async fn bucket_with_retention() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: Some(42300), + lock: true, + retention: Some(BucketRetention { + validity: 10, + r#type: RetentionType::Governance, + }), + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert_eq!( + service + .bucket_get_default_retention(TEST_BUCKET_NAME) + .await + .unwrap(), + Some(BucketRetention { + validity: 10, + r#type: RetentionType::Governance + }) + ); + + service + .bucket_set_default_retention(TEST_BUCKET_NAME, None) + .await + .unwrap(); + assert_eq!( + service + .bucket_get_default_retention(TEST_BUCKET_NAME) + .await + .unwrap(), + None + ); + + service + .bucket_set_default_retention( + TEST_BUCKET_NAME, + Some(BucketRetention { + validity: 42, + r#type: RetentionType::Compliance, + }), + ) + .await + .unwrap(); + assert_eq!( + service + .bucket_get_default_retention(TEST_BUCKET_NAME) + .await + .unwrap(), + Some(BucketRetention { + validity: 42, + r#type: RetentionType::Compliance + }) + ); + + service + .bucket_set_default_retention( + TEST_BUCKET_NAME, + Some(BucketRetention { + validity: 21, + r#type: RetentionType::Governance, + }), + ) + .await + .unwrap(); + assert_eq!( + service + .bucket_get_default_retention(TEST_BUCKET_NAME) + .await + .unwrap(), + Some(BucketRetention { + validity: 21, + r#type: RetentionType::Governance + }) + ); + } + + #[tokio::test] + async fn bucket_without_retention() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: TEST_BUCKET_NAME.to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: Some(42300), + lock: true, + retention: None, + }) + .await + .unwrap(); + + assert!(service.bucket_exists(TEST_BUCKET_NAME).await.unwrap()); + assert_eq!( + service + .bucket_get_default_retention(TEST_BUCKET_NAME) + .await + .unwrap(), + None + ); + } } diff --git a/src/minio_test_server.rs b/src/minio_test_server.rs index 5e8654a..29a655d 100644 --- a/src/minio_test_server.rs +++ b/src/minio_test_server.rs @@ -7,6 +7,7 @@ use crate::utils::rand_str; use rand::RngCore; use std::io::ErrorKind; use std::process::{Child, Command}; +use std::time::Duration; pub struct MinioTestServer { #[allow(dead_code)] @@ -62,6 +63,8 @@ impl MinioTestServer { } check_count += 1; + std::thread::sleep(Duration::from_millis(100)); + if instance.as_service().is_ready().await { break; } diff --git a/yaml/minio-bucket.yaml b/yaml/minio-bucket.yaml index 7a4795d..82b696b 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/minio-bucket.yaml @@ -50,7 +50,7 @@ spec: description: Limits the amount of data in the bucket, in bytes. By default it is unlimited example: 1000000000 lock: - description: Object locking prevent objects from being deleted. Will be considered as set to true when retention is defined. + description: Object locking prevent objects from being deleted. MUST be set to true when retention is defined. Cannot be changed. type: boolean default: false retention: -- 2.45.2 From 42e2ea5539ae1023abb560809c0eb7cb28f63848 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 15:28:41 +0200 Subject: [PATCH 12/21] Improve terminology --- src/minio.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/minio.rs b/src/minio.rs index 00b1d90..7189ac5 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -207,11 +207,11 @@ impl MinioService { .any(|b| b.bucket_name().eq(name))) } - /// Create a bucket - pub async fn create_bucket(&self, b: &MinioBucketSpec) -> anyhow::Result<()> { + /// Apply bucket desired configuration. If bucket already exists, it is not dropped + pub async fn apply_bucket(&self, b: &MinioBucketSpec) -> anyhow::Result<()> { // Set base parameters let bucket_name = format!("{}/{}", MC_ALIAS_NAME, b.name); - let mut args = ["mb", bucket_name.as_str()].to_vec(); + let mut args = ["mb", bucket_name.as_str(), "-p"].to_vec(); if b.lock { args.push("--with-lock"); @@ -441,7 +441,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -463,7 +463,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -498,7 +498,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -529,7 +529,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -582,7 +582,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -609,7 +609,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -636,7 +636,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -679,7 +679,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -724,7 +724,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -751,7 +751,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -841,7 +841,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .create_bucket(&MinioBucketSpec { + .apply_bucket(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), -- 2.45.2 From 1ae2cf7282674cbcf0a0763040ee60311381c1d1 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 16:00:53 +0200 Subject: [PATCH 13/21] Can create policies --- src/minio.rs | 90 +++++++++++++++++++ ...{bucket-policy.yaml => bucket-policy.json} | 0 test/test-policy1.json | 15 ++++ test/test-policy2.json | 11 +++ 4 files changed, 116 insertions(+) rename test/{bucket-policy.yaml => bucket-policy.json} (100%) create mode 100644 test/test-policy1.json create mode 100644 test/test-policy2.json diff --git a/src/minio.rs b/src/minio.rs index 7189ac5..94a992c 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -23,6 +23,8 @@ enum MinioError { SetQuotaFailed, #[error("Failed to set bucket retention!")] SetRetentionFailed, + #[error("Failed to set policy!")] + ApplyPolicyFailed, } #[derive(Debug, Clone)] @@ -91,6 +93,23 @@ struct MinioRetentionResult { pub validity: Option, } +#[derive(Debug, Clone, Deserialize)] +struct MinioPolicy { + pub policy: String, +} + +#[allow(non_snake_case)] +#[derive(Debug, Clone, Deserialize)] +struct MinioPolicyInfo { + pub policyInfo: PolicyInfo, +} + +#[allow(non_snake_case)] +#[derive(Debug, Clone, Deserialize)] +struct PolicyInfo { + Policy: serde_json::Value, +} + impl BasicMinioResult { pub fn success(&self) -> bool { self.status == "success" @@ -412,6 +431,49 @@ impl MinioService { } Ok(None) } + + /// Apply a bucket policy + pub async fn policy_apply(&self, name: &str, content: &str) -> anyhow::Result<()> { + let tmp_file = mktemp::Temp::new_file()?; + std::fs::write(&tmp_file, content)?; + + let res = self + .exec_mc_cmd::(&[ + "admin", + "policy", + "create", + MC_ALIAS_NAME, + name, + tmp_file.to_str().unwrap(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::ApplyPolicyFailed.into()); + } + + Ok(()) + } + + /// Get the list of existing policies + pub async fn policy_list(&self) -> anyhow::Result> { + Ok(self + .exec_mc_cmd::(&["admin", "policy", "list", MC_ALIAS_NAME]) + .await? + .iter() + .map(|p| p.policy.to_string()) + .collect()) + } + + /// Get the content of a given policy + pub async fn policy_content(&self, name: &str) -> anyhow::Result { + let policy = self + .exec_mc_cmd::(&["admin", "policy", "info", MC_ALIAS_NAME, name]) + .await? + .remove(0); + + Ok(serde_json::to_string(&policy.policyInfo.Policy)?) + } } #[cfg(test)] @@ -420,6 +482,7 @@ mod test { use crate::minio_test_server::MinioTestServer; const TEST_BUCKET_NAME: &str = "mybucket"; + const TEST_POLICY_NAME: &str = "mypolicy"; #[tokio::test] async fn list_buckets_empty_instance() { @@ -863,4 +926,31 @@ mod test { None ); } + + fn unify_policy(p: &str) -> String { + serde_json::to_string(&serde_json::from_str::(p).unwrap()).unwrap() + } + + #[tokio::test] + async fn policy_apply() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + + let policy_1 = unify_policy(include_str!("../test/test-policy1.json")); + let policy_2 = unify_policy(include_str!("../test/test-policy2.json")); + + assert_ne!(policy_1, policy_2); + + assert!(!service.policy_list().await.unwrap().contains(&TEST_POLICY_NAME.to_string())); + + service.policy_apply(TEST_POLICY_NAME, &policy_1).await.unwrap(); + assert!(service.policy_list().await.unwrap().contains(&TEST_POLICY_NAME.to_string())); + assert_eq!(unify_policy(&service.policy_content(TEST_POLICY_NAME).await.unwrap()), policy_1); + + service.policy_apply(TEST_POLICY_NAME, &policy_2).await.unwrap(); + assert!(service.policy_list().await.unwrap().contains(&TEST_POLICY_NAME.to_string())); + assert_eq!(unify_policy(&service.policy_content(TEST_POLICY_NAME).await.unwrap()), policy_2); + } } diff --git a/test/bucket-policy.yaml b/test/bucket-policy.json similarity index 100% rename from test/bucket-policy.yaml rename to test/bucket-policy.json diff --git a/test/test-policy1.json b/test/test-policy1.json new file mode 100644 index 0000000..acbf1c5 --- /dev/null +++ b/test/test-policy1.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListObjectsInBucket", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::bucket" + ] + } + ] +} \ No newline at end of file diff --git a/test/test-policy2.json b/test/test-policy2.json new file mode 100644 index 0000000..e45f5c0 --- /dev/null +++ b/test/test-policy2.json @@ -0,0 +1,11 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListObjectsInBucket", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::bucketdos"] + } + ] +} -- 2.45.2 From 328036b7b3db304f37b7af4face31b2c7c5583f8 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 16:04:23 +0200 Subject: [PATCH 14/21] Add basic drone configuration --- .drone.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..dcc7d9f --- /dev/null +++ b/.drone.yml @@ -0,0 +1,16 @@ +--- +kind: pipeline +type: docker +name: default + +steps: +- name: cargo_check + image: rust + commands: + - wget -O /usr/bin/minio https://dl.min.io/server/minio/release/linux-amd64/minio + - wget -O /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc + - chmod +x /usr/bin/minio /usr/bin/mc + - rustup component add clippy + - cargo clippy -- -D warnings + - cargo test + -- 2.45.2 From c90e46f0386a201a0209ec4e5c87b6cbffa04fed Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 16:18:15 +0200 Subject: [PATCH 15/21] Can create users --- src/minio.rs | 91 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 7 deletions(-) diff --git a/src/minio.rs b/src/minio.rs index 94a992c..ebb0fbc 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -25,6 +25,8 @@ enum MinioError { SetRetentionFailed, #[error("Failed to set policy!")] ApplyPolicyFailed, + #[error("Failed to create user!")] + CreateUserFailed, } #[derive(Debug, Clone)] @@ -110,6 +112,12 @@ struct PolicyInfo { Policy: serde_json::Value, } +#[allow(non_snake_case)] +#[derive(Debug, Clone, Deserialize)] +struct MinioUserListRes { + accessKey: String, +} + impl BasicMinioResult { pub fn success(&self) -> bool { self.status == "success" @@ -474,11 +482,42 @@ impl MinioService { Ok(serde_json::to_string(&policy.policyInfo.Policy)?) } + + /// Apply a user + pub async fn user_apply(&self, user: &MinioUser) -> anyhow::Result<()> { + let res = self + .exec_mc_cmd::(&[ + "admin", + "user", + "add", + MC_ALIAS_NAME, + user.username.as_str(), + user.password.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::CreateUserFailed.into()); + } + + Ok(()) + } + + /// Get the list of users + pub async fn user_list(&self) -> anyhow::Result> { + Ok(self + .exec_mc_cmd::(&["admin", "user", "list", MC_ALIAS_NAME]) + .await? + .iter() + .map(|p| p.accessKey.to_string()) + .collect()) + } } #[cfg(test)] mod test { use crate::crd::{BucketRetention, MinioBucketSpec, RetentionType}; + use crate::minio::MinioUser; use crate::minio_test_server::MinioTestServer; const TEST_BUCKET_NAME: &str = "mybucket"; @@ -943,14 +982,52 @@ mod test { assert_ne!(policy_1, policy_2); - assert!(!service.policy_list().await.unwrap().contains(&TEST_POLICY_NAME.to_string())); + assert!(!service + .policy_list() + .await + .unwrap() + .contains(&TEST_POLICY_NAME.to_string())); - service.policy_apply(TEST_POLICY_NAME, &policy_1).await.unwrap(); - assert!(service.policy_list().await.unwrap().contains(&TEST_POLICY_NAME.to_string())); - assert_eq!(unify_policy(&service.policy_content(TEST_POLICY_NAME).await.unwrap()), policy_1); + service + .policy_apply(TEST_POLICY_NAME, &policy_1) + .await + .unwrap(); + assert!(service + .policy_list() + .await + .unwrap() + .contains(&TEST_POLICY_NAME.to_string())); + assert_eq!( + unify_policy(&service.policy_content(TEST_POLICY_NAME).await.unwrap()), + policy_1 + ); - service.policy_apply(TEST_POLICY_NAME, &policy_2).await.unwrap(); - assert!(service.policy_list().await.unwrap().contains(&TEST_POLICY_NAME.to_string())); - assert_eq!(unify_policy(&service.policy_content(TEST_POLICY_NAME).await.unwrap()), policy_2); + service + .policy_apply(TEST_POLICY_NAME, &policy_2) + .await + .unwrap(); + assert!(service + .policy_list() + .await + .unwrap() + .contains(&TEST_POLICY_NAME.to_string())); + assert_eq!( + unify_policy(&service.policy_content(TEST_POLICY_NAME).await.unwrap()), + policy_2 + ); + } + + #[tokio::test] + async fn policy_user() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + + let user = MinioUser::gen_random(); + + assert!(!service.user_list().await.unwrap().contains(&user.username)); + service.user_apply(&user).await.unwrap(); + assert!(service.user_list().await.unwrap().contains(&user.username)); } } -- 2.45.2 From 073c91fe0dcc26725fa9a9937161a1715a3358a9 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 17:08:59 +0200 Subject: [PATCH 16/21] Can attach policies to users --- src/minio.rs | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/minio.rs b/src/minio.rs index ebb0fbc..9bc3e25 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -118,6 +118,22 @@ struct MinioUserListRes { accessKey: String, } +#[derive(Debug, Clone, Deserialize)] +struct MinioPoliciesUserEntities { + result: MinioPoliciesUserEntitiesInner, +} + +#[allow(non_snake_case)] +#[derive(Debug, Clone, Deserialize)] +struct MinioPoliciesUserEntitiesInner { + userMappings: Option>, +} + +#[derive(Debug, Clone, Deserialize)] +struct MinioPoliciesUserEntitiesInnerUser { + policies: Vec, +} + impl BasicMinioResult { pub fn success(&self) -> bool { self.status == "success" @@ -512,6 +528,55 @@ impl MinioService { .map(|p| p.accessKey.to_string()) .collect()) } + + /// Attach a user to a policy + pub async fn policy_attach_user(&self, user: &MinioUser, policy: &str) -> anyhow::Result<()> { + let res = self + .exec_mc_cmd::(&[ + "admin", + "policy", + "attach", + MC_ALIAS_NAME, + policy, + "--user", + user.username.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::CreateUserFailed.into()); + } + + Ok(()) + } + + /// Get the list of entities attached to a user + pub async fn policy_attach_get_user_list( + &self, + user: &MinioUser, + ) -> anyhow::Result> { + let res = self + .exec_mc_cmd::(&[ + "admin", + "policy", + "entities", + MC_ALIAS_NAME, + "--user", + user.username.as_str(), + ]) + .await? + .remove(0) + .result + .userMappings; + + if let Some(mapping) = res { + if let Some(e) = mapping.get(0) { + return Ok(e.policies.clone()); + } + } + + Ok(vec![]) + } } #[cfg(test)] @@ -1030,4 +1095,35 @@ mod test { service.user_apply(&user).await.unwrap(); assert!(service.user_list().await.unwrap().contains(&user.username)); } + + #[tokio::test] + async fn attach_policy_user() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + + let user = MinioUser::gen_random(); + + service.user_apply(&user).await.unwrap(); + service + .policy_apply(TEST_POLICY_NAME, include_str!("../test/test-policy1.json")) + .await + .unwrap(); + + assert!(!service + .policy_attach_get_user_list(&user) + .await + .unwrap() + .contains(&TEST_POLICY_NAME.to_string())); + service + .policy_attach_user(&user, TEST_POLICY_NAME) + .await + .unwrap(); + assert!(service + .policy_attach_get_user_list(&user) + .await + .unwrap() + .contains(&TEST_POLICY_NAME.to_string())); + } } -- 2.45.2 From 268f9a47cdd4f0629dd5803b97b225f3a4081ba1 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 17:20:54 +0200 Subject: [PATCH 17/21] Applied first configuration --- src/main.rs | 18 ++++++++++++++++-- src/minio.rs | 24 ++++++++++++------------ src/policy_template.json | 17 +++++++++++++++++ test/bucket-policy.json | 28 ---------------------------- test/test-outside-cluster.yaml | 25 +++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 42 deletions(-) create mode 100644 src/policy_template.json delete mode 100644 test/bucket-policy.json create mode 100644 test/test-outside-cluster.yaml diff --git a/src/main.rs b/src/main.rs index 3e1af02..f2279a3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -88,8 +88,22 @@ async fn apply_bucket(b: &MinioBucket, client: &Client) -> anyhow::Result<()> { password: read_secret_str(&user_secret, SECRET_MINIO_BUCKET_SECRET_KEY)?, }; - println!("{:?}", service); - println!("{:?}", user); + log::debug!("Create or update bucket..."); + service.bucket_apply(&b.spec).await?; + + let policy_name = format!("bucket-{}", b.spec.name); + log::debug!("Create or update policy '{policy_name}'..."); + let policy_content = + include_str!("policy_template.json").replace("{{ bucket }}", b.spec.name.as_str()); + service.policy_apply(&policy_name, &policy_content).await?; + + log::debug!("Create or update user '{}'...", user.username); + service.user_apply(&user).await?; + + log::debug!("Attach policy '{policy_name}' to user..."); + service.policy_attach_user(&user, &policy_name).await?; + + log::debug!("Successfully applied desired configuration!"); Ok(()) } diff --git a/src/minio.rs b/src/minio.rs index 9bc3e25..6e8c7de 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -251,7 +251,7 @@ impl MinioService { } /// Apply bucket desired configuration. If bucket already exists, it is not dropped - pub async fn apply_bucket(&self, b: &MinioBucketSpec) -> anyhow::Result<()> { + pub async fn bucket_apply(&self, b: &MinioBucketSpec) -> anyhow::Result<()> { // Set base parameters let bucket_name = format!("{}/{}", MC_ALIAS_NAME, b.name); let mut args = ["mb", bucket_name.as_str(), "-p"].to_vec(); @@ -608,7 +608,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -630,7 +630,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -665,7 +665,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -696,7 +696,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -749,7 +749,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -776,7 +776,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -803,7 +803,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -846,7 +846,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -891,7 +891,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -918,7 +918,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), @@ -1008,7 +1008,7 @@ mod test { let srv = MinioTestServer::start().await.unwrap(); let service = srv.as_service(); service - .apply_bucket(&MinioBucketSpec { + .bucket_apply(&MinioBucketSpec { instance: "".to_string(), name: TEST_BUCKET_NAME.to_string(), secret: "".to_string(), diff --git a/src/policy_template.json b/src/policy_template.json new file mode 100644 index 0000000..deb6ce3 --- /dev/null +++ b/src/policy_template.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListObjectsInBucket", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::{{ bucket }}"] + }, + { + "Sid": "AllObjectActions", + "Effect": "Allow", + "Action": ["s3:DeleteObject", "s3:Get*", "s3:PutObject", "s3:*Object"], + "Resource": ["arn:aws:s3:::{{ bucket }}/*"] + } + ] +} diff --git a/test/bucket-policy.json b/test/bucket-policy.json deleted file mode 100644 index 951f4d8..0000000 --- a/test/bucket-policy.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ListObjectsInBucket", - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::bucket" - ] - }, - { - "Sid": "AllObjectActions", - "Effect": "Allow", - "Action": [ - "s3:DeleteObject", - "s3:Get*", - "s3:PutObject", - "s3:*Object" - ], - "Resource": [ - "arn:aws:s3:::bucket/*" - ] - } - ] -} \ No newline at end of file diff --git a/test/test-outside-cluster.yaml b/test/test-outside-cluster.yaml new file mode 100644 index 0000000..5f9e072 --- /dev/null +++ b/test/test-outside-cluster.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-root +type: Opaque +data: + accessKey: bWluaW9hZG1pbg== + secretKey: bWluaW9hZG1pbg== +--- +apiVersion: "communiquons.org/v1" +kind: MinioInstance +metadata: + name: my-minio-instance +spec: + endpoint: http://localhost:9000/ + credentials: minio-root +--- +apiVersion: "communiquons.org/v1" +kind: MinioBucket +metadata: + name: first-bucket +spec: + instance: my-minio-instance + name: first-bucket + secret: first-bucket-secret \ No newline at end of file -- 2.45.2 From 765a8108fd2d014b4899701a2d19cb87c02345a5 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 17:26:42 +0200 Subject: [PATCH 18/21] Fix indempotence issue with policy attach --- src/minio.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/minio.rs b/src/minio.rs index 6e8c7de..aca6c70 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -531,6 +531,15 @@ impl MinioService { /// Attach a user to a policy pub async fn policy_attach_user(&self, user: &MinioUser, policy: &str) -> anyhow::Result<()> { + // Check if the policy has already been attached to the user + if self + .policy_attach_get_user_list(user) + .await? + .contains(&policy.to_string()) + { + return Ok(()); + } + let res = self .exec_mc_cmd::(&[ "admin", -- 2.45.2 From b25b7aa2a50bc2a7fd10529a073bad56acf24f80 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 18:04:11 +0200 Subject: [PATCH 19/21] Create docker image & test in docker --- Dockerfile | 6 +++ build_docker_image.sh | 14 +++++++ test/second-bucket.yaml | 3 +- test/test-inside-cluster.yaml | 25 +++++++++++++ yaml/deployment.yaml | 70 +++++++++++++++++++++++++++++++++++ yaml/service_account.yaml | 33 ----------------- 6 files changed, 117 insertions(+), 34 deletions(-) create mode 100644 Dockerfile create mode 100755 build_docker_image.sh create mode 100644 test/test-inside-cluster.yaml create mode 100644 yaml/deployment.yaml delete mode 100644 yaml/service_account.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..91ce89a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM debian:bullseye-slim + +COPY minio-operator /usr/local/bin/minio-operator +COPY mc /usr/local/bin/mc + +ENTRYPOINT /usr/local/bin/minio-operator diff --git a/build_docker_image.sh b/build_docker_image.sh new file mode 100755 index 0000000..f87ea57 --- /dev/null +++ b/build_docker_image.sh @@ -0,0 +1,14 @@ +#!/bin/bash +cargo build --release + +TEMP_DIR=$(mktemp -d) +cp target/release/minio-operator "$TEMP_DIR" + +# Download mc +wget -O "$TEMP_DIR/mc" https://dl.min.io/client/mc/release/linux-amd64/mc +chmod +x "$TEMP_DIR/mc" + +docker build -f Dockerfile "$TEMP_DIR" -t pierre42100/minio_operator + +rm -r $TEMP_DIR + diff --git a/test/second-bucket.yaml b/test/second-bucket.yaml index e14d20f..2050437 100644 --- a/test/second-bucket.yaml +++ b/test/second-bucket.yaml @@ -6,4 +6,5 @@ metadata: spec: instance: my-minio-instance name: second-bucket - secret: second-bucket-secret \ No newline at end of file + secret: second-bucket-secret + versioning: false \ No newline at end of file diff --git a/test/test-inside-cluster.yaml b/test/test-inside-cluster.yaml new file mode 100644 index 0000000..9bb37ad --- /dev/null +++ b/test/test-inside-cluster.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-root +type: Opaque +data: + accessKey: bWluaW9hZG1pbg== + secretKey: bWluaW9hZG1pbg== +--- +apiVersion: "communiquons.org/v1" +kind: MinioInstance +metadata: + name: my-minio-instance +spec: + endpoint: http://192.168.2.103:9000/ + credentials: minio-root +--- +apiVersion: "communiquons.org/v1" +kind: MinioBucket +metadata: + name: first-bucket +spec: + instance: my-minio-instance + name: first-bucket + secret: first-bucket-secret \ No newline at end of file diff --git a/yaml/deployment.yaml b/yaml/deployment.yaml new file mode 100644 index 0000000..5fee94a --- /dev/null +++ b/yaml/deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + name: minio-operator + namespace: default + labels: + app: minio-operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: minio-operator + namespace: default +rules: +- apiGroups: ["communiquons.org"] + resources: ["minioinstances", "miniobuckets"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: minio-operator + namespace: default +subjects: +- kind: ServiceAccount + name: minio-operator + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: minio-operator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio-operator + labels: + app: minio-operator +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: minio-operator + template: + metadata: + labels: + app: minio-operator + spec: + serviceAccountName: minio-operator + containers: + - name: minio-operator + image: pierre42100/minio_operator + resources: + limits: + memory: 300Mi + cpu: "0.1" + requests: + memory: 150Mi + cpu: "0.01" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL diff --git a/yaml/service_account.yaml b/yaml/service_account.yaml deleted file mode 100644 index 3d89363..0000000 --- a/yaml/service_account.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -automountServiceAccountToken: true -metadata: - name: minio-buckets - namespace: default - labels: - app: minio ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: minio-buckets - namespace: default -rules: -- apiGroups: ["communiquons.org"] - resources: ["minioinstances", "miniobuckets"] - verbs: ["get", "watch"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: minio-buckets - namespace: default -subjects: -- kind: ServiceAccount - name: minio-buckets - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: minio-buckets ---- -- 2.45.2 From f536b24c4a696c55e36c2dd8672e63d7623c3f1e Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 18:06:11 +0200 Subject: [PATCH 20/21] Grouped CRD into a single file --- yaml/{minio-bucket.yaml => crd.yaml} | 49 ++++++++++++++++++++++++++++ yaml/minio-instance.yaml | 49 ---------------------------- 2 files changed, 49 insertions(+), 49 deletions(-) rename yaml/{minio-bucket.yaml => crd.yaml} (66%) delete mode 100644 yaml/minio-instance.yaml diff --git a/yaml/minio-bucket.yaml b/yaml/crd.yaml similarity index 66% rename from yaml/minio-bucket.yaml rename to yaml/crd.yaml index 82b696b..d36fa60 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/crd.yaml @@ -1,5 +1,54 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + # name must match the spec fields below, and be in the form: . + name: minioinstances.communiquons.org +spec: + # group name to use for REST API: /apis// + group: communiquons.org + # list of versions supported by this CustomResourceDefinition + versions: + - name: v1 + # Each version can be enabled/disabled by Served flag. + served: true + # One and only one version must be marked as the storage version. + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + description: Information about how to reach the Minio bucket + properties: + endpoint: + description: The URL where the Minio API can be reached + example: https://minio.communiquons.org + type: string + credentials: + description: | + The name of the secret containings privilegied / root credentials of Minio instance + + The secret must contains two fields : + * An access key named `accessKey` + * A secret key named `secretKey` + type: string + example: minio-root + # either Namespaced or Cluster + scope: Namespaced + names: + # plural name to be used in the URL: /apis/// + plural: minioinstances + # singular name to be used as an alias on the CLI and for display + singular: minioinstance + # kind is normally the CamelCased singular type. Your resource manifests use this. + kind: MinioInstance + # shortNames allow shorter string to match your resource on the CLI + shortNames: + - mis +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: # name must match the spec fields below, and be in the form: . name: miniobuckets.communiquons.org diff --git a/yaml/minio-instance.yaml b/yaml/minio-instance.yaml deleted file mode 100644 index c87f77b..0000000 --- a/yaml/minio-instance.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - # name must match the spec fields below, and be in the form: . - name: minioinstances.communiquons.org -spec: - # group name to use for REST API: /apis// - group: communiquons.org - # list of versions supported by this CustomResourceDefinition - versions: - - name: v1 - # Each version can be enabled/disabled by Served flag. - served: true - # One and only one version must be marked as the storage version. - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - description: Information about how to reach the Minio bucket - properties: - endpoint: - description: The URL where the Minio API can be reached - example: https://minio.communiquons.org - type: string - credentials: - description: | - The name of the secret containings privilegied / root credentials of Minio instance - - The secret must contains two fields : - * An access key named `accessKey` - * A secret key named `secretKey` - type: string - example: minio-root - # either Namespaced or Cluster - scope: Namespaced - names: - # plural name to be used in the URL: /apis/// - plural: minioinstances - # singular name to be used as an alias on the CLI and for display - singular: minioinstance - # kind is normally the CamelCased singular type. Your resource manifests use this. - kind: MinioInstance - # shortNames allow shorter string to match your resource on the CLI - shortNames: - - mis ---- -- 2.45.2 From e4603b0f38d952d7494e9e341ae0b13233e67c5a Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Mon, 8 May 2023 18:09:59 +0200 Subject: [PATCH 21/21] Add some pause before the beginning of each test to let Minio start --- src/minio_test_server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/minio_test_server.rs b/src/minio_test_server.rs index 29a655d..1f61dfa 100644 --- a/src/minio_test_server.rs +++ b/src/minio_test_server.rs @@ -51,6 +51,7 @@ impl MinioTestServer { }; // Wait for Minio to become ready + std::thread::sleep(Duration::from_millis(500)); let mut check_count = 0; loop { if check_count >= 100 { -- 2.45.2