From a3f523410fd73978eab81c9e50793bfa6beaf872 Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Sat, 6 May 2023 16:41:36 +0200 Subject: [PATCH] Add buckets creation --- Cargo.toml | 2 - src/constants.rs | 2 + src/crd.rs | 2 + src/lib.rs | 2 +- src/minio.rs | 260 +++++++++++++++++- src/{minio_server.rs => minio_test_server.rs} | 10 +- yaml/minio-bucket.yaml | 4 + 7 files changed, 273 insertions(+), 9 deletions(-) rename src/{minio_server.rs => minio_test_server.rs} (93%) diff --git a/Cargo.toml b/Cargo.toml index d8af726..2971ef7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,5 @@ k8s-openapi = { version = "0.18.0", features = ["v1_26"] } # TODO : switch to v1 futures = "0.3.28" thiserror = "1.0.40" rand = "0.8.5" - -[dev-dependencies] mktemp = "0.5.0" reqwest = "0.11.17" \ No newline at end of file diff --git a/src/constants.rs b/src/constants.rs index 9ba1ead..f380b30 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -7,3 +7,5 @@ pub const SECRET_MINIO_BUCKET_SECRET_KEY: &str = "secretKey"; pub const SECRET_MINIO_BUCKET_ACCESS_LEN: usize = 20; pub const SECRET_MINIO_BUCKET_SECRET_LEN: usize = 35; + +pub const MC_EXE: &str = "mc"; diff --git a/src/crd.rs b/src/crd.rs index 5f0d147..e6374ec 100644 --- a/src/crd.rs +++ b/src/crd.rs @@ -45,5 +45,7 @@ pub struct MinioBucketSpec { #[serde(default)] pub versioning: bool, pub quota: Option, + #[serde(default)] + pub lock: bool, pub retention: Option, } diff --git a/src/lib.rs b/src/lib.rs index 9a6443a..86bbbb1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,6 @@ pub mod constants; pub mod crd; pub mod minio; #[cfg(test)] -pub mod minio_server; +pub mod minio_test_server; pub mod secrets; pub mod utils; diff --git a/src/minio.rs b/src/minio.rs index f672b06..9950a6d 100644 --- a/src/minio.rs +++ b/src/minio.rs @@ -1,5 +1,27 @@ -use crate::constants::{SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; +use crate::constants::{MC_EXE, SECRET_MINIO_BUCKET_ACCESS_LEN, SECRET_MINIO_BUCKET_SECRET_LEN}; +use crate::crd::{MinioBucketSpec, RetentionType}; use crate::utils::rand_str; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use std::process::Command; + +const MC_ALIAS_NAME: &str = "managedminioinst"; + +#[derive(thiserror::Error, Debug)] +enum MinioError { + #[error("Failed to set 'mc' alias!")] + SetMcAlias, + #[error("Failed to execute 'mc' command!")] + ExecMc, + #[error("Failed to execute 'mc mb' command!")] + MakeBucketFailed, + #[error("Failed to set anonymous access!")] + SetAnonymousAcccessFailed, + #[error("Failed to set bucket quota!")] + SetQuotaFailed, + #[error("Failed to set bucket retention!")] + SetRetentionFailed, +} #[derive(Debug, Clone)] pub struct MinioService { @@ -23,6 +45,29 @@ impl MinioUser { } } +#[derive(Debug, Clone, Deserialize)] +pub struct BucketEntry { + pub status: String, + key: String, +} + +impl BucketEntry { + pub fn bucket_name(&self) -> &str { + &self.key[0..self.key.len() - 1] + } +} + +#[derive(Debug, Clone, Deserialize)] +struct BasicMinioResult { + pub status: String, +} + +impl BasicMinioResult { + pub fn success(&self) -> bool { + self.status == "success" + } +} + impl MinioService { /// Check if the Minio Service is ready to respond to our requests pub async fn is_ready(&self) -> bool { @@ -43,4 +88,217 @@ impl MinioService { false } + + /// Execute a minio mc command + async fn exec_mc_cmd(&self, args: &[&str]) -> anyhow::Result> + where + A: DeserializeOwned, + { + log::debug!("exec_mc_cmd with args {:?}", args); + + let conf_dir = mktemp::Temp::new_dir()?; + let global_flags = ["--config-dir", conf_dir.to_str().unwrap(), "--json"]; + + // First, set our alias to mc in a temporary directory + let res = Command::new(MC_EXE) + .args(global_flags) + .args([ + "alias", + "set", + MC_ALIAS_NAME, + self.hostname.as_str(), + self.access_key.as_str(), + self.secret_key.as_str(), + ]) + .output()?; + if res.status.code() != Some(0) { + log::error!( + "Failed to configure mc alias! (status code {:?}, stderr={}, stdout={})", + res.status, + String::from_utf8_lossy(&res.stderr), + String::from_utf8_lossy(&res.stdout) + ); + return Err(MinioError::SetMcAlias.into()); + } + + // Execute requested command + let res = Command::new(MC_EXE) + .args(global_flags) + .args(args) + .output()?; + + if res.status.code() != Some(0) { + log::error!( + "Failed execute command! (status code {:?}, stderr={}, stdout={})", + res.status, + String::from_utf8_lossy(&res.stderr), + String::from_utf8_lossy(&res.stdout) + ); + return Err(MinioError::ExecMc.into()); + } + + let stdout = String::from_utf8_lossy(&res.stdout); + log::debug!( + "stdout='{}' stderr='{}'", + stdout, + String::from_utf8_lossy(&res.stderr) + ); + + if stdout.is_empty() { + log::info!("Command returned no result!"); + return Ok(vec![]); + } + + let mut out = vec![]; + for l in stdout.split('\n') { + if !l.trim().is_empty() { + out.push(serde_json::from_str(l)?); + } + } + Ok(out) + } + + /// Get the list of buckets + pub async fn buckets_list(&self) -> anyhow::Result> { + self.exec_mc_cmd::(&["ls", MC_ALIAS_NAME]) + .await + } + + /// Check if a bucket exists or not + pub async fn bucket_exists(&self, name: &str) -> anyhow::Result { + Ok(self + .buckets_list() + .await? + .iter() + .any(|b| b.bucket_name().eq(name))) + } + + /// Create a bucket + pub async fn create_bucket(&self, b: &MinioBucketSpec) -> anyhow::Result<()> { + // Set base parameters + let bucket_name = format!("{}/{}", MC_ALIAS_NAME, b.name); + let mut args = ["mb", bucket_name.as_str()].to_vec(); + + if b.versioning { + args.push("--with-versioning"); + } + + if b.lock || b.retention.is_some() { + args.push("--with-lock"); + } + + let res = self.exec_mc_cmd::(&args).await?; + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::MakeBucketFailed.into()); + } + + // Enable anonymous access, eg. public hosting + if b.anonymous_read_access { + let target = format!("{}/*", bucket_name); + + let res = self + .exec_mc_cmd::(&["anonymous", "set", "download", target.as_str()]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetAnonymousAcccessFailed.into()); + } + } + + // Set quota, if requested + if let Some(quota) = &b.quota { + let quota = format!("{}MB", quota); + + let res = self + .exec_mc_cmd::(&[ + "quota", + "set", + bucket_name.as_str(), + "--size", + quota.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetQuotaFailed.into()); + } + } + + // Set retention, if requested + if let Some(retention) = &b.retention { + let days = format!("{}d", retention.validity); + + let res = self + .exec_mc_cmd::(&[ + "retention", + "set", + "--default", + match retention.r#type { + RetentionType::Compliance => "compliance", + RetentionType::Governance => "governance", + }, + days.as_str(), + bucket_name.as_str(), + ]) + .await?; + + if res.get(0).map(|r| r.success()) != Some(true) { + return Err(MinioError::SetRetentionFailed.into()); + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use crate::crd::MinioBucketSpec; + use crate::minio_test_server::MinioTestServer; + + #[tokio::test] + async fn list_buckets_empty_instance() { + let srv = MinioTestServer::start().await.unwrap(); + let buckets = srv.as_service().buckets_list().await.unwrap(); + assert!(buckets.is_empty()); + } + + #[tokio::test] + async fn bucket_exists_no_bucket() { + let srv = MinioTestServer::start().await.unwrap(); + assert!(!srv.as_service().bucket_exists("mybucket").await.unwrap()); + } + + #[tokio::test] + async fn bucket_basic_creation() { + let _ = env_logger::builder().is_test(true).try_init(); + + let srv = MinioTestServer::start().await.unwrap(); + let service = srv.as_service(); + service + .create_bucket(&MinioBucketSpec { + instance: "".to_string(), + name: "mybucket".to_string(), + secret: "".to_string(), + anonymous_read_access: false, + versioning: false, + quota: None, + lock: false, + retention: None, + }) + .await + .unwrap(); + assert!(service.bucket_exists("mybucket").await.unwrap()); + } + + // TODO : with anonymous access + // TODO : without anonymous access + // TODO : with versioning + // TODO : without versioning + // TODO : with quota + // TODO : without quota + // TODO : with lock + // TODO : without lock + // TODO : with retention + // TODO : without retention } diff --git a/src/minio_server.rs b/src/minio_test_server.rs similarity index 93% rename from src/minio_server.rs rename to src/minio_test_server.rs index 103add0..5e8654a 100644 --- a/src/minio_server.rs +++ b/src/minio_test_server.rs @@ -8,7 +8,7 @@ use rand::RngCore; use std::io::ErrorKind; use std::process::{Child, Command}; -pub struct MinioServer { +pub struct MinioTestServer { #[allow(dead_code)] storage_base_dir: mktemp::Temp, child: Child, @@ -17,7 +17,7 @@ pub struct MinioServer { pub root_password: String, } -impl MinioServer { +impl MinioTestServer { pub async fn start() -> anyhow::Result { let storage_dir = mktemp::Temp::new_dir()?; @@ -84,7 +84,7 @@ impl MinioServer { } } -impl Drop for MinioServer { +impl Drop for MinioTestServer { fn drop(&mut self) { if let Err(e) = self.child.kill() { log::error!("Failed to kill child server! {}", e); @@ -94,13 +94,13 @@ impl Drop for MinioServer { #[cfg(test)] mod test { - use crate::minio_server::MinioServer; + use crate::minio_test_server::MinioTestServer; #[tokio::test] async fn start_minio() { let _ = env_logger::builder().is_test(true).try_init(); - let server = MinioServer::start().await.unwrap(); + let server = MinioTestServer::start().await.unwrap(); let service = server.as_service(); println!("{:?}", service); diff --git a/yaml/minio-bucket.yaml b/yaml/minio-bucket.yaml index d2067f6..8fa4a55 100644 --- a/yaml/minio-bucket.yaml +++ b/yaml/minio-bucket.yaml @@ -49,6 +49,10 @@ spec: type: integer description: Limits the amount of data in the bucket, in Megabytes. By default it is unlimited example: 100 + lock: + description: Object locking prevent objects from being deleted. Will be considered as set to true when retention is defined. + type: boolean + default: false retention: type: object description: Impose rules to prevent object deletion for a period of time. It requires versioning to be enabled/disabled