From 4cd75198906c251f8a60cd1c4ed30f676005f16f Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Sat, 5 Aug 2023 14:49:17 +0200 Subject: [PATCH] Add S3 bucket configuration --- geneit_backend/Cargo.lock | 300 +++++++++++++++++- geneit_backend/Cargo.toml | 3 +- geneit_backend/src/app_config.rs | 50 +++ geneit_backend/src/connections/mod.rs | 1 + .../src/connections/s3_connection.rs | 47 +++ geneit_backend/src/main.rs | 7 + 6 files changed, 400 insertions(+), 8 deletions(-) create mode 100644 geneit_backend/src/connections/s3_connection.rs diff --git a/geneit_backend/Cargo.lock b/geneit_backend/Cargo.lock index f25d179..6e0443d 100644 --- a/geneit_backend/Cargo.lock +++ b/geneit_backend/Cargo.lock @@ -45,7 +45,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash 0.8.3", - "base64", + "base64 0.21.2", "bitflags 1.3.2", "brotli", "bytes", @@ -347,12 +347,63 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" +[[package]] +name = "async-trait" +version = "0.1.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.23", +] + +[[package]] +name = "attohttpc" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +dependencies = [ + "http", + "log", + "native-tls", + "serde", + "serde_json", + "url", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-creds" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" +dependencies = [ + "attohttpc", + "dirs", + "log", + "quick-xml", + "rust-ini", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba" +dependencies = [ + "thiserror", +] + [[package]] name = "backtrace" version = "0.3.68" @@ -368,6 +419,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.2" @@ -380,7 +437,7 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d1c9c15093eb224f0baa400f38fcd713fc1391a6f1c389d886beef146d60a3" dependencies = [ - "base64", + "base64 0.21.2", "blowfish", "getrandom", "subtle", @@ -733,6 +790,16 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", ] [[package]] @@ -745,6 +812,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -756,13 +834,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "email-encoding" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbfb21b9878cf7a348dcb8559109aabc0ec40d69924bd706fa5149846c4fef75" dependencies = [ - "base64", + "base64 0.21.2", "memchr", ] @@ -879,6 +963,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -886,6 +985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -894,6 +994,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -929,9 +1040,11 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -959,6 +1072,7 @@ dependencies = [ "mailchecker", "rand", "redis", + "rust-s3", "rust_iso3166", "serde", "serde_json", @@ -1017,6 +1131,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] [[package]] name = "heck" @@ -1036,6 +1153,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hostname" version = "0.3.1" @@ -1272,7 +1398,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76bd09637ae3ec7bd605b8e135e757980b3968430ff2b1a4a94fb7769e50166d" dependencies = [ - "base64", + "base64 0.21.2", "email-encoding", "email_address", "fastrand", @@ -1301,7 +1427,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "608aa1b7148a6eeab631c6267deca33407ff851ab50eea115e52c13a9bb184ee" dependencies = [ - "base64", + "base64 0.21.2", "log", "reqwest", "serde", @@ -1371,6 +1497,23 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -1383,6 +1526,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minidom" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278" +dependencies = [ + "rxml", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1516,6 +1668,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -1649,6 +1811,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.29" @@ -1760,7 +1932,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -1783,14 +1955,59 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "winreg", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.1", + "bytes", + "cfg-if", + "futures", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "minidom", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_derive", + "sha2", + "thiserror", + "time", + "tokio", + "tokio-stream", + "url", +] + [[package]] name = "rust_iso3166" version = "0.1.10" @@ -1851,6 +2068,23 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "rxml" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7" +dependencies = [ + "bytes", + "rxml_validation", + "smartstring", +] + +[[package]] +name = "rxml_validation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530" + [[package]] name = "ryu" version = "1.0.14" @@ -1950,7 +2184,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21e47d95bc83ed33b2ecf84f4187ad1ab9685d18ff28db000c99deac8ce180e3" dependencies = [ - "base64", + "base64 0.21.2", "chrono", "hex", "indexmap", @@ -1989,6 +2223,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -2019,6 +2264,17 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "socket2" version = "0.4.9" @@ -2029,6 +2285,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" @@ -2187,6 +2449,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.8" @@ -2383,6 +2656,19 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.64" diff --git a/geneit_backend/Cargo.toml b/geneit_backend/Cargo.toml index d8084dc..3683a95 100644 --- a/geneit_backend/Cargo.toml +++ b/geneit_backend/Cargo.toml @@ -26,4 +26,5 @@ bcrypt = "0.15.0" light-openid = "1.0.1" thiserror = "1.0.40" serde_with = "3.1.0" -rust_iso3166 = "0.1.10" \ No newline at end of file +rust_iso3166 = "0.1.10" +rust-s3 = "0.33.0" \ No newline at end of file diff --git a/geneit_backend/src/app_config.rs b/geneit_backend/src/app_config.rs index fa9a0e9..5363d2b 100644 --- a/geneit_backend/src/app_config.rs +++ b/geneit_backend/src/app_config.rs @@ -1,4 +1,6 @@ use clap::Parser; +use s3::creds::Credentials; +use s3::{Bucket, Region}; /// GeneIT backend API #[derive(Parser, Debug, Clone)] @@ -115,6 +117,30 @@ pub struct AppConfig { /// OpenID login redirect URL #[arg(long, env, default_value = "APP_ORIGIN/oidc_cb")] oidc_redirect_url: String, + + /// S3 Bucket name + #[arg(long, env, default_value = "geneit-data")] + s3_bucket_name: String, + + /// S3 region (if not using Minio) + #[arg(long, env, default_value = "eu-central-1")] + s3_region: String, + + /// S3 API endpoint + #[arg(long, env, default_value = "http://localhost:9000")] + s3_endpoint: String, + + /// S3 access key + #[arg(long, env, default_value = "topsecret")] + s3_access_key: String, + + /// S3 secret key + #[arg(long, env, default_value = "topsecret")] + s3_secret_key: String, + + /// S3 skip auto create bucket if not existing + #[arg(long, env)] + pub s3_skip_auto_create_bucket: bool, } lazy_static::lazy_static! { @@ -183,6 +209,30 @@ impl AppConfig { self.oidc_redirect_url .replace("APP_ORIGIN", &self.website_origin) } + + /// Get s3 credentials + pub fn s3_credentials(&self) -> anyhow::Result { + Ok(Credentials::new( + Some(&self.s3_access_key), + Some(&self.s3_secret_key), + None, + None, + None, + )?) + } + + /// Get S3 bucket + pub fn s3_bucket(&self) -> anyhow::Result { + Ok(Bucket::new( + &self.s3_bucket_name, + Region::Custom { + region: self.s3_region.to_string(), + endpoint: self.s3_endpoint.to_string(), + }, + self.s3_credentials()?, + )? + .with_path_style()) + } } #[derive(Debug, Clone, serde::Serialize)] diff --git a/geneit_backend/src/connections/mod.rs b/geneit_backend/src/connections/mod.rs index b38c36c..4351064 100644 --- a/geneit_backend/src/connections/mod.rs +++ b/geneit_backend/src/connections/mod.rs @@ -2,3 +2,4 @@ pub mod db_connection; pub mod redis_connection; +pub mod s3_connection; diff --git a/geneit_backend/src/connections/s3_connection.rs b/geneit_backend/src/connections/s3_connection.rs new file mode 100644 index 0000000..4e75397 --- /dev/null +++ b/geneit_backend/src/connections/s3_connection.rs @@ -0,0 +1,47 @@ +use crate::app_config::AppConfig; +use s3::error::S3Error; +use s3::{Bucket, BucketConfiguration}; + +#[derive(thiserror::Error, Debug)] +enum BucketServiceError { + #[error("Failed to fetch bucket information!")] + FailedFetchBucketInfo, +} + +/// Create S3 bucket if required +pub async fn create_bucket_if_required() -> anyhow::Result<()> { + if AppConfig::get().s3_skip_auto_create_bucket { + log::debug!("Skipping bucket existence check"); + return Ok(()); + } + + let bucket = AppConfig::get().s3_bucket()?; + + match bucket.location().await { + Ok(_) => { + log::debug!("The bucket already exists."); + return Ok(()); + } + Err(S3Error::Http(404, s)) if s.contains("NoSuchKey") => { + log::warn!("Failed to fetch bucket location, but it seems that bucket exists."); + return Ok(()); + } + Err(S3Error::Http(404, s)) if s.contains("NoSuchBucket") => { + log::warn!("The bucket does not seem to exists, trying to create it!") + } + Err(e) => { + log::error!("Got unexpected error when querying bucket info: {}", e); + return Err(BucketServiceError::FailedFetchBucketInfo.into()); + } + } + + Bucket::create_with_path_style( + &bucket.name, + bucket.region, + AppConfig::get().s3_credentials()?, + BucketConfiguration::private(), + ) + .await?; + + Ok(()) +} diff --git a/geneit_backend/src/main.rs b/geneit_backend/src/main.rs index 88f1322..865c4e5 100644 --- a/geneit_backend/src/main.rs +++ b/geneit_backend/src/main.rs @@ -3,6 +3,7 @@ use actix_remote_ip::RemoteIPConfig; use actix_web::middleware::Logger; use actix_web::{web, App, HttpServer}; use geneit_backend::app_config::AppConfig; +use geneit_backend::connections::s3_connection; use geneit_backend::controllers::{ auth_controller, families_controller, members_controller, server_controller, users_controller, }; @@ -13,6 +14,12 @@ async fn main() -> std::io::Result<()> { log::info!("Start to listen on {}", AppConfig::get().listen_address); + // Initialize bucket + log::info!("Initialize bucket"); + s3_connection::create_bucket_if_required() + .await + .expect("Failed to initialize S3 bucket!"); + HttpServer::new(|| { App::new() .wrap(