Add S3 bucket configuration

geneit_backend/Cargo.lock

@@ -45,7 +45,7 @@ dependencies = [
  "actix-service",
  "actix-utils",
  "ahash 0.8.3",
- "base64",
+ "base64 0.21.2",
  "bitflags 1.3.2",
  "brotli",
  "bytes",
@@ -347,12 +347,63 @@ version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a"
 
+[[package]]
+name = "async-trait"
+version = "0.1.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.23",
+]
+
+[[package]]
+name = "attohttpc"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7"
+dependencies = [
+ "http",
+ "log",
+ "native-tls",
+ "serde",
+ "serde_json",
+ "url",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "aws-creds"
+version = "0.34.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5"
+dependencies = [
+ "attohttpc",
+ "dirs",
+ "log",
+ "quick-xml",
+ "rust-ini",
+ "serde",
+ "thiserror",
+ "time",
+ "url",
+]
+
+[[package]]
+name = "aws-region"
+version = "0.25.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba"
+dependencies = [
+ "thiserror",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.68"
@@ -368,6 +419,12 @@ dependencies = [
  "rustc-demangle",
 ]
 
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
 [[package]]
 name = "base64"
 version = "0.21.2"
@@ -380,7 +437,7 @@ version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28d1c9c15093eb224f0baa400f38fcd713fc1391a6f1c389d886beef146d60a3"
 dependencies = [
- "base64",
+ "base64 0.21.2",
  "blowfish",
  "getrandom",
  "subtle",
@@ -733,6 +790,16 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer",
  "crypto-common",
+ "subtle",
+]
+
+[[package]]
+name = "dirs"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
+dependencies = [
+ "dirs-sys",
 ]
 
 [[package]]
@@ -745,6 +812,17 @@ dependencies = [
  "dirs-sys-next",
 ]
 
+[[package]]
+name = "dirs-sys"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
 [[package]]
 name = "dirs-sys-next"
 version = "0.1.2"
@@ -756,13 +834,19 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "dlv-list"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
+
 [[package]]
 name = "email-encoding"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbfb21b9878cf7a348dcb8559109aabc0ec40d69924bd706fa5149846c4fef75"
 dependencies = [
- "base64",
+ "base64 0.21.2",
  "memchr",
 ]
 
@@ -879,6 +963,21 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "futures"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.28"
@@ -886,6 +985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
 dependencies = [
  "futures-core",
+ "futures-sink",
 ]
 
 [[package]]
@@ -894,6 +994,17 @@ version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
 
+[[package]]
+name = "futures-executor"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-io"
 version = "0.3.28"
@@ -929,9 +1040,11 @@ version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
 dependencies = [
+ "futures-channel",
  "futures-core",
  "futures-io",
  "futures-macro",
+ "futures-sink",
  "futures-task",
  "memchr",
  "pin-project-lite",
@@ -959,6 +1072,7 @@ dependencies = [
  "mailchecker",
  "rand",
  "redis",
+ "rust-s3",
  "rust_iso3166",
  "serde",
  "serde_json",
@@ -1017,6 +1131,9 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash 0.7.6",
+]
 
 [[package]]
 name = "heck"
@@ -1036,6 +1153,15 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
 [[package]]
 name = "hostname"
 version = "0.3.1"
@@ -1272,7 +1398,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "76bd09637ae3ec7bd605b8e135e757980b3968430ff2b1a4a94fb7769e50166d"
 dependencies = [
- "base64",
+ "base64 0.21.2",
  "email-encoding",
  "email_address",
  "fastrand",
@@ -1301,7 +1427,7 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "608aa1b7148a6eeab631c6267deca33407ff851ab50eea115e52c13a9bb184ee"
 dependencies = [
- "base64",
+ "base64 0.21.2",
  "log",
  "reqwest",
  "serde",
@@ -1371,6 +1497,23 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
 
+[[package]]
+name = "maybe-async"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "md5"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -1383,6 +1526,15 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "minidom"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278"
+dependencies = [
+ "rxml",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -1516,6 +1668,16 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "ordered-multimap"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
+dependencies = [
+ "dlv-list",
+ "hashbrown",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -1649,6 +1811,16 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.29"
@@ -1760,7 +1932,7 @@ version = "0.11.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
 dependencies = [
- "base64",
+ "base64 0.21.2",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -1783,14 +1955,59 @@ dependencies = [
  "serde_urlencoded",
  "tokio",
  "tokio-native-tls",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-streams",
  "web-sys",
  "winreg",
 ]
 
+[[package]]
+name = "rust-ini"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
+dependencies = [
+ "cfg-if",
+ "ordered-multimap",
+]
+
+[[package]]
+name = "rust-s3"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027"
+dependencies = [
+ "async-trait",
+ "aws-creds",
+ "aws-region",
+ "base64 0.13.1",
+ "bytes",
+ "cfg-if",
+ "futures",
+ "hex",
+ "hmac",
+ "http",
+ "log",
+ "maybe-async",
+ "md5",
+ "minidom",
+ "percent-encoding",
+ "quick-xml",
+ "reqwest",
+ "serde",
+ "serde_derive",
+ "sha2",
+ "thiserror",
+ "time",
+ "tokio",
+ "tokio-stream",
+ "url",
+]
+
 [[package]]
 name = "rust_iso3166"
 version = "0.1.10"
@@ -1851,6 +2068,23 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
+[[package]]
+name = "rxml"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7"
+dependencies = [
+ "bytes",
+ "rxml_validation",
+ "smartstring",
+]
+
+[[package]]
+name = "rxml_validation"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530"
+
 [[package]]
 name = "ryu"
 version = "1.0.14"
@@ -1950,7 +2184,7 @@ version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21e47d95bc83ed33b2ecf84f4187ad1ab9685d18ff28db000c99deac8ce180e3"
 dependencies = [
- "base64",
+ "base64 0.21.2",
  "chrono",
  "hex",
  "indexmap",
@@ -1989,6 +2223,17 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
 
+[[package]]
+name = "sha2"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.1"
@@ -2019,6 +2264,17 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
+[[package]]
+name = "smartstring"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
+dependencies = [
+ "autocfg",
+ "static_assertions",
+ "version_check",
+]
+
 [[package]]
 name = "socket2"
 version = "0.4.9"
@@ -2029,6 +2285,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -2187,6 +2449,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-stream"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.8"
@@ -2383,6 +2656,19 @@ version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
+[[package]]
+name = "wasm-streams"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.64"

geneit_backend/Cargo.toml

@@ -26,4 +26,5 @@ bcrypt = "0.15.0"
 light-openid = "1.0.1"
 thiserror = "1.0.40"
 serde_with = "3.1.0"
-rust_iso3166 = "0.1.10"
+rust_iso3166 = "0.1.10"
+rust-s3 = "0.33.0"

geneit_backend/src/app_config.rs

@@ -1,4 +1,6 @@
 use clap::Parser;
+use s3::creds::Credentials;
+use s3::{Bucket, Region};
 
 /// GeneIT backend API
 #[derive(Parser, Debug, Clone)]
@@ -115,6 +117,30 @@ pub struct AppConfig {
     /// OpenID login redirect URL
     #[arg(long, env, default_value = "APP_ORIGIN/oidc_cb")]
     oidc_redirect_url: String,
+
+    /// S3 Bucket name
+    #[arg(long, env, default_value = "geneit-data")]
+    s3_bucket_name: String,
+
+    /// S3 region (if not using Minio)
+    #[arg(long, env, default_value = "eu-central-1")]
+    s3_region: String,
+
+    /// S3 API endpoint
+    #[arg(long, env, default_value = "http://localhost:9000")]
+    s3_endpoint: String,
+
+    /// S3 access key
+    #[arg(long, env, default_value = "topsecret")]
+    s3_access_key: String,
+
+    /// S3 secret key
+    #[arg(long, env, default_value = "topsecret")]
+    s3_secret_key: String,
+
+    /// S3 skip auto create bucket if not existing
+    #[arg(long, env)]
+    pub s3_skip_auto_create_bucket: bool,
 }
 
 lazy_static::lazy_static! {
@@ -183,6 +209,30 @@ impl AppConfig {
         self.oidc_redirect_url
             .replace("APP_ORIGIN", &self.website_origin)
     }
+
+    /// Get s3 credentials
+    pub fn s3_credentials(&self) -> anyhow::Result<Credentials> {
+        Ok(Credentials::new(
+            Some(&self.s3_access_key),
+            Some(&self.s3_secret_key),
+            None,
+            None,
+            None,
+        )?)
+    }
+
+    /// Get S3 bucket
+    pub fn s3_bucket(&self) -> anyhow::Result<Bucket> {
+        Ok(Bucket::new(
+            &self.s3_bucket_name,
+            Region::Custom {
+                region: self.s3_region.to_string(),
+                endpoint: self.s3_endpoint.to_string(),
+            },
+            self.s3_credentials()?,
+        )?
+        .with_path_style())
+    }
 }
 
 #[derive(Debug, Clone, serde::Serialize)]

geneit_backend/src/connections/mod.rs

@@ -2,3 +2,4 @@
 
 pub mod db_connection;
 pub mod redis_connection;
+pub mod s3_connection;

geneit_backend/src/connections/s3_connection.rs

@@ -0,0 +1,47 @@
+use crate::app_config::AppConfig;
+use s3::error::S3Error;
+use s3::{Bucket, BucketConfiguration};
+
+#[derive(thiserror::Error, Debug)]
+enum BucketServiceError {
+    #[error("Failed to fetch bucket information!")]
+    FailedFetchBucketInfo,
+}
+
+/// Create S3 bucket if required
+pub async fn create_bucket_if_required() -> anyhow::Result<()> {
+    if AppConfig::get().s3_skip_auto_create_bucket {
+        log::debug!("Skipping bucket existence check");
+        return Ok(());
+    }
+
+    let bucket = AppConfig::get().s3_bucket()?;
+
+    match bucket.location().await {
+        Ok(_) => {
+            log::debug!("The bucket already exists.");
+            return Ok(());
+        }
+        Err(S3Error::Http(404, s)) if s.contains("<Code>NoSuchKey</Code>") => {
+            log::warn!("Failed to fetch bucket location, but it seems that bucket exists.");
+            return Ok(());
+        }
+        Err(S3Error::Http(404, s)) if s.contains("<Code>NoSuchBucket</Code>") => {
+            log::warn!("The bucket does not seem to exists, trying to create it!")
+        }
+        Err(e) => {
+            log::error!("Got unexpected error when querying bucket info: {}", e);
+            return Err(BucketServiceError::FailedFetchBucketInfo.into());
+        }
+    }
+
+    Bucket::create_with_path_style(
+        &bucket.name,
+        bucket.region,
+        AppConfig::get().s3_credentials()?,
+        BucketConfiguration::private(),
+    )
+    .await?;
+
+    Ok(())
+}

geneit_backend/src/main.rs

@@ -3,6 +3,7 @@ use actix_remote_ip::RemoteIPConfig;
 use actix_web::middleware::Logger;
 use actix_web::{web, App, HttpServer};
 use geneit_backend::app_config::AppConfig;
+use geneit_backend::connections::s3_connection;
 use geneit_backend::controllers::{
     auth_controller, families_controller, members_controller, server_controller, users_controller,
 };
@@ -13,6 +14,12 @@ async fn main() -> std::io::Result<()> {
 
     log::info!("Start to listen on {}", AppConfig::get().listen_address);
 
+    // Initialize bucket
+    log::info!("Initialize bucket");
+    s3_connection::create_bucket_if_required()
+        .await
+        .expect("Failed to initialize S3 bucket!");
+
     HttpServer::new(|| {
         App::new()
             .wrap(