pierre/GeneIT
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Add S3 bucket configuration

Browse Source
This commit is contained in:
Pierre HUBERT
2023-08-05 14:49:17 +02:00
parent aa351bfae9
commit 4cd7519890
6 changed files with 400 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

300
geneit_backend/Cargo.lock generated
View File

@@ -45,7 +45,7 @@ dependencies = [
"actix-service",
"actix-utils",
"ahash 0.8.3",
"base64",
"base64 0.21.2",
"bitflags 1.3.2",
"brotli",
"bytes",
@@ -347,12 +347,63 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a"
[[package]]
name = "async-trait"
version = "0.1.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.23",
]
[[package]]
name = "attohttpc"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7"
dependencies = [
"http",
"log",
"native-tls",
"serde",
"serde_json",
"url",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "aws-creds"
version = "0.34.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5"
dependencies = [
"attohttpc",
"dirs",
"log",
"quick-xml",
"rust-ini",
"serde",
"thiserror",
"time",
"url",
]
[[package]]
name = "aws-region"
version = "0.25.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba"
dependencies = [
"thiserror",
]
[[package]]
name = "backtrace"
version = "0.3.68"
@@ -368,6 +419,12 @@ dependencies = [
"rustc-demangle",
]
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.21.2"
@@ -380,7 +437,7 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d1c9c15093eb224f0baa400f38fcd713fc1391a6f1c389d886beef146d60a3"
dependencies = [
"base64",
"base64 0.21.2",
"blowfish",
"getrandom",
"subtle",
@@ -733,6 +790,16 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
name = "dirs"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
dependencies = [
"dirs-sys",
]
[[package]]
@@ -745,6 +812,17 @@ dependencies = [
"dirs-sys-next",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
@@ -756,13 +834,19 @@ dependencies = [
"winapi",
]
[[package]]
name = "dlv-list"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
[[package]]
name = "email-encoding"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbfb21b9878cf7a348dcb8559109aabc0ec40d69924bd706fa5149846c4fef75"
dependencies = [
"base64",
"base64 0.21.2",
"memchr",
]
@@ -879,6 +963,21 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "futures"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.28"
@@ -886,6 +985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
@@ -894,6 +994,17 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
[[package]]
name = "futures-executor"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.28"
@@ -929,9 +1040,11 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
@@ -959,6 +1072,7 @@ dependencies = [
"mailchecker",
"rand",
"redis",
"rust-s3",
"rust_iso3166",
"serde",
"serde_json",
@@ -1017,6 +1131,9 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash 0.7.6",
]
[[package]]
name = "heck"
@@ -1036,6 +1153,15 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hmac"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
"digest",
]
[[package]]
name = "hostname"
version = "0.3.1"
@@ -1272,7 +1398,7 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76bd09637ae3ec7bd605b8e135e757980b3968430ff2b1a4a94fb7769e50166d"
dependencies = [
"base64",
"base64 0.21.2",
"email-encoding",
"email_address",
"fastrand",
@@ -1301,7 +1427,7 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "608aa1b7148a6eeab631c6267deca33407ff851ab50eea115e52c13a9bb184ee"
dependencies = [
"base64",
"base64 0.21.2",
"log",
"reqwest",
"serde",
@@ -1371,6 +1497,23 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
[[package]]
name = "maybe-async"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.5.0"
@@ -1383,6 +1526,15 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "minidom"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278"
dependencies = [
"rxml",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@@ -1516,6 +1668,16 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "ordered-multimap"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
dependencies = [
"dlv-list",
"hashbrown",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
@@ -1649,6 +1811,16 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "quick-xml"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quote"
version = "1.0.29"
@@ -1760,7 +1932,7 @@ version = "0.11.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
dependencies = [
"base64",
"base64 0.21.2",
"bytes",
"encoding_rs",
"futures-core",
@@ -1783,14 +1955,59 @@ dependencies = [
"serde_urlencoded",
"tokio",
"tokio-native-tls",
"tokio-util",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"winreg",
]
[[package]]
name = "rust-ini"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
dependencies = [
"cfg-if",
"ordered-multimap",
]
[[package]]
name = "rust-s3"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027"
dependencies = [
"async-trait",
"aws-creds",
"aws-region",
"base64 0.13.1",
"bytes",
"cfg-if",
"futures",
"hex",
"hmac",
"http",
"log",
"maybe-async",
"md5",
"minidom",
"percent-encoding",
"quick-xml",
"reqwest",
"serde",
"serde_derive",
"sha2",
"thiserror",
"time",
"tokio",
"tokio-stream",
"url",
]
[[package]]
name = "rust_iso3166"
version = "0.1.10"
@@ -1851,6 +2068,23 @@ version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
[[package]]
name = "rxml"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7"
dependencies = [
"bytes",
"rxml_validation",
"smartstring",
]
[[package]]
name = "rxml_validation"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530"
[[package]]
name = "ryu"
version = "1.0.14"
@@ -1950,7 +2184,7 @@ version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21e47d95bc83ed33b2ecf84f4187ad1ab9685d18ff28db000c99deac8ce180e3"
dependencies = [
"base64",
"base64 0.21.2",
"chrono",
"hex",
"indexmap",
@@ -1989,6 +2223,17 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "sha2"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
@@ -2019,6 +2264,17 @@ version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
[[package]]
name = "smartstring"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
dependencies = [
"autocfg",
"static_assertions",
"version_check",
]
[[package]]
name = "socket2"
version = "0.4.9"
@@ -2029,6 +2285,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strsim"
version = "0.10.0"
@@ -2187,6 +2449,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.8"
@@ -2383,6 +2656,19 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "wasm-streams"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.64"

1
geneit_backend/Cargo.toml
View File

@@ -27,3 +27,4 @@ light-openid = "1.0.1"
thiserror = "1.0.40"
serde_with = "3.1.0"
rust_iso3166 = "0.1.10"
rust-s3 = "0.33.0"

50
geneit_backend/src/app_config.rs
View File

@@ -1,4 +1,6 @@
use clap::Parser;
use s3::creds::Credentials;
use s3::{Bucket, Region};
/// GeneIT backend API
#[derive(Parser, Debug, Clone)]
@@ -115,6 +117,30 @@ pub struct AppConfig {
/// OpenID login redirect URL
#[arg(long, env, default_value = "APP_ORIGIN/oidc_cb")]
oidc_redirect_url: String,
/// S3 Bucket name
#[arg(long, env, default_value = "geneit-data")]
s3_bucket_name: String,
/// S3 region (if not using Minio)
#[arg(long, env, default_value = "eu-central-1")]
s3_region: String,
/// S3 API endpoint
#[arg(long, env, default_value = "http://localhost:9000")]
s3_endpoint: String,
/// S3 access key
#[arg(long, env, default_value = "topsecret")]
s3_access_key: String,
/// S3 secret key
#[arg(long, env, default_value = "topsecret")]
s3_secret_key: String,
/// S3 skip auto create bucket if not existing
#[arg(long, env)]
pub s3_skip_auto_create_bucket: bool,
}
lazy_static::lazy_static! {
@@ -183,6 +209,30 @@ impl AppConfig {
self.oidc_redirect_url
.replace("APP_ORIGIN", &self.website_origin)
}
/// Get s3 credentials
pub fn s3_credentials(&self) -> anyhow::Result<Credentials> {
Ok(Credentials::new(
Some(&self.s3_access_key),
Some(&self.s3_secret_key),
None,
None,
None,
)?)
}
/// Get S3 bucket
pub fn s3_bucket(&self) -> anyhow::Result<Bucket> {
Ok(Bucket::new(
&self.s3_bucket_name,
Region::Custom {
region: self.s3_region.to_string(),
endpoint: self.s3_endpoint.to_string(),
},
self.s3_credentials()?,
)?
.with_path_style())
}
}
#[derive(Debug, Clone, serde::Serialize)]

1
geneit_backend/src/connections/mod.rs
View File

@@ -2,3 +2,4 @@
pub mod db_connection;
pub mod redis_connection;
pub mod s3_connection;

47
geneit_backend/src/connections/s3_connection.rs Normal file
View File

@@ -0,0 +1,47 @@
use crate::app_config::AppConfig;
use s3::error::S3Error;
use s3::{Bucket, BucketConfiguration};
#[derive(thiserror::Error, Debug)]
enum BucketServiceError {
#[error("Failed to fetch bucket information!")]
FailedFetchBucketInfo,
}
/// Create S3 bucket if required
pub async fn create_bucket_if_required() -> anyhow::Result<()> {
if AppConfig::get().s3_skip_auto_create_bucket {
log::debug!("Skipping bucket existence check");
return Ok(());
}
let bucket = AppConfig::get().s3_bucket()?;
match bucket.location().await {
Ok(_) => {
log::debug!("The bucket already exists.");
return Ok(());
}
Err(S3Error::Http(404, s)) if s.contains("<Code>NoSuchKey</Code>") => {
log::warn!("Failed to fetch bucket location, but it seems that bucket exists.");
return Ok(());
}
Err(S3Error::Http(404, s)) if s.contains("<Code>NoSuchBucket</Code>") => {
log::warn!("The bucket does not seem to exists, trying to create it!")
}
Err(e) => {
log::error!("Got unexpected error when querying bucket info: {}", e);
return Err(BucketServiceError::FailedFetchBucketInfo.into());
}
}
Bucket::create_with_path_style(
&bucket.name,
bucket.region,
AppConfig::get().s3_credentials()?,
BucketConfiguration::private(),
)
.await?;
Ok(())
}

7
geneit_backend/src/main.rs
View File

@@ -3,6 +3,7 @@ use actix_remote_ip::RemoteIPConfig;
use actix_web::middleware::Logger;
use actix_web::{web, App, HttpServer};
use geneit_backend::app_config::AppConfig;
use geneit_backend::connections::s3_connection;
use geneit_backend::controllers::{
auth_controller, families_controller, members_controller, server_controller, users_controller,
};
@@ -13,6 +14,12 @@ async fn main() -> std::io::Result<()> {
log::info!("Start to listen on {}", AppConfig::get().listen_address);
// Initialize bucket
log::info!("Initialize bucket");
s3_connection::create_bucket_if_required()
.await
.expect("Failed to initialize S3 bucket!");
HttpServer::new(|| {
App::new()
.wrap(