From 5fa3d79b4c1a65e4544bf516cfd8e33aed0f25dc Mon Sep 17 00:00:00 2001 From: Pierre Hubert Date: Fri, 18 Aug 2023 13:41:20 +0200 Subject: [PATCH] Can import photos --- geneit_backend/Cargo.lock | 53 +++----- geneit_backend/Cargo.toml | 3 +- .../src/controllers/couples_controller.rs | 2 +- .../src/controllers/data_controller.rs | 114 +++++++++++++++--- .../src/controllers/members_controller.rs | 2 +- geneit_backend/src/controllers/mod.rs | 6 + geneit_backend/src/models.rs | 2 +- geneit_backend/src/services/photos_service.rs | 36 +++++- 8 files changed, 159 insertions(+), 59 deletions(-) diff --git a/geneit_backend/Cargo.lock b/geneit_backend/Cargo.lock index 132ba66..a9a9f15 100644 --- a/geneit_backend/Cargo.lock +++ b/geneit_backend/Cargo.lock @@ -1129,6 +1129,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "fdeflate" version = "0.3.0" @@ -1310,6 +1316,7 @@ dependencies = [ "serde_json", "serde_with", "sha2", + "tempfile", "thiserror", "uuid", "zip", @@ -1603,17 +1610,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - [[package]] name = "ipnet" version = "2.8.0" @@ -1627,7 +1623,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" dependencies = [ "hermit-abi", - "rustix 0.38.2", + "rustix", "windows-sys", ] @@ -1691,7 +1687,7 @@ dependencies = [ "base64 0.21.2", "email-encoding", "email_address", - "fastrand", + "fastrand 1.9.0", "futures-util", "hostname", "httpdate", @@ -1725,12 +1721,6 @@ dependencies = [ "urlencoding", ] -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - [[package]] name = "linux-raw-sys" version = "0.4.3" @@ -2468,20 +2458,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.37.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8818fa822adcc98b18fedbb3632a6a33213c070556b5aa7c4c8cc21cff565c4c" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys", -] - [[package]] name = "rustix" version = "0.38.2" @@ -2491,7 +2467,7 @@ dependencies = [ "bitflags 2.3.3", "errno", "libc", - "linux-raw-sys 0.4.3", + "linux-raw-sys", "windows-sys", ] @@ -2784,15 +2760,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ - "autocfg", "cfg-if", - "fastrand", + "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.37.22", + "rustix", "windows-sys", ] diff --git a/geneit_backend/Cargo.toml b/geneit_backend/Cargo.toml index c163969..c6750f2 100644 --- a/geneit_backend/Cargo.toml +++ b/geneit_backend/Cargo.toml @@ -34,4 +34,5 @@ image = "0.24.6" uuid = { version = "1.4.1", features = ["v4"] } httpdate = "1.0.2" zip = "0.6.6" -mime_guess = "2.0.4" \ No newline at end of file +mime_guess = "2.0.4" +tempfile = "3.7.1" \ No newline at end of file diff --git a/geneit_backend/src/controllers/couples_controller.rs b/geneit_backend/src/controllers/couples_controller.rs index 362dd09..50da9a9 100644 --- a/geneit_backend/src/controllers/couples_controller.rs +++ b/geneit_backend/src/controllers/couples_controller.rs @@ -164,7 +164,7 @@ pub async fn set_photo( m: FamilyAndCoupleInPath, MultipartForm(form): MultipartForm, ) -> HttpResult { - let photo = photos_service::finalize_upload(form.photo).await?; + let photo = photos_service::finalize_upload(form.photo.into()).await?; let mut couple = m.to_couple(); couples_service::remove_photo(&mut couple).await?; diff --git a/geneit_backend/src/controllers/data_controller.rs b/geneit_backend/src/controllers/data_controller.rs index 342b6a9..e2b7a9c 100644 --- a/geneit_backend/src/controllers/data_controller.rs +++ b/geneit_backend/src/controllers/data_controller.rs @@ -5,10 +5,12 @@ use crate::controllers::members_controller::MemberRequest; use crate::controllers::HttpResult; use crate::extractors::family_extractor::{FamilyInPath, FamilyInPathWithAdminMembership}; use crate::models::{CoupleID, MemberID, PhotoID}; +use crate::services::photos_service::UploadedFile; use crate::services::{couples_service, members_service, photos_service}; use actix_multipart::form::tempfile::TempFile; use actix_multipart::form::MultipartForm; use actix_web::HttpResponse; +use mime_guess::Mime; use std::collections::HashMap; use std::io; use std::io::{Cursor, Read, Write}; @@ -17,6 +19,7 @@ use zip::{CompressionMethod, ZipArchive}; const MEMBERS_FILE: &str = "members.json"; const COUPLES_FILE: &str = "couples.json"; +const PHOTOS_DIR: &str = "photos/"; #[derive(serde::Deserialize)] struct ImportMemberRequest { @@ -28,7 +31,6 @@ struct ImportMemberRequest { #[derive(serde::Deserialize)] struct ImportCoupleRequest { - id: CoupleID, photo_id: Option, #[serde(flatten)] data: CoupleRequest, @@ -70,7 +72,7 @@ pub async fn export_family(f: FamilyInPath) -> HttpResult { let ext = photo.mime_extension().unwrap_or("bad"); let file = s3_connection::get_file(&photo.photo_path()).await?; - zip_file.start_file(format!("photos/{}.{ext}", id.0), files_opt)?; + zip_file.start_file(format!("{PHOTOS_DIR}{}.{ext}", id.0), files_opt)?; zip_file.write_all(&file)?; } @@ -87,6 +89,21 @@ pub struct UploadFamilyDataForm { archive: TempFile, } +struct Photo { + path: String, + mimetype: Mime, +} + +enum PhotoTarget { + Member(MemberID), + Couple(CoupleID), +} + +struct PhotoToProcess { + id: PhotoID, + target: PhotoTarget, +} + /// Import whole family data pub async fn import_family( f: FamilyInPathWithAdminMembership, @@ -94,6 +111,23 @@ pub async fn import_family( ) -> HttpResult { let mut zip = ZipArchive::new(form.archive.file)?; + // Pre-process photos list + let mut photos = HashMap::new(); + for file in zip.file_names() { + let (id, ext) = match file.strip_prefix(PHOTOS_DIR).map(|f| f.split_once('.')) { + Some(Some((id, ext))) => (id, ext), + _ => continue, + }; + + photos.insert( + PhotoID(id.parse()?), + Photo { + path: file.to_string(), + mimetype: mime_guess::from_ext(ext).first_or_octet_stream(), + }, + ); + } + // Parse general information let members_list = serde_json::from_slice::>(&read_zip_file( &mut zip, @@ -115,6 +149,8 @@ pub async fn import_family( let mut rev_members_id_mapping = HashMap::new(); let mut new_members = Vec::with_capacity(members_list.len()); + let mut photos_to_insert = Vec::with_capacity(photos.len()); + for req_m in members_list { // Create member entry in database let new_m = members_service::create(f.family_id()).await?; @@ -146,7 +182,14 @@ pub async fn import_family( req_member_data.mother = members_id_mapping.get(&i).copied(); } - req_member_data.to_member(member, true).await?; + req_member_data.to_member(member).await?; + + if let Some(id) = req_member.photo_id { + photos_to_insert.push(PhotoToProcess { + id, + target: PhotoTarget::Member(db_id), + }) + } } // Check for loops @@ -161,29 +204,72 @@ pub async fn import_family( } // Extract and insert couples information - let mut couple_mapping = HashMap::new(); - for c in &mut couples_list { + let mut new_couples = HashMap::new(); + for req_couple in &mut couples_list { // Map wife and husband - if let Some(i) = c.data.wife { - c.data.wife = members_id_mapping.get(&i).copied(); + if let Some(i) = req_couple.data.wife { + req_couple.data.wife = members_id_mapping.get(&i).copied(); } - if let Some(i) = c.data.husband { - c.data.husband = members_id_mapping.get(&i).copied(); + if let Some(i) = req_couple.data.husband { + req_couple.data.husband = members_id_mapping.get(&i).copied(); } let mut db_couple = couples_service::create(f.family_id()).await?; - couple_mapping.insert(c.id, db_couple.id()); - c.data.clone().to_couple(&mut db_couple, true).await?; + req_couple.data.clone().to_couple(&mut db_couple).await?; couples_service::update(&mut db_couple).await?; + + if let Some(id) = req_couple.photo_id { + photos_to_insert.push(PhotoToProcess { + id, + target: PhotoTarget::Couple(db_couple.id()), + }) + } + + new_couples.insert(db_couple.id(), db_couple); } - // Insert photos - // TODO + // Insert member photos + for photo_to_process in photos_to_insert { + let photo = match photos.get(&photo_to_process.id) { + None => continue, + Some(photo) => photo, + }; - Ok(HttpResponse::Ok().body("go on")) + let mut file = zip.by_name(&photo.path)?; + if file.size() > constants::PHOTOS_MAX_SIZE as u64 { + return Ok(HttpResponse::BadRequest().body("File is too large!")); + } + let mut buff = Vec::with_capacity(file.size() as usize); + file.read_to_end(&mut buff)?; + + let photo = photos_service::finalize_upload(UploadedFile::from_memory( + &buff, + Some(photo.mimetype.clone()), + )?) + .await?; + + // Update appropriate database entry + match photo_to_process.target { + PhotoTarget::Member(member_id) => { + let member = new_members + .iter_mut() + .find(|m| m.id().eq(&member_id)) + .unwrap(); + member.set_photo_id(Some(photo.id())); + members_service::update(member).await?; + } + PhotoTarget::Couple(couple_id) => { + let couple = new_couples.get_mut(&couple_id).unwrap(); + couple.set_photo_id(Some(photo.id())); + couples_service::update(couple).await?; + } + } + } + + Ok(HttpResponse::Accepted().finish()) } fn read_zip_file( diff --git a/geneit_backend/src/controllers/members_controller.rs b/geneit_backend/src/controllers/members_controller.rs index dcb9035..3749706 100644 --- a/geneit_backend/src/controllers/members_controller.rs +++ b/geneit_backend/src/controllers/members_controller.rs @@ -325,7 +325,7 @@ pub async fn set_photo( m: FamilyAndMemberInPath, MultipartForm(form): MultipartForm, ) -> HttpResult { - let photo = photos_service::finalize_upload(form.photo).await?; + let photo = photos_service::finalize_upload(form.photo.into()).await?; let mut member = m.to_member(); members_service::remove_photo(&mut member).await?; diff --git a/geneit_backend/src/controllers/mod.rs b/geneit_backend/src/controllers/mod.rs index c5ba702..0d22c59 100644 --- a/geneit_backend/src/controllers/mod.rs +++ b/geneit_backend/src/controllers/mod.rs @@ -57,4 +57,10 @@ impl From for HttpErr { } } +impl From for HttpErr { + fn from(value: std::num::ParseIntError) -> Self { + HttpErr { err: value.into() } + } +} + pub type HttpResult = Result; diff --git a/geneit_backend/src/models.rs b/geneit_backend/src/models.rs index e2a562e..9b39897 100644 --- a/geneit_backend/src/models.rs +++ b/geneit_backend/src/models.rs @@ -227,7 +227,7 @@ impl Sex { } } -#[derive(Queryable, Debug, serde::Serialize)] +#[derive(Queryable, Debug, serde::Serialize, Clone)] pub struct Member { id: i32, family_id: i32, diff --git a/geneit_backend/src/services/photos_service.rs b/geneit_backend/src/services/photos_service.rs index e7ed50c..5400e65 100644 --- a/geneit_backend/src/services/photos_service.rs +++ b/geneit_backend/src/services/photos_service.rs @@ -8,7 +8,9 @@ use actix_multipart::form::tempfile::TempFile; use diesel::prelude::*; use image::imageops::FilterType; use image::ImageOutputFormat; -use std::io::{Cursor, Read}; +use mime_guess::Mime; +use std::fs::File; +use std::io::{Cursor, Read, Seek, Write}; use uuid::Uuid; #[derive(thiserror::Error, Debug)] @@ -23,8 +25,38 @@ enum PhotoServiceError { MimeTypeForbidden(String), } +pub struct UploadedFile { + pub size: usize, + pub content_type: Option, + pub file: File, +} + +impl From for UploadedFile { + fn from(value: TempFile) -> Self { + Self { + size: value.size, + content_type: value.content_type, + file: value.file.into_file(), + } + } +} + +impl UploadedFile { + pub fn from_memory(buff: &[u8], content_type: Option) -> anyhow::Result { + let mut file = tempfile::tempfile()?; + file.write_all(buff)?; + file.rewind()?; + + Ok(Self { + size: buff.len(), + content_type, + file, + }) + } +} + /// Finalize upload of a photo -pub async fn finalize_upload(mut file: TempFile) -> anyhow::Result { +pub async fn finalize_upload(mut file: UploadedFile) -> anyhow::Result { // Prerequisite checks if file.size > PHOTOS_MAX_SIZE { return Err(PhotoServiceError::FileToLarge(file.size).into());