From afcce8463f665cab1becd68e2b35fe491e298722 Mon Sep 17 00:00:00 2001 From: Pierre HUBERT Date: Sat, 6 Mar 2021 09:35:36 +0100 Subject: [PATCH] Add support for multiple files types --- Cargo.lock | 84 ++++++++++++- Cargo.toml | 5 +- config.yaml | 2 +- src/api_data/server_config.rs | 6 +- src/constants.rs | 27 +++- src/controllers/conversations_controller.rs | 104 ++++++++++++++-- src/data/base_request_handler.rs | 130 ++++++++++++++++++-- src/utils/mod.rs | 5 +- src/utils/mp3_utils.rs | 16 +++ src/utils/mp4_utils.rs | 17 +++ src/utils/zip_utils.rs | 16 +++ 11 files changed, 377 insertions(+), 35 deletions(-) create mode 100644 src/utils/mp3_utils.rs create mode 100644 src/utils/mp4_utils.rs create mode 100644 src/utils/zip_utils.rs diff --git a/Cargo.lock b/Cargo.lock index 448fc36..a409eb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -500,7 +500,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1374191e2dd25f9ae02e3aa95041ed5d747fc77b3c102b49fe2dd9a8117a6244" dependencies = [ - "num-bigint", + "num-bigint 0.2.6", "num-integer", "num-traits", "serde", @@ -640,6 +640,27 @@ dependencies = [ "bytes 1.0.1", ] +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.10+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17fa3d1ac1ca21c5c4e36a97f3c3eb25084576f6fc47bf0139c1123434216c6c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.66" @@ -707,6 +728,8 @@ dependencies = [ "lazy_static", "mailchecker", "mime_guess", + "mp3-metadata", + "mp4", "mysql", "pdf", "percent-encoding", @@ -718,6 +741,7 @@ dependencies = [ "sha1", "webrtc-sdp", "yaml-rust", + "zip", ] [[package]] @@ -957,15 +981,15 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.20" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" dependencies = [ - "cfg-if 1.0.0", + "cfg-if 0.1.10", "crc32fast", "libc", "libz-sys", - "miniz_oxide 0.4.3", + "miniz_oxide 0.3.7", ] [[package]] @@ -1527,6 +1551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] @@ -1727,6 +1752,26 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "mp3-metadata" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb969bc3573726b0bf60238d5d70f1aa6cc0f9e87f8db3e047b8309317319699" + +[[package]] +name = "mp4" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "999db0cca8fd8c1b2352f362691bdafbd90069efd581144066a7909895d23b86" +dependencies = [ + "byteorder", + "bytes 0.5.6", + "num-rational", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "mutate_once" version = "0.1.1" @@ -1771,7 +1816,7 @@ dependencies = [ "flate2", "lazy_static", "lexical", - "num-bigint", + "num-bigint 0.2.6", "num-traits", "rand 0.7.3", "regex", @@ -1855,6 +1900,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e9a41747ae4633fce5adffb4d2e81ffc5e89593cb19917f8fb2cc5ff76507bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1883,8 +1939,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ "autocfg", + "num-bigint 0.3.1", "num-integer", "num-traits", + "serde", ] [[package]] @@ -3249,3 +3307,17 @@ checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" dependencies = [ "linked-hash-map", ] + +[[package]] +name = "zip" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a8977234acab718eb2820494b2f96cbb16004c19dddf88b7445b27381450997" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time 0.1.44", +] diff --git a/Cargo.toml b/Cargo.toml index 89f3816..df8fb15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,4 +33,7 @@ regex = "1.4.3" dashmap = "4.0.2" reqwest = { version = "0.11.1", features = ["json", "blocking"] } webrtc-sdp = "0.3.8" -bcrypt = "0.9.0" \ No newline at end of file +bcrypt = "0.9.0" +mp3-metadata = "0.3.3" +mp4 = "0.8.1" +zip = "0.5.10" \ No newline at end of file diff --git a/config.yaml b/config.yaml index 74ec49b..2cadd92 100644 --- a/config.yaml +++ b/config.yaml @@ -39,7 +39,7 @@ database: password: pierre # If set to true, every requests made on the database will be shown on the terminal - log-all-queries: true + log-all-queries: false # Video calls configuration diff --git a/src/api_data/server_config.rs b/src/api_data/server_config.rs index b24aab8..80bfe87 100644 --- a/src/api_data/server_config.rs +++ b/src/api_data/server_config.rs @@ -3,7 +3,7 @@ //! @author Pierre Hubert use serde::Serialize; -use crate::constants::{conservation_policy, MIN_SUPPORTED_MOBILE_VERSION, password_policy, MIN_CONVERSATION_MESSAGE_LENGTH, MAX_CONVERSATION_MESSAGE_LENGTH}; +use crate::constants::{ALLOWED_CONVERSATION_FILES_TYPES, conservation_policy, CONVERSATION_FILES_MAX_SIZE, MAX_CONVERSATION_MESSAGE_LENGTH, MIN_CONVERSATION_MESSAGE_LENGTH, MIN_SUPPORTED_MOBILE_VERSION, password_policy}; use crate::data::config::conf; #[derive(Serialize)] @@ -39,6 +39,8 @@ pub struct ServerConfig { data_conservation_policy: DataConservationPolicy, min_conversation_message_len: usize, max_conversation_message_len: usize, + allowed_conversation_files_type: [&'static str; 17], + conversation_files_max_size: usize, } impl ServerConfig { @@ -52,6 +54,8 @@ impl ServerConfig { min_conversation_message_len: MIN_CONVERSATION_MESSAGE_LENGTH, max_conversation_message_len: MAX_CONVERSATION_MESSAGE_LENGTH, + allowed_conversation_files_type: ALLOWED_CONVERSATION_FILES_TYPES, + conversation_files_max_size: CONVERSATION_FILES_MAX_SIZE, password_policy: PasswordPolicy { allow_email_in_password: password_policy::ALLOW_EMAIL_IN_PASSWORD, diff --git a/src/constants.rs b/src/constants.rs index 39a9e68..c4848de 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -166,4 +166,29 @@ pub const MIN_SUPPORTED_MOBILE_VERSION: &str = "1.1.1"; /// Minimum message length pub const MIN_CONVERSATION_MESSAGE_LENGTH: usize = 1; -pub const MAX_CONVERSATION_MESSAGE_LENGTH: usize = 16000; \ No newline at end of file +pub const MAX_CONVERSATION_MESSAGE_LENGTH: usize = 16000; + +/// Allowed files type in conversations +pub const ALLOWED_CONVERSATION_FILES_TYPES: [&str; 17] = [ + "image/png", "image/jpeg", "image/gif", "image/bmp", + "application/pdf", + "audio/mpeg", + "video/mp4", + "application/zip", + + // MS Office docs + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + + // Open Office docs + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.oasis.opendocument.spreadsheet", + + // Source code docs (UTF-8 encoded) + "text/x-csrc", "text/plain", "text/x-c++src" +]; + +/// File maximum size in conversations (10 Mb) +pub const CONVERSATION_FILES_MAX_SIZE: usize = 10 * 1024 * 1024; \ No newline at end of file diff --git a/src/controllers/conversations_controller.rs b/src/controllers/conversations_controller.rs index 65e9139..3eb2af2 100644 --- a/src/controllers/conversations_controller.rs +++ b/src/controllers/conversations_controller.rs @@ -2,16 +2,15 @@ //! //! @author Pierre Hubert - use crate::api_data::conversation_api::ConversationAPI; use crate::api_data::conversation_message_api::ConversationMessageAPI; use crate::api_data::list_unread_conversations_api::UnreadConversationAPI; use crate::api_data::res_count_unread_conversations::ResultCountUnreadConversations; use crate::api_data::res_create_conversation::ResCreateConversation; use crate::api_data::res_find_private_conversations::ResFindPrivateConversations; -use crate::constants::{MAX_CONVERSATION_MESSAGE_LENGTH, MIN_CONVERSATION_MESSAGE_LENGTH}; +use crate::constants::{ALLOWED_CONVERSATION_FILES_TYPES, CONVERSATION_FILES_MAX_SIZE, MAX_CONVERSATION_MESSAGE_LENGTH, MIN_CONVERSATION_MESSAGE_LENGTH}; use crate::controllers::user_ws_controller; -use crate::data::base_request_handler::BaseRequestHandler; +use crate::data::base_request_handler::{BaseRequestHandler, RequestValue}; use crate::data::conversation::{ConversationMemberSetting, NewConversationSettings}; use crate::data::conversation_message::ConversationMessageFile; use crate::data::error::Res; @@ -25,6 +24,7 @@ use crate::helpers::{conversations_helper, events_helper, user_helper}; use crate::helpers::events_helper::Event; use crate::routes::RequestResult; use crate::utils::string_utils::remove_html_nodes; +use crate::utils::user_data_utils::{delete_user_data_file_if_exists, user_data_path}; /// Create a new conversation pub fn create(r: &mut HttpRequestHandler) -> RequestResult { @@ -251,22 +251,100 @@ pub fn send_message(r: &mut HttpRequestHandler) -> RequestResult { // TODO : add support for other files type - // Get image - let file = match r.has_file("image") { - false => None, - true => { - let path = r.save_post_image("image", "conversations", 1200, 1200)?; + // Get associated file + let file = match r.post_parameter_opt("file") { + Some(RequestValue::File(file)) => { + + // File name + let mut name = file.name.to_string(); + + if file.buff.len() > CONVERSATION_FILES_MAX_SIZE { + r.bad_request("File is too big!".to_string())?; + } + + // Determine file mime type + let mut mime_type = r.post_file_type("file")?; + + // Check for thumbnail + let mut thumbnail = match r.has_file("thumbnail") { + false => None, + true => Some("thumbnail".to_string()) + }; + + let path; + + if !ALLOWED_CONVERSATION_FILES_TYPES.contains(&mime_type.as_str()) { + r.bad_request("File type is not allowed!".to_string())?; + } + + // Images + if mime_type.starts_with("image/") { + if let None = thumbnail { + thumbnail = Some("file".to_string()); + } + + path = r.save_post_image("file", "conversation", 2000, 2000)?; + mime_type = "image/png".to_string(); + name = "picture.png".to_string(); + } + + // PDF + else if mime_type.eq("application/pdf") { + path = r.save_post_pdf("file", "conversation")?; + } + + // MP3 + else if mime_type.eq("audio/mpeg") { + path = r.save_post_mp3("file", "conversation")?; + } + + // MP4 + else if mime_type.eq("video/mp4") { + path = r.save_post_mp4("file", "conversation")?; + } + + // ZIP archive + else if mime_type.eq("application/zip") { + path = r.save_post_zip("file", "conversation")?; + } + + // Office document + else if mime_type.starts_with("application/") { + path = r.save_post_office_doc("file", "conversation")?; + } + + // Text files + else { + path = r.save_post_txt_doc("file", "conversation")?; + } + + + // Attempt to save thumbnail, if it fails we can not save message + let thumbnail = match thumbnail { + None => None, + Some(f) => Some(match r.save_post_image(&f, "conversations-thumb", 200, 200) { + Ok(s) => Ok(s), + Err(e) => { + eprintln!("Failed to save conversation thumbnail! {:#?}", e); + delete_user_data_file_if_exists(&path).unwrap(); + Err(e) + } + }?) + }; + + Some(ConversationMessageFile { path: path.clone(), - size: std::fs::metadata(&path)?.len(), - name: "picture.png".to_string(), - thumbnail: Some(r.save_post_image("image", "conversations", 50, 50)?), - r#type: "image/png".to_string(), + size: std::fs::metadata(user_data_path(path.as_ref()))?.len(), + name, + thumbnail, + r#type: mime_type, }) } + _ => None, }; - // Get message, if there is no image + // Get message, if there is no file let message = if let None = file { let msg = r.post_string_without_html("message", MIN_CONVERSATION_MESSAGE_LENGTH, true)?; diff --git a/src/data/base_request_handler.rs b/src/data/base_request_handler.rs index 3718870..ea3f732 100644 --- a/src/data/base_request_handler.rs +++ b/src/data/base_request_handler.rs @@ -3,6 +3,7 @@ //! Base handling code for all user requests +use std::collections::HashSet; use std::error::Error; use exif::In; @@ -23,11 +24,14 @@ use crate::data::user_token::UserAccessToken; use crate::helpers::{account_helper, comments_helper, conversations_helper, custom_emojies_helper, friends_helper, groups_helper, posts_helper, user_helper, virtual_directory_helper}; use crate::helpers::virtual_directory_helper::VirtualDirType; use crate::routes::RequestResult; +use crate::utils::mp3_utils::is_valid_mp3; +use crate::utils::mp4_utils::is_valid_mp4; use crate::utils::pdf_utils::is_valid_pdf; -use crate::utils::string_utils::{check_emoji_code, check_string_before_insert, check_url, remove_html_nodes, check_html_color}; +use crate::utils::string_utils::{check_emoji_code, check_html_color, check_string_before_insert, check_url, remove_html_nodes}; use crate::utils::user_data_utils::{generate_new_user_data_file_name, prepare_file_creation, user_data_path}; use crate::utils::virtual_directories_utils; -use std::collections::HashSet; +use crate::utils::zip_utils::is_valid_zip; +use std::str::from_utf8; #[derive(Serialize)] struct SuccessMessage { @@ -272,6 +276,34 @@ pub trait BaseRequestHandler { unreachable!(); } + /// Get the mime type of a file included in the request + fn post_file_type(&mut self, name: &str) -> Res { + let file = self.post_file(name)?; + + let filetype = mime_guess::from_path(&file.name) + .first() + .map(|m| format!("{}/{}", m.type_(), m.subtype())); + + if let None = filetype { + self.bad_request(format!("Could not determine file type in '{}' !", name))?; + unreachable!(); + } + + Ok(filetype.unwrap()) + } + + /// Get the extension of a file included in the request + fn post_file_ext(&mut self, name: &str, default: &str) -> Res { + let suffix = self.post_file_type(name)? + .parse::()? + .suffix() + .map(|s| s.as_str()) + .unwrap_or(default) + .to_string(); + + Ok(suffix) + } + /// Save an image in user data directory fn save_post_image(&mut self, name: &str, folder: &str, max_w: u32, max_h: u32) -> ResultBoxError { @@ -329,8 +361,25 @@ pub trait BaseRequestHandler { Ok(target_file_path.to_string_lossy().to_string()) } + /// Save a file included in the request + fn save_post_file(&mut self, name: &str, folder: &str, ext: &str) -> Res { + let file = self.post_file(name)?; + + // Avoid memory warnings + let copied_buff = file.buff.clone(); + + // Determine pdf file destination + let target_user_data_folder = prepare_file_creation(self.user_id_ref()?, folder)?; + let target_file_path = generate_new_user_data_file_name(target_user_data_folder.as_path(), ext)?; + let target_sys_path = user_data_path(target_file_path.as_path()); + + std::fs::write(target_sys_path, &copied_buff.as_ref())?; + + Ok(target_file_path.to_string_lossy().to_string()) + } + /// Save a pdf included in the request - fn save_post_pdf(&mut self, name: &str, folder: &str) -> ResultBoxError { + fn save_post_pdf(&mut self, name: &str, folder: &str) -> Res { let file = self.post_file(name)?; if !is_valid_pdf(&file.buff)? { @@ -338,17 +387,76 @@ pub trait BaseRequestHandler { unreachable!(); } - // Avoid memory warnings - let copied_buff = file.buff.clone(); + self.save_post_file(name, folder, "pdf") + } - // Determine pdf file destination - let target_user_data_folder = prepare_file_creation(self.user_id_ref()?, folder)?; - let target_file_path = generate_new_user_data_file_name(target_user_data_folder.as_path(), "pdf")?; - let target_sys_path = user_data_path(target_file_path.as_path()); + /// Save a mp3 file included in the request + fn save_post_mp3(&mut self, name: &str, folder: &str) -> Res { + let file = self.post_file(name)?; - std::fs::write(target_sys_path, &copied_buff.as_ref())?; + if !is_valid_mp3(&file.buff) { + self.bad_request(format!("Invalid MP3 file specified in {} !", name))?; + unreachable!(); + } - Ok(target_file_path.to_string_lossy().to_string()) + self.save_post_file(name, folder, "mp3") + } + + /// Save a mp4 file included in the request + fn save_post_mp4(&mut self, name: &str, folder: &str) -> Res { + let file = self.post_file(name)?; + + if !is_valid_mp4(&file.buff) { + self.bad_request(format!("Invalid MP4 file specified in {} !", name))?; + unreachable!(); + } + + self.save_post_file(name, folder, "mp4") + } + + /// Save a zip file included in the request + fn save_post_zip(&mut self, name: &str, folder: &str) -> Res { + let file = self.post_file(name)?; + + if !is_valid_zip(&file.buff) { + self.bad_request(format!("Invalid ZIP archive file specified in {} !", name))?; + unreachable!(); + } + + self.save_post_file(name, folder, "zip") + } + + /// Save an office document included in the request + fn save_post_office_doc(&mut self, name: &str, folder: &str) -> Res { + let file = self.post_file(name)?; + + if !is_valid_zip(&file.buff) { + self.bad_request(format!("Invalid ZIP archive file specified in {} !", name))?; + unreachable!(); + } + + let mime = self.post_file_type(name)?; + + if !mime.starts_with("application/") { + self.bad_request(format!("Invalid file name in {} !", name))?; + unreachable!(); + } + + let ext = &self.post_file_ext(name, "zip")?; + self.save_post_file(name, folder, ext) + } + + /// Save a simple text document included in the request + fn save_post_txt_doc(&mut self, name: &str, folder: &str) -> Res { + let file = self.post_file(name)?; + + if from_utf8(&file.buff).is_err() { + self.bad_request(format!("File in {} is not UTF-8!", name))?; + unreachable!(); + } + + let ext = &self.post_file_ext(name, "txt")?; + self.save_post_file(name, folder, ext) } /// Get an integer included in the POST request diff --git a/src/utils/mod.rs b/src/utils/mod.rs index a9559af..422f08e 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -7,4 +7,7 @@ pub mod user_data_utils; pub mod virtual_directories_utils; pub mod date_utils; pub mod string_utils; -pub mod pdf_utils; \ No newline at end of file +pub mod pdf_utils; +pub mod mp3_utils; +pub mod mp4_utils; +pub mod zip_utils; \ No newline at end of file diff --git a/src/utils/mp3_utils.rs b/src/utils/mp3_utils.rs new file mode 100644 index 0000000..f6f93b5 --- /dev/null +++ b/src/utils/mp3_utils.rs @@ -0,0 +1,16 @@ +//! # MP3 utilities +//! +//! @author Pierre Hubert + +/// Check out whether a file is a valid MP3 file or not +pub fn is_valid_mp3(file: &[u8]) -> bool { + let res = mp3_metadata::read_from_slice(file); + + match res { + Ok(_) => true, + Err(e) => { + eprintln!("Error while parsing MP3 file ! {:#?}", e); + false + } + } +} \ No newline at end of file diff --git a/src/utils/mp4_utils.rs b/src/utils/mp4_utils.rs new file mode 100644 index 0000000..91a28f3 --- /dev/null +++ b/src/utils/mp4_utils.rs @@ -0,0 +1,17 @@ +//! # MP utilities +//! +//! @author Pierre Hubert + +/// Check out whether an MP4 file is valid or not +pub fn is_valid_mp4(file: &[u8]) -> bool { + let cursor = std::io::Cursor::new(file); + let reader = mp4::Mp4Reader::read_header(cursor, file.len() as u64); + + match reader { + Ok(_) => true, + Err(e) => { + eprintln!("Failed to read MP4! {:#?}", e); + false + } + } +} \ No newline at end of file diff --git a/src/utils/zip_utils.rs b/src/utils/zip_utils.rs new file mode 100644 index 0000000..6ce5f21 --- /dev/null +++ b/src/utils/zip_utils.rs @@ -0,0 +1,16 @@ +//! # ZIP utilities +//! +//! @author Pierre Hubert + +/// Check out whether a given file is a valid ZIP archive or not +pub fn is_valid_zip(file: &[u8]) -> bool { + let cursor = std::io::Cursor::new(file); + + match zip::ZipArchive::new(cursor) { + Ok(_) => true, + Err(e) => { + eprintln!("Failed to read ZIP archive! {:#?}", e); + false + } + } +} \ No newline at end of file