From bb0226577d4ff784f21e6f3bced75672bfc19dd4 Mon Sep 17 00:00:00 2001 From: Pierre HUBERT Date: Tue, 19 Nov 2024 20:38:19 +0100 Subject: [PATCH] Can download a copy of storage --- central_backend/Cargo.lock | 170 ++++++++++++++++++ central_backend/Cargo.toml | 4 +- central_backend/src/app_config.rs | 4 +- central_backend/src/server/custom_error.rs | 13 ++ central_backend/src/server/servers.rs | 5 + .../server/web_api/management_controller.rs | 66 +++++++ central_backend/src/server/web_api/mod.rs | 1 + central_backend/src/utils/time_utils.rs | 6 + central_frontend/src/App.tsx | 2 + .../src/routes/ManagementRoute.tsx | 31 ++++ .../src/widgets/SolarEnergyNavList.tsx | 6 + 11 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 central_backend/src/server/web_api/management_controller.rs create mode 100644 central_frontend/src/routes/ManagementRoute.tsx diff --git a/central_backend/Cargo.lock b/central_backend/Cargo.lock index c01f9a5..ebd569a 100644 --- a/central_backend/Cargo.lock +++ b/central_backend/Cargo.lock @@ -496,6 +496,15 @@ version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "asn1" version = "0.19.0" @@ -644,6 +653,27 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.1.31" @@ -697,6 +727,8 @@ dependencies = [ "tokio", "tokio_schedule", "uuid", + "walkdir", + "zip", ] [[package]] @@ -775,6 +807,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -824,6 +862,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -924,6 +977,12 @@ dependencies = [ "syn", ] +[[package]] +name = "deflate64" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + [[package]] name = "deranged" version = "0.3.11" @@ -933,6 +992,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_more" version = "0.99.18" @@ -999,6 +1069,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -1678,12 +1759,28 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1879,6 +1976,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem" version = "3.0.4" @@ -2389,6 +2496,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simple_asn1" version = "0.6.2" @@ -3097,6 +3210,63 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "rand", + "sha1", + "thiserror 1.0.69", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] [[package]] name = "zstd" diff --git a/central_backend/Cargo.toml b/central_backend/Cargo.toml index af5353e..4e17ad9 100644 --- a/central_backend/Cargo.toml +++ b/central_backend/Cargo.toml @@ -26,7 +26,7 @@ actix = "0.13.5" actix-identity = "0.8.0" actix-session = { version = "0.10.1", features = ["cookie-session"] } actix-cors = "0.7.0" -actix-multipart = { version ="0.7.2", features = ["derive"] } +actix-multipart = { version = "0.7.2", features = ["derive"] } actix-remote-ip = "0.1.0" futures-util = "0.3.31" uuid = { version = "1.11.0", features = ["v4", "serde"] } @@ -42,3 +42,5 @@ chrono = "0.4.38" serde_yml = "0.0.12" bincode = "=2.0.0-rc.3" fs4 = { version = "0.11.0", features = ["sync"] } +zip = { version = "2.2.0", features = ["bzip2"] } +walkdir = "2.5.0" \ No newline at end of file diff --git a/central_backend/src/app_config.rs b/central_backend/src/app_config.rs index a90e671..70dda46 100644 --- a/central_backend/src/app_config.rs +++ b/central_backend/src/app_config.rs @@ -10,7 +10,7 @@ pub enum ConsumptionHistoryType { } /// Electrical consumption fetcher backend -#[derive(Subcommand, Debug, Clone)] +#[derive(Subcommand, Debug, Clone, serde::Serialize)] pub enum ConsumptionBackend { /// Constant consumption value Constant { @@ -49,7 +49,7 @@ pub enum ConsumptionBackend { } /// Solar system central backend -#[derive(Parser, Debug)] +#[derive(Parser, Debug, serde::Serialize)] #[command(version, about, long_about = None)] pub struct AppConfig { /// Read arguments from env file diff --git a/central_backend/src/server/custom_error.rs b/central_backend/src/server/custom_error.rs index 0bdda98..8f3d044 100644 --- a/central_backend/src/server/custom_error.rs +++ b/central_backend/src/server/custom_error.rs @@ -4,6 +4,7 @@ use actix_web::HttpResponse; use std::error::Error; use std::fmt::{Display, Formatter}; use std::io::ErrorKind; +use zip::result::ZipError; /// Custom error to ease controller writing #[derive(Debug)] @@ -109,6 +110,18 @@ impl From for HttpErr { } } +impl From for HttpErr { + fn from(value: ZipError) -> Self { + HttpErr::Err(std::io::Error::new(ErrorKind::Other, value.to_string()).into()) + } +} + +impl From for HttpErr { + fn from(value: walkdir::Error) -> Self { + HttpErr::Err(std::io::Error::new(ErrorKind::Other, value.to_string()).into()) + } +} + impl From for HttpErr { fn from(value: HttpResponse) -> Self { HttpErr::HTTPResponse(value) diff --git a/central_backend/src/server/servers.rs b/central_backend/src/server/servers.rs index 76d7b82..ca0a7cb 100644 --- a/central_backend/src/server/servers.rs +++ b/central_backend/src/server/servers.rs @@ -243,6 +243,11 @@ pub async fn secure_server(energy_actor: EnergyActorAddr) -> anyhow::Result<()> "/web_api/relay/{id}/status", web::get().to(relays_controller::status_single), ) + // Management API + .route( + "/web_api/management/download_storage", + web::get().to(management_controller::download_storage), + ) // Devices API .route( "/devices_api/utils/time", diff --git a/central_backend/src/server/web_api/management_controller.rs b/central_backend/src/server/web_api/management_controller.rs new file mode 100644 index 0000000..3c0798b --- /dev/null +++ b/central_backend/src/server/web_api/management_controller.rs @@ -0,0 +1,66 @@ +use crate::app_config::AppConfig; +use crate::server::custom_error::HttpResult; +use crate::utils::time_utils::current_day; +use actix_web::HttpResponse; +use anyhow::Context; +use std::fs::File; +use std::io::{Cursor, Read, Write}; +use walkdir::WalkDir; +use zip::write::SimpleFileOptions; + +/// Download a full copy of the storage data +pub async fn download_storage() -> HttpResult { + let mut zip_buff = Cursor::new(Vec::new()); + let mut zip = zip::ZipWriter::new(&mut zip_buff); + + let options = SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Bzip2) + .unix_permissions(0o700); + + let storage = AppConfig::get().storage_path(); + + let mut file_buff = Vec::new(); + for entry in WalkDir::new(&storage) { + let entry = entry?; + + let path = entry.path(); + let name = path.strip_prefix(&storage).unwrap(); + let path_as_string = name + .to_str() + .map(str::to_owned) + .with_context(|| format!("{name:?} Is a Non UTF-8 Path"))?; + + // Write file or directory explicitly + // Some unzip tools unzip files with directory paths correctly, some do not! + if path.is_file() { + log::debug!("adding file {path:?} as {name:?} ..."); + zip.start_file(path_as_string, options)?; + let mut f = File::open(path)?; + + f.read_to_end(&mut file_buff)?; + zip.write_all(&file_buff)?; + file_buff.clear(); + } else if !name.as_os_str().is_empty() { + // Only if not root! Avoids path spec / warning + // and mapname conversion failed error on unzip + log::debug!("adding dir {path_as_string:?} as {name:?} ..."); + zip.add_directory(path_as_string, options)?; + } + } + + // Inject runtime configuration + zip.start_file("/app_config.json", options)?; + zip.write_all(&serde_json::to_vec_pretty(&AppConfig::get())?)?; + + zip.finish()?; + + let filename = format!("storage-{}.zip", current_day()); + + Ok(HttpResponse::Ok() + .content_type("application/zip") + .insert_header(( + "content-disposition", + format!("attachment; filename=\"{filename}\""), + )) + .body(zip_buff.into_inner())) +} diff --git a/central_backend/src/server/web_api/mod.rs b/central_backend/src/server/web_api/mod.rs index a967058..24dd33e 100644 --- a/central_backend/src/server/web_api/mod.rs +++ b/central_backend/src/server/web_api/mod.rs @@ -2,6 +2,7 @@ pub mod auth_controller; pub mod devices_controller; pub mod energy_controller; pub mod logging_controller; +pub mod management_controller; pub mod ota_controller; pub mod relays_controller; pub mod server_controller; diff --git a/central_backend/src/utils/time_utils.rs b/central_backend/src/utils/time_utils.rs index acdc994..90df441 100644 --- a/central_backend/src/utils/time_utils.rs +++ b/central_backend/src/utils/time_utils.rs @@ -41,6 +41,12 @@ pub fn time_start_of_day() -> anyhow::Result { Ok(local.timestamp() as u64) } +/// Get formatted string containing current day information +pub fn current_day() -> String { + let dt = Local::now(); + format!("{}-{:0>2}-{:0>2}", dt.year(), dt.month(), dt.day()) +} + #[cfg(test)] mod test { use crate::utils::time_utils::day_number; diff --git a/central_frontend/src/App.tsx b/central_frontend/src/App.tsx index fdee53f..a7236ff 100644 --- a/central_frontend/src/App.tsx +++ b/central_frontend/src/App.tsx @@ -16,6 +16,7 @@ import { PendingDevicesRoute } from "./routes/PendingDevicesRoute"; import { RelaysListRoute } from "./routes/RelaysListRoute"; import { BaseAuthenticatedPage } from "./widgets/BaseAuthenticatedPage"; import { OTARoute } from "./routes/OTARoute"; +import { ManagementRoute } from "./routes/ManagementRoute"; export function App() { if (!AuthApi.SignedIn && !ServerApi.Config.auth_disabled) @@ -31,6 +32,7 @@ export function App() { } /> } /> } /> + } /> } /> ) diff --git a/central_frontend/src/routes/ManagementRoute.tsx b/central_frontend/src/routes/ManagementRoute.tsx new file mode 100644 index 0000000..f8672bc --- /dev/null +++ b/central_frontend/src/routes/ManagementRoute.tsx @@ -0,0 +1,31 @@ +import { Button } from "@mui/material"; +import { useConfirm } from "../hooks/context_providers/ConfirmDialogProvider"; +import { SolarEnergyRouteContainer } from "../widgets/SolarEnergyRouteContainer"; +import { APIClient } from "../api/ApiClient"; + +export function ManagementRoute(): React.ReactElement { + const confirm = useConfirm(); + + const downloadBackup = async () => { + try { + if ( + !(await confirm( + `Do you really want to download a copy of the storage? It will contain sensitive information!` + )) + ) + return; + + location.href = APIClient.backendURL() + "/management/download_storage"; + } catch (e) { + console.error(`Failed to donwload a backup of the storage! Error: ${e}`); + } + }; + + return ( + + + + ); +} diff --git a/central_frontend/src/widgets/SolarEnergyNavList.tsx b/central_frontend/src/widgets/SolarEnergyNavList.tsx index e2310f3..5f5e6c9 100644 --- a/central_frontend/src/widgets/SolarEnergyNavList.tsx +++ b/central_frontend/src/widgets/SolarEnergyNavList.tsx @@ -1,5 +1,6 @@ import { mdiChip, + mdiCog, mdiElectricSwitch, mdiHome, mdiMonitorArrowDown, @@ -54,6 +55,11 @@ export function SolarEnergyNavList(): React.ReactElement { uri="/logs" icon={} /> + } + />