From dd91c50d502c0f02a387ea9320ee00cae23f7b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhun=20Parmaks=C4=B1z?= Date: Wed, 23 Mar 2022 11:38:32 +0300 Subject: [PATCH] feat(server): support auto-deletion of expired files (#17) feat(server): support auto-deletion of expired files (#17) chore(ci): set the number of test threads to 1 feat(config): allow the real-time update of cleanup routine docs(readme): update README.md about deleting expired files --- .github/workflows/ci.yml | 2 +- README.md | 5 ++++ config.toml | 1 + src/config.rs | 12 +++++++++ src/main.rs | 53 ++++++++++++++++++++++++++++++++++++---- src/util.rs | 53 ++++++++++++++++++++++++++++++++++++---- 6 files changed, 115 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a4c45e..4555f8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: tar -xzf cargo-tarpaulin-*.tar.gz mv cargo-tarpaulin ~/.cargo/bin/ - name: Run tests - run: cargo tarpaulin --out Xml --verbose + run: cargo tarpaulin --out Xml --verbose -- --test-threads 1 - name: Upload reports to codecov uses: codecov/codecov-action@v1 with: diff --git a/README.md b/README.md index ade7e30..333be40 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ some text - pet name (e.g. `capital-mosquito.txt`) - alphanumeric string (e.g. `yB84D2Dv.txt`) - supports expiring links + - auto-deletion of expired files (optional) - supports one shot links (can only be viewed once) - guesses MIME types - supports overriding and blacklisting @@ -113,6 +114,10 @@ $ curl -F "remote=https://example.com/file.png" "" #### Cleaning up expired files +Configure `delete_expired_files` to set an interval for deleting the expired files automatically. + +On the other hand, following script can be used as [cron](https://en.wikipedia.org/wiki/Cron) for cleaning up the expired files manually: + ```sh #!/bin/env sh now=$(date +%s) diff --git a/config.toml b/config.toml index 305b2b0..cfbab40 100644 --- a/config.toml +++ b/config.toml @@ -27,3 +27,4 @@ mime_blacklist = [ "application/java-vm" ] duplicate_files = false +delete_expired_files = { enabled = true, interval = "1h" } diff --git a/src/config.rs b/src/config.rs index f8e70e4..1664fa6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -56,6 +56,18 @@ pub struct PasteConfig { pub mime_blacklist: Vec, /// Allow duplicate uploads pub duplicate_files: Option, + /// Delete expired files. + pub delete_expired_files: Option, +} + +/// Cleanup configuration. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct CleanupConfig { + /// Enable cleaning up. + pub enabled: bool, + /// Interval between clean-ups. + #[serde(default, with = "humantime_serde")] + pub interval: Duration, } impl Config { diff --git a/src/main.rs b/src/main.rs index 2176f51..09f4758 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,12 +6,14 @@ use hotwatch::{Event, Hotwatch}; use rustypaste::config::Config; use rustypaste::paste::PasteType; use rustypaste::server; +use rustypaste::util; use rustypaste::CONFIG_ENV; use std::env; use std::fs; use std::io::Result as IoResult; use std::path::PathBuf; -use std::sync::RwLock; +use std::sync::{mpsc, RwLock}; +use std::thread; use std::time::Duration; #[actix_web::main] @@ -30,6 +32,8 @@ async fn main() -> IoResult<()> { }; let config = Config::parse(&config_path).expect("failed to parse config"); let server_config = config.server.clone(); + let paste_config = RwLock::new(config.paste.clone()); + let (config_sender, config_receiver) = mpsc::channel::(); // Create necessary directories. fs::create_dir_all(&server_config.upload_path)?; @@ -55,15 +59,18 @@ async fn main() -> IoResult<()> { match Config::parse(&path) { Ok(config) => match cloned_config.write() { Ok(mut cloned_config) => { - *cloned_config = config; + *cloned_config = config.clone(); log::info!("Configuration has been updated."); + if let Err(e) = config_sender.send(config) { + log::error!("Failed to send config for the cleanup routine: {}", e) + } } Err(e) => { - log::error!("Failed to acquire configuration: {}", e); + log::error!("Failed to acquire config: {}", e); } }, Err(e) => { - log::error!("Failed to update configuration: {}", e); + log::error!("Failed to update config: {}", e); } } } @@ -72,7 +79,43 @@ async fn main() -> IoResult<()> { .watch(&config_path, config_watcher) .unwrap_or_else(|_| panic!("failed to watch {:?}", config_path)); - // Create a HTTP server. + // Create a thread for cleaning up expired files. + thread::spawn(move || loop { + let mut enabled = false; + if let Some(ref cleanup_config) = paste_config + .read() + .ok() + .and_then(|v| v.delete_expired_files.clone()) + { + if cleanup_config.enabled { + log::debug!("Running cleanup..."); + for file in util::get_expired_files(&server_config.upload_path) { + match fs::remove_file(&file) { + Ok(()) => log::info!("Removed expired file: {:?}", file), + Err(e) => log::error!("Cannot remove expired file: {}", e), + } + } + thread::sleep(cleanup_config.interval); + } + enabled = cleanup_config.enabled; + } + if let Some(new_config) = if enabled { + config_receiver.try_recv().ok() + } else { + config_receiver.recv().ok() + } { + match paste_config.write() { + Ok(mut paste_config) => { + *paste_config = new_config.paste; + } + Err(e) => { + log::error!("Failed to update config for the cleanup routine: {}", e); + } + } + } + }); + + // Create an HTTP server. let mut http_server = HttpServer::new(move || { let http_client = ClientBuilder::new() .timeout( diff --git a/src/util.rs b/src/util.rs index bb4d1f1..93fce27 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,9 +1,10 @@ +use crate::paste::PasteType; use actix_web::{error, Error as ActixError}; use glob::glob; use lazy_regex::{lazy_regex, Lazy, Regex}; use ring::digest::{Context, SHA256}; use std::io::{BufReader, Read}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; @@ -50,6 +51,30 @@ pub fn glob_match_file(mut path: PathBuf) -> Result { Ok(path) } +/// Returns the found expired files in the possible upload locations. +/// +/// Fail-safe, omits errors. +pub fn get_expired_files(base_path: &Path) -> Vec { + [PasteType::File, PasteType::Oneshot, PasteType::Url] + .into_iter() + .filter_map(|v| glob(&v.get_path(base_path).join("*.[0-9]*").to_string_lossy()).ok()) + .flat_map(|glob| glob.filter_map(|v| v.ok()).collect::>()) + .filter(|path| { + if let Some(extension) = path + .extension() + .and_then(|v| v.to_str()) + .and_then(|v| v.parse().ok()) + { + get_system_time() + .map(|system_time| system_time > Duration::from_millis(extension)) + .unwrap_or(false) + } else { + false + } + }) + .collect() +} + /// Returns the SHA256 digest of the given input. pub fn sha256_digest(input: R) -> Result { let mut reader = BufReader::new(input); @@ -76,6 +101,7 @@ pub fn sha256_digest(input: R) -> Result { #[cfg(test)] mod tests { use super::*; + use std::env; use std::fs; use std::thread; #[test] @@ -89,16 +115,16 @@ mod tests { #[test] fn test_glob_match() -> Result<(), ActixError> { let path = PathBuf::from(format!( - "expired.file.{}", + "expired.file1.{}", get_system_time()?.as_millis() + 50 )); fs::write(&path, String::new())?; - assert_eq!(path, glob_match_file(PathBuf::from("expired.file"))?); + assert_eq!(path, glob_match_file(PathBuf::from("expired.file1"))?); thread::sleep(Duration::from_millis(75)); assert_eq!( - PathBuf::from("expired.file"), - glob_match_file(PathBuf::from("expired.file"))? + PathBuf::from("expired.file1"), + glob_match_file(PathBuf::from("expired.file1"))? ); fs::remove_file(path)?; @@ -117,4 +143,21 @@ mod tests { ); Ok(()) } + + #[test] + fn test_get_expired_files() -> Result<(), ActixError> { + let current_dir = env::current_dir()?; + let expiration_time = get_system_time()?.as_millis() + 50; + let path = PathBuf::from(format!("expired.file2.{}", expiration_time)); + fs::write(&path, String::new())?; + assert_eq!(Vec::::new(), get_expired_files(¤t_dir)); + thread::sleep(Duration::from_millis(75)); + assert_eq!( + vec![current_dir.join(&path)], + get_expired_files(¤t_dir) + ); + fs::remove_file(path)?; + assert_eq!(Vec::::new(), get_expired_files(¤t_dir)); + Ok(()) + } }