feat(server): support auto-deletion of expired files (#17)

feat(server): support auto-deletion of expired files (#17)

chore(ci): set the number of test threads to 1

feat(config): allow the real-time update of cleanup routine

docs(readme): update README.md about deleting expired files

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
           tar -xzf cargo-tarpaulin-*.tar.gz
           mv cargo-tarpaulin ~/.cargo/bin/
       - name: Run tests
-        run: cargo tarpaulin --out Xml --verbose
+        run: cargo tarpaulin --out Xml --verbose -- --test-threads 1
       - name: Upload reports to codecov
         uses: codecov/codecov-action@v1
         with:

README.md

@@ -20,6 +20,7 @@ some text
     - pet name (e.g. `capital-mosquito.txt`)
     - alphanumeric string (e.g. `yB84D2Dv.txt`)
   - supports expiring links
+    - auto-deletion of expired files (optional)
   - supports one shot links (can only be viewed once)
   - guesses MIME types
     - supports overriding and blacklisting
@@ -113,6 +114,10 @@ $ curl -F "remote=https://example.com/file.png" "<server_address>"
 
 #### Cleaning up expired files
 
+Configure `delete_expired_files` to set an interval for deleting the expired files automatically.
+
+On the other hand, following script can be used as [cron](https://en.wikipedia.org/wiki/Cron) for cleaning up the expired files manually:
+
 ```sh
 #!/bin/env sh
 now=$(date +%s)

config.toml

@@ -27,3 +27,4 @@ mime_blacklist = [
     "application/java-vm"
 ]
 duplicate_files = false
+delete_expired_files = { enabled = true, interval = "1h" }

src/config.rs

@@ -56,6 +56,18 @@ pub struct PasteConfig {
     pub mime_blacklist: Vec<String>,
     /// Allow duplicate uploads
     pub duplicate_files: Option<bool>,
+    /// Delete expired files.
+    pub delete_expired_files: Option<CleanupConfig>,
+}
+
+/// Cleanup configuration.
+#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
+pub struct CleanupConfig {
+    /// Enable cleaning up.
+    pub enabled: bool,
+    /// Interval between clean-ups.
+    #[serde(default, with = "humantime_serde")]
+    pub interval: Duration,
 }
 
 impl Config {

src/main.rs

@@ -6,12 +6,14 @@ use hotwatch::{Event, Hotwatch};
 use rustypaste::config::Config;
 use rustypaste::paste::PasteType;
 use rustypaste::server;
+use rustypaste::util;
 use rustypaste::CONFIG_ENV;
 use std::env;
 use std::fs;
 use std::io::Result as IoResult;
 use std::path::PathBuf;
-use std::sync::RwLock;
+use std::sync::{mpsc, RwLock};
+use std::thread;
 use std::time::Duration;
 
 #[actix_web::main]
@@ -30,6 +32,8 @@ async fn main() -> IoResult<()> {
     };
     let config = Config::parse(&config_path).expect("failed to parse config");
     let server_config = config.server.clone();
+    let paste_config = RwLock::new(config.paste.clone());
+    let (config_sender, config_receiver) = mpsc::channel::<Config>();
 
     // Create necessary directories.
     fs::create_dir_all(&server_config.upload_path)?;
@@ -55,15 +59,18 @@ async fn main() -> IoResult<()> {
             match Config::parse(&path) {
                 Ok(config) => match cloned_config.write() {
                     Ok(mut cloned_config) => {
-                        *cloned_config = config;
+                        *cloned_config = config.clone();
                         log::info!("Configuration has been updated.");
+                        if let Err(e) = config_sender.send(config) {
+                            log::error!("Failed to send config for the cleanup routine: {}", e)
+                        }
                     }
                     Err(e) => {
-                        log::error!("Failed to acquire configuration: {}", e);
+                        log::error!("Failed to acquire config: {}", e);
                     }
                 },
                 Err(e) => {
-                    log::error!("Failed to update configuration: {}", e);
+                    log::error!("Failed to update config: {}", e);
                 }
             }
         }
@@ -72,7 +79,43 @@ async fn main() -> IoResult<()> {
         .watch(&config_path, config_watcher)
         .unwrap_or_else(|_| panic!("failed to watch {:?}", config_path));
 
-    // Create a HTTP server.
+    // Create a thread for cleaning up expired files.
+    thread::spawn(move || loop {
+        let mut enabled = false;
+        if let Some(ref cleanup_config) = paste_config
+            .read()
+            .ok()
+            .and_then(|v| v.delete_expired_files.clone())
+        {
+            if cleanup_config.enabled {
+                log::debug!("Running cleanup...");
+                for file in util::get_expired_files(&server_config.upload_path) {
+                    match fs::remove_file(&file) {
+                        Ok(()) => log::info!("Removed expired file: {:?}", file),
+                        Err(e) => log::error!("Cannot remove expired file: {}", e),
+                    }
+                }
+                thread::sleep(cleanup_config.interval);
+            }
+            enabled = cleanup_config.enabled;
+        }
+        if let Some(new_config) = if enabled {
+            config_receiver.try_recv().ok()
+        } else {
+            config_receiver.recv().ok()
+        } {
+            match paste_config.write() {
+                Ok(mut paste_config) => {
+                    *paste_config = new_config.paste;
+                }
+                Err(e) => {
+                    log::error!("Failed to update config for the cleanup routine: {}", e);
+                }
+            }
+        }
+    });
+
+    // Create an HTTP server.
     let mut http_server = HttpServer::new(move || {
         let http_client = ClientBuilder::new()
             .timeout(

src/util.rs

@@ -1,9 +1,10 @@
+use crate::paste::PasteType;
 use actix_web::{error, Error as ActixError};
 use glob::glob;
 use lazy_regex::{lazy_regex, Lazy, Regex};
 use ring::digest::{Context, SHA256};
 use std::io::{BufReader, Read};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -50,6 +51,30 @@ pub fn glob_match_file(mut path: PathBuf) -> Result<PathBuf, ActixError> {
     Ok(path)
 }
 
+/// Returns the found expired files in the possible upload locations.
+///
+/// Fail-safe, omits errors.
+pub fn get_expired_files(base_path: &Path) -> Vec<PathBuf> {
+    [PasteType::File, PasteType::Oneshot, PasteType::Url]
+        .into_iter()
+        .filter_map(|v| glob(&v.get_path(base_path).join("*.[0-9]*").to_string_lossy()).ok())
+        .flat_map(|glob| glob.filter_map(|v| v.ok()).collect::<Vec<PathBuf>>())
+        .filter(|path| {
+            if let Some(extension) = path
+                .extension()
+                .and_then(|v| v.to_str())
+                .and_then(|v| v.parse().ok())
+            {
+                get_system_time()
+                    .map(|system_time| system_time > Duration::from_millis(extension))
+                    .unwrap_or(false)
+            } else {
+                false
+            }
+        })
+        .collect()
+}
+
 /// Returns the SHA256 digest of the given input.
 pub fn sha256_digest<R: Read>(input: R) -> Result<String, ActixError> {
     let mut reader = BufReader::new(input);
@@ -76,6 +101,7 @@ pub fn sha256_digest<R: Read>(input: R) -> Result<String, ActixError> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::env;
     use std::fs;
     use std::thread;
     #[test]
@@ -89,16 +115,16 @@ mod tests {
     #[test]
     fn test_glob_match() -> Result<(), ActixError> {
         let path = PathBuf::from(format!(
-            "expired.file.{}",
+            "expired.file1.{}",
             get_system_time()?.as_millis() + 50
         ));
         fs::write(&path, String::new())?;
-        assert_eq!(path, glob_match_file(PathBuf::from("expired.file"))?);
+        assert_eq!(path, glob_match_file(PathBuf::from("expired.file1"))?);
 
         thread::sleep(Duration::from_millis(75));
         assert_eq!(
-            PathBuf::from("expired.file"),
-            glob_match_file(PathBuf::from("expired.file"))?
+            PathBuf::from("expired.file1"),
+            glob_match_file(PathBuf::from("expired.file1"))?
         );
         fs::remove_file(path)?;
 
@@ -117,4 +143,21 @@ mod tests {
         );
         Ok(())
     }
+
+    #[test]
+    fn test_get_expired_files() -> Result<(), ActixError> {
+        let current_dir = env::current_dir()?;
+        let expiration_time = get_system_time()?.as_millis() + 50;
+        let path = PathBuf::from(format!("expired.file2.{}", expiration_time));
+        fs::write(&path, String::new())?;
+        assert_eq!(Vec::<PathBuf>::new(), get_expired_files(&current_dir));
+        thread::sleep(Duration::from_millis(75));
+        assert_eq!(
+            vec![current_dir.join(&path)],
+            get_expired_files(&current_dir)
+        );
+        fs::remove_file(path)?;
+        assert_eq!(Vec::<PathBuf>::new(), get_expired_files(&current_dir));
+        Ok(())
+    }
 }