feat(server): add an endpoint for retrieving a list of files (#94)

* Start

* Wip

* Implement path based JSON index

* Remove json_index_path

* Return datetime stamp instead of relative time

* Add file size to list item

* Add auth check when retrieving JSON index

* Make json index path hardcoded

* Test (currently failing)

* Fix test for test_json_list

* Clippy fix

* Revert cargo to original versions with only needed changes

* Add detail about auth guard affecting list route

* Change json_index_path to expose_list

* Remove unneeded linebreak

* Remove unnecessary import

* Remove unnecessary space at end of line

* Move config check after auth check

* Use new auth check syntax, add docs to struct, rename test_json_list to test_list

* Replace chrono usage with uts2ts

* Check list result in test

* Add example to README

* Upgrade serde_json to 1.0.103

* Add linebreak

* Remove unneeded clone

* Remove extra nl

* Update README.md

* Update README.md

Co-authored-by: Orhun Parmaksız <orhunparmaksiz@gmail.com>

* Update README.md

Co-authored-by: Orhun Parmaksız <orhunparmaksiz@gmail.com>

* Remove serde_json

* Set default config to false for expose_list

* Apply suggestions from code review

Co-authored-by: Orhun Parmaksız <orhunparmaksiz@gmail.com>

* Check that option is value in test_list

* Update Cargo.toml

Co-authored-by: Helmut K. C. Tessarek <tessarek@evermeet.cx>

* Update cargo.lock

* Use expect() to check file name

* Remove underscore from list item struct

* Keep comma after last line

* refactor(server): rename ListItem fields

* test(fixtures): add fixture test for listing files

* test push

* remove file again

* chop off ts from filename and minor refactor

* update README

* docs(readme): fix capitalization

* refactor(server): clean up list implementation

---------

Co-authored-by: Orhun Parmaksız <orhunparmaksiz@gmail.com>
Co-authored-by: Helmut K. C. Tessarek <tessarek@evermeet.cx>

Cargo.lock

@@ -2242,6 +2242,7 @@ dependencies = [
  "shuttle-static-folder",
  "tokio",
  "url",
+ "uts2ts",
 ]
 
 [[package]]
@@ -2303,9 +2304,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.99"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
+checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
 dependencies = [
  "itoa",
  "ryu",
@@ -3053,6 +3054,12 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1"
 
+[[package]]
+name = "uts2ts"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "018ca105ca58080880634723c71f26f89591c7de0ca600d9b851270b9980b44f"
+
 [[package]]
 name = "uuid"
 version = "1.4.0"

Cargo.toml

@@ -52,6 +52,7 @@ shuttle-actix-web = { version = "0.22.0", optional = true }
 shuttle-runtime = { version = "0.22.0", optional = true }
 shuttle-static-folder = { version = "0.22.0", optional = true }
 tokio = { version = "1.29.1", optional = true }
+uts2ts = "0.4.0"
 
 [dependencies.config]
 version = "0.13.3"

README.md

@@ -51,6 +51,7 @@ Here you can read the blog post about how it is deployed on Shuttle: [https://bl
     - [Paste file from remote URL](#paste-file-from-remote-url)
     - [Cleaning up expired files](#cleaning-up-expired-files)
   - [Server](#server)
+  - [List endpoint](#list-endpoint)
     - [HTML Form](#html-form)
     - [Docker](#docker)
     - [Nginx](#nginx)
@@ -257,6 +258,18 @@ You can also set multiple auth tokens via the array field `[server].auth_tokens`
 
 See [config.toml](./config.toml) for configuration options.
 
+### List endpoint
+
+Set `expose_list` to true in [config.toml](./config.toml) to be able to retrieve a JSON formatted list of files in your uploads directory. This will not include oneshot files, oneshot URLs, or URLs.
+
+```sh
+$ curl "http://<server_address>/list"
+
+[{"file_name":"accepted-cicada.txt","file_size":241,"expires_at_utc":null}]
+```
+
+This route will require an `AUTH_TOKEN` if one is set.
+
 #### HTML Form
 
 It is possible to use an HTML form for uploading files. To do so, you need to update two fields in your `config.toml`:

config.toml

@@ -9,6 +9,7 @@ max_content_length = "10MB"
 upload_path = "./upload"
 timeout = "30s"
 expose_version = false
+expose_list = false
 #auth_tokens = [
 #  "super_secret_token1",
 #  "super_secret_token2",

fixtures/test-list-files/config.toml

@@ -0,0 +1,11 @@
+[server]
+address = "127.0.0.1:8000"
+max_content_length = "10MB"
+upload_path = "./upload"
+auth_token = "rustypasteisawesome"
+expose_list = true
+
+[paste]
+random_url = { enabled = true, type = "petname", words = 2, separator = "-" }
+default_extension = "txt"
+duplicate_files = true

fixtures/test-list-files/test.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+auth_token="rustypasteisawesome"
+content="test data"
+file_count=3
+
+setup() {
+  echo "$content" > file
+}
+
+run_test() {
+  seq $file_count | xargs -I -- curl -s -F "file=@file" -H "Authorization: $auth_token" localhost:8000 >/dev/null
+  test "$file_count" = "$(curl -s -H "Authorization: $auth_token" localhost:8000/list | grep -o 'file_name' | wc -l)"
+  test "unauthorized" = "$(curl -s localhost:8000/list)"
+}
+
+teardown() {
+  rm file
+  rm -r upload
+}

src/config.rs

@@ -58,6 +58,8 @@ pub struct ServerConfig {
     /// Landing page content-type.
     #[deprecated(note = "use the [landing_page] table")]
     pub landing_page_content_type: Option<String>,
+    /// Path of the JSON index.
+    pub expose_list: Option<bool>,
 }
 
 /// Landing page configuration.

src/server.rs

@@ -12,11 +12,13 @@ use awc::Client;
 use byte_unit::Byte;
 use futures_util::stream::StreamExt;
 use mime::TEXT_PLAIN_UTF_8;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 use std::env;
 use std::fs;
+use std::path::PathBuf;
 use std::sync::RwLock;
+use uts2ts;
 
 /// Shows the landing page.
 #[get("/")]
@@ -283,10 +285,77 @@ async fn upload(
     Ok(HttpResponse::Ok().body(urls.join("")))
 }
 
+/// File entry item for list endpoint.
+#[derive(Serialize, Deserialize)]
+pub struct ListItem {
+    /// Uploaded file name.
+    pub file_name: PathBuf,
+    /// Size of the file in bytes.
+    pub file_size: u64,
+    /// ISO8601 formatted date-time string of the expiration timestamp if one exists for this file.
+    pub expires_at_utc: Option<String>,
+}
+
+/// Returns the list of files.
+#[get("/list")]
+async fn list(
+    request: HttpRequest,
+    config: web::Data<RwLock<Config>>,
+) -> Result<HttpResponse, Error> {
+    let config = config
+        .read()
+        .map_err(|_| error::ErrorInternalServerError("cannot acquire config"))?
+        .clone();
+    let connection = request.connection_info().clone();
+    let host = connection.realip_remote_addr().unwrap_or("unknown host");
+    let tokens = config.get_tokens();
+    auth::check(host, request.headers(), tokens)?;
+    if !config.server.expose_list.unwrap_or(false) {
+        log::warn!("server is not configured to expose list endpoint");
+        Err(error::ErrorForbidden("endpoint is not exposed\n"))?;
+    }
+    let entries: Vec<ListItem> = fs::read_dir(config.server.upload_path)?
+        .filter_map(|entry| {
+            entry.ok().and_then(|e| {
+                let metadata = match e.metadata() {
+                    Ok(metadata) => {
+                        if metadata.is_dir() {
+                            return None;
+                        }
+                        metadata
+                    }
+                    Err(e) => {
+                        log::error!("failed to read metadata: {e}");
+                        return None;
+                    }
+                };
+                let mut file_name = PathBuf::from(e.file_name());
+                let expires_at_utc = if let Some(expiration) = file_name
+                    .extension()
+                    .and_then(|ext| ext.to_str())
+                    .and_then(|v| v.parse::<i64>().ok())
+                {
+                    file_name.set_extension("");
+                    Some(uts2ts::uts2ts(expiration / 1000).as_string())
+                } else {
+                    None
+                };
+                Some(ListItem {
+                    file_name,
+                    file_size: metadata.len(),
+                    expires_at_utc,
+                })
+            })
+        })
+        .collect();
+    Ok(HttpResponse::Ok().json(entries))
+}
+
 /// Configures the server routes.
 pub fn configure_routes(cfg: &mut web::ServiceConfig) {
     cfg.service(index)
         .service(version)
+        .service(list)
         .service(serve)
         .service(upload)
         .route("", web::head().to(HttpResponse::MethodNotAllowed));
@@ -513,6 +582,48 @@ mod tests {
         Ok(())
     }
 
+    #[actix_web::test]
+    async fn test_list() -> Result<(), Error> {
+        let mut config = Config::default();
+        config.server.expose_list = Some(true);
+
+        let test_upload_dir = "test_upload";
+        fs::create_dir(test_upload_dir)?;
+        config.server.upload_path = PathBuf::from(test_upload_dir);
+
+        let app = test::init_service(
+            App::new()
+                .app_data(Data::new(RwLock::new(config)))
+                .app_data(Data::new(Client::default()))
+                .configure(configure_routes),
+        )
+        .await;
+
+        let filename = "test_file.txt";
+        let timestamp = util::get_system_time()?.as_secs().to_string();
+        test::call_service(
+            &app,
+            get_multipart_request(&timestamp, "file", filename).to_request(),
+        )
+        .await;
+
+        let request = TestRequest::default()
+            .insert_header(("content-type", "text/plain"))
+            .uri("/list")
+            .to_request();
+        let result: Vec<ListItem> = test::call_and_read_body_json(&app, request).await;
+
+        assert_eq!(result.len(), 1);
+        assert_eq!(
+            result.first().expect("json object").file_name,
+            PathBuf::from(filename)
+        );
+
+        fs::remove_dir_all(test_upload_dir)?;
+
+        Ok(())
+    }
+
     #[actix_web::test]
     async fn test_auth() -> Result<(), Error> {
         let mut config = Config::default();