fediverse
/
kitsune
mirror of https://github.com/kitsune-soc/kitsune
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WASM-based MRF (#490) 

* add skeleton

* use wasmtime

* disable unused features

* add wit and codegen

* add initial component compilation

* update authors name

* rebuild if wit changed

* add config options and docs

* split into files

* add metadata functions

* stuff

* add test binary, add explainer

* progress

* enable async support

* fmt

* fix

* remove unwrap

* embed mrf into kitsune for incoming activities

* rename dir

* updates

* fmt

* disallow any network access

* up

* add manifest for mrf

* add manifest parser/serializer

* AAAAAAAAAAAAAAAAAAAA

* fix test, better diagnostics

* restrict version field to semver

* make schemars optional

* derive default

* convert `activityTypes` to set

* add `configSchema` field

* move mrf-manifest crate

* add license

* add mrf-tool to tree

* fix comment

* remove cfg flags

* add encode function

* add docs

* use simdutf8 where possible

* add description

* restructure commands

* restructure cli layout

* split up functions

* to_owned for manifest

* up

* rename to mrf, version in source

* load mrf module config and pass to the module

* log that config has been found

* add span instead of log level fields

* ensure span is used properly

* filter by activity types

* fix url encoding for international actors

* stuff

* fix

* add submodules

* add wit deps

* add wit readme

* exclude wit deps

* add logging to example

* add logging facade to wasm modules

* add tracing subscriber to wasm mrf test

* remove wasi-keyvalue

* create own keyvalue def

* add kv example

* add bucket example

* add storage backend support

* finish kv storage impl for fs

* up

* up yarn

* flake.lock: Update

Flake lock file updates:

• Updated input 'devenv':
    'github:cachix/devenv/5a30b9e5ac7c6167e61b1f4193d5130bb9f8defa' (2024-02-13)
  → 'github:cachix/devenv/4eccee9a19ad9be42a7859211b456b281d704313' (2024-03-05)
• Updated input 'flake-utils':
    'github:numtide/flake-utils/1ef2e671c3b0c19053962c07dbda38332dcebf26' (2024-01-15)
  → 'github:numtide/flake-utils/d465f4819400de7c8d874d50b982301f28a84605' (2024-02-28)
• Updated input 'nixpkgs':
    'github:nixos/nixpkgs/a4d4fe8c5002202493e87ec8dbc91335ff55552c' (2024-02-15)
  → 'github:nixos/nixpkgs/b8697e57f10292a6165a20f03d2f42920dfaf973' (2024-03-03)
• Updated input 'rust-overlay':
    'github:oxalica/rust-overlay/4ee92bf124fbc4e157cbce1bc2a35499866989fc' (2024-02-16)
  → 'github:oxalica/rust-overlay/e86c0fb5d3a22a5f30d7f64ecad88643fe26449d' (2024-03-05)

* up fuzz

* add licenses

* get rid of futures-retry-policies

* up

* up

* fix warning

* update ci

* add redis skeleton

* impl redis backend

* introduce module name to backend

* add configurable redis backend

* rename

* progress

* remove from fetcher
This commit is contained in:
Aumetra Weisman 2024-03-09 13:21:15 +01:00 committed by GitHub
parent cfd8a7636b
commit d24354f69e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
122 changed files with 4854 additions and 1439 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/audit.yml vendored
View File

@ -14,7 +14,7 @@ jobs:
security_audit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: rustsec/audit-check@v1.4.1
with:
token: ${{ secrets.GITHUB_TOKEN }}

2
.github/workflows/cargo-machete.yml vendored
View File

@ -15,6 +15,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Machete
uses: bnjbvr/cargo-machete@main

52
.github/workflows/release.yml vendored
View File

@ -62,7 +62,7 @@ jobs:
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.7.2/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.11.1/cargo-dist-installer.sh | sh"
# sure would be cool if github gave us proper conditionals...
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
# functionality based on whether this is a pull_request, and whether it's from a fork.
@ -70,15 +70,15 @@ jobs:
# but also really annoying to build CI around when it needs secrets to work right.)
- id: plan
run: |
cargo dist ${{ !github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name) || (github.event.pull_request.head.repo.fork && 'plan' || 'host --steps=check') }} --output-format=json > dist-manifest.json
cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
echo "cargo dist ran successfully"
cat dist-manifest.json
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
cat plan-dist-manifest.json
echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: artifacts
path: dist-manifest.json
name: artifacts-plan-dist-manifest
path: plan-dist-manifest.json
# Build and packages all the platform-specific things
build-local-artifacts:
@ -113,10 +113,11 @@ jobs:
run: ${{ matrix.install_dist }}
# Get the dist-manifest
- name: Fetch local artifacts
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: artifacts
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- name: Install dependencies
run: |
${{ matrix.packages_install }}
@ -139,9 +140,9 @@ jobs:
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: artifacts
name: artifacts-build-local-${{ join(matrix.targets, '_') }}
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
@ -160,13 +161,15 @@ jobs:
with:
submodules: recursive
- name: Install cargo-dist
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.7.2/cargo-dist-installer.sh | sh"
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.11.1/cargo-dist-installer.sh | sh"
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
- name: Fetch local artifacts
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: artifacts
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- id: cargo-dist
shell: bash
run: |
@ -180,9 +183,9 @@ jobs:
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: artifacts
name: artifacts-build-global
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
@ -204,13 +207,14 @@ jobs:
with:
submodules: recursive
- name: Install cargo-dist
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.7.2/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.11.1/cargo-dist-installer.sh | sh"
# Fetch artifacts from scratch-storage
- name: Fetch artifacts
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: artifacts
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
# This is a harmless no-op for Github Releases, hosting for that happens in "announce"
- id: host
shell: bash
@ -220,9 +224,10 @@ jobs:
cat dist-manifest.json
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: artifacts
# Overwrite the previous copy
name: artifacts-dist-manifest
path: dist-manifest.json
# Create a Github Release while uploading all files to it
@ -242,10 +247,11 @@ jobs:
with:
submodules: recursive
- name: "Download Github Artifacts"
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: artifacts
pattern: artifacts-*
path: artifacts
merge-multiple: true
- name: Cleanup
run: |
# Remove the granular manifests

8
.github/workflows/rust.yml vendored
View File

@ -17,7 +17,7 @@ jobs:
name: Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: DeterminateSystems/nix-installer-action@v4
- uses: DeterminateSystems/magic-nix-cache-action@main
- uses: taiki-e/install-action@cargo-hack
@ -29,11 +29,11 @@ jobs:
name: Formatting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: DeterminateSystems/nix-installer-action@v4
- uses: DeterminateSystems/magic-nix-cache-action@main
- run: nix develop --impure -c cargo fmt --all -- --check
test:
name: Test
runs-on: ubuntu-latest
@ -61,7 +61,7 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: DeterminateSystems/nix-installer-action@v4
- uses: DeterminateSystems/magic-nix-cache-action@main
- uses: Swatinem/rust-cache@v2

2
.github/workflows/web.yml vendored
View File

@ -54,7 +54,7 @@ jobs:
runs-on: ubuntu-latest
steps:
# Setup
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: dtolnay/rust-toolchain@stable

3
.gitignore vendored
View File

@ -18,6 +18,9 @@ target-analyzer
# Production configuration file
/config.toml
# MRF directory
/mrf-modules
# Devenv stuff
/result
/.devenv

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "crates/kitsune-wasm-mrf/wit/wasi-logging"]
path = crates/kitsune-wasm-mrf/wit/wasi-logging
url = https://github.com/WebAssembly/wasi-logging.git

1788
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

10
Cargo.toml
View File

@ -40,6 +40,8 @@ members = [
"crates/kitsune-type",
"crates/kitsune-url",
"crates/kitsune-util",
"crates/kitsune-wasm-mrf",
"crates/kitsune-wasm-mrf/example-mrf",
"crates/kitsune-webfinger",
"kitsune",
"kitsune-cli",
@ -49,8 +51,10 @@ members = [
"lib/cursiv",
"lib/http-compat",
"lib/http-signatures",
"lib/kitsune-retry-policies",
"lib/just-retry",
"lib/masto-id-convert",
"lib/mrf-manifest",
"lib/mrf-tool",
"lib/multiplex-pool",
"lib/post-process",
"lib/speedy-uuid",
@ -81,7 +85,7 @@ rust_2018_idioms = "forbid"
unsafe_code = "deny"
[workspace.package]
authors = ["Kitsune developers"]
authors = ["The Kitsune authors"]
edition = "2021"
version = "0.0.1-pre.5"
license = "AGPL-3.0-or-later"
@ -91,7 +95,7 @@ license = "AGPL-3.0-or-later"
# Whether to pass --all-features to cargo build
all-features = true
# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
cargo-dist-version = "0.7.2"
cargo-dist-version = "0.11.1"
# CI backends to support
ci = ["github"]
# The installers to generate for each app

6
_typos.toml
View File

@ -2,6 +2,12 @@
extend-exclude = [
"crates/kitsune-language/examples/basic.rs",
"crates/kitsune-language/src/map.rs",
# Exclude all WIT dependencies since we don't really have control over that
"crates/kitsune-wasm-mrf/wit/deps/*",
"crates/kitsune-wasm-mrf/wit/wasi-keyvalue",
"crates/kitsune-wasm-mrf/wit/wasi-logging",
"lib/http-signatures/tests/data.rs",
"lib/post-process/tests/input/*",
]

3
config.example.toml
View File

@ -136,6 +136,9 @@ default-language = "en"
[messaging]
type = "in-process"
[mrf]
module-dir = "mrf-modules"
# OIDC configuration
#
# Kitsune can use an OIDC service to manage logins

3
crates/kitsune-activitypub/Cargo.toml
View File

@ -13,7 +13,7 @@ diesel = "2.1.4"
diesel-async = "0.4.1"
futures-util = "0.3.30"
headers = "0.4.0"
http = "1.0.0"
http = "1.1.0"
iso8601-timestamp = "0.2.17"
kitsune-cache = { path = "../kitsune-cache" }
kitsune-config = { path = "../kitsune-config" }
@ -28,6 +28,7 @@ kitsune-service = { path = "../kitsune-service" }
kitsune-type = { path = "../kitsune-type" }
kitsune-url = { path = "../kitsune-url" }
kitsune-util = { path = "../kitsune-util" }
kitsune-wasm-mrf = { path = "../kitsune-wasm-mrf" }
mime = "0.3.17"
mime_guess = { version = "2.0.4", default-features = false }
rsa = "0.9.6"

8
crates/kitsune-activitypub/src/deliverer/core.rs
View File

@ -7,6 +7,7 @@ use kitsune_db::model::{account::Account, user::User};
use kitsune_federation_filter::FederationFilter;
use kitsune_http_client::Client;
use kitsune_type::ap::Activity;
use kitsune_wasm_mrf::{MrfService, Outcome};
use sha2::{Digest, Sha256};
use std::pin::pin;
use typed_builder::TypedBuilder;
@ -20,6 +21,7 @@ pub struct Deliverer {
#[builder(default = Client::builder().user_agent(USER_AGENT).unwrap().build())]
client: Client,
federation_filter: FederationFilter,
mrf_service: MrfService,
}
impl Deliverer {
@ -40,7 +42,11 @@ impl Deliverer {
return Ok(());
}
let body = simd_json::to_string(&activity)?;
let body = match self.mrf_service.handle_outgoing(activity).await? {
Outcome::Accept(body) => body,
Outcome::Reject => todo!(),
};
let body_digest = base64_simd::STANDARD.encode_to_string(Sha256::digest(body.as_bytes()));
let digest_header = format!("sha-256={body_digest}");

3
crates/kitsune-activitypub/src/error.rs
View File

@ -59,6 +59,9 @@ pub enum Error {
#[error("Missing host")]
MissingHost,
#[error(transparent)]
Mrf(#[from] kitsune_wasm_mrf::Error),
#[error("Not found")]
NotFound,

2
crates/kitsune-cache/Cargo.toml
View File

@ -7,7 +7,7 @@ license.workspace = true
[dependencies]
enum_dispatch = "0.3.12"
moka = { version = "0.12.5", features = ["sync"] }
moka = { version = "0.12.5", features = ["future"] }
multiplex-pool = { path = "../../lib/multiplex-pool" }
redis = { version = "0.24.0", default-features = false, features = [
"connection-manager",

10
crates/kitsune-cache/src/in_memory.rs
View File

@ -1,5 +1,5 @@
use crate::{CacheBackend, CacheResult};
use moka::sync::Cache;
use moka::future::Cache;
use std::{fmt::Display, marker::PhantomData, time::Duration};
pub struct InMemory<K, V>
@ -35,16 +35,16 @@ where
V: Clone + Send + Sync + 'static,
{
async fn delete(&self, key: &K) -> CacheResult<()> {
self.inner.remove(&key.to_string());
self.inner.remove(&key.to_string()).await;
Ok(())
}
async fn get(&self, key: &K) -> CacheResult<Option<V>> {
Ok(self.inner.get(&key.to_string()))
Ok(self.inner.get(&key.to_string()).await)
}
async fn set(&self, key: &K, value: &V) -> CacheResult<()> {
self.inner.insert(key.to_string(), value.clone());
self.inner.insert(key.to_string(), value.clone()).await;
Ok(())
}
}
@ -69,7 +69,7 @@ mod test {
cache.set(&"hello", &"world").await.unwrap();
cache.set(&"another", &"pair").await.unwrap();
cache.inner.run_pending_tasks();
cache.inner.run_pending_tasks().await;
assert_eq!(cache.inner.entry_count(), 1);
}

2
crates/kitsune-captcha/Cargo.toml
View File

@ -7,7 +7,7 @@ license.workspace = true
[dependencies]
enum_dispatch = "0.3.12"
http = "1.0.0"
http = "1.1.0"
kitsune-http-client = { path = "../kitsune-http-client" }
serde = { version = "1.0.197", features = ["derive"] }
serde_urlencoded = "0.7.1"

2
crates/kitsune-config/Cargo.toml
View File

@ -7,7 +7,7 @@ license.workspace = true
[dependencies]
isolang = { version = "2.4.0", features = ["serde"] }
miette = "7.1.0"
miette = "7.2.0"
serde = { version = "1.0.197", features = ["derive"] }
smol_str = { version = "0.2.1", features = ["serde"] }
tokio = { version = "1.36.0", features = ["fs"] }

2
crates/kitsune-config/src/lib.rs
View File

@ -7,6 +7,7 @@ pub mod instance;
pub mod job_queue;
pub mod language_detection;
pub mod messaging;
pub mod mrf;
pub mod oidc;
pub mod open_telemetry;
pub mod search;
@ -31,6 +32,7 @@ pub struct Configuration {
pub job_queue: job_queue::Configuration,
pub language_detection: language_detection::Configuration,
pub messaging: messaging::Configuration,
pub mrf: mrf::Configuration,
pub opentelemetry: Option<open_telemetry::Configuration>,
pub server: server::Configuration,
pub search: search::Configuration,

31
crates/kitsune-config/src/mrf.rs Normal file
View File

@ -0,0 +1,31 @@
use serde::{Deserialize, Serialize};
use smol_str::SmolStr;
use std::{collections::HashMap, num::NonZeroUsize};
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct FsKvStorage {
pub path: SmolStr,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct RedisKvStorage {
pub url: SmolStr,
pub pool_size: NonZeroUsize,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case", tag = "type")]
pub enum KvStorage {
Fs(FsKvStorage),
Redis(RedisKvStorage),
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct Configuration {
pub module_dir: SmolStr,
pub module_config: HashMap<SmolStr, SmolStr>,
pub storage: KvStorage,
}

2
crates/kitsune-core/Cargo.toml
View File

@ -9,7 +9,7 @@ build = "build.rs"
[dependencies]
async-trait = "0.1.77"
const_format = "0.2.32"
http = "1.0.0"
http = "1.1.0"
kitsune-db = { path = "../kitsune-db" }
kitsune-messaging = { path = "../kitsune-messaging" }
serde = { version = "1.0.197", features = ["derive"] }

4
crates/kitsune-db/Cargo.toml
View File

@ -24,10 +24,10 @@ iso8601-timestamp = { version = "0.2.17", features = ["diesel-pg"] }
kitsune-config = { path = "../kitsune-config" }
kitsune-language = { path = "../kitsune-language" }
kitsune-type = { path = "../kitsune-type" }
miette = "7.1.0"
miette = "7.2.0"
num-derive = "0.4.2"
num-traits = "0.2.18"
rustls = "0.22.2"
rustls = "=0.22.2"
rustls-native-certs = "0.7.0"
serde = { version = "1.0.197", features = ["derive"] }
simd-json = "0.13.8"

4
crates/kitsune-email/Cargo.toml
View File

@ -24,8 +24,8 @@ lettre = { version = "0.11.4", default-features = false, features = [
"tokio1-rustls-tls",
"tracing",
] }
miette = "7.1.0"
mrml = { version = "3.0.1", default-features = false, features = [
miette = "7.2.0"
mrml = { version = "3.0.4", default-features = false, features = [
"orderedmap",
"parse",
"render",

4
crates/kitsune-embed/Cargo.toml
View File

@ -9,12 +9,12 @@ license.workspace = true
diesel = "2.1.4"
diesel-async = "0.4.1"
embed-sdk = { git = "https://github.com/Lantern-chat/embed-service.git", rev = "0d43394bb2514f57edc402a83f69b171705c3650" }
http = "1.0.0"
http = "1.1.0"
iso8601-timestamp = "0.2.17"
kitsune-db = { path = "../kitsune-db" }
kitsune-http-client = { path = "../kitsune-http-client" }
once_cell = "1.19.0"
scraper = { version = "0.18.1", default-features = false }
scraper = { version = "0.19.0", default-features = false }
smol_str = "0.2.1"
thiserror = "1.0.57"
typed-builder = "0.18.1"

2
crates/kitsune-federation-filter/Cargo.toml
View File

@ -9,7 +9,7 @@ license.workspace = true
globset = "0.4.14"
kitsune-config = { path = "../kitsune-config" }
kitsune-type = { path = "../kitsune-type" }
miette = "7.1.0"
miette = "7.2.0"
thiserror = "1.0.57"
url = "2.5.0"

1
crates/kitsune-federation/Cargo.toml
View File

@ -16,6 +16,7 @@ kitsune-federation-filter = { path = "../kitsune-federation-filter" }
kitsune-search = { path = "../kitsune-search" }
kitsune-service = { path = "../kitsune-service" }
kitsune-url = { path = "../kitsune-url" }
kitsune-wasm-mrf = { path = "../kitsune-wasm-mrf" }
kitsune-webfinger = { path = "../kitsune-webfinger" }
typed-builder = "0.18.1"

3
crates/kitsune-federation/src/activitypub.rs
View File

@ -13,6 +13,7 @@ use kitsune_federation_filter::FederationFilter;
use kitsune_search::AnySearchBackend;
use kitsune_service::attachment::AttachmentService;
use kitsune_url::UrlService;
use kitsune_wasm_mrf::MrfService;
use kitsune_webfinger::Webfinger;
use std::sync::Arc;
use typed_builder::TypedBuilder;
@ -22,6 +23,7 @@ pub struct PrepareDeliverer {
attachment_service: AttachmentService,
db_pool: PgPool,
federation_filter: FederationFilter,
mrf_service: MrfService,
url_service: UrlService,
}
@ -41,6 +43,7 @@ pub struct PrepareFetcher {
pub(crate) fn prepare_deliverer(prepare: PrepareDeliverer) -> Arc<dyn Deliverer> {
let core_deliverer = kitsune_activitypub::CoreDeliverer::builder()
.federation_filter(prepare.federation_filter)
.mrf_service(prepare.mrf_service)
.build();
let inbox_resolver = InboxResolver::new(prepare.db_pool.clone());

3
crates/kitsune-http-client/Cargo.toml
View File

@ -23,8 +23,9 @@ hyper-util = { version = "0.1.3", features = [
] }
hyper-rustls = { version = "0.26.0", features = ["http2"] }
kitsune-type = { path = "../kitsune-type" }
pin-project = "1.1.4"
pin-project = "1.1.5"
serde = "1.0.197"
simdutf8 = { version = "0.1.4", features = ["aarch64_neon"] }
simd-json = "0.13.8"
tower = { version = "0.4.13", features = ["util"] }
tower-http = { version = "0.5.2", features = [

7
crates/kitsune-http-client/src/body.rs
View File

@ -4,6 +4,7 @@ use http_body::Frame;
use http_body_util::StreamBody;
use pin_project::pin_project;
use std::{
borrow::Cow,
fmt::{self, Debug},
pin::Pin,
task::{self, Poll},
@ -69,6 +70,12 @@ impl From<Bytes> for Body {
}
}
impl From<Cow<'_, str>> for Body {
fn from(value: Cow<'_, str>) -> Self {
Self::data(value.into_owned())
}
}
impl From<String> for Body {
fn from(value: String) -> Self {
Self::data(value)

6
crates/kitsune-http-client/src/lib.rs
View File

@ -337,7 +337,11 @@ impl Response {
/// - The body isn't a UTF-8 encoded string
pub async fn text(self) -> Result<String> {
let body = self.bytes().await?;
String::from_utf8(body.to_vec()).map_err(Error::new)
// `.to_owned()` as the same performance overhead as calling `.to_vec()` on the `Bytes` body.
// Therefore we can circumvent unsafe usage here by simply calling `.to_owned()` on the string slice at no extra cost.
simdutf8::basic::from_utf8(&body)
.map(ToOwned::to_owned)
.map_err(Error::new)
}
/// Read the body and deserialise it as JSON into a `serde` enabled structure

2
crates/kitsune-jobs/Cargo.toml
View File

@ -14,7 +14,7 @@ futures-util = "0.3.30"
kitsune-core = { path = "../kitsune-core" }
kitsune-db = { path = "../kitsune-db" }
kitsune-email = { path = "../kitsune-email" }
miette = "7.1.0"
miette = "7.2.0"
scoped-futures = "0.1.3"
serde = { version = "1.0.197", features = ["derive"] }
speedy-uuid = { path = "../../lib/speedy-uuid" }

4
crates/kitsune-messaging/Cargo.toml
View File

@ -6,10 +6,10 @@ edition.workspace = true
license.workspace = true
[dependencies]
ahash = "0.8.9"
ahash = "0.8.11"
derive_more = { version = "1.0.0-beta.6", features = ["from"] }
futures-util = "0.3.30"
kitsune-retry-policies = { path = "../../lib/kitsune-retry-policies" }
just-retry = { path = "../../lib/just-retry" }
pin-project-lite = "0.2.13"
redis = { version = "0.24.0", features = ["connection-manager", "tokio-comp"] }
serde = "1.0.197"

4
crates/kitsune-messaging/src/redis.rs
View File

@ -5,7 +5,7 @@
use crate::{util::TransparentDebug, MessagingBackend, Result};
use ahash::AHashMap;
use futures_util::{future, Stream, StreamExt, TryStreamExt};
use kitsune_retry_policies::{futures_backoff_policy, RetryFutureExt};
use just_retry::RetryExt;
use redis::{
aio::{ConnectionManager, PubSub},
AsyncCommands, RedisError,
@ -87,7 +87,7 @@ impl MultiplexActor {
.map(|conn| TransparentDebug(conn.into_pubsub()))
}
})
.retry(futures_backoff_policy())
.retry(just_retry::backoff_policy())
.await
.map(|conn| conn.0)
.unwrap();

14
crates/kitsune-observability/Cargo.toml
View File

@ -13,15 +13,15 @@ hyper = { version = "1.2.0", default-features = false }
kitsune-config = { path = "../kitsune-config" }
kitsune-http-client = { path = "../kitsune-http-client" }
metrics = "=0.22.0"
metrics-opentelemetry = { git = "https://github.com/aumetra/metrics-opentelemetry.git", rev = "b2e2586da553ebd62abdcd3bfd04b5f41a32449a" }
metrics-opentelemetry = { git = "https://github.com/aumetra/metrics-opentelemetry.git", rev = "95537b16370e595981e195be52f98ea5983a7a8e" }
metrics-tracing-context = "0.15.0"
metrics-util = "0.16.2"
miette = "7.1.0"
opentelemetry = { version = "0.21.0", default-features = false, features = [
miette = "7.2.0"
opentelemetry = { version = "0.22.0", default-features = false, features = [
"trace",
] }
opentelemetry-http = "0.10.0"
opentelemetry-otlp = { version = "0.14.0", default-features = false, features = [
opentelemetry-http = "0.11.0"
opentelemetry-otlp = { version = "0.15.0", default-features = false, features = [
"grpc-tonic",
"http-proto",
"metrics",
@ -29,12 +29,12 @@ opentelemetry-otlp = { version = "0.14.0", default-features = false, features =
"tls-roots",
"trace",
] }
opentelemetry_sdk = { version = "0.21.2", default-features = false, features = [
opentelemetry_sdk = { version = "0.22.1", default-features = false, features = [
"rt-tokio",
] }
tracing = "0.1.40"
tracing-error = "0.2.0"
tracing-opentelemetry = { version = "0.22.0", default-features = false }
tracing-opentelemetry = { version = "0.23.0", default-features = false }
tracing-subscriber = "0.3.18"
[lints]

6
crates/kitsune-oidc/Cargo.toml
View File

@ -7,12 +7,12 @@ license.workspace = true
[dependencies]
enum_dispatch = "0.3.12"
http = "1.0.0"
http = "1.1.0"
http-compat = { path = "../../lib/http-compat" }
kitsune-config = { path = "../kitsune-config" }
kitsune-http-client = { path = "../kitsune-http-client" }
miette = "7.1.0"
moka = { version = "0.12.5", features = ["sync"] }
miette = "7.2.0"
moka = { version = "0.12.5", features = ["future"] }
multiplex-pool = { path = "../../lib/multiplex-pool" }
once_cell = "1.19.0"
openidconnect = { version = "3.5.0", default-features = false, features = [

6
crates/kitsune-oidc/src/state/store/in_memory.rs
View File

@ -3,7 +3,7 @@ use crate::{
error::{Error, Result},
state::LoginState,
};
use moka::sync::Cache;
use moka::future::Cache;
#[derive(Clone)]
pub struct InMemory {
@ -20,11 +20,11 @@ impl InMemory {
impl Store for InMemory {
async fn get_and_remove(&self, key: &str) -> Result<LoginState> {
self.inner.remove(key).ok_or(Error::MissingLoginState)
self.inner.remove(key).await.ok_or(Error::MissingLoginState)
}
async fn set(&self, key: &str, value: LoginState) -> Result<()> {
self.inner.insert(key.to_string(), value);
self.inner.insert(key.to_string(), value).await;
Ok(())
}
}

2
crates/kitsune-search/Cargo.toml
View File

@ -17,7 +17,7 @@ futures-util = "0.3.30"
kitsune-config = { path = "../kitsune-config" }
kitsune-db = { path = "../kitsune-db" }
kitsune-language = { path = "../kitsune-language" }
miette = "7.1.0"
miette = "7.2.0"
serde = { version = "1.0.197", features = ["derive"] }
speedy-uuid = { path = "../../lib/speedy-uuid" }
strum = { version = "0.26.1", features = ["derive"] }

8
crates/kitsune-service/Cargo.toml
View File

@ -6,7 +6,7 @@ version.workspace = true
license.workspace = true
[dependencies]
ahash = "0.8.9"
ahash = "0.8.11"
argon2 = "0.5.3"
async-stream = "0.3.5"
athena = { path = "../../lib/athena" }
@ -23,7 +23,7 @@ garde = { version = "0.18.0", default-features = false, features = [
"regex",
"serde",
] }
http = "1.0.0"
http = "1.1.0"
img-parts = "0.3.0"
iso8601-timestamp = "0.2.17"
kitsune-cache = { path = "../kitsune-cache" }
@ -41,7 +41,7 @@ kitsune-search = { path = "../kitsune-search" }
kitsune-storage = { path = "../kitsune-storage" }
kitsune-url = { path = "../kitsune-url" }
kitsune-util = { path = "../kitsune-util" }
miette = "7.1.0"
miette = "7.2.0"
mime = "0.3.17"
multiplex-pool = { path = "../../lib/multiplex-pool" }
password-hash = { version = "0.5.0", features = ["std"] }
@ -81,7 +81,7 @@ kitsune-test = { path = "../kitsune-test" }
kitsune-webfinger = { path = "../kitsune-webfinger" }
pretty_assertions = "1.4.0"
serial_test = "3.0.0"
tempfile = "3.10.0"
tempfile = "3.10.1"
tower = { version = "0.4.13", default-features = false, features = ["util"] }
[lints]

4
crates/kitsune-storage/Cargo.toml
View File

@ -9,14 +9,14 @@ license.workspace = true
bytes = "1.5.0"
derive_more = { version = "1.0.0-beta.6", features = ["from"] }
futures-util = "0.3.30"
http = "1.0.0"
http = "1.1.0"
kitsune-http-client = { path = "../kitsune-http-client" }
rusty-s3 = { version = "0.5.0", default-features = false }
tokio = { version = "1.36.0", features = ["fs", "io-util"] }
tokio-util = { version = "0.7.10", features = ["io"] }
[dev-dependencies]
tempfile = "3.10.0"
tempfile = "3.10.1"
tokio = { version = "1.36.0", features = ["macros", "rt"] }
[lints]

2
crates/kitsune-test/Cargo.toml
View File

@ -10,7 +10,7 @@ bytes = "1.5.0"
diesel = "2.1.4"
diesel-async = "0.4.1"
futures-util = "0.3.30"
http = "1.0.0"
http = "1.1.0"
http-body-util = "0.1.0"
isolang = "2.4.0"
kitsune-config = { path = "../kitsune-config" }

1
crates/kitsune-type/Cargo.toml
View File

@ -11,6 +11,7 @@ serde = { version = "1.0.197", features = ["derive"] }
simd-json = "0.13.8"
smol_str = { version = "0.2.1", features = ["serde"] }
speedy-uuid = { path = "../../lib/speedy-uuid", features = ["serde"] }
strum = { version = "0.26.1", features = ["derive"] }
utoipa = { version = "4.2.0", features = ["chrono", "uuid"] }
[dev-dependencies]

3
crates/kitsune-type/src/ap/mod.rs
View File

@ -3,6 +3,7 @@ use crate::jsonld::{self, RdfNode};
use iso8601_timestamp::Timestamp;
use serde::{Deserialize, Serialize};
use simd_json::{json, OwnedValue};
use strum::AsRefStr;
pub const PUBLIC_IDENTIFIER: &str = "https://www.w3.org/ns/activitystreams#Public";
@ -33,7 +34,7 @@ pub fn ap_context() -> OwnedValue {
])
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(AsRefStr, Clone, Debug, Deserialize, Serialize)]
pub enum ActivityType {
Accept,
Announce,

2
crates/kitsune-type/src/jsonld/serde/set.rs
View File

@ -213,7 +213,9 @@ mod tests {
#[test]
fn unit() {
#[allow(clippy::let_unit_value)]
let data = ();
assert_eq!(
Set::deserialize(into_deserializer(data)),
Ok(Vec::<u32>::new())

53
crates/kitsune-wasm-mrf/Cargo.toml Normal file
View File

@ -0,0 +1,53 @@
[package]
name = "kitsune-wasm-mrf"
authors.workspace = true
edition.workspace = true
version.workspace = true
license.workspace = true
build = "build.rs"
[dependencies]
async-trait = "0.1.77"
derive_more = { version = "1.0.0-beta.6", features = ["from"] }
enum_dispatch = "0.3.12"
futures-util = { version = "0.3.30", default-features = false, features = [
"alloc",
] }
kitsune-config = { path = "../kitsune-config" }
kitsune-type = { path = "../kitsune-type" }
miette = "7.2.0"
mrf-manifest = { path = "../../lib/mrf-manifest", features = ["parse"] }
multiplex-pool = { path = "../../lib/multiplex-pool" }
redis = { version = "0.24.0", default-features = false, features = [
"connection-manager",
"tokio-rustls-comp",
] }
simd-json = "0.13.8"
slab = "0.4.9"
sled = "0.34.7"
smol_str = "0.2.1"
thiserror = "1.0.57"
tokio = { version = "1.36.0", features = ["fs"] }
tracing = "0.1.40"
typed-builder = "0.18.1"
walkdir = "2.5.0"
wasmtime = { version = "18.0.2", default-features = false, features = [
"addr2line",
"async",
"component-model",
"cranelift",
"parallel-compilation",
"pooling-allocator",
"runtime",
] }
wasmtime-wasi = { version = "18.0.2", default-features = false, features = [
"preview2",
] }
[dev-dependencies]
tempfile = "3.10.1"
tokio = { version = "1.36.0", features = ["macros", "rt"] }
tracing-subscriber = "0.3.18"
[lints]
workspace = true

1
crates/kitsune-wasm-mrf/LICENSE-AGPL-3.0 Symbolic link
View File

@ -0,0 +1 @@
../../LICENSE-AGPL-3.0

13
crates/kitsune-wasm-mrf/README.md Normal file
View File

@ -0,0 +1,13 @@
# kitsune-wasm-mrf
Kitsune's implementation of the FEP draft for WASM-based MRFs
## Note on the WASM binary inside the `tests/` directory
The binary is used to verify whether the library can run MRF modules.
To reproduce the binary (or rather the function of the binary, codegen might differ), compile the `example-mrf` project like so:
- Download `wasi_snapshot_preview1.reactor.wasm` from the latest release page of wasmtime
- Install `wasm-tools`
- Compile the project with `cargo build --target wasm32-wasi --profile=dist`
- Link the snapshot to the build artifact with `wasm-tools component new example_mrf.wasm -o example_mrf.component.wasm --adapt wasi_snapshot_preview1.reactor.wasm`

3
crates/kitsune-wasm-mrf/build.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() {
println!("cargo:rerun-if-changed=wit");
}

17
crates/kitsune-wasm-mrf/example-mrf/Cargo.toml Normal file
View File

@ -0,0 +1,17 @@
[package]
name = "example-mrf"
authors.workspace = true
edition.workspace = true
version.workspace = true
license.workspace = true
build = "build.rs"
[lib]
crate-type = ["cdylib"]
[dependencies]
rand = "0.8.5"
wit-bindgen = "0.21.0"
[lints]
workspace = true

3
crates/kitsune-wasm-mrf/example-mrf/build.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() {
println!("cargo:rerun-if-changed=wit");
}

7
crates/kitsune-wasm-mrf/example-mrf/manifest.json Normal file
View File

@ -0,0 +1,7 @@
{
"manifestVersion": "v1",
"apiVersion": "v1",
"name": "example-mrf",
"version": "1.0.0",
"activityTypes": ["*"]
}

45
crates/kitsune-wasm-mrf/example-mrf/src/lib.rs Normal file
View File

@ -0,0 +1,45 @@
#![allow(clippy::missing_safety_doc, clippy::transmute_int_to_bool, unsafe_code)]
use self::{
fep::mrf::keyvalue::{self, Bucket},
wasi::logging::logging::{self, Level},
};
use rand::{distributions::Alphanumeric, Rng};
wit_bindgen::generate!();
fn generate_random_key() -> String {
rand::thread_rng()
.sample_iter(Alphanumeric)
.take(50)
.map(|byte| byte as char)
.collect()
}
struct Mrf;
impl Guest for Mrf {
fn transform(
_config: String,
_direction: Direction,
activity: String,
) -> Result<String, Error> {
logging::log(
Level::Debug,
"example-mrf",
"we got an activity! that's cool!",
);
// We even have a key-value store! Check this out:
let key = generate_random_key();
let bucket = Bucket::open_bucket("example-bucket").unwrap();
keyvalue::set(&bucket, &key, b"world").unwrap();
assert!(keyvalue::exists(&bucket, &key).unwrap());
keyvalue::delete(&bucket, &key).unwrap();
Ok(activity)
}
}
export!(Mrf);

1
crates/kitsune-wasm-mrf/example-mrf/wit Symbolic link
View File

@ -0,0 +1 @@
../wit

52
crates/kitsune-wasm-mrf/src/ctx.rs Normal file
View File

@ -0,0 +1,52 @@
use crate::kv_storage;
use slab::Slab;
use std::sync::Arc;
use wasmtime::{component::ResourceTable, Engine, Store};
use wasmtime_wasi::preview2::{WasiCtx, WasiCtxBuilder, WasiView};
pub struct KvContext {
pub module_name: Option<String>,
pub storage: Arc<kv_storage::BackendDispatch>,
pub buckets: Slab<kv_storage::BucketBackendDispatch>,
}
pub struct Context {
pub kv_ctx: KvContext,
pub resource_table: ResourceTable,
pub wasi_ctx: WasiCtx,
}
impl WasiView for Context {
fn ctx(&mut self) -> &mut WasiCtx {
&mut self.wasi_ctx
}
fn table(&mut self) -> &mut ResourceTable {
&mut self.resource_table
}
}
#[inline]
pub fn construct_store(
engine: &Engine,
storage: Arc<kv_storage::BackendDispatch>,
) -> Store<Context> {
let wasi_ctx = WasiCtxBuilder::new()
.allow_ip_name_lookup(false)
.allow_tcp(false)
.allow_udp(false)
.build();
Store::new(
engine,
Context {
kv_ctx: KvContext {
module_name: None,
storage,
buckets: Slab::new(),
},
resource_table: ResourceTable::new(),
wasi_ctx,
},
)
}

14
crates/kitsune-wasm-mrf/src/error.rs Normal file
View File

@ -0,0 +1,14 @@
use miette::Diagnostic;
use thiserror::Error;
#[derive(Debug, Diagnostic, Error)]
pub enum Error {
#[error(transparent)]
Json(#[from] simd_json::Error),
#[error(transparent)]
ManifestParse(#[from] mrf_manifest::DecodeError),
#[error(transparent)]
Runtime(wasmtime::Error),
}

56
crates/kitsune-wasm-mrf/src/kv_storage/fs.rs Normal file
View File

@ -0,0 +1,56 @@
use super::{Backend, BoxError, BucketBackend};
use miette::IntoDiagnostic;
use std::path::Path;
pub struct FsBackend {
inner: sled::Db,
}
impl FsBackend {
pub fn from_path<P>(path: P) -> miette::Result<Self>
where
P: AsRef<Path>,
{
Ok(Self {
inner: sled::open(path).into_diagnostic()?,
})
}
}
impl Backend for FsBackend {
type Bucket = FsBucketBackend;
async fn open(&self, module_name: &str, name: &str) -> Result<Self::Bucket, BoxError> {
self.inner
.open_tree(format!("{module_name}:{name}"))
.map(|tree| FsBucketBackend { inner: tree })
.map_err(Into::into)
}
}
pub struct FsBucketBackend {
inner: sled::Tree,
}
impl BucketBackend for FsBucketBackend {
async fn exists(&self, key: &str) -> Result<bool, BoxError> {
self.inner.contains_key(key).map_err(Into::into)
}
async fn delete(&self, key: &str) -> Result<(), BoxError> {
self.inner.remove(key)?;
Ok(())
}
async fn get(&self, key: &str) -> Result<Option<Vec<u8>>, BoxError> {
self.inner
.get(key)
.map(|maybe_val| maybe_val.map(|val| val.to_vec()))
.map_err(Into::into)
}
async fn set(&self, key: &str, value: &[u8]) -> Result<(), BoxError> {
self.inner.insert(key, value)?;
Ok(())
}
}

159
crates/kitsune-wasm-mrf/src/kv_storage/mod.rs Normal file
View File

@ -0,0 +1,159 @@
use crate::mrf_wit::v1::fep::mrf::keyvalue;
use async_trait::async_trait;
use derive_more::From;
use enum_dispatch::enum_dispatch;
use std::{error::Error, future::Future};
use wasmtime::component::Resource;
pub use self::{
fs::{FsBackend, FsBucketBackend},
redis::{RedisBackend, RedisBucketBackend},
};
mod fs;
mod redis;
type BoxError = Box<dyn Error + Send + Sync>;
pub trait Backend {
type Bucket: BucketBackend;
fn open(
&self,
module_name: &str,
name: &str,
) -> impl Future<Output = Result<Self::Bucket, BoxError>> + Send;
}
#[enum_dispatch]
#[allow(async_fn_in_trait)]
pub trait BucketBackend {
async fn exists(&self, key: &str) -> Result<bool, BoxError>;
async fn delete(&self, key: &str) -> Result<(), BoxError>;
async fn get(&self, key: &str) -> Result<Option<Vec<u8>>, BoxError>;
async fn set(&self, key: &str, value: &[u8]) -> Result<(), BoxError>;
}
#[derive(From)]
pub enum BackendDispatch {
Fs(FsBackend),
Redis(RedisBackend),
}
impl Backend for BackendDispatch {
type Bucket = BucketBackendDispatch;
async fn open(&self, module_name: &str, name: &str) -> Result<Self::Bucket, BoxError> {
match self {
Self::Fs(fs) => fs.open(module_name, name).await.map(Into::into),
Self::Redis(redis) => redis.open(module_name, name).await.map(Into::into),
}
}
}
#[enum_dispatch(BucketBackend)]
pub enum BucketBackendDispatch {
Fs(FsBucketBackend),
Redis(RedisBucketBackend),
}
#[async_trait]
impl keyvalue::HostBucket for crate::ctx::Context {
async fn open_bucket(
&mut self,
name: String,
) -> wasmtime::Result<Result<Resource<keyvalue::Bucket>, Resource<keyvalue::Error>>> {
let module_name = self
.kv_ctx
.module_name
.as_ref()
.expect("[Bug] Module name not set");
let bucket = match self.kv_ctx.storage.open(&name, module_name).await {
Ok(bucket) => bucket,
Err(error) => {
error!(?error, "failed to open bucket");
return Ok(Err(Resource::new_own(0)));
}
};
let idx = self.kv_ctx.buckets.insert(bucket);
Ok(Ok(Resource::new_own(idx as u32)))
}
fn drop(&mut self, rep: Resource<keyvalue::Bucket>) -> wasmtime::Result<()> {
self.kv_ctx.buckets.remove(rep.rep() as usize);
Ok(())
}
}
#[async_trait]
impl keyvalue::HostError for crate::ctx::Context {
fn drop(&mut self, _rep: Resource<keyvalue::Error>) -> wasmtime::Result<()> {
Ok(())
}
}
#[async_trait]
impl keyvalue::Host for crate::ctx::Context {
async fn get(
&mut self,
bucket: Resource<keyvalue::Bucket>,
key: String,
) -> wasmtime::Result<Result<Option<Vec<u8>>, Resource<keyvalue::Error>>> {
let bucket = &self.kv_ctx.buckets[bucket.rep() as usize];
match bucket.get(&key).await {
Ok(val) => Ok(Ok(val)),
Err(error) => {
error!(?error, %key, "failed to get key from storage");
Ok(Err(Resource::new_own(0)))
}
}
}
async fn set(
&mut self,
bucket: Resource<keyvalue::Bucket>,
key: String,
value: Vec<u8>,
) -> wasmtime::Result<Result<(), Resource<keyvalue::Error>>> {
let bucket = &self.kv_ctx.buckets[bucket.rep() as usize];
match bucket.set(&key, &value).await {
Ok(()) => Ok(Ok(())),
Err(error) => {
error!(?error, %key, "failed to set key in storage");
Ok(Err(Resource::new_own(0)))
}
}
}
async fn delete(
&mut self,
bucket: Resource<keyvalue::Bucket>,
key: String,
) -> wasmtime::Result<Result<(), Resource<keyvalue::Error>>> {
let bucket = &self.kv_ctx.buckets[bucket.rep() as usize];
match bucket.delete(&key).await {
Ok(()) => Ok(Ok(())),
Err(error) => {
error!(?error, %key, "failed to delete key from storage");
Ok(Err(Resource::new_own(0)))
}
}
}
async fn exists(
&mut self,
bucket: Resource<keyvalue::Bucket>,
key: String,
) -> wasmtime::Result<Result<bool, Resource<keyvalue::Error>>> {
let bucket = &self.kv_ctx.buckets[bucket.rep() as usize];
match bucket.exists(&key).await {
Ok(exists) => Ok(Ok(exists)),
Err(error) => {
error!(?error, %key, "failed to check existence of key in storage");
Ok(Err(Resource::new_own(0)))
}
}
}
}

73
crates/kitsune-wasm-mrf/src/kv_storage/redis.rs Normal file
View File

@ -0,0 +1,73 @@
use super::BoxError;
use miette::IntoDiagnostic;
use redis::{aio::ConnectionManager, AsyncCommands};
const REDIS_NAMESPACE: &str = "MRF-KV-STORE";
pub struct RedisBackend {
pool: multiplex_pool::Pool<ConnectionManager>,
}
impl RedisBackend {
pub async fn from_client(client: redis::Client, pool_size: usize) -> miette::Result<Self> {
let pool = multiplex_pool::Pool::from_producer(
|| client.get_connection_manager(),
pool_size,
multiplex_pool::RoundRobinStrategy::default(),
)
.await
.into_diagnostic()?;
Ok(Self { pool })
}
}
impl super::Backend for RedisBackend {
type Bucket = RedisBucketBackend;
async fn open(&self, module_name: &str, name: &str) -> Result<Self::Bucket, BoxError> {
Ok(RedisBucketBackend {
name: format!("{REDIS_NAMESPACE}:{module_name}:{name}"),
pool: self.pool.clone(),
})
}
}
pub struct RedisBucketBackend {
name: String,
pool: multiplex_pool::Pool<ConnectionManager>,
}
impl super::BucketBackend for RedisBucketBackend {
async fn exists(&self, key: &str) -> Result<bool, BoxError> {
self.pool
.get()
.hexists(&self.name, key)
.await
.map_err(Into::into)
}
async fn delete(&self, key: &str) -> Result<(), BoxError> {
self.pool
.get()
.hdel(&self.name, key)
.await
.map_err(Into::into)
}
async fn get(&self, key: &str) -> Result<Option<Vec<u8>>, BoxError> {
self.pool
.get()
.hget(&self.name, key)
.await
.map_err(Into::into)
}
async fn set(&self, key: &str, value: &[u8]) -> Result<(), BoxError> {
self.pool
.get()
.hset(&self.name, key, value)
.await
.map_err(Into::into)
}
}

287
crates/kitsune-wasm-mrf/src/lib.rs Normal file
View File

@ -0,0 +1,287 @@
#[macro_use]
extern crate tracing;
use self::{
ctx::{construct_store, Context},
mrf_wit::v1::fep::mrf::types::{Direction, Error as MrfError},
};
use futures_util::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt, TryStreamExt};
use kitsune_config::mrf::{
Configuration as MrfConfiguration, FsKvStorage, KvStorage, RedisKvStorage,
};
use kitsune_type::ap::Activity;
use miette::{Diagnostic, IntoDiagnostic};
use mrf_manifest::{Manifest, ManifestV1};
use smol_str::SmolStr;
use std::{
borrow::Cow,
fmt::Debug,
io,
path::{Path, PathBuf},
sync::Arc,
};
use thiserror::Error;
use tokio::fs;
use typed_builder::TypedBuilder;
use walkdir::WalkDir;
use wasmtime::{
component::{Component, Linker},
Config, Engine, InstanceAllocationStrategy,
};
pub use self::error::Error;
mod ctx;
mod error;
mod logging;
mod mrf_wit;
pub mod kv_storage;
#[inline]
fn find_mrf_modules<P>(dir: P) -> impl Stream<Item = Result<(PathBuf, Vec<u8>), io::Error>>
where
P: AsRef<Path>,
{
// Read all the `.wasm` files from the disk
// Recursively traverse the entire directory tree doing so and follow all symlinks
// Also run the I/O operations inside a `FuturesUnordered` to enable concurrent reading
WalkDir::new(dir)
.follow_links(true)
.into_iter()
.filter_map(|entry| {
let entry = entry.ok()?;
(entry.path().is_file() && entry.path().extension() == Some("wasm".as_ref()))
.then(|| entry.into_path())
})
.inspect(|path| debug!(?path, "discovered WASM module"))
.map(|path| fs::read(path.clone()).map_ok(|data| (path, data)))
.collect::<FuturesUnordered<_>>()
}
#[inline]
fn load_mrf_module(
engine: &Engine,
module_path: &Path,
bytes: &[u8],
) -> miette::Result<Option<(ManifestV1<'static>, Component)>> {
let component = Component::new(engine, bytes).map_err(|err| {
miette::Report::new(ComponentParseError {
path_help: format!("path to the module: {}", module_path.display()),
advice: "Did you make the WASM file a component via `wasm-tools`?",
})
.wrap_err(err)
})?;
let Some((manifest, _section_range)) = mrf_manifest::decode(bytes)? else {
error!("missing manifest. skipping load.");
return Ok(None);
};
let Manifest::V1(ref manifest_v1) = manifest else {
error!("invalid manifest version. expected v1");
return Ok(None);
};
info!(name = %manifest_v1.name, version = %manifest_v1.version, "loaded MRF module");
Ok(Some((manifest_v1.to_owned(), component)))
}
#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)]
pub enum Outcome<'a> {
Accept(Cow<'a, str>),
Reject,
}
#[derive(Debug, Diagnostic, Error)]
#[error("{path_help}")]
struct ComponentParseError {
path_help: String,
#[help]
advice: &'static str,
}
pub struct MrfModule {
pub component: Component,
pub config: SmolStr,
pub manifest: ManifestV1<'static>,
}
#[derive(Clone, TypedBuilder)]
pub struct MrfService {
engine: Engine,
linker: Arc<Linker<Context>>,
modules: Arc<[MrfModule]>,
storage: Arc<kv_storage::BackendDispatch>,
}
impl MrfService {
#[inline]
pub fn from_components(
engine: Engine,
modules: Vec<MrfModule>,
storage: kv_storage::BackendDispatch,
) -> miette::Result<Self> {
let mut linker = Linker::<Context>::new(&engine);
mrf_wit::v1::Mrf::add_to_linker(&mut linker, |ctx| ctx).map_err(miette::Report::msg)?;
wasmtime_wasi::preview2::command::add_to_linker(&mut linker)
.map_err(miette::Report::msg)?;
Ok(Self {
engine,
linker: Arc::new(linker),
modules: modules.into(),
storage: Arc::new(storage),
})
}
#[instrument(skip_all, fields(module_dir = %config.module_dir))]
pub async fn from_config(config: &MrfConfiguration) -> miette::Result<Self> {
let storage = match config.storage {
KvStorage::Fs(FsKvStorage { ref path }) => {
kv_storage::FsBackend::from_path(path.as_str())?.into()
}
KvStorage::Redis(RedisKvStorage { ref url, pool_size }) => {
let client = redis::Client::open(url.as_str()).into_diagnostic()?;
kv_storage::RedisBackend::from_client(client, pool_size.get())
.await?
.into()
}
};
let mut engine_config = Config::new();
engine_config
.allocation_strategy(InstanceAllocationStrategy::pooling())
.async_support(true)
.wasm_component_model(true);
let engine = Engine::new(&engine_config).map_err(miette::Report::msg)?;
let wasm_data_stream = find_mrf_modules(config.module_dir.as_str())
.map(IntoDiagnostic::into_diagnostic)
.and_then(|(module_path, wasm_data)| {
let engine = &engine;
async move { load_mrf_module(engine, &module_path, &wasm_data) }
});
tokio::pin!(wasm_data_stream);
let mut modules = Vec::new();
while let Some((manifest, component)) = wasm_data_stream.try_next().await?.flatten() {
// TODO: permission grants, etc.
let span = info_span!(
"load_mrf_module_config",
name = %manifest.name,
version = %manifest.version,
);
let config = span.in_scope(|| {
config
.module_config
.get(&*manifest.name)
.cloned()
.inspect(|_| debug!("found configuration"))
.unwrap_or_else(|| {
debug!("didn't find configuration. defaulting to empty string");
SmolStr::default()
})
});
let module = MrfModule {
component,
config,
manifest,
};
modules.push(module);
}
Self::from_components(engine, modules, storage)
}
#[must_use]
pub fn module_count(&self) -> usize {
self.modules.len()
}
async fn handle<'a>(
&self,
direction: Direction,
activity_type: &str,
activity: &'a str,
) -> Result<Outcome<'a>, Error> {
let mut store = construct_store(&self.engine, self.storage.clone());
let mut activity = Cow::Borrowed(activity);
for module in self.modules.iter() {
let activity_types = &module.manifest.activity_types;
if !activity_types.all_activities() && !activity_types.contains(activity_type) {
continue;
}
let (mrf, _) =
mrf_wit::v1::Mrf::instantiate_async(&mut store, &module.component, &self.linker)
.await
.map_err(Error::Runtime)?;
store.data_mut().kv_ctx.module_name = Some(module.manifest.name.to_string());
let result = mrf
.call_transform(&mut store, &module.config, direction, &activity)
.await
.map_err(Error::Runtime)?;
match result {
Ok(transformed) => {
activity = Cow::Owned(transformed);
}
Err(MrfError::ErrorContinue(msg)) => {
error!(%msg, "MRF errored out. Continuing.");
}
Err(MrfError::ErrorReject(msg)) => {
error!(%msg, "MRF errored out. Aborting.");
return Ok(Outcome::Reject);
}
Err(MrfError::Reject) => {
error!("MRF rejected activity. Aborting.");
return Ok(Outcome::Reject);
}
}
}
Ok(Outcome::Accept(activity))
}
#[inline]
pub async fn handle_incoming<'a>(
&self,
activity_type: &str,
activity: &'a str,
) -> Result<Outcome<'a>, Error> {
self.handle(Direction::Incoming, activity_type, activity)
.await
}
#[inline]
pub async fn handle_outgoing(&self, activity: &Activity) -> Result<Outcome<'static>, Error> {
let serialised = simd_json::to_string(activity)?;
let outcome = self
.handle(Direction::Outgoing, activity.r#type.as_ref(), &serialised)
.await?;
let outcome: Outcome<'static> = match outcome {
Outcome::Accept(Cow::Borrowed(..)) => {
// As per the logic in the previous function, we can assume that if the Cow is owned, it has been modified
// If it hasn't been modified it is in its borrowed state
//
// Therefore we don't need to allocate again here, simply reconstruct a new `Outcome` with an owned Cow.
Outcome::Accept(Cow::Owned(serialised))
}
Outcome::Accept(Cow::Owned(owned)) => Outcome::Accept(Cow::Owned(owned)),
Outcome::Reject => Outcome::Reject,
};
Ok(outcome)
}
}

35
crates/kitsune-wasm-mrf/src/logging.rs Normal file
View File

@ -0,0 +1,35 @@
use crate::mrf_wit::v1::wasi::logging::logging::{self, Level};
use async_trait::async_trait;
macro_rules! event_dispatch {
($level:ident, $context:ident, $message:ident, {
$($left:path => $right:path),+$(,)?
}) => {{
match $level {
$(
$left => event!($right, %$context, "{}", $message),
)+
}
}};
}
#[async_trait]
impl logging::Host for crate::ctx::Context {
async fn log(
&mut self,
level: Level,
context: String,
message: String,
) -> wasmtime::Result<()> {
event_dispatch!(level, context, message, {
Level::Trace => tracing::Level::TRACE,
Level::Debug => tracing::Level::DEBUG,
Level::Info => tracing::Level::INFO,
Level::Warn => tracing::Level::WARN,
Level::Error => tracing::Level::ERROR,
Level::Critical => tracing::Level::ERROR,
});
Ok(())
}
}

8
crates/kitsune-wasm-mrf/src/mrf_wit.rs Normal file
View File

@ -0,0 +1,8 @@
pub mod v1 {
wasmtime::component::bindgen!({
async: true,
world: "mrf",
});
}
impl v1::fep::mrf::types::Host for crate::ctx::Context {}

BIN
crates/kitsune-wasm-mrf/tests/example_mrf.component.wasm Executable file
View File

Binary file not shown.

52
crates/kitsune-wasm-mrf/tests/load_and_run.rs Normal file
View File

@ -0,0 +1,52 @@
use kitsune_wasm_mrf::{MrfModule, MrfService, Outcome};
use mrf_manifest::{ActivitySet, ApiVersion, ManifestV1};
use smol_str::SmolStr;
use std::{borrow::Cow, collections::HashSet};
use wasmtime::{component::Component, Config, Engine};
const WASM_COMPONENT: &[u8] = include_bytes!("example_mrf.component.wasm");
fn dummy_manifest() -> ManifestV1<'static> {
ManifestV1 {
api_version: ApiVersion::V1,
name: "dummy".into(),
version: "1.0.0".parse().unwrap(),
activity_types: ActivitySet::from(
[Cow::Borrowed("*")].into_iter().collect::<HashSet<_, _>>(),
),
config_schema: None,
}
}
#[tokio::test]
async fn basic() {
tracing_subscriber::fmt::init();
let dir = tempfile::tempdir().unwrap();
let fs_backend = kitsune_wasm_mrf::kv_storage::FsBackend::from_path(dir.path()).unwrap();
let mut config = Config::new();
config.async_support(true).wasm_component_model(true);
let engine = Engine::new(&config).unwrap();
let component = Component::new(&engine, WASM_COMPONENT).unwrap();
let service = MrfService::from_components(
engine,
vec![MrfModule {
component,
config: SmolStr::default(),
manifest: dummy_manifest(),
}],
fs_backend.into(),
)
.unwrap();
let result = service
.handle_incoming("[anything]", "[imagine activity here]")
.await
.unwrap();
assert_eq!(
result,
Outcome::Accept(Cow::Borrowed("[imagine activity here]"))
);
}

4
crates/kitsune-wasm-mrf/wit/README.md Normal file
View File

@ -0,0 +1,4 @@
# WIT
Dependencies here are managed using [`wit-deps`](https://github.com/bytecodealliance/wit-deps).
Make sure to `wit-deps lock` after updating the submodules.

4
crates/kitsune-wasm-mrf/wit/deps.lock Normal file
View File

@ -0,0 +1,4 @@
[logging]
path = "wasi-logging/wit"
sha256 = "9676b482485bb0fd2751a390374c1108865a096b7037f4b5dbe524f066bfb06e"
sha512 = "30a621a6d48a0175e8047c062e618523a85f69c45a7c31918da2b888f7527fce1aca67fa132552222725d0f6cdcaed95be7f16c28488d9468c0fad00cb7450b9"

1
crates/kitsune-wasm-mrf/wit/deps.toml Normal file
View File

@ -0,0 +1 @@
logging = "wasi-logging/wit"

35
crates/kitsune-wasm-mrf/wit/deps/logging/logging.wit Normal file
View File

@ -0,0 +1,35 @@
/// WASI Logging is a logging API intended to let users emit log messages with
/// simple priority levels and context values.
interface logging {
/// A log level, describing a kind of message.
enum level {
/// Describes messages about the values of variables and the flow of
/// control within a program.
trace,
/// Describes messages likely to be of interest to someone debugging a
/// program.
debug,
/// Describes messages likely to be of interest to someone monitoring a
/// program.
info,
/// Describes messages indicating hazardous situations.
warn,
/// Describes messages indicating serious errors.
error,
/// Describes messages indicating fatal errors.
critical,
}
/// Emit a log message.
///
/// A log message has a `level` describing what kind of message is being
/// sent, a context, which is an uninterpreted string meant to help
/// consumers group similar messages, and a string containing the message
/// text.
log: func(level: level, context: string, message: string);
}

5
crates/kitsune-wasm-mrf/wit/deps/logging/world.wit Normal file
View File

@ -0,0 +1,5 @@
package wasi:logging;
world imports {
import logging;
}

62
crates/kitsune-wasm-mrf/wit/mrf.wit Normal file
View File

@ -0,0 +1,62 @@
package fep:mrf@1.0.0;
/// Home-grown version of wasi-keyvalue
///
/// Built around a synchronous interface since MRFs are synchronous in their current representation
interface keyvalue {
/// Opaque representation of some error
resource error {}
/// Logical collection of Key-Value pairs
resource bucket {
/// Open or create a new bucket
open-bucket: static func(name: string) -> result<bucket, error>;
}
/// Get a value from a bucket
get: func(bucket: borrow<bucket>, key: string) -> result<option<list<u8>>, error>;
/// Set the value inside a bucket
///
/// Overwrites existing values
set: func(bucket: borrow<bucket>, key: string, value: list<u8>) -> result<_, error>;
/// Delete the value from a bucket
delete: func(bucket: borrow<bucket>, key: string) -> result<_, error>;
/// Check if a key exists in the bucket
exists: func(bucket: borrow<bucket>, key: string) -> result<bool, error>;
}
interface types {
/// The direction the activity is going
enum direction {
/// The activity is being received
incoming,
/// The activity is being sent out
outgoing,
}
/// Error types
variant error {
/// Reject the activity
reject,
/// An error occurred but the processing can continue
error-continue(string),
/// An error occurred and the processing should not continue
error-reject(string),
}
}
world mrf {
use types.{direction, error};
import keyvalue;
import wasi:logging/logging;
/// Transform an ActivityPub activity
export transform: func(configuration: string, direction: direction, activity: string) -> result<string, error>;
}

1
crates/kitsune-wasm-mrf/wit/wasi-logging Submodule

@ -0,0 +1 @@
Subproject commit 3293e84de91a1ead98a1b4362f95ac8af5a16ddd

3
crates/kitsune-webfinger/Cargo.toml
View File

@ -9,7 +9,7 @@ license.workspace = true
async-trait = "0.1.77"
autometrics = { version = "1.0.1", default-features = false }
futures-util = "0.3.30"
http = "1.0.0"
http = "1.1.0"
kitsune-cache = { path = "../kitsune-cache" }
kitsune-core = { path = "../kitsune-core" }
kitsune-http-client = { path = "../kitsune-http-client" }
@ -21,6 +21,7 @@ redis = { version = "0.24.0", default-features = false, features = [
"tokio-comp",
] }
tracing = "0.1.40"
urlencoding = "2.1.3"
[dev-dependencies]
http-body-util = "0.1.0"

2
crates/kitsune-webfinger/src/lib.rs
View File

@ -81,7 +81,7 @@ impl Resolver for Webfinger {
let mut username = username;
let mut domain = domain;
let original_acct = format!("acct:{username}@{domain}");
let original_acct = format!("acct:{}@{domain}", urlencoding::encode(username));
let mut acct_buf: String;
let mut acct = original_acct.as_str();

1
docs/src/SUMMARY.md
View File

@ -16,6 +16,7 @@
- [Job Scheduler](./configuring/job-scheduler.md)
- [Language Detection](./configuring/language-detection.md)
- [Link embedding](./configuring/link-embedding.md)
- [MRF (Message Rewrite Facility)](./configuring/mrf.md)
- [Messaging](./configuring/messaging.md)
- [OIDC (OpenID Connect)](./configuring/oidc.md)
- [OpenTelemetry](./configuring/opentelemetry.md)

24
docs/src/configuring/mrf.md Normal file
View File

@ -0,0 +1,24 @@
# MRF (Message Rewrite Facility)
The idea of a message rewrite facility was originally popularized by Pleroma/Akkoma.
Essentially it enables you to add small filters/transformers into your ActivityPub federation.
The MRF module sits at the very beginning of processing an incoming activity.
In this position, the MRF can transform and reject activities based on criteria defined by the developers of the module.
For example, you can use it to:
- detect spam
- mark media attachments as sensitive
- nya-ify every incoming post
## Configuration
### `module-dir`
This configuration option tells Kitsune where to scan for WASM modules to load and compile.
```toml
[mrf]
module-dir = "mrf-modules"
```

24
flake.lock
View File

@ -10,11 +10,11 @@
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 1707817777,
"narHash": "sha256-vHyIs1OULQ3/91wD6xOiuayfI71JXALGA5KLnDKAcy0=",
"lastModified": 1709596918,
"narHash": "sha256-X8tp7nYunRZds8GdSEp+ZBMPf3ym9e6VjZWN8fmzBrc=",
"owner": "cachix",
"repo": "devenv",
"rev": "5a30b9e5ac7c6167e61b1f4193d5130bb9f8defa",
"rev": "4eccee9a19ad9be42a7859211b456b281d704313",
"type": "github"
},
"original": {
@ -62,11 +62,11 @@
"systems": "systems_2"
},
"locked": {
"lastModified": 1705309234,
"narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
"lastModified": 1709126324,
"narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
"rev": "d465f4819400de7c8d874d50b982301f28a84605",
"type": "github"
},
"original": {
@ -139,11 +139,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1707956935,
"narHash": "sha256-ZL2TrjVsiFNKOYwYQozpbvQSwvtV/3Me7Zwhmdsfyu4=",
"lastModified": 1709479366,
"narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "a4d4fe8c5002202493e87ec8dbc91335ff55552c",
"rev": "b8697e57f10292a6165a20f03d2f42920dfaf973",
"type": "github"
},
"original": {
@ -231,11 +231,11 @@
]
},
"locked": {
"lastModified": 1708049456,
"narHash": "sha256-8qGWZTQPPBhcF5dsl1KSWF+H7RX8C3BZGvqYWKBtLjQ=",
"lastModified": 1709604635,
"narHash": "sha256-le4fwmWmjGRYWwkho0Gr7mnnZndOOe4XGbLw68OvF40=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "4ee92bf124fbc4e157cbce1bc2a35499866989fc",
"rev": "e86c0fb5d3a22a5f30d7f64ecad88643fe26449d",
"type": "github"
},
"original": {

4
kitsune-cli/Cargo.toml
View File

@ -13,14 +13,14 @@ license = false
eula = false
[dependencies]
clap = { version = "4.5.1", features = ["derive", "wrap_help"] }
clap = { version = "4.5.2", features = ["derive", "wrap_help"] }
diesel = "2.1.4"
diesel-async = "0.4.1"
dotenvy = "0.15.7"
envy = "0.4.2"
kitsune-config = { path = "../crates/kitsune-config" }
kitsune-db = { path = "../crates/kitsune-db" }
miette = { version = "7.1.0", features = ["fancy"] }
miette = { version = "7.2.0", features = ["fancy"] }
serde = { version = "1.0.197", features = ["derive"] }
speedy-uuid = { path = "../lib/speedy-uuid" }
tokio = { version = "1.36.0", features = ["full"] }

36
kitsune-fe/package.json
View File

@ -21,12 +21,12 @@
"@fortawesome/vue-fontawesome": "^3.0.6",
"@hcaptcha/vue3-hcaptcha": "^1.3.0",
"@mcaptcha/vanilla-glue": "^0.1.0-alpha-3",
"@tiptap/pm": "^2.2.3",
"@tiptap/starter-kit": "^2.2.3",
"@tiptap/vue-3": "^2.2.3",
"@urql/exchange-graphcache": "^6.4.1",
"@tiptap/pm": "^2.2.4",
"@tiptap/starter-kit": "^2.2.4",
"@tiptap/vue-3": "^2.2.4",
"@urql/exchange-graphcache": "^6.5.0",
"@urql/vue": "^1.1.2",
"@vueuse/core": "^10.7.2",
"@vueuse/core": "^10.9.0",
"@zxcvbn-ts/core": "^3.0.4",
"@zxcvbn-ts/language-common": "^3.0.4",
"@zxcvbn-ts/language-en": "^3.0.2",
@ -38,34 +38,34 @@
"pinia-plugin-persistedstate": "^3.2.1",
"rollup": "npm:@rollup/wasm-node",
"tiptap-markdown": "^0.8.9",
"unhead": "^1.8.10",
"vue": "^3.4.19",
"unhead": "^1.8.11",
"vue": "^3.4.21",
"vue-powerglitch": "^1.0.0",
"vue-router": "^4.2.5",
"vue-router": "^4.3.0",
"vue-virtual-scroller": "^2.0.0-beta.8"
},
"devDependencies": {
"@graphql-codegen/cli": "^5.0.2",
"@graphql-codegen/client-preset": "^4.2.2",
"@parcel/watcher": "^2.4.0",
"@graphql-codegen/client-preset": "^4.2.4",
"@parcel/watcher": "^2.4.1",
"@trivago/prettier-plugin-sort-imports": "^4.3.0",
"@types/lodash": "^4.14.202",
"@typescript-eslint/eslint-plugin": "^7.0.1",
"@typescript-eslint/parser": "^7.0.1",
"@typescript-eslint/eslint-plugin": "^7.1.1",
"@typescript-eslint/parser": "^7.1.1",
"@vitejs/plugin-vue": "^5.0.4",
"@vue/eslint-config-prettier": "^9.0.0",
"@vue/eslint-config-typescript": "^12.0.0",
"eslint": "^8.56.0",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-vue": "^9.21.1",
"eslint-plugin-vue": "^9.22.0",
"prettier": "^3.2.5",
"prettier-plugin-css-order": "^2.0.1",
"sass": "^1.71.0",
"sass": "^1.71.1",
"typescript": "^5.3.3",
"unplugin-fluent-vue": "^1.1.4",
"vite": "^5.1.3",
"vue-tsc": "^1.8.27"
"unplugin-fluent-vue": "^1.2.0",
"vite": "^5.1.5",
"vue-tsc": "^2.0.5"
},
"resolutions": {
"rollup": "npm:@rollup/wasm-node"

1465
kitsune-fe/yarn.lock
View File

File diff suppressed because it is too large Load Diff

7
kitsune-job-runner/Cargo.toml
View File

@ -13,7 +13,8 @@ eula = false
[dependencies]
athena = { path = "../lib/athena" }
clap = { version = "4.5.1", features = ["derive", "wrap_help"] }
clap = { version = "4.5.2", features = ["derive", "wrap_help"] }
just-retry = { path = "../lib/just-retry" }
kitsune-config = { path = "../crates/kitsune-config" }
kitsune-core = { path = "../crates/kitsune-core" }
kitsune-db = { path = "../crates/kitsune-db" }
@ -22,10 +23,10 @@ kitsune-federation = { path = "../crates/kitsune-federation" }
kitsune-federation-filter = { path = "../crates/kitsune-federation-filter" }
kitsune-jobs = { path = "../crates/kitsune-jobs" }
kitsune-observability = { path = "../crates/kitsune-observability" }
kitsune-retry-policies = { path = "../lib/kitsune-retry-policies" }
kitsune-service = { path = "../crates/kitsune-service" }
kitsune-url = { path = "../crates/kitsune-url" }
miette = { version = "7.1.0", features = ["fancy"] }
kitsune-wasm-mrf = { path = "../crates/kitsune-wasm-mrf" }
miette = { version = "7.2.0", features = ["fancy"] }
mimalloc = "0.1.39"
multiplex-pool = { path = "../lib/multiplex-pool" }
redis = { version = "0.24.0", default-features = false, features = [

47
kitsune-job-runner/src/lib.rs
View File

@ -1,7 +1,8 @@
#[macro_use]
extern crate tracing;
use athena::JobQueue;
use athena::{JobQueue, TaskTracker};
use just_retry::RetryExt;
use kitsune_config::job_queue::Configuration;
use kitsune_db::PgPool;
use kitsune_email::{
@ -13,13 +14,12 @@ use kitsune_federation::{
};
use kitsune_federation_filter::FederationFilter;
use kitsune_jobs::{JobRunnerContext, KitsuneContextRepo, Service};
use kitsune_retry_policies::{futures_backoff_policy, RetryPolicy};
use kitsune_service::attachment::AttachmentService;
use kitsune_url::UrlService;
use kitsune_wasm_mrf::MrfService;
use multiplex_pool::RoundRobinStrategy;
use redis::RedisResult;
use std::{ops::ControlFlow, sync::Arc, time::Duration};
use tokio::task::JoinSet;
use std::{sync::Arc, time::Duration};
use typed_builder::TypedBuilder;
const EXECUTION_TIMEOUT_DURATION: Duration = Duration::from_secs(30);
@ -30,6 +30,7 @@ pub struct JobDispatcherState {
db_pool: PgPool,
federation_filter: FederationFilter,
mail_sender: Option<MailSender<AsyncSmtpTransport<Tokio1Executor>>>,
mrf_service: MrfService,
url_service: UrlService,
}
@ -66,6 +67,7 @@ pub async fn run_dispatcher(
.attachment_service(state.attachment_service)
.db_pool(state.db_pool.clone())
.federation_filter(state.federation_filter)
.mrf_service(state.mrf_service)
.url_service(state.url_service.clone())
.build();
let prepare_deliverer = PrepareDeliverer::builder()
@ -86,28 +88,29 @@ pub async fn run_dispatcher(
},
});
let mut job_joinset = JoinSet::new();
let job_queue = Arc::new(job_queue);
let job_tracker = TaskTracker::new();
job_tracker.close();
loop {
let mut backoff_policy = futures_backoff_policy();
loop {
let result = job_queue
.spawn_jobs(
num_job_workers - job_joinset.len(),
Arc::clone(&ctx),
&mut job_joinset,
)
.await;
let _ = (|| {
let job_queue = Arc::clone(&job_queue);
let ctx = Arc::clone(&ctx);
let job_tracker = job_tracker.clone();
if let ControlFlow::Continue(duration) = backoff_policy.should_retry(result) {
tokio::time::sleep(duration).await;
} else {
break;
async move {
job_queue
.spawn_jobs(
num_job_workers - job_tracker.len(),
Arc::clone(&ctx),
&job_tracker,
)
.await
}
}
let _ = tokio::time::timeout(EXECUTION_TIMEOUT_DURATION, async {
while job_joinset.join_next().await.is_some() {}
})
.retry(just_retry::backoff_policy())
.await;
let _ = tokio::time::timeout(EXECUTION_TIMEOUT_DURATION, job_tracker.wait()).await;
}
}

3
kitsune-job-runner/src/main.rs
View File

@ -5,6 +5,7 @@ use kitsune_federation_filter::FederationFilter;
use kitsune_job_runner::JobDispatcherState;
use kitsune_service::{attachment::AttachmentService, prepare};
use kitsune_url::UrlService;
use kitsune_wasm_mrf::MrfService;
use miette::IntoDiagnostic;
use std::path::PathBuf;
@ -34,6 +35,7 @@ async fn main() -> miette::Result<()> {
.await
.into_diagnostic()?;
let mrf_service = MrfService::from_config(&config.mrf).await?;
let url_service = UrlService::builder()
.domain(config.url.domain)
.scheme(config.url.scheme)
@ -58,6 +60,7 @@ async fn main() -> miette::Result<()> {
.map(prepare::mail_sender)
.transpose()?,
)
.mrf_service(mrf_service)
.url_service(url_service)
.build();

14
kitsune/Cargo.toml
View File

@ -32,15 +32,15 @@ axum-extra = { version = "0.9.2", features = [
axum-flash = "0.8.0"
blowocking = { path = "../lib/blowocking" }
bytes = "1.5.0"
chrono = { version = "0.4.34", default-features = false }
clap = { version = "4.5.1", features = ["derive", "wrap_help"] }
chrono = { version = "0.4.35", default-features = false }
clap = { version = "4.5.2", features = ["derive", "wrap_help"] }
cursiv = { path = "../lib/cursiv", features = ["axum"] }
der = { version = "0.7.8", features = ["std"] }
diesel = "2.1.4"
diesel-async = "0.4.1"
futures-util = "0.3.30"
headers = "0.4.0"
http = "1.0.0"
http = "1.1.0"
http-body-util = "0.1.0"
http-signatures = { path = "../lib/http-signatures" }
iso8601-timestamp = "0.2.17"
@ -66,9 +66,10 @@ kitsune-storage = { path = "../crates/kitsune-storage" }
kitsune-type = { path = "../crates/kitsune-type" }
kitsune-url = { path = "../crates/kitsune-url" }
kitsune-util = { path = "../crates/kitsune-util" }
kitsune-wasm-mrf = { path = "../crates/kitsune-wasm-mrf" }
kitsune-webfinger = { path = "../crates/kitsune-webfinger" }
metrics = "=0.22.0"
miette = { version = "7.1.0", features = ["fancy"] }
miette = { version = "7.2.0", features = ["fancy"] }
mimalloc = "0.1.39"
mime = "0.3.17"
mime_guess = { version = "2.0.4", default-features = false }
@ -78,14 +79,15 @@ oxide-auth-axum = "0.4.0"
redis = { version = "0.24.0", default-features = false, features = [
"tokio-rustls-comp",
] }
rust-embed = { version = "8.2.0", features = ["include-exclude"] }
rust-embed = { version = "8.3.0", features = ["include-exclude"] }
scoped-futures = "0.1.3"
serde = { version = "1.0.197", features = ["derive"] }
serde_urlencoded = "0.7.1"
simd-json = "0.13.8"
simdutf8 = { version = "0.1.4", features = ["aarch64_neon"] }
speedy-uuid = { path = "../lib/speedy-uuid" }
strum = { version = "0.26.1", features = ["derive", "phf"] }
tempfile = "3.10.0"
tempfile = "3.10.1"
thiserror = "1.0.57"
time = "0.3.34"
tokio = { version = "1.36.0", features = ["full"] }

3
kitsune/src/error.rs
View File

@ -56,6 +56,9 @@ pub enum Error {
#[error(transparent)]
Messaging(kitsune_messaging::BoxError),
#[error(transparent)]
Mrf(#[from] kitsune_wasm_mrf::Error),
#[error(transparent)]
Multipart(#[from] MultipartError),

31
kitsune/src/http/extractor/signed_activity.rs
View File

@ -17,6 +17,7 @@ use http_body_util::BodyExt;
use kitsune_core::{error::HttpError, traits::fetcher::AccountFetchOptions};
use kitsune_db::{model::account::Account, schema::accounts, PgPool};
use kitsune_type::ap::Activity;
use kitsune_wasm_mrf::Outcome;
use scoped_futures::ScopedFutureExt;
/// Parses the body into an ActivityPub activity and verifies the HTTP signature
@ -43,14 +44,40 @@ impl FromRequest<Zustand, Body> for SignedActivity {
let (mut parts, body) = req.with_limited_body().into_parts();
parts.uri = original_uri;
let activity: Activity = match body.collect().await {
Ok(body) => simd_json::from_reader(body.aggregate().reader()).map_err(Error::from)?,
let aggregated_body = match body.collect().await {
Ok(body) => body.to_bytes(),
Err(error) => {
debug!(?error, "Failed to buffer body");
return Err(StatusCode::INTERNAL_SERVER_ERROR.into_response());
}
};
let activity: Activity =
simd_json::from_reader(aggregated_body.clone().reader()).map_err(Error::from)?;
let Ok(str_body) = simdutf8::basic::from_utf8(&aggregated_body) else {
debug!("Malformed body. Not UTF-8");
return Err(StatusCode::BAD_REQUEST.into_response());
};
let Outcome::Accept(str_body) = state
.service
.mrf
.handle_incoming(activity.r#type.as_ref(), str_body)
.await
.map_err(Error::from)?
else {
debug!("sending rejection");
return Err(StatusCode::BAD_REQUEST.into_response());
};
let activity: Activity = match simd_json::from_reader(str_body.as_ref().as_bytes()) {
Ok(activity) => activity,
Err(error) => {
debug!(?error, "Malformed activity");
return Err(StatusCode::BAD_REQUEST.into_response());
}
};
let ap_id = activity.actor.as_str();
let Some(remote_user) = state
.fetcher

4
kitsune/src/lib.rs
View File

@ -40,6 +40,7 @@ use kitsune_service::{
user::UserService,
};
use kitsune_url::UrlService;
use kitsune_wasm_mrf::MrfService;
#[cfg(feature = "oidc")]
use {futures_util::future::OptionFuture, kitsune_oidc::OidcService};
@ -152,6 +153,8 @@ pub async fn initialise_state(
.build()
.unwrap();
let mrf_service = MrfService::from_config(&config.mrf).await?;
let notification_service = NotificationService::builder()
.db_pool(db_pool.clone())
.build();
@ -225,6 +228,7 @@ pub async fn initialise_state(
custom_emoji: custom_emoji_service,
job: job_service,
mailing: mailing_service,
mrf: mrf_service,
notification: notification_service,
post: post_service,
instance: instance_service,

1
kitsune/src/main.rs
View File

@ -40,6 +40,7 @@ async fn boot() -> miette::Result<()> {
.db_pool(state.db_pool.clone())
.federation_filter(state.federation_filter.clone())
.mail_sender(state.service.mailing.sender())
.mrf_service(state.service.mrf.clone())
.url_service(state.service.url.clone())
.build();

4
kitsune/src/oauth2/mod.rs
View File

@ -38,9 +38,7 @@ fn timestamp_to_chrono(ts: iso8601_timestamp::Timestamp) -> chrono::DateTime<Utc
let secs = ts
.duration_since(iso8601_timestamp::Timestamp::UNIX_EPOCH)
.whole_seconds();
chrono::NaiveDateTime::from_timestamp_opt(secs, ts.nanosecond())
.unwrap()
.and_utc()
chrono::DateTime::from_timestamp(secs, ts.nanosecond()).unwrap()
}
#[inline]

4
kitsune/src/oauth2/registrar.rs
View File

@ -106,8 +106,8 @@ impl Registrar for OAuthRegistrar {
let mut client_query = oauth2_applications::table.find(client_id).into_boxed();
if let Some(passphrase) = passphrase {
let passphrase =
str::from_utf8(passphrase).map_err(|_| RegistrarError::PrimitiveError)?;
let passphrase = simdutf8::basic::from_utf8(passphrase)
.map_err(|_| RegistrarError::PrimitiveError)?;
client_query = client_query.filter(oauth2_applications::secret.eq(passphrase));
}

3
kitsune/src/state.rs
View File

@ -13,6 +13,7 @@ use kitsune_service::{
timeline::TimelineService, user::UserService,
};
use kitsune_url::UrlService;
use kitsune_wasm_mrf::MrfService;
use std::{ops::Deref, sync::Arc};
#[cfg(feature = "mastodon-api")]
@ -59,6 +60,7 @@ impl_from_ref! {
FederationFilter => |input: &Zustand| input.federation_filter.clone(),
JobService => |input: &Zustand| input.service.job.clone(),
MailingService => |input: &Zustand| input.service.mailing.clone(),
MrfService => |input: &Zustand| input.service.mrf.clone(),
NotificationService => |input: &Zustand| input.service.notification.clone(),
PostService => |input: &Zustand| input.service.post.clone(),
SearchService => |input: &Zustand| input.service.search.clone(),
@ -141,6 +143,7 @@ pub struct Service {
pub custom_emoji: CustomEmojiService,
pub job: JobService,
pub mailing: MailingService,
pub mrf: MrfService,
pub notification: NotificationService,
pub post: PostService,
pub instance: InstanceService,

6
lib/athena/Cargo.toml
View File

@ -6,11 +6,11 @@ version.workspace = true
license = "MIT OR Apache-2.0"
[dependencies]
ahash = "0.8.9"
ahash = "0.8.11"
either = { version = "1.10.0", default-features = false }
futures-util = { version = "0.3.30", default-features = false }
iso8601-timestamp = { version = "0.2.17", features = ["diesel-pg"] }
kitsune-retry-policies = { path = "../kitsune-retry-policies" }
just-retry = { path = "../just-retry" }
multiplex-pool = { path = "../multiplex-pool" }
once_cell = "1.19.0"
rand = "0.8.5"
@ -21,13 +21,13 @@ redis = { version = "0.24.0", default-features = false, features = [
"streams",
"tokio-comp",
] }
retry-policies = "0.2.1"
serde = { version = "1.0.197", features = ["derive"] }
simd-json = "0.13.8"
smol_str = "0.2.1"
speedy-uuid = { path = "../speedy-uuid", features = ["redis", "serde"] }
thiserror = "1.0.57"
tokio = { version = "1.36.0", features = ["macros", "rt", "sync"] }
tokio-util = { version = "0.7.10", features = ["rt"] }
tracing = "0.1.40"
typed-builder = "0.18.1"

11
lib/athena/examples/basic_queue.rs
View File

@ -14,7 +14,7 @@ use std::{
},
time::Duration,
};
use tokio::task::JoinSet;
use tokio_util::task::TaskTracker;
#[derive(Clone)]
struct JobCtx;
@ -96,11 +96,13 @@ async fn main() {
.unwrap();
}
let mut jobs = JoinSet::new();
let jobs = TaskTracker::new();
jobs.close();
loop {
if tokio::time::timeout(
Duration::from_secs(5),
queue.spawn_jobs(20, Arc::new(()), &mut jobs),
queue.spawn_jobs(20, Arc::new(()), &jobs),
)
.await
.is_err()
@ -108,6 +110,7 @@ async fn main() {
return;
}
while jobs.join_next().await.is_some() {}
jobs.wait().await;
println!("spawned");
}
}

1
lib/athena/src/lib.rs
View File

@ -9,6 +9,7 @@ pub use self::{
error::Error,
queue::{JobDetails, JobQueue},
};
pub use tokio_util::task::TaskTracker;
mod error;
mod macros;

22
lib/athena/src/queue/mod.rs
View File

@ -4,23 +4,27 @@ use ahash::AHashMap;
use either::Either;
use futures_util::StreamExt;
use iso8601_timestamp::Timestamp;
use kitsune_retry_policies::{futures_backoff_policy, RetryFutureExt};
use just_retry::{
retry_policies::{policies::ExponentialBackoff, Jitter},
JustRetryPolicy, RetryExt, StartTime,
};
use redis::{
aio::ConnectionLike,
streams::{StreamReadOptions, StreamReadReply},
AsyncCommands, RedisResult,
};
use retry_policies::{policies::ExponentialBackoff, Jitter, RetryDecision, RetryPolicy};
use serde::{Deserialize, Serialize};
use smol_str::SmolStr;
use speedy_uuid::Uuid;
use std::{
ops::ControlFlow,
pin::pin,
str::FromStr,
sync::Arc,
time::{Duration, SystemTime},
};
use tokio::{sync::OnceCell, task::JoinSet, time::Instant};
use tokio::{sync::OnceCell, time::Instant};
use tokio_util::task::TaskTracker;
use typed_builder::TypedBuilder;
mod scheduled;
@ -263,13 +267,15 @@ where
.jitter(Jitter::Bounded)
.build_with_max_retries(self.max_retries);
if let RetryDecision::Retry { execute_after } = backoff.should_retry(*fail_count) {
if let ControlFlow::Continue(delta) =
backoff.should_retry(StartTime::Irrelevant, *fail_count)
{
let job_meta = JobMeta {
job_id: *job_id,
fail_count: fail_count + 1,
};
let backoff_timestamp = Timestamp::from(SystemTime::from(execute_after));
let backoff_timestamp = Timestamp::from(SystemTime::now() + delta);
let enqueue_cmd = self.enqueue_redis_cmd(&job_meta, Some(backoff_timestamp))?;
pipeline.add_command(enqueue_cmd);
@ -315,7 +321,7 @@ where
&self,
max_jobs: usize,
run_ctx: Arc<<CR::JobContext as Runnable>::Context>,
join_set: &mut JoinSet<()>,
join_set: &TaskTracker,
) -> Result<()> {
let job_data = self.fetch_job_data(max_jobs).await?;
let context_stream = self
@ -356,7 +362,7 @@ where
result = &mut run_fut => break result,
_ = tick_interval.tick() => {
(|| this.reclaim_job(job_data))
.retry(futures_backoff_policy())
.retry(just_retry::backoff_policy())
.await
.expect("Failed to reclaim job");
}
@ -378,7 +384,7 @@ where
};
(|| this.complete_job(&job_state))
.retry(futures_backoff_policy())
.retry(just_retry::backoff_policy())
.await
.expect("Failed to mark job as completed");
});

2
lib/blowocking/Cargo.toml
View File

@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0"
[dependencies]
once_cell = "1.19.0"
rayon = "1.8.1"
rayon = "1.9.0"
thiserror = "1.0.57"
tokio = { version = "1.36.0", features = ["sync"] }
tracing = "0.1.40"

2
lib/cursiv/Cargo.toml
View File

@ -10,7 +10,7 @@ aliri_braid = "0.4.0"
blake3 = "1.5.0"
cookie = { version = "0.18.0", features = ["percent-encode"] }
hex-simd = "0.8.0"
http = "1.0.0"
http = "1.1.0"
pin-project-lite = "0.2.13"
rand = "0.8.5"
tower = { version = "0.4.13", default-features = false }

4
lib/http-compat/Cargo.toml
View File

@ -6,8 +6,8 @@ version.workspace = true
license = "MIT OR Apache-2.0"
[dependencies]
http02 = { package = "http", version = "0.2.11" }
http1 = { package = "http", version = "1.0.0" }
http02 = { package = "http", version = "0.2.12" }
http1 = { package = "http", version = "1.1.0" }
[lints]
workspace = true

4
lib/http-signatures/Cargo.toml
View File

@ -23,11 +23,11 @@ base64-simd = "0.8.0"
blowocking = { path = "../blowocking", default-features = false, optional = true }
const-oid = { version = "0.9.6", features = ["db"] }
derive_builder = "0.20.0"
http = "1.0.0"
http = "1.1.0"
httpdate = "1.0.3"
itertools = { version = "0.12.1", default-features = false }
logos = "0.14.0"
miette = "7.1.0"
miette = "7.2.0"
pkcs8 = { version = "0.10.2", features = ["pem", "std"] }
ring = { version = "0.17.8", features = ["std"] }
scoped-futures = { version = "0.1.3", default-features = false }

15
lib/just-retry/Cargo.toml Normal file
View File

@ -0,0 +1,15 @@
[package]
name = "just-retry"
authors.workspace = true
edition.workspace = true
version.workspace = true
license = "MIT OR Apache-2.0"
[dependencies]
chrono = { version = "0.4.35", default-features = false, features = ["std"] }
retry-policies = "0.3.0"
tokio = { version = "1.36.0", features = ["time"] }
tracing = "0.1.40"
[lints]
workspace = true

0
lib/kitsune-retry-policies/LICENSE-APACHE-2.0 → lib/just-retry/LICENSE-APACHE-2.0
View File

0
lib/kitsune-retry-policies/LICENSE-MIT → lib/just-retry/LICENSE-MIT
View File

115
lib/just-retry/src/lib.rs Normal file
View File

@ -0,0 +1,115 @@
#[macro_use]
extern crate tracing;
use retry_policies::{policies::ExponentialBackoff, Jitter, RetryDecision};
use std::{
fmt::Debug,
future::Future,
ops::ControlFlow,
time::{Duration, SystemTime},
};
pub use retry_policies;
/// Start time of the request
pub enum StartTime {
/// Implies the start time is at `n`
At(SystemTime),
/// Implies the start time is irrelevant to the policy and we will imply pass
/// [`SystemTime::UNIX_EPOCH`] to it to avoid syscalls
Irrelevant,
}
impl StartTime {
fn as_time(&self) -> SystemTime {
match self {
Self::At(at) => *at,
Self::Irrelevant => SystemTime::UNIX_EPOCH,
}
}
}
pub trait JustRetryPolicy: retry_policies::RetryPolicy {
fn should_retry(
&self,
request_start_time: StartTime,
n_past_retries: u32,
) -> ControlFlow<(), Duration>;
}
impl<T> JustRetryPolicy for T
where
T: retry_policies::RetryPolicy,
{
fn should_retry(
&self,
request_start_time: StartTime,
n_past_retries: u32,
) -> ControlFlow<(), Duration> {
if let RetryDecision::Retry { execute_after } =
self.should_retry(request_start_time.as_time().into(), n_past_retries)
{
let now = chrono::DateTime::from(SystemTime::now());
let delta = (execute_after - now)
.to_std()
.expect("Some major clock fuckery happened");
ControlFlow::Continue(delta)
} else {
ControlFlow::Break(())
}
}
}
pub trait RetryExt<T> {
fn retry<R>(&mut self, retry_policy: R) -> impl Future<Output = T> + Send
where
R: JustRetryPolicy + Send;
}
impl<F, Fut, T, E> RetryExt<Fut::Output> for F
where
F: FnMut() -> Fut + Send,
Fut: Future<Output = Result<T, E>> + Send,
T: Send,
E: Debug + Send,
{
#[instrument(skip_all)]
async fn retry<R>(&mut self, retry_policy: R) -> Fut::Output
where
R: JustRetryPolicy + Send,
{
let start_time = SystemTime::now();
let mut retry_count = 0;
loop {
let result = match (self)().await {
val @ Ok(..) => break val,
Err(error) => {
debug!(?error, retry_count, "run failed");
Err(error)
}
};
if let ControlFlow::Continue(delta) =
JustRetryPolicy::should_retry(&retry_policy, StartTime::At(start_time), retry_count)
{
debug!(?delta, "retrying after backoff");
tokio::time::sleep(delta).await;
} else {
debug!("not retrying");
break result;
}
retry_count += 1;
}
}
}
#[must_use]
pub fn backoff_policy() -> impl JustRetryPolicy {
ExponentialBackoff::builder()
.jitter(Jitter::Bounded)
.build_with_total_retry_duration(Duration::from_secs(24 * 3600)) // Kill the retrying after 24 hours
}

Some files were not shown because too many files have changed in this diff Show More