From b63370118d4a707db112421b4073d22b91628fba Mon Sep 17 00:00:00 2001 From: Zeerooth Date: Sat, 11 Nov 2023 20:49:36 +0100 Subject: [PATCH] Custom and remote emojis (#405) * initial custom emoji schema * some more progress idk * emoji process * better emoji processing * it just works * implement mastodon's /api/v1/custom_emojis * proper url and shortcode in /api/v1/custom_emojis * add emoji list to statues * add some tests and stand against clippy * handle some unsound unwraps * make emoji a resolvable AP type * fix tests * add a missing test fixture * address PR review --------- Co-authored-by: aumetra --- .../kitsune-core/src/activitypub/fetcher.rs | 159 ++++++++++++++- crates/kitsune-core/src/activitypub/mod.rs | 51 ++++- crates/kitsune-core/src/consts.rs | 1 + crates/kitsune-core/src/error.rs | 3 + crates/kitsune-core/src/lib.rs | 10 + .../src/mapping/activitypub/object.rs | 101 ++++++++-- .../src/mapping/mastodon/sealed.rs | 151 +++++++++----- crates/kitsune-core/src/resolve/post.rs | 153 ++++++++++++-- crates/kitsune-core/src/service/attachment.rs | 11 +- .../kitsune-core/src/service/custom_emoji.rs | 188 ++++++++++++++++++ crates/kitsune-core/src/service/mod.rs | 1 + crates/kitsune-core/src/service/post.rs | 59 +++++- crates/kitsune-core/src/service/url.rs | 5 + crates/kitsune-core/src/state.rs | 8 +- .../up.sql | 2 +- .../down.sql | 6 + .../up.sql | 32 +++ crates/kitsune-db/src/model/custom_emoji.rs | 42 ++++ .../kitsune-db/src/model/media_attachment.rs | 4 +- crates/kitsune-db/src/model/mod.rs | 1 + crates/kitsune-db/src/schema.rs | 97 ++++++++- crates/kitsune-type/src/ap/emoji.rs | 24 +++ crates/kitsune-type/src/ap/mod.rs | 2 + .../kitsune-type/src/mastodon/custom_emoji.rs | 11 + crates/kitsune-type/src/mastodon/mod.rs | 2 + crates/kitsune-type/src/mastodon/status.rs | 3 +- .../http/graphql/types/media_attachment.rs | 2 +- kitsune/src/http/handler/custom_emojis.rs | 21 ++ .../handler/mastodon/api/v1/custom_emojis.rs | 42 ++++ .../src/http/handler/mastodon/api/v1/mod.rs | 2 + kitsune/src/http/handler/mod.rs | 1 + kitsune/src/http/mod.rs | 3 +- kitsune/src/state.rs | 3 +- lib/post-process/src/lib.rs | 41 +++- .../tests/snapshots/emote__parse_emote.snap | 10 +- lib/post-process/tests/transformation.rs | 2 +- .../corteximplant.com_emoji_7952.json | 1 + .../corteximplant.com_emoji_8933.json | 1 + 38 files changed, 1131 insertions(+), 125 deletions(-) create mode 100644 crates/kitsune-core/src/service/custom_emoji.rs create mode 100644 crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/down.sql create mode 100644 crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/up.sql create mode 100644 crates/kitsune-db/src/model/custom_emoji.rs create mode 100644 crates/kitsune-type/src/ap/emoji.rs create mode 100644 crates/kitsune-type/src/mastodon/custom_emoji.rs create mode 100644 kitsune/src/http/handler/custom_emojis.rs create mode 100644 kitsune/src/http/handler/mastodon/api/v1/custom_emojis.rs create mode 100644 test-fixtures/corteximplant.com_emoji_7952.json create mode 100644 test-fixtures/corteximplant.com_emoji_8933.json diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 4459d3c3..f533277d 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -14,25 +14,30 @@ use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; use headers::{ContentType, HeaderMapExt}; use http::HeaderValue; +use iso8601_timestamp::Timestamp; use kitsune_cache::{ArcCache, CacheBackend}; use kitsune_db::{ model::{ account::{Account, AccountConflictChangeset, NewAccount, UpdateAccountMedia}, + custom_emoji::CustomEmoji, + media_attachment::{MediaAttachment, NewMediaAttachment}, post::Post, }, - schema::{accounts, posts}, + schema::{accounts, custom_emojis, media_attachments, posts}, PgPool, }; use kitsune_embed::Client as EmbedClient; use kitsune_http_client::Client; + use kitsune_search::SearchBackend; use kitsune_type::{ - ap::{actor::Actor, Object}, + ap::{actor::Actor, emoji::Emoji, Object}, jsonld::RdfNode, }; use mime::Mime; use scoped_futures::ScopedFutureExt; use serde::de::DeserializeOwned; +use speedy_uuid::Uuid; use typed_builder::TypedBuilder; use url::Url; @@ -175,7 +180,7 @@ impl Fetcher { let mut actor: Actor = self.fetch_ap_resource(url.as_str()).await?; - let mut domain = url.host_str().unwrap(); + let mut domain = url.host_str().ok_or(ApiError::MissingHost)?; let domain_buf; let fetch_webfinger = opts .acct @@ -203,7 +208,7 @@ impl Fetcher { }; if !used_webfinger && actor.id != url.as_str() { url = Url::parse(&actor.id)?; - domain = url.host_str().unwrap(); + domain = url.host_str().ok_or(ApiError::MissingHost)?; } actor.clean_html(); @@ -300,6 +305,88 @@ impl Fetcher { Ok(account) } + pub async fn fetch_emoji(&self, url: &str) -> Result { + let existing_emoji = self + .db_pool + .with_connection(|db_conn| { + async move { + custom_emojis::table + .filter(custom_emojis::remote_id.eq(url)) + .select(CustomEmoji::as_select()) + .first(db_conn) + .await + .optional() + } + .scoped() + }) + .await?; + + if let Some(emoji) = existing_emoji { + return Ok(emoji); + } + + let mut url = Url::parse(url)?; + if !self.federation_filter.is_url_allowed(&url)? { + return Err(ApiError::Unauthorised.into()); + } + + let emoji: Emoji = self.client.get(url.as_str()).await?.jsonld().await?; + + let mut domain = url.host_str().ok_or(ApiError::MissingHost)?; + + if emoji.id != url.as_str() { + url = Url::parse(&emoji.id)?; + domain = url.host_str().ok_or(ApiError::MissingHost)?; + } + + let content_type = emoji + .icon + .media_type + .as_deref() + .or_else(|| mime_guess::from_path(&emoji.icon.url).first_raw()) + .ok_or(ApiError::UnsupportedMediaType)?; + + let name_pure = emoji.name.replace(':', ""); + + let emoji: CustomEmoji = self + .db_pool + .with_transaction(|tx| { + async move { + let media_attachment = diesel::insert_into(media_attachments::table) + .values(NewMediaAttachment { + id: Uuid::now_v7(), + account_id: None, + content_type, + description: None, + blurhash: None, + file_path: None, + remote_url: Some(&emoji.icon.url), + }) + .returning(MediaAttachment::as_returning()) + .get_result::(tx) + .await?; + let emoji = diesel::insert_into(custom_emojis::table) + .values(CustomEmoji { + id: Uuid::now_v7(), + remote_id: emoji.id, + shortcode: name_pure.to_string(), + domain: Some(domain.to_string()), + media_attachment_id: media_attachment.id, + endorsed: false, + created_at: Timestamp::now_utc(), + updated_at: Timestamp::now_utc(), + }) + .returning(CustomEmoji::as_returning()) + .get_result::(tx) + .await?; + Ok::<_, Error>(emoji) + } + .scope_boxed() + }) + .await?; + Ok(emoji) + } + #[async_recursion] pub(super) async fn fetch_object_inner( &self, @@ -382,7 +469,10 @@ mod test { use iso8601_timestamp::Timestamp; use kitsune_cache::NoopCache; use kitsune_config::instance::FederationFilterConfiguration; - use kitsune_db::{model::account::Account, schema::accounts}; + use kitsune_db::{ + model::{account::Account, media_attachment::MediaAttachment}, + schema::{accounts, media_attachments}, + }; use kitsune_http_client::Client; use kitsune_search::NoopSearchService; use kitsune_test::{build_ap_response, database_test}; @@ -915,6 +1005,55 @@ mod test { .await; } + #[tokio::test] + #[serial_test::serial] + async fn fetch_emoji() { + database_test(|db_pool| async move { + let client = Client::builder().service(service_fn(handle)); + + let fetcher = Fetcher::builder() + .client(client.clone()) + .db_pool(db_pool.clone()) + .embed_client(None) + .federation_filter( + FederationFilterService::new(&FederationFilterConfiguration::Deny { + domains: Vec::new(), + }) + .unwrap(), + ) + .search_backend(NoopSearchService) + .webfinger(Webfinger::with_client(client, Arc::new(NoopCache.into()))) + .post_cache(Arc::new(NoopCache.into())) + .user_cache(Arc::new(NoopCache.into())) + .build(); + + let emoji = fetcher + .fetch_emoji("https://corteximplant.com/emojis/7952") + .await + .expect("Fetch emoji"); + assert_eq!(emoji.shortcode, "Blobhaj"); + assert_eq!(emoji.domain, Some(String::from("corteximplant.com"))); + + let media_attachment = db_pool + .with_connection(|db_conn| { + media_attachments::table + .find(emoji.media_attachment_id) + .select(MediaAttachment::as_select()) + .get_result::(db_conn) + .scoped() + }) + .await + .expect("Get media attachment"); + + assert_eq!( + media_attachment.remote_url, + Some(String::from( + "https://corteximplant.com/system/custom_emojis/images/000/007/952/original/33b7f12bd094b815.png" + ))); + }) + .await; + } + async fn handle(req: Request) -> Result, Infallible> { match req.uri().path_and_query().unwrap().as_str() { "/users/0x0" => { @@ -933,6 +1072,16 @@ mod test { ); Ok::<_, Infallible>(build_ap_response(body)) } + "/emojis/7952" => { + let body = + include_str!("../../../../test-fixtures/corteximplant.com_emoji_7952.json"); + Ok::<_, Infallible>(build_ap_response(body)) + } + "/emojis/8933" => { + let body = + include_str!("../../../../test-fixtures/corteximplant.com_emoji_8933.json"); + Ok::<_, Infallible>(build_ap_response(body)) + } "/.well-known/webfinger?resource=acct:0x0@corteximplant.com" => { let body = include_str!("../../../../test-fixtures/0x0_jrd.json"); Ok::<_, Infallible>(Response::new(Body::from(body))) diff --git a/crates/kitsune-core/src/activitypub/mod.rs b/crates/kitsune-core/src/activitypub/mod.rs index b7d396e7..e60a5ec1 100644 --- a/crates/kitsune-core/src/activitypub/mod.rs +++ b/crates/kitsune-core/src/activitypub/mod.rs @@ -5,17 +5,20 @@ use crate::{ }; use diesel::{ExpressionMethods, SelectableHelper}; use diesel_async::{AsyncPgConnection, RunQueryDsl}; -use futures_util::FutureExt; +use futures_util::{future::try_join_all, FutureExt, TryFutureExt}; use http::Uri; use iso8601_timestamp::Timestamp; use kitsune_db::{ model::{ account::Account, + custom_emoji::PostCustomEmoji, media_attachment::{NewMediaAttachment, NewPostMediaAttachment}, mention::NewMention, post::{FullPostChangeset, NewPost, Post, PostConflictChangeset, Visibility}, }, - schema::{media_attachments, posts, posts_media_attachments, posts_mentions}, + schema::{ + media_attachments, posts, posts_custom_emojis, posts_media_attachments, posts_mentions, + }, PgPool, }; use kitsune_embed::Client as EmbedClient; @@ -64,6 +67,43 @@ async fn handle_mentions( Ok(()) } +async fn handle_custom_emojis( + db_conn: &mut AsyncPgConnection, + post_id: Uuid, + fetcher: &Fetcher, + tags: &[Tag], +) -> Result<()> { + let emoji_iter = tags.iter().filter(|tag| tag.r#type == TagType::Emoji); + + let emoji_count = emoji_iter.clone().count(); + if emoji_count == 0 { + return Ok(()); + } + + let futures = emoji_iter.clone().filter_map(|emoji| { + let remote_id = emoji.id.as_ref()?; + Some(fetcher.fetch_emoji(remote_id).map_ok(move |f| (f, emoji))) + }); + + let emojis = try_join_all(futures) + .await? + .iter() + .map(|(resolved_emoji, emoji_tag)| PostCustomEmoji { + post_id, + custom_emoji_id: resolved_emoji.id, + emoji_text: emoji_tag.name.to_string(), + }) + .collect::>(); + + diesel::insert_into(posts_custom_emojis::table) + .values(emojis) + .on_conflict_do_nothing() + .execute(db_conn) + .await?; + + Ok(()) +} + /// Process a bunch of ActivityPub attachments /// /// # Returns @@ -92,7 +132,7 @@ pub async fn process_attachments( Some(NewMediaAttachment { id: attachment_id, - account_id: author.id, + account_id: Some(author.id), content_type, description: attachment.name.as_deref(), blurhash: attachment.blurhash.as_deref(), @@ -130,6 +170,7 @@ struct PreprocessedObject<'a> { content_lang: Language, db_pool: &'a PgPool, object: Box, + fetcher: &'a Fetcher, search_backend: &'a AnySearchBackend, } @@ -201,6 +242,7 @@ async fn preprocess_object( content_lang, db_pool, object, + fetcher, search_backend, }) } @@ -215,6 +257,7 @@ pub async fn process_new_object(process_data: ProcessNewObject<'_>) -> Result) -> Result(new_post) } @@ -287,6 +331,7 @@ pub async fn update_object(process_data: ProcessNewObject<'_>) -> Result { content_lang, db_pool, object, + fetcher: _, search_backend, } = preprocess_object(process_data).await?; diff --git a/crates/kitsune-core/src/consts.rs b/crates/kitsune-core/src/consts.rs index 2720102f..2f97a475 100644 --- a/crates/kitsune-core/src/consts.rs +++ b/crates/kitsune-core/src/consts.rs @@ -1,6 +1,7 @@ use const_format::concatcp; pub const API_MAX_LIMIT: usize = 40; +pub const MAX_EMOJI_SHORTCODE_LENGTH: usize = 64; pub const MAX_MEDIA_DESCRIPTION_LENGTH: usize = 5000; pub const USER_AGENT: &str = concatcp!(env!("CARGO_PKG_NAME"), "/", VERSION); pub const VERSION: &str = concatcp!(env!("CARGO_PKG_VERSION"), "-", env!("VERGEN_GIT_SHA")); diff --git a/crates/kitsune-core/src/error.rs b/crates/kitsune-core/src/error.rs index 19106088..769e5966 100644 --- a/crates/kitsune-core/src/error.rs +++ b/crates/kitsune-core/src/error.rs @@ -20,6 +20,9 @@ pub enum ApiError { #[error("Invalid captcha")] InvalidCaptcha, + #[error("Missing host")] + MissingHost, + #[error("Not found")] NotFound, diff --git a/crates/kitsune-core/src/lib.rs b/crates/kitsune-core/src/lib.rs index 6d0ece28..f35dcf11 100644 --- a/crates/kitsune-core/src/lib.rs +++ b/crates/kitsune-core/src/lib.rs @@ -56,7 +56,9 @@ use kitsune_search::{AnySearchBackend, NoopSearchService, SqlSearchService}; use kitsune_storage::{fs::Storage as FsStorage, s3::Storage as S3Storage, AnyStorageBackend}; use rusty_s3::{Bucket as S3Bucket, Credentials as S3Credentials}; use serde::{de::DeserializeOwned, Serialize}; +use service::custom_emoji::CustomEmojiService; use service::search::SearchService; + use std::{ fmt::Display, str::FromStr, @@ -272,6 +274,12 @@ pub async fn prepare_state( let captcha_backend = config.captcha.as_ref().map(prepare_captcha); let captcha_service = CaptchaService::builder().backend(captcha_backend).build(); + let custom_emoji_service = CustomEmojiService::builder() + .attachment_service(attachment_service.clone()) + .db_pool(db_pool.clone()) + .url_service(url_service.clone()) + .build(); + let instance_service = InstanceService::builder() .db_pool(db_pool.clone()) .name(config.instance.name.as_str()) @@ -292,6 +300,7 @@ pub async fn prepare_state( let post_resolver = PostResolver::builder() .account(account_service.clone()) + .custom_emoji(custom_emoji_service.clone()) .build(); let post_service = PostService::builder() @@ -344,6 +353,7 @@ pub async fn prepare_state( service: Service { account: account_service, captcha: captcha_service, + custom_emoji: custom_emoji_service, federation_filter: federation_filter_service, instance: instance_service, job: job_service, diff --git a/crates/kitsune-core/src/mapping/activitypub/object.rs b/crates/kitsune-core/src/mapping/activitypub/object.rs index 11b4f642..b351a0c5 100644 --- a/crates/kitsune-core/src/mapping/activitypub/object.rs +++ b/crates/kitsune-core/src/mapping/activitypub/object.rs @@ -4,21 +4,23 @@ use crate::{ try_join, util::BaseToCc, }; -use diesel::{BelongingToDsl, QueryDsl, SelectableHelper}; +use diesel::{BelongingToDsl, ExpressionMethods, QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; use futures_util::{future::OptionFuture, FutureExt, TryFutureExt, TryStreamExt}; use kitsune_db::{ model::{ account::Account, + custom_emoji::{CustomEmoji, PostCustomEmoji}, media_attachment::{MediaAttachment as DbMediaAttachment, PostMediaAttachment}, mention::Mention, post::Post, }, - schema::{accounts, media_attachments, posts}, + schema::{accounts, custom_emojis, media_attachments, posts, posts_custom_emojis}, }; use kitsune_type::ap::{ actor::{Actor, PublicKey}, ap_context, + emoji::Emoji, object::{MediaAttachment, MediaAttachmentType}, AttributedToField, Object, ObjectType, Tag, TagType, }; @@ -56,6 +58,42 @@ impl IntoObject for DbMediaAttachment { } } +fn build_post_tags( + mentions: Vec<(Mention, Account)>, + to: &mut Vec, + emojis: Vec<(CustomEmoji, PostCustomEmoji, DbMediaAttachment)>, +) -> Vec { + let mut tag = Vec::new(); + for (mention, mentioned) in mentions { + to.push(mentioned.url.clone()); + tag.push(Tag { + id: None, + r#type: TagType::Mention, + name: mention.mention_text, + href: Some(mentioned.url), + icon: None, + }); + } + for (custom_emoji, post_emoji, attachment) in emojis { + if let Some(attachment_url) = attachment.remote_url { + tag.push(Tag { + id: Some(custom_emoji.remote_id), + r#type: TagType::Emoji, + name: post_emoji.emoji_text, + href: None, + icon: Some(MediaAttachment { + r#type: MediaAttachmentType::Image, + name: None, + media_type: Some(attachment.content_type), + blurhash: None, + url: attachment_url, + }), + }); + } + } + tag +} + impl IntoObject for Post { type Output = Object; @@ -67,7 +105,7 @@ impl IntoObject for Post { return Err(ApiError::NotFound.into()); } - let (account, in_reply_to, mentions, attachment_stream) = state + let (account, in_reply_to, mentions, emojis, attachment_stream) = state .db_pool .with_connection(|db_conn| { async { @@ -90,6 +128,17 @@ impl IntoObject for Post { .select((Mention::as_select(), Account::as_select())) .load::<(Mention, Account)>(db_conn); + let custom_emojis_fut = custom_emojis::table + .inner_join(posts_custom_emojis::table) + .inner_join(media_attachments::table) + .filter(posts_custom_emojis::post_id.eq(self.id)) + .select(( + CustomEmoji::as_select(), + PostCustomEmoji::as_select(), + DbMediaAttachment::as_select(), + )) + .load::<(CustomEmoji, PostCustomEmoji, DbMediaAttachment)>(db_conn); + let attachment_stream_fut = PostMediaAttachment::belonging_to(&self) .inner_join(media_attachments::table) .select(DbMediaAttachment::as_select()) @@ -99,6 +148,7 @@ impl IntoObject for Post { account_fut, in_reply_to_fut, mentions_fut, + custom_emojis_fut, attachment_stream_fut ) } @@ -122,17 +172,9 @@ impl IntoObject for Post { .try_collect() .await?; - let mut tag = Vec::new(); let (mut to, cc) = self.visibility.base_to_cc(state, &account); - for (mention, mentioned) in mentions { - to.push(mentioned.url.clone()); - tag.push(Tag { - r#type: TagType::Mention, - name: mention.mention_text, - href: Some(mentioned.url), - icon: None, - }); - } + let tag = build_post_tags(mentions, &mut to, emojis); + let account_url = state.service.url.user_url(account.id); Ok(Object { @@ -219,3 +261,36 @@ impl IntoObject for Account { }) } } + +impl IntoObject for CustomEmoji { + type Output = Emoji; + + async fn into_object(self, state: &State) -> Result { + // Officially we don't have any info about remote emojis as we're not the origin + // Let's pretend we're not home and do not answer + let name = match self.domain { + None => Ok(format!(":{}:", self.shortcode)), + Some(_) => Err(ApiError::NotFound), + }?; + let icon = state + .db_pool + .with_connection(|db_conn| { + media_attachments::table + .find(self.media_attachment_id) + .get_result::(db_conn) + .map_err(Error::from) + .and_then(|media_attachment| media_attachment.into_object(state)) + .scoped() + }) + .await?; + + Ok(Emoji { + context: ap_context(), + id: self.remote_id, + r#type: String::from("Emoji"), + name, + icon, + updated: self.updated_at, + }) + } +} diff --git a/crates/kitsune-core/src/mapping/mastodon/sealed.rs b/crates/kitsune-core/src/mapping/mastodon/sealed.rs index 15588030..d990a8d3 100644 --- a/crates/kitsune-core/src/mapping/mastodon/sealed.rs +++ b/crates/kitsune-core/src/mapping/mastodon/sealed.rs @@ -9,9 +9,11 @@ use diesel::{ }; use diesel_async::RunQueryDsl; use futures_util::{future::OptionFuture, FutureExt, TryFutureExt, TryStreamExt}; +use iso8601_timestamp::Timestamp; use kitsune_db::{ model::{ account::Account as DbAccount, + custom_emoji::{CustomEmoji as DbCustomEmoji, PostCustomEmoji as DbPostCustomEmoji}, favourite::Favourite as DbFavourite, follower::Follow, link_preview::LinkPreview, @@ -23,7 +25,8 @@ use kitsune_db::{ post::{Post as DbPost, PostSource}, }, schema::{ - accounts, accounts_follows, media_attachments, notifications, posts, posts_favourites, + accounts, accounts_follows, custom_emojis, media_attachments, notifications, posts, + posts_favourites, }, PgPool, }; @@ -35,7 +38,7 @@ use kitsune_type::mastodon::{ preview_card::PreviewType, relationship::Relationship, status::{Mention, StatusSource}, - Account, MediaAttachment, Notification, PreviewCard, Status, + Account, CustomEmoji, MediaAttachment, Notification, PreviewCard, Status, }; use mime::Mime; use scoped_futures::ScopedFutureExt; @@ -332,56 +335,69 @@ impl IntoMastodon for DbPost { state: MapperState<'_>, ) -> impl Future> + Send { async move { - let (account, reblog_count, favourites_count, media_attachments, mentions_stream) = - state - .db_pool - .with_connection(|db_conn| { - async { - let account_fut = accounts::table - .find(self.account_id) - .select(DbAccount::as_select()) - .get_result::(db_conn) - .map_err(Error::from) - .and_then(|db_account| db_account.into_mastodon(state)); + let ( + account, + reblog_count, + favourites_count, + media_attachments, + mentions_stream, + custom_emojis_stream, + ) = state + .db_pool + .with_connection(|db_conn| { + async { + let account_fut = accounts::table + .find(self.account_id) + .select(DbAccount::as_select()) + .get_result::(db_conn) + .map_err(Error::from) + .and_then(|db_account| db_account.into_mastodon(state)); - let reblog_count_fut = posts::table - .filter(posts::reposted_post_id.eq(self.id)) - .count() - .get_result::(db_conn) - .map_err(Error::from); + let reblog_count_fut = posts::table + .filter(posts::reposted_post_id.eq(self.id)) + .count() + .get_result::(db_conn) + .map_err(Error::from); - let favourites_count_fut = DbFavourite::belonging_to(&self) - .count() - .get_result::(db_conn) - .map_err(Error::from); + let favourites_count_fut = DbFavourite::belonging_to(&self) + .count() + .get_result::(db_conn) + .map_err(Error::from); - let media_attachments_fut = DbPostMediaAttachment::belonging_to(&self) - .inner_join(media_attachments::table) - .select(DbMediaAttachment::as_select()) - .load_stream::(db_conn) - .map_err(Error::from) - .and_then(|attachment_stream| { - attachment_stream - .map_err(Error::from) - .and_then(|attachment| attachment.into_mastodon(state)) - .try_collect() - }); + let media_attachments_fut = DbPostMediaAttachment::belonging_to(&self) + .inner_join(media_attachments::table) + .select(DbMediaAttachment::as_select()) + .load_stream::(db_conn) + .map_err(Error::from) + .and_then(|attachment_stream| { + attachment_stream + .map_err(Error::from) + .and_then(|attachment| attachment.into_mastodon(state)) + .try_collect() + }); - let mentions_stream_fut = DbMention::belonging_to(&self) - .load_stream::(db_conn) - .map_err(Error::from); + let mentions_stream_fut = DbMention::belonging_to(&self) + .load_stream::(db_conn) + .map_err(Error::from); - try_join!( - account_fut, - reblog_count_fut, - favourites_count_fut, - media_attachments_fut, - mentions_stream_fut, - ) - } - .scoped() - }) - .await?; + let custom_emojis_stream_fut = DbPostCustomEmoji::belonging_to(&self) + .inner_join(custom_emojis::table.inner_join(media_attachments::table)) + .select((DbCustomEmoji::as_select(), DbMediaAttachment::as_select())) + .load_stream::<(DbCustomEmoji, DbMediaAttachment)>(db_conn) + .map_err(Error::from); + + try_join!( + account_fut, + reblog_count_fut, + favourites_count_fut, + media_attachments_fut, + mentions_stream_fut, + custom_emojis_stream_fut + ) + } + .scoped() + }) + .await?; let link_preview = OptionFuture::from( self.link_preview_url @@ -402,6 +418,12 @@ impl IntoMastodon for DbPost { .try_collect() .await?; + let emojis = custom_emojis_stream + .map_err(Error::from) + .and_then(|(emoji, attachment)| (emoji, attachment, None).into_mastodon(state)) + .try_collect() + .await?; + let reblog = state .db_pool .with_connection(|db_conn| { @@ -447,6 +469,7 @@ impl IntoMastodon for DbPost { account, media_attachments, mentions, + emojis, reblog, favourited: false, reblogged: false, @@ -588,3 +611,37 @@ impl IntoMastodon for DbNotification { }) } } + +impl IntoMastodon for (DbCustomEmoji, DbMediaAttachment, Option) { + type Output = CustomEmoji; + + fn id(&self) -> Option { + Some(self.0.id) + } + + async fn into_mastodon(self, state: MapperState<'_>) -> Result { + let (emoji, attachment, last_used) = self; + let shortcode = if let Some(ref domain) = emoji.domain { + format!(":{}@{}:", emoji.shortcode, domain) + } else { + format!(":{}:", emoji.shortcode) + }; + let url = state.url_service.media_url(attachment.id); + let category = if last_used.is_some() { + Some(String::from("recently used")) + } else if emoji.endorsed { + Some(String::from("endorsed")) + } else if emoji.domain.is_none() { + Some(String::from("local")) + } else { + Some(emoji.domain.unwrap()) + }; + Ok(CustomEmoji { + shortcode, + url: url.clone(), + static_url: url, + visible_in_picker: true, + category, + }) + } +} diff --git a/crates/kitsune-core/src/resolve/post.rs b/crates/kitsune-core/src/resolve/post.rs index 21fc7489..c1a2d3aa 100644 --- a/crates/kitsune-core/src/resolve/post.rs +++ b/crates/kitsune-core/src/resolve/post.rs @@ -1,6 +1,9 @@ use crate::{ error::{Error, Result}, - service::account::{AccountService, GetUser}, + service::{ + account::{AccountService, GetUser}, + custom_emoji::{CustomEmojiService, GetEmoji}, + }, }; use post_process::{BoxError, Element, Html, Render}; use speedy_uuid::Uuid; @@ -10,6 +13,13 @@ use typed_builder::TypedBuilder; #[derive(Clone, TypedBuilder)] pub struct PostResolver { account: AccountService, + custom_emoji: CustomEmojiService, +} + +pub struct ResolvedPost { + pub mentioned_accounts: Vec<(Uuid, String)>, + pub custom_emojis: Vec<(Uuid, String)>, + pub content: String, } impl PostResolver { @@ -17,6 +27,7 @@ impl PostResolver { &self, element: Element<'a>, mentioned_accounts: mpsc::Sender<(Uuid, String)>, + custom_emojis: mpsc::Sender<(Uuid, String)>, ) -> Result, BoxError> { let element = match element { Element::Mention(mention) => { @@ -47,6 +58,19 @@ impl PostResolver { attributes: vec![(Cow::Borrowed("href"), link.content.clone())], content: Box::new(Element::Link(link)), }), + Element::Emote(emote) => { + let get_emoji = GetEmoji::builder() + .shortcode(&emote.shortcode) + .domain(emote.domain.as_deref()) + .build(); + + if let Some(emoji) = self.custom_emoji.get(get_emoji).await? { + let mut emoji_text = String::new(); + Element::Emote(emote.clone()).render(&mut emoji_text); + let _ = custom_emojis.send((emoji.id, emoji_text)); + } + Element::Emote(emote) + } elem => elem, }; @@ -60,16 +84,25 @@ impl PostResolver { /// - List of mentioned accounts, represented as `(Account ID, Mention text)` /// - Content with the mentions replaced by links #[instrument(skip_all)] - pub async fn resolve(&self, content: &str) -> Result<(Vec<(Uuid, String)>, String)> { + pub async fn resolve(&self, content: &str) -> Result { let (mentioned_account_ids_acc, mentioned_account_ids) = mpsc::channel(); + let (custom_emoji_ids_sen, custom_emoji_ids_rec) = mpsc::channel(); let content = post_process::transform(content, |elem| { - self.transform(elem, mentioned_account_ids_acc.clone()) + self.transform( + elem, + mentioned_account_ids_acc.clone(), + custom_emoji_ids_sen.clone(), + ) }) .await .map_err(Error::PostProcessing)?; - Ok((mentioned_account_ids.try_iter().collect(), content)) + Ok(ResolvedPost { + mentioned_accounts: mentioned_account_ids.try_iter().collect(), + custom_emojis: custom_emoji_ids_rec.try_iter().collect(), + content, + }) } } @@ -81,8 +114,10 @@ mod test { job::KitsuneContextRepo, service::{ account::AccountService, attachment::AttachmentService, - federation_filter::FederationFilterService, job::JobService, url::UrlService, + custom_emoji::CustomEmojiService, federation_filter::FederationFilterService, + job::JobService, url::UrlService, }, + try_join, webfinger::Webfinger, }; use athena::JobQueue; @@ -90,24 +125,31 @@ mod test { use diesel::{QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; use hyper::{Body, Request, Response}; + use iso8601_timestamp::Timestamp; use kitsune_cache::NoopCache; use kitsune_config::instance::FederationFilterConfiguration; - use kitsune_db::{model::account::Account, schema::accounts}; + use kitsune_db::{ + model::{ + account::Account, custom_emoji::CustomEmoji, media_attachment::NewMediaAttachment, + }, + schema::{accounts, custom_emojis, media_attachments}, + }; use kitsune_http_client::Client; use kitsune_search::NoopSearchService; use kitsune_storage::fs::Storage as FsStorage; use kitsune_test::{build_ap_response, database_test, redis_test}; use pretty_assertions::assert_eq; use scoped_futures::ScopedFutureExt; + use speedy_uuid::Uuid; use std::sync::Arc; use tower::service_fn; #[tokio::test] #[serial_test::serial] - async fn parse_mentions() { + async fn parse_post() { redis_test(|redis_pool| async move { database_test(|db_pool| async move { - let post = "Hello @0x0@corteximplant.com! How are you doing?"; + let post = "Hello @0x0@corteximplant.com! How are you doing? :blobhaj_happy: :blobhaj_sad@example.com:"; let client = service_fn(|req: Request<_>| async move { match req.uri().path_and_query().unwrap().as_str() { @@ -158,13 +200,13 @@ mod test { let attachment_service = AttachmentService::builder() .db_pool(db_pool.clone()) - .media_proxy_enabled(false) + .media_proxy_enabled(true) .storage_backend(FsStorage::new("uploads".into())) .url_service(url_service.clone()) .build(); let account_service = AccountService::builder() - .attachment_service(attachment_service) + .attachment_service(attachment_service.clone()) .db_pool(db_pool.clone()) .fetcher(fetcher) .job_service(job_service) @@ -172,19 +214,93 @@ mod test { .webfinger(webfinger) .build(); - let mention_resolver = PostResolver::builder() - .account(account_service) + let custom_emoji_service = CustomEmojiService::builder() + .attachment_service(attachment_service.clone()) + .db_pool(db_pool.clone()) + .url_service(url_service.clone()) .build(); - let (mentioned_account_ids, content) = mention_resolver + let emoji_ids = (Uuid::now_v7(), Uuid::now_v7()); + let media_attachment_ids = (Uuid::now_v7(), Uuid::now_v7()); + db_pool + .with_connection(|db_conn| { + async { + let media_fut = diesel::insert_into(media_attachments::table) + .values(NewMediaAttachment { + id: media_attachment_ids.0, + content_type: "image/jpeg", + account_id: None, + description: None, + blurhash: None, + file_path: None, + remote_url: None, + }) + .execute(db_conn); + let emoji_fut = diesel::insert_into(custom_emojis::table) + .values(CustomEmoji { + id: emoji_ids.0, + shortcode: String::from("blobhaj_happy"), + domain: None, + remote_id: String::from("https://local.domain/emoji/blobhaj_happy"), + media_attachment_id: media_attachment_ids.0, + endorsed: false, + created_at: Timestamp::now_utc(), + updated_at: Timestamp::now_utc() + }) + .execute(db_conn); + try_join!(media_fut, emoji_fut) + }.scoped() + }) + .await + .expect("Failed to insert the local emoji"); + + db_pool + .with_connection(|db_conn| { + async { + let media_fut = diesel::insert_into(media_attachments::table) + .values(NewMediaAttachment { + id: media_attachment_ids.1, + content_type: "image/jpeg", + account_id: None, + description: None, + blurhash: None, + file_path: None, + remote_url: Some("https://media.example.com/emojis/blobhaj.jpeg"), + }) + .execute(db_conn); + let emoji_fut = diesel::insert_into(custom_emojis::table) + .values(CustomEmoji { + id: emoji_ids.1, + shortcode: String::from("blobhaj_sad"), + domain: Some(String::from("example.com")), + remote_id: String::from("https://example.com/emojis/1"), + media_attachment_id: media_attachment_ids.1, + endorsed: false, + created_at: Timestamp::now_utc(), + updated_at: Timestamp::now_utc(), + }) + .execute(db_conn); + try_join!(media_fut, emoji_fut) + }.scoped() + }) + .await + .expect("Failed to insert the remote emoji"); + + let post_resolver = PostResolver::builder() + .account(account_service) + .custom_emoji(custom_emoji_service) + .build(); + + let resolved = post_resolver .resolve(post) .await - .expect("Failed to resolve mentions"); + .expect("Failed to resolve the post"); - assert_eq!(content, "Hello @0x0@corteximplant.com! How are you doing?"); - assert_eq!(mentioned_account_ids.len(), 1); + assert_eq!(resolved.content, "Hello @0x0@corteximplant.com! How are you doing? :blobhaj_happy: :blobhaj_sad__example_com:"); + assert_eq!(resolved.mentioned_accounts.len(), 1); + assert_eq!(resolved.custom_emojis.len(), 2); - let (account_id, _mention_text) = &mentioned_account_ids[0]; + let (account_id, _mention_text) = &resolved.mentioned_accounts[0]; let mentioned_account = db_pool .with_connection(|db_conn| { accounts::table @@ -202,6 +318,9 @@ mod test { mentioned_account.url, "https://corteximplant.com/users/0x0" ); + + assert_eq!(resolved.custom_emojis[0], (emoji_ids.1, String::from(":blobhaj_sad__example_com:"))); + assert_eq!(resolved.custom_emojis[1], (emoji_ids.0, String::from(":blobhaj_happy:"))); }).await; }).await; } diff --git a/crates/kitsune-core/src/service/attachment.rs b/crates/kitsune-core/src/service/attachment.rs index 9c7a0eb3..0e3c0134 100644 --- a/crates/kitsune-core/src/service/attachment.rs +++ b/crates/kitsune-core/src/service/attachment.rs @@ -54,8 +54,9 @@ pub struct Update { #[derive(Builder, Validate)] #[builder(pattern = "owned")] pub struct Upload { + #[builder(default, setter(strip_option))] #[garde(skip)] - account_id: Uuid, + account_id: Option, #[garde(custom(is_allowed_filetype))] content_type: String, #[builder(default, setter(strip_option))] @@ -115,10 +116,10 @@ impl AttachmentService { pub async fn get_url(&self, id: Uuid) -> Result { let media_attachment = self.get_by_id(id).await?; if self.media_proxy_enabled || media_attachment.file_path.is_some() { - return Ok(self.url_service.media_url(id)); + return Ok(self.url_service.media_url(media_attachment.id)); } - Ok(media_attachment.remote_url.unwrap()) + Ok(media_attachment.remote_url.as_ref().unwrap().to_string()) } /// Return a stream that yields the file's contents @@ -224,8 +225,8 @@ impl AttachmentService { diesel::insert_into(media_attachments::table) .values(NewMediaAttachment { id: Uuid::now_v7(), - account_id: upload.account_id, content_type: upload.content_type.as_str(), + account_id: upload.account_id, description: upload.description.as_deref(), blurhash: None, file_path: Some(upload.path.as_str()), @@ -324,7 +325,7 @@ mod test { let attachment = MediaAttachment { id: Uuid::now_v7(), - account_id, + account_id: Some(account_id), content_type: String::from("image/jpeg"), description: None, blurhash: None, diff --git a/crates/kitsune-core/src/service/custom_emoji.rs b/crates/kitsune-core/src/service/custom_emoji.rs new file mode 100644 index 00000000..d5731737 --- /dev/null +++ b/crates/kitsune-core/src/service/custom_emoji.rs @@ -0,0 +1,188 @@ +use crate::{ + consts::MAX_EMOJI_SHORTCODE_LENGTH, + error::{BoxError, Error, Result}, +}; + +use bytes::Bytes; +use diesel::{ + BoolExpressionMethods, ExpressionMethods, JoinOnDsl, NullableExpressionMethods, + OptionalExtension, QueryDsl, SelectableHelper, +}; +use diesel_async::RunQueryDsl; +use futures_util::{Stream, TryStreamExt}; +use garde::Validate; +use iso8601_timestamp::Timestamp; +use kitsune_db::{ + model::{custom_emoji::CustomEmoji, media_attachment::MediaAttachment}, + schema::{custom_emojis, media_attachments, posts, posts_custom_emojis}, + PgPool, +}; +use scoped_futures::ScopedFutureExt; +use speedy_uuid::Uuid; +use typed_builder::TypedBuilder; + +use super::{ + attachment::{AttachmentService, Upload}, + url::UrlService, +}; + +const ALLOWED_FILETYPES: &[mime::Name<'_>] = &[mime::IMAGE]; + +#[allow(clippy::trivially_copy_pass_by_ref)] +fn is_allowed_filetype(value: &str, _ctx: &()) -> garde::Result { + let content_type: mime::Mime = value + .parse() + .map_err(|err: mime::FromStrError| garde::Error::new(err.to_string()))?; + + if !ALLOWED_FILETYPES.contains(&content_type.type_()) { + return Err(garde::Error::new("Invalid file type")); + } + + Ok(()) +} + +#[derive(TypedBuilder)] +pub struct GetEmoji<'a> { + shortcode: &'a str, + #[builder(default)] + domain: Option<&'a str>, +} + +#[derive(TypedBuilder)] +pub struct GetEmojiList { + #[builder(default)] + fetching_account_id: Option, + #[builder(default = 5000)] + limit: i64, +} + +#[derive(TypedBuilder, Validate)] +pub struct EmojiUpload { + #[garde(custom(is_allowed_filetype))] + content_type: String, + #[garde(length(max = MAX_EMOJI_SHORTCODE_LENGTH))] + #[garde(pattern("^([a-zA-Z0-9]_?)*[a-zA-Z0-9]$"))] + shortcode: String, + #[garde(skip)] + stream: S, +} + +#[derive(Clone, TypedBuilder)] +pub struct CustomEmojiService { + attachment_service: AttachmentService, + db_pool: PgPool, + url_service: UrlService, +} + +impl CustomEmojiService { + pub async fn get(&self, get_emoji: GetEmoji<'_>) -> Result> { + let mut query = custom_emojis::table + .filter(custom_emojis::shortcode.eq(get_emoji.shortcode)) + .inner_join(media_attachments::table) + .select(CustomEmoji::as_select()) + .into_boxed(); + + if let Some(domain) = get_emoji.domain { + query = query.filter(custom_emojis::domain.eq(domain)); + } + + self.db_pool + .with_connection(|db_conn| { + async move { query.first(db_conn).await.optional() }.scoped() + }) + .await + .map_err(Error::from) + } + + pub async fn get_by_id(&self, id: Uuid) -> Result { + let query = custom_emojis::table + .find(id) + .select(CustomEmoji::as_select()); + + self.db_pool + .with_connection(|db_conn| async move { query.get_result(db_conn).await }.scoped()) + .await + .map_err(Error::from) + } + + pub async fn get_list( + &self, + get_emoji_list: GetEmojiList, + ) -> Result)>> + '_> + { + let query = custom_emojis::table + .left_join( + posts_custom_emojis::table.inner_join( + posts::table.on(posts::account_id + .nullable() + .eq(get_emoji_list.fetching_account_id)), + ), + ) + .inner_join(media_attachments::table) + .filter( + posts::account_id.is_null().or(posts::account_id + .nullable() + .eq(get_emoji_list.fetching_account_id)), + ) + .filter( + custom_emojis::endorsed + .eq(true) + .or(custom_emojis::domain.is_null()) + .or(posts::created_at.is_not_null()), + ) + .distinct_on(custom_emojis::id) + .select(( + CustomEmoji::as_select(), + MediaAttachment::as_select(), + posts::created_at.nullable(), + )) + .limit(get_emoji_list.limit); + + self.db_pool + .with_connection(|db_conn| { + async move { Ok::<_, Error>(query.load_stream(db_conn).await?.map_err(Error::from)) } + .scoped() + }) + .await + .map_err(Error::from) + } + + pub async fn add_emoji(&self, emoji_upload: EmojiUpload) -> Result + where + S: Stream> + Send + 'static, + { + emoji_upload.validate(&())?; + + let attachment_upload = Upload::builder() + .content_type(emoji_upload.content_type) + .stream(emoji_upload.stream) + .build() + .unwrap(); + + let attachment = self.attachment_service.upload(attachment_upload).await?; + + let id = Uuid::now_v7(); + let remote_id = self.url_service.custom_emoji_url(id); + + let custom_emoji = self + .db_pool + .with_connection(|db_conn| { + diesel::insert_into(custom_emojis::table) + .values(CustomEmoji { + id, + remote_id, + shortcode: emoji_upload.shortcode, + domain: None, + media_attachment_id: attachment.id, + endorsed: false, + created_at: Timestamp::now_utc(), + updated_at: Timestamp::now_utc(), + }) + .get_result(db_conn) + .scoped() + }) + .await?; + + Ok(custom_emoji) + } +} diff --git a/crates/kitsune-core/src/service/mod.rs b/crates/kitsune-core/src/service/mod.rs index e3f0c982..799df296 100644 --- a/crates/kitsune-core/src/service/mod.rs +++ b/crates/kitsune-core/src/service/mod.rs @@ -3,6 +3,7 @@ use crate::consts::API_MAX_LIMIT; pub mod account; pub mod attachment; pub mod captcha; +pub mod custom_emoji; pub mod federation_filter; pub mod instance; pub mod job; diff --git a/crates/kitsune-core/src/service/post.rs b/crates/kitsune-core/src/service/post.rs index 250ed419..23f1523d 100644 --- a/crates/kitsune-core/src/service/post.rs +++ b/crates/kitsune-core/src/service/post.rs @@ -31,6 +31,7 @@ use iso8601_timestamp::Timestamp; use kitsune_db::{ model::{ account::Account, + custom_emoji::PostCustomEmoji, favourite::{Favourite, NewFavourite}, media_attachment::NewPostMediaAttachment, mention::NewMention, @@ -40,8 +41,9 @@ use kitsune_db::{ }, post_permission_check::{PermissionCheck, PostPermissionCheckExt}, schema::{ - accounts, accounts_preferences, media_attachments, notifications, posts, posts_favourites, - posts_media_attachments, posts_mentions, users_roles, + accounts, accounts_preferences, media_attachments, notifications, posts, + posts_custom_emojis, posts_favourites, posts_media_attachments, posts_mentions, + users_roles, }, PgPool, }; @@ -402,11 +404,39 @@ impl PostService { Ok(()) } + async fn process_custom_emojis( + conn: &mut AsyncPgConnection, + post_id: Uuid, + custom_emojis: Vec<(Uuid, String)>, + ) -> Result<()> { + if custom_emojis.is_empty() { + return Ok(()); + } + + diesel::insert_into(posts_custom_emojis::table) + .values( + custom_emojis + .iter() + .map(|(emoji_id, emoji_text)| PostCustomEmoji { + post_id, + custom_emoji_id: *emoji_id, + emoji_text: emoji_text.to_string(), + }) + .collect::>(), + ) + .on_conflict_do_nothing() + .execute(conn) + .await?; + + Ok(()) + } + /// Create a new post and deliver it to the followers /// /// # Panics /// /// This should never ever panic. If it does, create a bug report. + #[allow(clippy::too_many_lines)] pub async fn create(&self, create_post: CreatePost) -> Result { create_post.validate(&PostValidationContext { character_limit: self.instance_service.character_limit(), @@ -432,7 +462,7 @@ impl PostService { |lang| Language::from_639_1(&lang).unwrap_or_else(|| detect_language(&content)), ); - let (mentioned_account_ids, content) = self.post_resolver.resolve(&content).await?; + let resolved = self.post_resolver.resolve(&content).await?; let link_preview_url = if let Some(ref embed_client) = self.embed_client { embed_client .fetch_embed_for_fragment(&content) @@ -468,7 +498,7 @@ impl PostService { in_reply_to_id, reposted_post_id: None, subject: subject.as_deref(), - content: content.as_str(), + content: resolved.content.as_str(), content_source: content_source.as_str(), content_lang: content_lang.into(), link_preview_url: link_preview_url.as_deref(), @@ -482,8 +512,14 @@ impl PostService { .get_result(tx) .await?; - Self::process_mentions(tx, post.account_id, post.id, mentioned_account_ids) - .await?; + Self::process_mentions( + tx, + post.account_id, + post.id, + resolved.mentioned_accounts, + ) + .await?; + Self::process_custom_emojis(tx, post.id, resolved.custom_emojis).await?; Self::process_media_attachments(tx, post.id, &create_post.media_ids).await?; NotificationService::notify_on_new_post(tx, post.account_id, post.id).await?; @@ -591,12 +627,16 @@ impl PostService { .map(|c| kitsune_language::detect_language(DetectionBackend::default(), c)), }; - let (mentioned_account_ids, content) = match content.as_ref() { + let (mentioned_account_ids, custom_emojis, content) = match content.as_ref() { Some(content) => { let resolved = self.post_resolver.resolve(content).await?; - (resolved.0, Some(resolved.1)) + ( + resolved.mentioned_accounts, + resolved.custom_emojis, + Some(resolved.content), + ) } - None => (Vec::new(), None), + None => (Vec::new(), Vec::new(), None), }; let link_preview_url = if let (Some(embed_client), Some(content)) = @@ -631,6 +671,7 @@ impl PostService { Self::process_mentions(tx, post.account_id, post.id, mentioned_account_ids) .await?; + Self::process_custom_emojis(tx, post.id, custom_emojis).await?; Self::process_media_attachments(tx, post.id, &update_post.media_ids).await?; NotificationService::notify_on_update_post(tx, post.account_id, post.id) .await?; diff --git a/crates/kitsune-core/src/service/url.rs b/crates/kitsune-core/src/service/url.rs index 07277836..349f5fa7 100644 --- a/crates/kitsune-core/src/service/url.rs +++ b/crates/kitsune-core/src/service/url.rs @@ -32,6 +32,11 @@ impl UrlService { format!("{}/confirm-account/{token}", self.base_url()) } + #[must_use] + pub fn custom_emoji_url(&self, custom_emoji_id: Uuid) -> String { + format!("{}/emojis/{}", self.base_url(), custom_emoji_id) + } + #[must_use] pub fn default_avatar_url(&self) -> String { format!("{}/public/assets/default-avatar.png", self.base_url()) diff --git a/crates/kitsune-core/src/state.rs b/crates/kitsune-core/src/state.rs index 926d4396..94caf4d0 100644 --- a/crates/kitsune-core/src/state.rs +++ b/crates/kitsune-core/src/state.rs @@ -3,9 +3,10 @@ use crate::{ event::PostEventEmitter, service::{ account::AccountService, attachment::AttachmentService, captcha::CaptchaService, - federation_filter::FederationFilterService, instance::InstanceService, job::JobService, - mailing::MailingService, notification::NotificationService, post::PostService, - search::SearchService, timeline::TimelineService, url::UrlService, user::UserService, + custom_emoji::CustomEmojiService, federation_filter::FederationFilterService, + instance::InstanceService, job::JobService, mailing::MailingService, + notification::NotificationService, post::PostService, search::SearchService, + timeline::TimelineService, url::UrlService, user::UserService, }, webfinger::Webfinger, }; @@ -30,6 +31,7 @@ pub struct Service { pub account: AccountService, pub attachment: AttachmentService, pub captcha: CaptchaService, + pub custom_emoji: CustomEmojiService, pub federation_filter: FederationFilterService, pub job: JobService, pub mailing: MailingService, diff --git a/crates/kitsune-db/migrations/2023-05-19-230408_create_media_attachment_tables/up.sql b/crates/kitsune-db/migrations/2023-05-19-230408_create_media_attachment_tables/up.sql index 48d0ad8d..b27000aa 100644 --- a/crates/kitsune-db/migrations/2023-05-19-230408_create_media_attachment_tables/up.sql +++ b/crates/kitsune-db/migrations/2023-05-19-230408_create_media_attachment_tables/up.sql @@ -1,6 +1,6 @@ CREATE TABLE media_attachments ( id UUID PRIMARY KEY, - account_id UUID NOT NULL, + account_id UUID, content_type TEXT NOT NULL, description TEXT, blurhash TEXT, diff --git a/crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/down.sql b/crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/down.sql new file mode 100644 index 00000000..8cc3ada5 --- /dev/null +++ b/crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/down.sql @@ -0,0 +1,6 @@ +DROP INDEX "idx-custom_emojis-remote_id"; +DROP INDEX "idx-custom_emojis-shortcode"; +DROP INDEX "idx-custom_emojis-domain"; + +DROP TABLE posts_custom_emojis; +DROP TABLE custom_emojis; diff --git a/crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/up.sql b/crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/up.sql new file mode 100644 index 00000000..2ea0d8d8 --- /dev/null +++ b/crates/kitsune-db/migrations/2023-10-14-102320_create_emoji_tables/up.sql @@ -0,0 +1,32 @@ +CREATE TABLE custom_emojis ( + id UUID PRIMARY KEY, + shortcode TEXT NOT NULL, + domain TEXT, + remote_id TEXT NOT NULL UNIQUE, + media_attachment_id UUID NOT NULL, + endorsed BOOLEAN NOT NULL DEFAULT FALSE, + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- UNIQUE constraints + UNIQUE (shortcode, domain), + + -- Foreign key constraints + FOREIGN KEY (media_attachment_id) REFERENCES media_attachments(id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE TABLE posts_custom_emojis ( + post_id UUID NOT NULL, + custom_emoji_id UUID NOT NULL, + emoji_text TEXT NOT NULL, + PRIMARY KEY (post_id, custom_emoji_id), + + -- Foreign key constraints + FOREIGN KEY (post_id) REFERENCES posts(id) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (custom_emoji_id) REFERENCES custom_emojis(id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX "idx-custom_emojis-remote_id" ON custom_emojis (remote_id); +CREATE INDEX "idx-custom_emojis-shortcode" ON custom_emojis (shortcode); +CREATE INDEX "idx-custom_emojis-domain" ON custom_emojis (domain); diff --git a/crates/kitsune-db/src/model/custom_emoji.rs b/crates/kitsune-db/src/model/custom_emoji.rs new file mode 100644 index 00000000..fd9f4c77 --- /dev/null +++ b/crates/kitsune-db/src/model/custom_emoji.rs @@ -0,0 +1,42 @@ +use super::post::Post; +use crate::schema::posts_custom_emojis; +use diesel::{AsChangeset, Associations, Identifiable, Insertable, Queryable, Selectable}; +use iso8601_timestamp::Timestamp; +use serde::{Deserialize, Serialize}; +use speedy_uuid::Uuid; + +use crate::schema::custom_emojis; + +#[derive(Clone, Deserialize, Serialize, Identifiable, Insertable, Selectable, Queryable)] +#[diesel(table_name = custom_emojis)] +pub struct CustomEmoji { + pub id: Uuid, + pub shortcode: String, + pub domain: Option, + pub remote_id: String, + pub media_attachment_id: Uuid, + pub endorsed: bool, + pub created_at: Timestamp, + pub updated_at: Timestamp, +} + +#[derive(AsChangeset)] +#[diesel(table_name = custom_emojis)] +pub struct CustomEmojiConflictChangeset { + pub media_attachment_id: Uuid, +} + +#[derive( + Associations, Clone, Deserialize, Identifiable, Insertable, Queryable, Selectable, Serialize, +)] +#[diesel( + belongs_to(CustomEmoji), + belongs_to(Post), + primary_key(custom_emoji_id, post_id), + table_name = posts_custom_emojis, +)] +pub struct PostCustomEmoji { + pub post_id: Uuid, + pub custom_emoji_id: Uuid, + pub emoji_text: String, +} diff --git a/crates/kitsune-db/src/model/media_attachment.rs b/crates/kitsune-db/src/model/media_attachment.rs index 4a2cc4f2..9282ee2f 100644 --- a/crates/kitsune-db/src/model/media_attachment.rs +++ b/crates/kitsune-db/src/model/media_attachment.rs @@ -9,7 +9,7 @@ use speedy_uuid::Uuid; #[diesel(belongs_to(Account), table_name = media_attachments)] pub struct MediaAttachment { pub id: Uuid, - pub account_id: Uuid, + pub account_id: Option, pub content_type: String, pub description: Option, pub blurhash: Option, @@ -29,7 +29,7 @@ pub struct UpdateMediaAttachment<'a> { #[diesel(table_name = media_attachments)] pub struct NewMediaAttachment<'a> { pub id: Uuid, - pub account_id: Uuid, + pub account_id: Option, pub content_type: &'a str, pub description: Option<&'a str>, pub blurhash: Option<&'a str>, diff --git a/crates/kitsune-db/src/model/mod.rs b/crates/kitsune-db/src/model/mod.rs index bdbfb77d..5d0ac44e 100644 --- a/crates/kitsune-db/src/model/mod.rs +++ b/crates/kitsune-db/src/model/mod.rs @@ -1,4 +1,5 @@ pub mod account; +pub mod custom_emoji; pub mod favourite; pub mod follower; pub mod job_context; diff --git a/crates/kitsune-db/src/schema.rs b/crates/kitsune-db/src/schema.rs index f3c69703..aeb0d594 100644 --- a/crates/kitsune-db/src/schema.rs +++ b/crates/kitsune-db/src/schema.rs @@ -267,6 +267,65 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use diesel_full_text_search::Tsvector; + + /// Representation of the `custom_emojis` table. + /// + /// (Automatically generated by Diesel.) + custom_emojis (id) { + /// The `id` column of the `custom_emojis` table. + /// + /// Its SQL type is `Uuid`. + /// + /// (Automatically generated by Diesel.) + id -> Uuid, + /// The `shortcode` column of the `custom_emojis` table. + /// + /// Its SQL type is `Text`. + /// + /// (Automatically generated by Diesel.) + shortcode -> Text, + /// The `domain` column of the `custom_emojis` table. + /// + /// Its SQL type is `Nullable`. + /// + /// (Automatically generated by Diesel.) + domain -> Nullable, + /// The `remote_id` column of the `custom_emojis` table. + /// + /// Its SQL type is `Text`. + /// + /// (Automatically generated by Diesel.) + remote_id -> Text, + /// The `media_attachment_id` column of the `custom_emojis` table. + /// + /// Its SQL type is `Uuid`. + /// + /// (Automatically generated by Diesel.) + media_attachment_id -> Uuid, + /// The `endorsed` column of the `custom_emojis` table. + /// + /// Its SQL type is `Bool`. + /// + /// (Automatically generated by Diesel.) + endorsed -> Bool, + /// The `created_at` column of the `custom_emojis` table. + /// + /// Its SQL type is `Timestamptz`. + /// + /// (Automatically generated by Diesel.) + created_at -> Timestamptz, + /// The `updated_at` column of the `custom_emojis` table. + /// + /// Its SQL type is `Timestamptz`. + /// + /// (Automatically generated by Diesel.) + updated_at -> Timestamptz, + } +} + diesel::table! { use diesel::sql_types::*; use diesel_full_text_search::Tsvector; @@ -359,10 +418,10 @@ diesel::table! { id -> Uuid, /// The `account_id` column of the `media_attachments` table. /// - /// Its SQL type is `Uuid`. + /// Its SQL type is `Nullable`. /// /// (Automatically generated by Diesel.) - account_id -> Uuid, + account_id -> Nullable, /// The `content_type` column of the `media_attachments` table. /// /// Its SQL type is `Text`. @@ -751,6 +810,35 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use diesel_full_text_search::Tsvector; + + /// Representation of the `posts_custom_emojis` table. + /// + /// (Automatically generated by Diesel.) + posts_custom_emojis (post_id, custom_emoji_id) { + /// The `post_id` column of the `posts_custom_emojis` table. + /// + /// Its SQL type is `Uuid`. + /// + /// (Automatically generated by Diesel.) + post_id -> Uuid, + /// The `custom_emoji_id` column of the `posts_custom_emojis` table. + /// + /// Its SQL type is `Uuid`. + /// + /// (Automatically generated by Diesel.) + custom_emoji_id -> Uuid, + /// The `emoji_text` column of the `posts_custom_emojis` table. + /// + /// Its SQL type is `Text`. + /// + /// (Automatically generated by Diesel.) + emoji_text -> Text, + } +} + diesel::table! { use diesel::sql_types::*; use diesel_full_text_search::Tsvector; @@ -963,6 +1051,7 @@ diesel::table! { } diesel::joinable!(accounts_preferences -> accounts (account_id)); +diesel::joinable!(custom_emojis -> media_attachments (media_attachment_id)); diesel::joinable!(notifications -> posts (post_id)); diesel::joinable!(oauth2_access_tokens -> oauth2_applications (application_id)); diesel::joinable!(oauth2_access_tokens -> users (user_id)); @@ -972,6 +1061,8 @@ diesel::joinable!(oauth2_refresh_tokens -> oauth2_access_tokens (access_token)); diesel::joinable!(oauth2_refresh_tokens -> oauth2_applications (application_id)); diesel::joinable!(posts -> accounts (account_id)); diesel::joinable!(posts -> link_previews (link_preview_url)); +diesel::joinable!(posts_custom_emojis -> custom_emojis (custom_emoji_id)); +diesel::joinable!(posts_custom_emojis -> posts (post_id)); diesel::joinable!(posts_favourites -> accounts (account_id)); diesel::joinable!(posts_favourites -> posts (post_id)); diesel::joinable!(posts_media_attachments -> media_attachments (media_attachment_id)); @@ -985,6 +1076,7 @@ diesel::allow_tables_to_appear_in_same_query!( accounts, accounts_follows, accounts_preferences, + custom_emojis, job_context, link_previews, media_attachments, @@ -994,6 +1086,7 @@ diesel::allow_tables_to_appear_in_same_query!( oauth2_authorization_codes, oauth2_refresh_tokens, posts, + posts_custom_emojis, posts_favourites, posts_media_attachments, posts_mentions, diff --git a/crates/kitsune-type/src/ap/emoji.rs b/crates/kitsune-type/src/ap/emoji.rs new file mode 100644 index 00000000..a5b3df61 --- /dev/null +++ b/crates/kitsune-type/src/ap/emoji.rs @@ -0,0 +1,24 @@ +use super::object::MediaAttachment; +use crate::jsonld::RdfNode; +use iso8601_timestamp::Timestamp; +use serde::{Deserialize, Serialize}; +use simd_json::OwnedValue; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Emoji { + #[serde(default, rename = "@context")] + pub context: OwnedValue, + pub id: String, + pub r#type: String, + pub name: String, + pub icon: MediaAttachment, + #[serde(default = "Timestamp::now_utc")] + pub updated: Timestamp, +} + +impl RdfNode for Emoji { + fn id(&self) -> Option<&str> { + Some(&self.id) + } +} diff --git a/crates/kitsune-type/src/ap/mod.rs b/crates/kitsune-type/src/ap/mod.rs index 99632d2e..17b9b3b9 100644 --- a/crates/kitsune-type/src/ap/mod.rs +++ b/crates/kitsune-type/src/ap/mod.rs @@ -12,6 +12,7 @@ pub const PUBLIC_IDENTIFIER: &str = "https://www.w3.org/ns/activitystreams#Publi pub mod actor; pub mod collection; +pub mod emoji; pub mod helper; pub mod object; @@ -210,6 +211,7 @@ pub enum TagType { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Tag { + pub id: Option, pub r#type: TagType, pub name: String, pub href: Option, diff --git a/crates/kitsune-type/src/mastodon/custom_emoji.rs b/crates/kitsune-type/src/mastodon/custom_emoji.rs new file mode 100644 index 00000000..654e602d --- /dev/null +++ b/crates/kitsune-type/src/mastodon/custom_emoji.rs @@ -0,0 +1,11 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Clone, Deserialize, Serialize, ToSchema)] +pub struct CustomEmoji { + pub shortcode: String, + pub url: String, + pub static_url: String, + pub visible_in_picker: bool, + pub category: Option, +} diff --git a/crates/kitsune-type/src/mastodon/mod.rs b/crates/kitsune-type/src/mastodon/mod.rs index cb3c68e2..1bc7e442 100644 --- a/crates/kitsune-type/src/mastodon/mod.rs +++ b/crates/kitsune-type/src/mastodon/mod.rs @@ -3,6 +3,7 @@ use speedy_uuid::Uuid; use utoipa::ToSchema; pub mod account; +pub mod custom_emoji; pub mod instance; pub mod media_attachment; pub mod notification; @@ -12,6 +13,7 @@ pub mod search; pub mod status; pub use self::account::Account; +pub use self::custom_emoji::CustomEmoji; pub use self::instance::Instance; pub use self::media_attachment::MediaAttachment; pub use self::notification::Notification; diff --git a/crates/kitsune-type/src/mastodon/status.rs b/crates/kitsune-type/src/mastodon/status.rs index 5c80665d..ea2406a3 100644 --- a/crates/kitsune-type/src/mastodon/status.rs +++ b/crates/kitsune-type/src/mastodon/status.rs @@ -1,4 +1,4 @@ -use super::{Account, MediaAttachment, PreviewCard}; +use super::{Account, CustomEmoji, MediaAttachment, PreviewCard}; use iso8601_timestamp::Timestamp; use serde::{Deserialize, Serialize}; use speedy_uuid::Uuid; @@ -49,6 +49,7 @@ pub struct Status { pub account: Account, pub media_attachments: Vec, pub mentions: Vec, + pub emojis: Vec, pub reblog: Option>, pub favourited: bool, pub reblogged: bool, diff --git a/kitsune/src/http/graphql/types/media_attachment.rs b/kitsune/src/http/graphql/types/media_attachment.rs index e92633ac..14fd36e0 100644 --- a/kitsune/src/http/graphql/types/media_attachment.rs +++ b/kitsune/src/http/graphql/types/media_attachment.rs @@ -44,7 +44,7 @@ impl From for MediaAttachment { fn from(value: DbMediaAttachment) -> Self { Self { id: value.id, - account_id: value.account_id, + account_id: value.account_id.unwrap(), content_type: value.content_type, description: value.description, blurhash: value.blurhash, diff --git a/kitsune/src/http/handler/custom_emojis.rs b/kitsune/src/http/handler/custom_emojis.rs new file mode 100644 index 00000000..2942d28f --- /dev/null +++ b/kitsune/src/http/handler/custom_emojis.rs @@ -0,0 +1,21 @@ +use crate::{error::Result, http::responder::ActivityPubJson, state::Zustand}; +use axum::{debug_handler, extract::Path, extract::State, routing, Router}; +use kitsune_core::{mapping::IntoObject, service::custom_emoji::CustomEmojiService}; +use kitsune_type::ap::emoji::Emoji; +use speedy_uuid::Uuid; + +#[debug_handler(state = Zustand)] +async fn get( + State(state): State, + State(emoji_service): State, + Path(id): Path, +) -> Result> { + let custom_emoji = emoji_service.get_by_id(id).await?; + Ok(ActivityPubJson( + custom_emoji.into_object(&state.core).await?, + )) +} + +pub fn routes() -> Router { + Router::new().route("/:id", routing::get(get)) +} diff --git a/kitsune/src/http/handler/mastodon/api/v1/custom_emojis.rs b/kitsune/src/http/handler/mastodon/api/v1/custom_emojis.rs new file mode 100644 index 00000000..c7cb1ae9 --- /dev/null +++ b/kitsune/src/http/handler/mastodon/api/v1/custom_emojis.rs @@ -0,0 +1,42 @@ +use crate::{error::Result, http::extractor::MastodonAuthExtractor, state::Zustand}; +use axum::{debug_handler, extract::State, routing, Json, Router}; +use futures_util::TryStreamExt; +use kitsune_core::{ + mapping::MastodonMapper, + service::custom_emoji::{CustomEmojiService, GetEmojiList}, +}; +use kitsune_type::mastodon::CustomEmoji; + +#[debug_handler(state = crate::state::Zustand)] +#[utoipa::path( + get, + path = "/api/v1/custom_emojis", + security( + ("oauth_token" = []) + ), + responses( + (status = 200, description = "List of custom emojis available on the server", body = Vec) + ), +)] +pub async fn get( + State(custom_emoji_service): State, + State(mastodon_mapper): State, + user_data: Option, +) -> Result>> { + let get_emoji_list = GetEmojiList::builder() + .fetching_account_id(user_data.map(|x| x.0.account.id)) + .build(); + + let custom_emojis: Vec = custom_emoji_service + .get_list(get_emoji_list) + .await? + .and_then(|acc| mastodon_mapper.map(acc)) + .try_collect() + .await?; + + Ok(Json(custom_emojis)) +} + +pub fn routes() -> Router { + Router::new().route("/", routing::get(get)) +} diff --git a/kitsune/src/http/handler/mastodon/api/v1/mod.rs b/kitsune/src/http/handler/mastodon/api/v1/mod.rs index 85fe8b02..4110acdb 100644 --- a/kitsune/src/http/handler/mastodon/api/v1/mod.rs +++ b/kitsune/src/http/handler/mastodon/api/v1/mod.rs @@ -3,6 +3,7 @@ use axum::Router; pub mod accounts; pub mod apps; +pub mod custom_emojis; pub mod follow_requests; pub mod instance; pub mod media; @@ -14,6 +15,7 @@ pub fn routes() -> Router { Router::new() .nest("/apps", apps::routes()) .nest("/accounts", accounts::routes()) + .nest("/custom_emojis", custom_emojis::routes()) .nest("/follow_requests", follow_requests::routes()) .nest("/instance", instance::routes()) .nest("/media", media::routes()) diff --git a/kitsune/src/http/handler/mod.rs b/kitsune/src/http/handler/mod.rs index 356a220f..c584339c 100644 --- a/kitsune/src/http/handler/mod.rs +++ b/kitsune/src/http/handler/mod.rs @@ -1,4 +1,5 @@ pub mod confirm_account; +pub mod custom_emojis; #[cfg(feature = "mastodon-api")] pub mod mastodon; pub mod media; diff --git a/kitsune/src/http/mod.rs b/kitsune/src/http/mod.rs index 0c90e75f..4e04af1d 100644 --- a/kitsune/src/http/mod.rs +++ b/kitsune/src/http/mod.rs @@ -1,5 +1,5 @@ use self::{ - handler::{confirm_account, media, nodeinfo, oauth, posts, users, well_known}, + handler::{confirm_account, custom_emojis, media, nodeinfo, oauth, posts, users, well_known}, openapi::api_docs, }; use crate::state::Zustand; @@ -44,6 +44,7 @@ pub fn create_router( #[allow(unused_mut)] let mut router = Router::new() .nest("/confirm-account", confirm_account::routes()) + .nest("/emojis", custom_emojis::routes()) .nest("/media", media::routes()) .nest("/nodeinfo", nodeinfo::routes()) .nest( diff --git a/kitsune/src/state.rs b/kitsune/src/state.rs index ee0076fd..ef8d66dd 100644 --- a/kitsune/src/state.rs +++ b/kitsune/src/state.rs @@ -4,7 +4,7 @@ use kitsune_core::{ activitypub::Fetcher, event::PostEventEmitter, service::{ - account::AccountService, attachment::AttachmentService, + account::AccountService, attachment::AttachmentService, custom_emoji::CustomEmojiService, federation_filter::FederationFilterService, instance::InstanceService, job::JobService, notification::NotificationService, post::PostService, search::SearchService, timeline::TimelineService, url::UrlService, user::UserService, @@ -55,6 +55,7 @@ impl_from_ref! { [ AccountService => |input: &Zustand| input.core.service.account.clone(), AttachmentService => |input: &Zustand| input.core.service.attachment.clone(), + CustomEmojiService => |input: &Zustand| input.core.service.custom_emoji.clone(), FederationFilterService => |input: &Zustand| input.core.service.federation_filter.clone(), JobService => |input: &Zustand| input.core.service.job.clone(), NotificationService => |input: &Zustand| input.core.service.notification.clone(), diff --git a/lib/post-process/src/lib.rs b/lib/post-process/src/lib.rs index a104ab8a..00e96722 100644 --- a/lib/post-process/src/lib.rs +++ b/lib/post-process/src/lib.rs @@ -41,6 +41,19 @@ fn enforce_prefix<'a>(lexer: &Lexer<'a, PostElement<'a>>) -> bool { } } +#[inline] +fn emoji_split<'a>(lexer: &Lexer<'a, PostElement<'a>>) -> (&'a str, Option<&'a str>) { + let slice = lexer.slice().trim_matches(':'); + + let emoji_data = if let Some((shortcode, domain)) = slice.split_once('@') { + (shortcode, Some(domain)) + } else { + (slice, None) + }; + + emoji_data +} + #[inline] fn mention_split<'a>(lexer: &Lexer<'a, PostElement<'a>>) -> Option<(&'a str, Option<&'a str>)> { if !enforce_prefix(lexer) || !enforce_postfix(lexer) { @@ -61,11 +74,8 @@ fn mention_split<'a>(lexer: &Lexer<'a, PostElement<'a>>) -> Option<(&'a str, Opt #[derive(Debug, Logos, PartialEq)] pub enum PostElement<'a> { - #[regex( - r":[\w\d-]+:", - |lexer| lexer.slice().trim_matches(':'), - )] - Emote(&'a str), + #[regex(r":[\w\d_-]+(@[\w\-_]+\.[\.\w]+)?:", emoji_split)] + Emote((&'a str, Option<&'a str>)), #[regex( r"#[\w_-]+", @@ -150,8 +160,9 @@ impl<'a> Element<'a> { ) -> impl Iterator, Span)> { pairs.map(|(item, span)| { let element = match item { - PostElement::Emote(name) => Self::Emote(Emote { - content: Cow::Borrowed(name), + PostElement::Emote((name, domain)) => Self::Emote(Emote { + shortcode: Cow::Borrowed(name), + domain: domain.map(Cow::Borrowed), }), PostElement::Hashtag(content) => Self::Hashtag(Hashtag { content: Cow::Borrowed(content), @@ -187,12 +198,22 @@ impl Render for Element<'_> { #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct Emote<'a> { /// Name of an emote - pub content: Cow<'a, str>, + pub shortcode: Cow<'a, str>, + /// Domain + pub domain: Option>, } impl Render for Emote<'_> { fn render(&self, out: &mut impl fmt::Write) { - let _ = write!(out, ":{}:", self.content); + let _ = match &self.domain { + Some(domain) => write!( + out, + ":{}__{}:", + self.shortcode, + domain.replace(['.', '-'], "_") + ), + None => write!(out, ":{}:", self.shortcode), + }; } } @@ -229,9 +250,7 @@ impl Render for Html<'_> { let _ = write!(out, " {name}=\"{value}\""); } let _ = out.write_char('>'); - self.content.render(out); - let _ = write!(out, "", self.tag); } } diff --git a/lib/post-process/tests/snapshots/emote__parse_emote.snap b/lib/post-process/tests/snapshots/emote__parse_emote.snap index 757bc041..ffabb8b8 100644 --- a/lib/post-process/tests/snapshots/emote__parse_emote.snap +++ b/lib/post-process/tests/snapshots/emote__parse_emote.snap @@ -6,14 +6,20 @@ input_file: lib/post-process/tests/input/emote/full_post_1 [ ( Emote( - "blobfoxcoffee", + ( + "blobfoxcoffee", + None, + ), ), 6..21, ":blobfoxcoffee:", ), ( Emote( - "blobcatpeek", + ( + "blobcatpeek", + None, + ), ), 40..53, ":blobcatpeek:", diff --git a/lib/post-process/tests/transformation.rs b/lib/post-process/tests/transformation.rs index 96d37d19..3667e4a4 100644 --- a/lib/post-process/tests/transformation.rs +++ b/lib/post-process/tests/transformation.rs @@ -12,7 +12,7 @@ fn link_transformation() { tag: Cow::Borrowed("a"), attributes: vec![( Cow::Borrowed("href"), - Cow::Owned(format!("https://example.com/emote/{}", emote.content)), + Cow::Owned(format!("https://example.com/emote/{}", emote.shortcode)), )], content: Box::new(Element::Emote(emote)), }), diff --git a/test-fixtures/corteximplant.com_emoji_7952.json b/test-fixtures/corteximplant.com_emoji_7952.json new file mode 100644 index 00000000..0687e531 --- /dev/null +++ b/test-fixtures/corteximplant.com_emoji_7952.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams",{"toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","focalPoint":{"@container":"@list","@id":"toot:focalPoint"}}],"id":"https://corteximplant.com/emojis/7952","type":"Emoji","name":":Blobhaj:","updated":"2022-11-10T18:48:57Z","icon":{"type":"Image","mediaType":"image/png","url":"https://corteximplant.com/system/custom_emojis/images/000/007/952/original/33b7f12bd094b815.png"}} diff --git a/test-fixtures/corteximplant.com_emoji_8933.json b/test-fixtures/corteximplant.com_emoji_8933.json new file mode 100644 index 00000000..02408730 --- /dev/null +++ b/test-fixtures/corteximplant.com_emoji_8933.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams",{"toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","focalPoint":{"@container":"@list","@id":"toot:focalPoint"}}],"id":"https://corteximplant.com/emojis/8933","type":"Emoji","name":":blobcat:","updated":"2022-11-10T19:10:49Z","icon":{"type":"Image","mediaType":"image/png","url":"https://corteximplant.com/system/custom_emojis/images/000/008/933/original/ee14c9edeec55f08.png"}}