Store activitypub endpoints in database #162

Merged
dessalines merged 2 commits from apub-endpoints-in-db into main 1 year ago
  1. 1
      Cargo.lock
  2. 24
      crates/api/src/comment.rs
  3. 25
      crates/api/src/community.rs
  4. 10
      crates/api/src/post.rs
  5. 48
      crates/api/src/user.rs
  6. 4
      crates/apub/src/activities/send/comment.rs
  7. 59
      crates/apub/src/activities/send/community.rs
  8. 8
      crates/apub/src/activities/send/private_message.rs
  9. 15
      crates/apub/src/activities/send/user.rs
  10. 4
      crates/apub/src/activity_queue.rs
  11. 14
      crates/apub/src/fetcher/community.rs
  12. 2
      crates/apub/src/http/community.rs
  13. 25
      crates/apub/src/inbox/mod.rs
  14. 95
      crates/apub/src/lib.rs
  15. 23
      crates/apub/src/objects/community.rs
  16. 13
      crates/apub/src/objects/user.rs
  17. 7
      crates/db_queries/src/aggregates/comment_aggregates.rs
  18. 10
      crates/db_queries/src/aggregates/community_aggregates.rs
  19. 7
      crates/db_queries/src/aggregates/post_aggregates.rs
  20. 5
      crates/db_queries/src/aggregates/site_aggregates.rs
  21. 7
      crates/db_queries/src/aggregates/user_aggregates.rs
  22. 2
      crates/db_queries/src/source/activity.rs
  23. 9
      crates/db_queries/src/source/comment.rs
  24. 19
      crates/db_queries/src/source/community.rs
  25. 7
      crates/db_queries/src/source/moderator.rs
  26. 2
      crates/db_queries/src/source/password_reset_request.rs
  27. 9
      crates/db_queries/src/source/post.rs
  28. 8
      crates/db_queries/src/source/private_message.rs
  29. 8
      crates/db_queries/src/source/user.rs
  30. 7
      crates/db_queries/src/source/user_mention.rs
  31. 5
      crates/db_schema/src/schema.rs
  32. 6
      crates/db_schema/src/source/community.rs
  33. 6
      crates/db_schema/src/source/user.rs
  34. 1
      crates/db_views/Cargo.toml
  35. 7
      crates/db_views/src/comment_view.rs
  36. 7
      crates/db_views/src/post_view.rs
  37. 29
      crates/utils/src/apub.rs
  38. 526
      lemmy_db/src/schema.rs
  39. 6
      migrations/2021-02-02-153240_apub_columns/down.sql
  40. 10
      migrations/2021-02-02-153240_apub_columns/up.sql
  41. 3
      scripts/test.sh
  42. 75
      src/code_migrations.rs
  43. 5
      tests/integration_test.rs

1
Cargo.lock

@ -1887,6 +1887,7 @@ dependencies = [
"lemmy_db_schema",
"log",
"serde 1.0.123",
"url",
]
[[package]]

24
crates/api/src/comment.rs

@ -9,7 +9,7 @@ use crate::{
Perform,
};
use actix_web::web::Data;
use lemmy_apub::{ApubLikeableType, ApubObjectType};
use lemmy_apub::{generate_apub_endpoint, ApubLikeableType, ApubObjectType, EndpointType};
use lemmy_db_queries::{
source::comment::Comment_,
Crud,
@ -26,7 +26,6 @@ use lemmy_db_views::{
};
use lemmy_structs::{blocking, comment::*, send_local_notifs};
use lemmy_utils::{
apub::{make_apub_endpoint, EndpointType},
utils::{remove_slurs, scrape_text_for_mentions},
APIError,
ConnectionId,
@ -104,16 +103,17 @@ impl Perform for CreateComment {
// Necessary to update the ap_id
let inserted_comment_id = inserted_comment.id;
let updated_comment: Comment = match blocking(context.pool(), move |conn| {
let apub_id =
make_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string()).to_string();
Comment::update_ap_id(&conn, inserted_comment_id, apub_id)
})
.await?
{
Ok(comment) => comment,
Err(_e) => return Err(APIError::err("couldnt_create_comment").into()),
};
let updated_comment: Comment =
match blocking(context.pool(), move |conn| -> Result<Comment, LemmyError> {
let apub_id =
generate_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string())?;
Ok(Comment::update_ap_id(&conn, inserted_comment_id, apub_id)?)
})
.await?
{
Ok(comment) => comment,
Err(_e) => return Err(APIError::err("couldnt_create_comment").into()),
};
updated_comment.send_create(&user, context).await?;

25
crates/api/src/community.rs

@ -9,7 +9,14 @@ use crate::{
};
use actix_web::web::Data;
use anyhow::Context;
use lemmy_apub::ActorType;
use lemmy_apub::{
generate_apub_endpoint,
generate_followers_url,
generate_inbox_url,
generate_shared_inbox_url,
ActorType,
EndpointType,
};
use lemmy_db_queries::{
diesel_option_overwrite,
source::{
@ -38,7 +45,7 @@ use lemmy_db_views_actor::{
};
use lemmy_structs::{blocking, community::*};
use lemmy_utils::{
apub::{generate_actor_keypair, make_apub_endpoint, EndpointType},
apub::generate_actor_keypair,
location_info,
utils::{check_slurs, check_slurs_opt, is_valid_community_name, naive_from_unix},
APIError,
@ -137,10 +144,10 @@ impl Perform for CreateCommunity {
}
// Double check for duplicate community actor_ids
let actor_id = make_apub_endpoint(EndpointType::Community, &data.name);
let actor_id_cloned = actor_id.to_owned();
let community_actor_id = generate_apub_endpoint(EndpointType::Community, &data.name)?;
let actor_id_cloned = community_actor_id.to_owned();
let community_dupe = blocking(context.pool(), move |conn| {
Community::read_from_apub_id(conn, &actor_id_cloned.into())
Community::read_from_apub_id(conn, &actor_id_cloned)
})
.await?;
if community_dupe.is_ok() {
@ -169,12 +176,15 @@ impl Perform for CreateCommunity {
deleted: None,
nsfw: data.nsfw,
updated: None,
actor_id: Some(actor_id.into()),
actor_id: Some(community_actor_id.to_owned()),
local: true,
private_key: Some(keypair.private_key),
public_key: Some(keypair.public_key),
last_refreshed_at: None,
published: None,
followers_url: Some(generate_followers_url(&community_actor_id)?),
inbox_url: Some(generate_inbox_url(&community_actor_id)?),
shared_inbox_url: Some(Some(generate_shared_inbox_url(&community_actor_id)?)),
dessalines marked this conversation as resolved
Review

Why Some(Some( for this one but not the others? I think since this is a required column, some(some( doesn't make sense. IE you can never use some(none

Why Some(Some( for this one but not the others? I think since this is a required column, some(some( doesn't make sense. IE you can never use some(none
Review

shared_inbox is optional in activitypub, so i also made it optional in our code. So this new code should work fine with software that doesnt have shared_inbox (though I didnt test that).

shared_inbox is optional in activitypub, so i also made it optional in our code. So this new code should work fine with software that doesnt have shared_inbox (though I didnt test that).
Review

Ahh... okay that's fine.

Ahh... okay that's fine.
};
let inserted_community = match blocking(context.pool(), move |conn| {
@ -275,6 +285,9 @@ impl Perform for EditCommunity {
public_key: read_community.public_key,
last_refreshed_at: None,
published: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let community_id = data.community_id;

10
crates/api/src/post.rs

@ -9,7 +9,7 @@ use crate::{
Perform,
};
use actix_web::web::Data;
use lemmy_apub::{ApubLikeableType, ApubObjectType};
use lemmy_apub::{generate_apub_endpoint, ApubLikeableType, ApubObjectType, EndpointType};
use lemmy_db_queries::{
source::post::Post_,
Crud,
@ -38,7 +38,6 @@ use lemmy_db_views_actor::{
};
use lemmy_structs::{blocking, post::*};
use lemmy_utils::{
apub::{make_apub_endpoint, EndpointType},
request::fetch_iframely_and_pictrs_data,
utils::{check_slurs, check_slurs_opt, is_valid_post_title},
APIError,
@ -115,10 +114,9 @@ impl Perform for CreatePost {
};
let inserted_post_id = inserted_post.id;
let updated_post = match blocking(context.pool(), move |conn| {
let apub_id =
make_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string()).to_string();
Post::update_ap_id(conn, inserted_post_id, apub_id)
let updated_post = match blocking(context.pool(), move |conn| -> Result<Post, LemmyError> {
let apub_id = generate_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string())?;
Ok(Post::update_ap_id(conn, inserted_post_id, apub_id)?)
})
.await?
{

48
crates/api/src/user.rs

@ -13,7 +13,14 @@ use anyhow::Context;
use bcrypt::verify;
use captcha::{gen, Difficulty};
use chrono::Duration;
use lemmy_apub::ApubObjectType;
use lemmy_apub::{
generate_apub_endpoint,
generate_followers_url,
generate_inbox_url,
generate_shared_inbox_url,
ApubObjectType,
EndpointType,
};
use lemmy_db_queries::{
diesel_option_overwrite,
source::{
@ -61,7 +68,7 @@ use lemmy_db_views_actor::{
};
use lemmy_structs::{blocking, send_email_to_user, user::*};
use lemmy_utils::{
apub::{generate_actor_keypair, make_apub_endpoint, EndpointType},
apub::generate_actor_keypair,
email::send_email,
location_info,
settings::Settings,
@ -179,6 +186,7 @@ impl Perform for Register {
if !is_valid_username(&data.username) {
return Err(APIError::err("invalid_username").into());
}
let user_actor_id = generate_apub_endpoint(EndpointType::User, &data.username)?;
// Register the new user
let user_form = UserForm {
@ -200,12 +208,14 @@ impl Perform for Register {
lang: "browser".into(),
show_avatars: true,
send_notifications_to_email: false,
actor_id: Some(make_apub_endpoint(EndpointType::User, &data.username).into()),
actor_id: Some(user_actor_id.clone()),
bio: None,
local: true,
private_key: Some(user_keypair.private_key),
public_key: Some(user_keypair.public_key),
last_refreshed_at: None,
inbox_url: Some(generate_inbox_url(&user_actor_id)?),
shared_inbox_url: Some(Some(generate_shared_inbox_url(&user_actor_id)?)),
dessalines marked this conversation as resolved
Review

hrm... this missing followers_url?

hrm... this missing followers_url?
Review

We only have the /c/main/followers endpoint for communities but not for users, because we dont support user following (yet). So I dont see any reason to add it at this time.

We only have the `/c/main/followers` endpoint for communities but not for users, because we dont support user following (yet). So I dont see any reason to add it at this time.
Review

We could add it later, but then we'll have to do yet another advanced_code migration when we could just handle that here.

We could add it later, but then we'll have to do yet another advanced_code migration when we could just handle that here.
};
// Create the user
@ -236,6 +246,7 @@ impl Perform for Register {
Ok(c) => c,
Err(_e) => {
let default_community_name = "main";
let actor_id = generate_apub_endpoint(EndpointType::Community, default_community_name)?;
let community_form = CommunityForm {
name: default_community_name.to_string(),
title: "The Default Community".to_string(),
@ -246,9 +257,7 @@ impl Perform for Register {
removed: None,
deleted: None,
updated: None,
actor_id: Some(
make_apub_endpoint(EndpointType::Community, default_community_name).into(),
),
actor_id: Some(actor_id.to_owned()),
local: true,
private_key: Some(main_community_keypair.private_key),
public_key: Some(main_community_keypair.public_key),
@ -256,6 +265,9 @@ impl Perform for Register {
published: None,
icon: None,
banner: None,
followers_url: Some(generate_followers_url(&actor_id)?),
inbox_url: Some(generate_inbox_url(&actor_id)?),
shared_inbox_url: Some(Some(generate_shared_inbox_url(&actor_id)?)),
dessalines marked this conversation as resolved
Review

These should probably all be generated as a tuple or struct output, since they're all based on the actor_id anyway. IE (followers_url, inbox_url, shared_inbox_url) = generate_bleh_urls(&actor_id) or inboxes: Inboxes = generate...

These should probably all be generated as a tuple or struct output, since they're all based on the `actor_id` anyway. IE `(followers_url, inbox_url, shared_inbox_url) = generate_bleh_urls(&actor_id)` or `inboxes: Inboxes = generate...`
Review

I dont think that would be better. Right now the code is neatly split into separate functions, and its very clear which one does what. If we put it into a single method, thats gonna be a rather long one. And tuples suck as return values, especially when all items have the same type. It would be very confusing to figure out which value is where. Plus in some places (like advanced migrations) we dont need to generate the actor_id.

I dont think that would be better. Right now the code is neatly split into separate functions, and its very clear which one does what. If we put it into a single method, thats gonna be a rather long one. And tuples suck as return values, especially when all items have the same type. It would be very confusing to figure out which value is where. Plus in some places (like advanced migrations) we dont need to generate the actor_id.
Review

Okay that's fine.

Okay that's fine.
};
blocking(context.pool(), move |conn| {
Community::create(conn, &community_form)
@ -420,6 +432,7 @@ impl Perform for SaveUserSettings {
matrix_user_id,
avatar,
banner,
inbox_url: None,
password_encrypted,
preferred_username,
published: Some(user.published),
@ -439,6 +452,7 @@ impl Perform for SaveUserSettings {
private_key: user.private_key,
public_key: user.public_key,
last_refreshed_at: None,
shared_inbox_url: None,
};
let res = blocking(context.pool(), move |conn| {
@ -1036,14 +1050,20 @@ impl Perform for CreatePrivateMessage {
};
let inserted_private_message_id = inserted_private_message.id;
let updated_private_message = match blocking(context.pool(), move |conn| {
let apub_id = make_apub_endpoint(
EndpointType::PrivateMessage,
&inserted_private_message_id.to_string(),
)
.to_string();
PrivateMessage::update_ap_id(&conn, inserted_private_message_id, apub_id)
})
let updated_private_message = match blocking(
context.pool(),
move |conn| -> Result<PrivateMessage, LemmyError> {
let apub_id = generate_apub_endpoint(
EndpointType::PrivateMessage,
&inserted_private_message_id.to_string(),
)?;
Ok(PrivateMessage::update_ap_id(
&conn,
inserted_private_message_id,
apub_id,
)?)
},
)
.await?
{
Ok(private_message) => private_message,

4
crates/apub/src/activities/send/comment.rs

@ -351,7 +351,7 @@ async fn collect_non_local_mentions(
let parent_creator = get_comment_parent_creator(context.pool(), comment).await?;
let mut addressed_ccs = vec![community.actor_id(), parent_creator.actor_id()];
// Note: dont include community inbox here, as we send to it separately with `send_to_community()`
let mut inboxes = vec![parent_creator.get_shared_inbox_url()?];
let mut inboxes = vec![parent_creator.get_shared_inbox_or_inbox_url()];
// Add the mention tag
let mut tags = Vec::new();
@ -370,7 +370,7 @@ async fn collect_non_local_mentions(
addressed_ccs.push(actor_id.to_owned().to_string().parse()?);
let mention_user = get_or_fetch_and_upsert_user(&actor_id, context, &mut 0).await?;
inboxes.push(mention_user.get_shared_inbox_url()?);
inboxes.push(mention_user.get_shared_inbox_or_inbox_url());
let mut mention_tag = Mention::new();
mention_tag.set_href(actor_id).set_name(mention.full_name());

59
crates/apub/src/activities/send/community.rs

@ -27,16 +27,18 @@ use lemmy_db_queries::DbPool;
use lemmy_db_schema::source::community::Community;
use lemmy_db_views_actor::community_follower_view::CommunityFollowerView;
use lemmy_structs::blocking;
use lemmy_utils::{location_info, settings::Settings, LemmyError};
use lemmy_utils::{location_info, LemmyError};
use lemmy_websocket::LemmyContext;
use url::Url;
#[async_trait::async_trait(?Send)]
impl ActorType for Community {
fn is_local(&self) -> bool {
self.local
}
fn actor_id(&self) -> Url {
self.actor_id.to_owned().into_inner()
}
fn public_key(&self) -> Option<String> {
self.public_key.to_owned()
}
@ -44,6 +46,14 @@ impl ActorType for Community {
self.private_key.to_owned()
}
fn get_shared_inbox_or_inbox_url(&self) -> Url {
self
.shared_inbox_url
.clone()
.unwrap_or_else(|| self.inbox_url.to_owned())
.into()
}
dessalines marked this conversation as resolved
Review

This is dangerous right? assert will panic, which we shouldn't do. Maybe use a LemmyError and have this throw one of those.

This is dangerous right? assert will panic, which we shouldn't do. Maybe use a LemmyError and have this throw one of those.
Review

Fixed.

Fixed.
async fn send_follow(
&self,
_follow_actor_id: &Url,
@ -81,7 +91,7 @@ impl ActorType for Community {
.set_id(generate_activity_id(AcceptType::Accept)?)
.set_to(user.actor_id());
send_activity_single_dest(accept, self, user.get_inbox_url()?, context).await?;
send_activity_single_dest(accept, self, user.inbox_url.into(), context).await?;
Ok(())
}
@ -92,7 +102,7 @@ impl ActorType for Community {
.set_many_contexts(lemmy_context()?)
.set_id(generate_activity_id(DeleteType::Delete)?)
.set_to(public())
.set_many_ccs(vec![self.get_followers_url()?]);
.set_many_ccs(vec![self.followers_url.clone().into_inner()]);
send_to_community_followers(delete, self, context).await?;
Ok(())
@ -105,14 +115,14 @@ impl ActorType for Community {
.set_many_contexts(lemmy_context()?)
.set_id(generate_activity_id(DeleteType::Delete)?)
.set_to(public())
.set_many_ccs(vec![self.get_followers_url()?]);
.set_many_ccs(vec![self.followers_url.clone().into_inner()]);
let mut undo = Undo::new(self.actor_id(), delete.into_any_base()?);
undo
.set_many_contexts(lemmy_context()?)
.set_id(generate_activity_id(UndoType::Undo)?)
.set_to(public())
.set_many_ccs(vec![self.get_followers_url()?]);
.set_many_ccs(vec![self.followers_url.clone().into_inner()]);
send_to_community_followers(undo, self, context).await?;
Ok(())
@ -125,7 +135,7 @@ impl ActorType for Community {
.set_many_contexts(lemmy_context()?)
.set_id(generate_activity_id(RemoveType::Remove)?)
.set_to(public())
.set_many_ccs(vec![self.get_followers_url()?]);
.set_many_ccs(vec![self.followers_url.clone().into_inner()]);
send_to_community_followers(remove, self, context).await?;
Ok(())
@ -138,7 +148,7 @@ impl ActorType for Community {
.set_many_contexts(lemmy_context()?)
.set_id(generate_activity_id(RemoveType::Remove)?)
.set_to(public())
.set_many_ccs(vec![self.get_followers_url()?]);
.set_many_ccs(vec![self.followers_url.clone().into_inner()]);
// Undo that fake activity
let mut undo = Undo::new(self.actor_id(), remove.into_any_base()?);
@ -146,7 +156,7 @@ impl ActorType for Community {
.set_many_contexts(lemmy_context()?)
.set_id(generate_activity_id(LikeType::Like)?)
.set_to(public())
.set_many_ccs(vec![self.get_followers_url()?]);
.set_many_ccs(vec![self.followers_url.clone().into_inner()]);
send_to_community_followers(undo, self, context).await?;
Ok(())
@ -164,7 +174,7 @@ impl ActorType for Community {
.set_many_contexts(lemmy_context()?)
.set_id(generate_activity_id(AnnounceType::Announce)?)
.set_to(public())
.set_many_ccs(vec![self.get_followers_url()?]);
.set_many_ccs(vec![self.followers_url.clone().into_inner()]);
send_to_community_followers(announce, self, context).await?;
@ -172,38 +182,21 @@ impl ActorType for Community {
}
/// For a given community, returns the inboxes of all followers.
///
/// TODO: this function is very badly implemented, we should just store shared_inbox_url in
/// CommunityFollowerView
async fn get_follower_inboxes(&self, pool: &DbPool) -> Result<Vec<Url>, LemmyError> {
let id = self.id;
let inboxes = blocking(pool, move |conn| {
let follows = blocking(pool, move |conn| {
CommunityFollowerView::for_community(conn, id)
})
.await??;
let inboxes = inboxes
let inboxes = follows
.into_iter()
.filter(|i| !i.follower.local)
.map(|u| -> Result<Url, LemmyError> {
let url = u.follower.actor_id.into_inner();
let domain = url.domain().context(location_info!())?;
let port = if let Some(port) = url.port() {
format!(":{}", port)
} else {
"".to_string()
};
Ok(Url::parse(&format!(
"{}://{}{}/inbox",
Settings::get().get_protocol_string(),
domain,
port,
))?)
})
.filter_map(Result::ok)
.filter(|f| !f.follower.local)
.map(|f| f.follower.shared_inbox_url.unwrap_or(f.follower.inbox_url))
.map(|i| i.into_inner())
.unique()
// Don't send to blocked instances
.filter(|inbox| check_is_apub_id_valid(inbox).is_ok())
.unique()
.collect();
Ok(inboxes)

8
crates/apub/src/activities/send/private_message.rs

@ -41,7 +41,7 @@ impl ApubObjectType for PrivateMessage {
.set_id(generate_activity_id(CreateType::Create)?)
.set_to(recipient.actor_id());
send_activity_single_dest(create, creator, recipient.get_inbox_url()?, context).await?;
send_activity_single_dest(create, creator, recipient.inbox_url.into(), context).await?;
Ok(())
}
@ -61,7 +61,7 @@ impl ApubObjectType for PrivateMessage {
.set_id(generate_activity_id(UpdateType::Update)?)
.set_to(recipient.actor_id());
send_activity_single_dest(update, creator, recipient.get_inbox_url()?, context).await?;
send_activity_single_dest(update, creator, recipient.inbox_url.into(), context).await?;
Ok(())
}
@ -78,7 +78,7 @@ impl ApubObjectType for PrivateMessage {
.set_id(generate_activity_id(DeleteType::Delete)?)
.set_to(recipient.actor_id());
send_activity_single_dest(delete, creator, recipient.get_inbox_url()?, context).await?;
send_activity_single_dest(delete, creator, recipient.inbox_url.into(), context).await?;
Ok(())
}
@ -109,7 +109,7 @@ impl ApubObjectType for PrivateMessage {
.set_id(generate_activity_id(UndoType::Undo)?)
.set_to(recipient.actor_id());
send_activity_single_dest(undo, creator, recipient.get_inbox_url()?, context).await?;
send_activity_single_dest(undo, creator, recipient.inbox_url.into(), context).await?;
Ok(())
}

15
crates/apub/src/activities/send/user.rs

@ -25,6 +25,9 @@ use url::Url;
#[async_trait::async_trait(?Send)]
impl ActorType for User_ {
fn is_local(&self) -> bool {
self.local
}
fn actor_id(&self) -> Url {
self.actor_id.to_owned().into_inner()
}
@ -37,6 +40,14 @@ impl ActorType for User_ {
self.private_key.to_owned()
}
fn get_shared_inbox_or_inbox_url(&self) -> Url {
self
.shared_inbox_url
.clone()
.unwrap_or_else(|| self.inbox_url.to_owned())
.into()
}
/// As a given local user, send out a follow request to a remote community.
async fn send_follow(
&self,
@ -65,7 +76,7 @@ impl ActorType for User_ {
.set_id(generate_activity_id(FollowType::Follow)?)
.set_to(community.actor_id());
send_activity_single_dest(follow, self, community.get_inbox_url()?, context).await?;
send_activity_single_dest(follow, self, community.inbox_url.into(), context).await?;
Ok(())
}
@ -96,7 +107,7 @@ impl ActorType for User_ {
.set_id(generate_activity_id(UndoType::Undo)?)
.set_to(community.actor_id());
send_activity_single_dest(undo, self, community.get_inbox_url()?, context).await?;
send_activity_single_dest(undo, self, community.inbox_url.into(), context).await?;
Ok(())
}

4
crates/apub/src/activity_queue.rs

@ -94,7 +94,7 @@ where
.collect();
debug!(
"Sending activity {:?} to followers of {}",
&activity.id_unchecked(),
&activity.id_unchecked().map(|i| i.to_string()),
&community.actor_id
);
@ -135,7 +135,7 @@ where
.send_announce(activity.into_any_base()?, context)
.await?;
} else {
let inbox = community.get_shared_inbox_url()?;
let inbox = community.get_shared_inbox_or_inbox_url();
check_is_apub_id_valid(&inbox)?;
debug!(
"Sending activity {:?} to community {}",

14
crates/apub/src/fetcher/community.rs

@ -7,10 +7,10 @@ use crate::{
},
inbox::user_inbox::receive_announce,
objects::FromApub,
ActorType,
GroupExt,
};
use activitystreams::{
actor::ApActorExt,
collection::{CollectionExt, OrderedCollection},
object::ObjectExt,
};
@ -116,7 +116,8 @@ async fn fetch_remote_community(
// only fetch outbox for new communities, otherwise this can create an infinite loop
if old_community.is_none() {
fetch_community_outbox(context, &community, recursion_counter).await?
let outbox = group.inner.outbox()?.context(location_info!())?;
fetch_community_outbox(context, outbox, &community, recursion_counter).await?
}
Ok(community)
@ -124,15 +125,12 @@ async fn fetch_remote_community(
async fn fetch_community_outbox(
context: &LemmyContext,
outbox: &Url,
community: &Community,
recursion_counter: &mut i32,
) -> Result<(), LemmyError> {
let outbox = fetch_remote_object::<OrderedCollection>(
context.client(),
&community.get_outbox_url()?,
recursion_counter,
)
.await?;
let outbox =
fetch_remote_object::<OrderedCollection>(context.client(), outbox, recursion_counter).await?;
let outbox_activities = outbox.items().context(location_info!())?.clone();
let mut outbox_activities = outbox_activities.many().context(location_info!())?;
if outbox_activities.len() > 20 {

2
crates/apub/src/http/community.rs

@ -60,7 +60,7 @@ pub async fn get_apub_community_followers(
let mut collection = UnorderedCollection::new();
collection
.set_many_contexts(lemmy_context()?)
.set_id(community.get_followers_url()?)
.set_id(community.followers_url.into())
.set_total_items(community_followers.len() as u64);
Ok(create_apub_response(&collection))
}

25
crates/apub/src/inbox/mod.rs

@ -12,7 +12,11 @@ use activitystreams::{
};
use actix_web::HttpRequest;
use anyhow::{anyhow, Context};
use lemmy_db_queries::{source::activity::Activity_, ApubObject, DbPool};
use lemmy_db_queries::{
source::{activity::Activity_, community::Community_},
ApubObject,
DbPool,
};
use lemmy_db_schema::source::{activity::Activity, community::Community, user::User_};
use lemmy_structs::blocking;
use lemmy_utils::{location_info, settings::Settings, LemmyError};
@ -141,16 +145,15 @@ pub(crate) async fn is_addressed_to_community_followers(
pool: &DbPool,
) -> Result<Option<Community>, LemmyError> {
for url in to_and_cc {
let url = url.to_string();
// TODO: extremely hacky, we should just store the followers url for each community in the db
if url.ends_with("/followers") {
let community_url = Url::parse(&url.replace("/followers", ""))?;
let community = blocking(&pool, move |conn| {
Community::read_from_apub_id(&conn, &community_url.into())
})
.await??;
if !community.local {
return Ok(Some(community));
let url = url.to_owned().into();
let community = blocking(&pool, move |conn| {
// ignore errors here, because the current url might not actually be a followers url
Community::read_from_followers_url(&conn, &url).ok()
})
.await?;
if let Some(c) = community {
if !c.local {
return Ok(Some(c));
}
}
}

95
crates/apub/src/lib.rs

@ -140,6 +140,7 @@ pub trait ApubLikeableType {
/// implemented by all actors.
#[async_trait::async_trait(?Send)]
pub trait ActorType {
fn is_local(&self) -> bool;
fn actor_id(&self) -> Url;
// TODO: every actor should have a public key, so this shouldnt be an option (needs to be fixed in db)
@ -178,32 +179,15 @@ pub trait ActorType {
/// For a given community, returns the inboxes of all followers.
async fn get_follower_inboxes(&self, pool: &DbPool) -> Result<Vec<Url>, LemmyError>;
// TODO move these to the db rows
fn get_inbox_url(&self) -> Result<Url, ParseError> {
Url::parse(&format!("{}/inbox", &self.actor_id()))
}
fn get_shared_inbox_url(&self) -> Result<Url, LemmyError> {
let actor_id = self.actor_id();
let url = format!(
"{}://{}{}/inbox",
&actor_id.scheme(),
&actor_id.host_str().context(location_info!())?,
if let Some(port) = actor_id.port() {
format!(":{}", port)
} else {
"".to_string()
},
);
Ok(Url::parse(&url)?)
}
fn get_outbox_url(&self) -> Result<Url, ParseError> {
Url::parse(&format!("{}/outbox", &self.actor_id()))
}
fn get_shared_inbox_or_inbox_url(&self) -> Url;
fn get_followers_url(&self) -> Result<Url, ParseError> {
Url::parse(&format!("{}/followers", &self.actor_id()))
/// Outbox URL is not generally used by Lemmy, so it can be generated on the fly (but only for
/// local actors).
dessalines marked this conversation as resolved
Review

Yeah these ones, check to see if you can do a default here.

Yeah these ones, check to see if you can do a default here.
Review

How do you mean?

How do you mean?
Review

You should be able to do a default function on the trait, so you don't have to do impls twice if its doing the same thing: https://doc.rust-lang.org/book/ch10-02-traits.html#default-implementations

You should be able to do a default function on the trait, so you don't have to do impls twice if its doing the same thing: https://doc.rust-lang.org/book/ch10-02-traits.html#default-implementations
Review

nm I see you got it.

nm I see you got it.
fn get_outbox_url(&self) -> Result<Url, LemmyError> {
if !self.is_local() {
return Err(anyhow!("get_outbox_url() called for remote actor").into());
}
Ok(Url::parse(&format!("{}/outbox", &self.actor_id()))?)
}
fn get_public_key_ext(&self) -> Result<PublicKeyExtension, LemmyError> {
@ -218,6 +202,67 @@ pub trait ActorType {
}
}
pub enum EndpointType {
Community,
User,
Post,
Comment,
PrivateMessage,
}
/// Generates the ActivityPub ID for a given object type and ID.
pub fn generate_apub_endpoint(
endpoint_type: EndpointType,
name: &str,
) -> Result<lemmy_db_schema::Url, ParseError> {
let point = match endpoint_type {
EndpointType::Community => "c",
EndpointType::User => "u",
EndpointType::Post => "post",
EndpointType::Comment => "comment",
EndpointType::PrivateMessage => "private_message",
};
Ok(
Url::parse(&format!(
"{}/{}/{}",
Settings::get().get_protocol_and_hostname(),
dessalines marked this conversation as resolved
Review

This function here could generate a tuple rather than splitting them out. I don't think you have to use every value in the tuple.

This function here could generate a tuple rather than splitting them out. I don't think you have to use every value in the tuple.
point,
name
))?
.into(),
)
}
pub fn generate_followers_url(
actor_id: &lemmy_db_schema::Url,
) -> Result<lemmy_db_schema::Url, ParseError> {
Ok(Url::parse(&format!("{}/followers", actor_id))?.into())
}
pub fn generate_inbox_url(
actor_id: &lemmy_db_schema::Url,
) -> Result<lemmy_db_schema::Url, ParseError> {
Ok(Url::parse(&format!("{}/inbox", actor_id))?.into())
}
pub fn generate_shared_inbox_url(
actor_id: &lemmy_db_schema::Url,
) -> Result<lemmy_db_schema::Url, LemmyError> {
let actor_id = actor_id.clone().into_inner();
let url = format!(
"{}://{}{}/inbox",
&actor_id.scheme(),
&actor_id.host_str().context(location_info!())?,
if let Some(port) = actor_id.port() {
format!(":{}", port)
} else {
"".to_string()
},
);
Ok(Url::parse(&url)?.into())
}
/// Store a sent or received activity in the database, for logging purposes. These records are not
/// persistent.
pub(crate) async fn insert_activity<T>(

23
crates/apub/src/objects/community.rs

@ -83,13 +83,13 @@ impl ToApub for Community {
group.set_image(image.into_any_base()?);
}
let mut ap_actor = ApActor::new(self.get_inbox_url()?, group);
let mut ap_actor = ApActor::new(self.inbox_url.clone().into(), group);
ap_actor
.set_preferred_username(self.name.to_owned())
.set_outbox(self.get_outbox_url()?)
.set_followers(self.get_followers_url()?)
.set_followers(self.followers_url.clone().into())
.set_endpoints(Endpoints {
shared_inbox: Some(self.get_shared_inbox_url()?),
shared_inbox: Some(self.get_shared_inbox_or_inbox_url()),
..Default::default()
});
@ -184,7 +184,6 @@ impl FromApubToForm<GroupExt> for CommunityForm {
),
None => None,
};
let banner = match group.image() {
Some(any_image) => Some(
Image::from_any_base(any_image.as_one().context(location_info!())?.clone())
@ -197,6 +196,12 @@ impl FromApubToForm<GroupExt> for CommunityForm {
),
None => None,
};
let shared_inbox = group
.inner
.endpoints()?
.map(|e| e.shared_inbox)
.flatten()
.map(|s| s.to_owned().into());
Ok(CommunityForm {
name,
@ -216,6 +221,16 @@ impl FromApubToForm<GroupExt> for CommunityForm {
last_refreshed_at: Some(naive_now()),
icon,
banner,
followers_url: Some(
group
.inner
.followers()?
.context(location_info!())?
.to_owned()
.into(),
),
inbox_url: Some(group.inner.inbox()?.to_owned().into()),
shared_inbox_url: Some(shared_inbox),
})
}
}

13
crates/apub/src/objects/user.rs

@ -71,12 +71,12 @@ impl ToApub for User_ {
person.set_name(i);
}
let mut ap_actor = ApActor::new(self.get_inbox_url()?, person);
let mut ap_actor = ApActor::new(self.inbox_url.clone().into(), person);
ap_actor
.set_preferred_username(self.name.to_owned())
.set_outbox(self.get_outbox_url()?)
.set_endpoints(Endpoints {
shared_inbox: Some(self.get_shared_inbox_url()?),
shared_inbox: Some(self.get_shared_inbox_or_inbox_url()),
..Default::default()
});
@ -158,8 +158,13 @@ impl FromApubToForm<PersonExt> for UserForm {
.flatten()
.map(|n| n.to_owned().xsd_string())
.flatten();
let bio = get_source_markdown_value(person)?;
let shared_inbox = person
.inner
.endpoints()?
.map(|e| e.shared_inbox)
.flatten()
.map(|s| s.to_owned().into());
check_slurs(&name)?;
check_slurs_opt(&preferred_username)?;
@ -190,6 +195,8 @@ impl FromApubToForm<PersonExt> for UserForm {
private_key: None,
public_key: Some(person.ext_one.public_key.to_owned().public_key_pem),
last_refreshed_at: Some(naive_now()),
inbox_url: Some(person.inner.inbox()?.to_owned().into()),
shared_inbox_url: Some(shared_inbox),
})
}
}

7
crates/db_queries/src/aggregates/comment_aggregates.rs

@ -67,6 +67,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -96,6 +98,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let another_inserted_user = User_::create(&conn, &another_user).unwrap();
@ -118,6 +122,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

10
crates/db_queries/src/aggregates/community_aggregates.rs

@ -71,6 +71,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -100,6 +102,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let another_inserted_user = User_::create(&conn, &another_user).unwrap();
@ -122,6 +126,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -144,6 +151,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let another_inserted_community = Community::create(&conn, &another_community).unwrap();

7
crates/db_queries/src/aggregates/post_aggregates.rs

@ -70,6 +70,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -99,6 +101,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let another_inserted_user = User_::create(&conn, &another_user).unwrap();
@ -121,6 +125,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

5
crates/db_queries/src/aggregates/site_aggregates.rs

@ -69,6 +69,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -105,6 +107,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

7
crates/db_queries/src/aggregates/user_aggregates.rs

@ -67,6 +67,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -96,6 +98,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let another_inserted_user = User_::create(&conn, &another_user).unwrap();
@ -118,6 +122,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

2
crates/db_queries/src/source/activity.rs

@ -162,6 +162,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_creator = User_::create(&conn, &creator_form).unwrap();

9
crates/db_queries/src/source/comment.rs

@ -14,7 +14,7 @@ use lemmy_db_schema::{
};
pub trait Comment_ {
fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: String) -> Result<Comment, Error>;
fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: Url) -> Result<Comment, Error>;
fn permadelete_for_creator(
conn: &PgConnection,
for_creator_id: i32,
@ -43,7 +43,7 @@ pub trait Comment_ {
}
impl Comment_ for Comment {
fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: String) -> Result<Self, Error> {
fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: Url) -> Result<Self, Error> {
use lemmy_db_schema::schema::comment::dsl::*;
diesel::update(comment.find(comment_id))
@ -242,6 +242,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -264,6 +266,9 @@ mod tests {
published: None,
banner: None,
icon: None,
inbox_url: None,
shared_inbox_url: None,
followers_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

19
crates/db_queries/src/source/community.rs

@ -133,6 +133,7 @@ pub trait Community_ {
new_creator_id: i32,
) -> Result<Community, Error>;
fn distinct_federated_communities(conn: &PgConnection) -> Result<Vec<String>, Error>;
fn read_from_followers_url(conn: &PgConnection, followers_url: &Url) -> Result<Community, Error>;
}
impl Community_ for Community {
@ -192,6 +193,16 @@ impl Community_ for Community {
use lemmy_db_schema::schema::community::dsl::*;
community.select(actor_id).distinct().load::<String>(conn)
}
fn read_from_followers_url(
conn: &PgConnection,
followers_url_: &Url,
) -> Result<Community, Error> {
use lemmy_db_schema::schema::community::dsl::*;
community
.filter(followers_url.eq(followers_url_))
.first::<Self>(conn)
}
}
impl Joinable<CommunityModeratorForm> for CommunityModerator {
@ -361,6 +372,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -383,6 +396,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();
@ -406,6 +422,9 @@ mod tests {
last_refreshed_at: inserted_community.published,
icon: None,
banner: None,
followers_url: inserted_community.followers_url.to_owned(),
inbox_url: inserted_community.inbox_url.to_owned(),
shared_inbox_url: None,
};
let community_follower_form = CommunityFollowerForm {

7
crates/db_queries/src/source/moderator.rs

@ -230,6 +230,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_mod = User_::create(&conn, &new_mod).unwrap();
@ -259,6 +261,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_user = User_::create(&conn, &new_user).unwrap();
@ -281,6 +285,9 @@ mod tests {
published: None,
icon: None,
banner: None,
followers_url: None,
inbox_url: None,
shared_inbox_url: None,
};
let inserted_community = Community::create(&conn, &new_community).unwrap();

2
crates/db_queries/src/source/password_reset_request.rs

@ -110,6 +110,8 @@ mod tests {
private_key: None,
public_key: None,
last_refreshed_at: None,
inbox_url: None,
shared_inbox_url: None,
};