Browse Source

Split code into cargo workspaces (#67)

More fixes

- fixed docker builds
- fixed mentions regex test
- fixed DATABASE_URL stuff
- change schema path in diesel.toml

Address review comments

- add jsonb column back into activity table
- remove authors field from cargo.toml
- adjust LEMMY_DATABASE_URL env var usage
- rename all occurences of LEMMY_DATABASE_URL to DATABASE_URL

Decouple utils and db

Split code into cargo workspaces

Co-authored-by: Felix Ableitner <me@nutomic.com>
Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/67
admin_sort_order
nutomic 9 months ago
committed by dessalines
parent
commit
80aef61aed
63 changed files with 1071 additions and 921 deletions
  1. +8
    -1
      docker/dev/Dockerfile
  2. +8
    -2
      docker/prod/Dockerfile
  3. +1
    -1
      docs/src/contributing_tests.md
  4. +1
    -1
      install.sh
  5. +37
    -12
      server/Cargo.lock
  6. +9
    -8
      server/Cargo.toml
  7. +2
    -1
      server/db-init.sh
  8. +1
    -1
      server/diesel.toml
  9. +15
    -0
      server/lemmy_db/Cargo.toml
  10. +25
    -25
      server/lemmy_db/src/activity.rs
  11. +2
    -3
      server/lemmy_db/src/category.rs
  12. +9
    -12
      server/lemmy_db/src/comment.rs
  13. +10
    -4
      server/lemmy_db/src/comment_view.rs
  14. +5
    -3
      server/lemmy_db/src/community.rs
  15. +1
    -1
      server/lemmy_db/src/community_view.rs
  16. +33
    -5
      server/lemmy_db/src/lib.rs
  17. +10
    -5
      server/lemmy_db/src/moderator.rs
  18. +1
    -1
      server/lemmy_db/src/moderator_views.rs
  19. +2
    -2
      server/lemmy_db/src/password_reset_request.rs
  20. +13
    -9
      server/lemmy_db/src/post.rs
  21. +9
    -4
      server/lemmy_db/src/post_view.rs
  22. +14
    -14
      server/lemmy_db/src/private_message.rs
  23. +1
    -1
      server/lemmy_db/src/private_message_view.rs
  24. +0
    -0
      server/lemmy_db/src/schema.rs
  25. +1
    -1
      server/lemmy_db/src/site.rs
  26. +0
    -0
      server/lemmy_db/src/site_view.rs
  27. +6
    -77
      server/lemmy_db/src/user.rs
  28. +10
    -5
      server/lemmy_db/src/user_mention.rs
  29. +1
    -1
      server/lemmy_db/src/user_mention_view.rs
  30. +1
    -1
      server/lemmy_db/src/user_view.rs
  31. +22
    -0
      server/lemmy_utils/Cargo.toml
  32. +324
    -0
      server/lemmy_utils/src/lib.rs
  33. +15
    -18
      server/lemmy_utils/src/settings.rs
  34. +73
    -0
      server/src/api/claims.rs
  35. +29
    -23
      server/src/api/comment.rs
  36. +12
    -14
      server/src/api/community.rs
  37. +3
    -6
      server/src/api/mod.rs
  38. +29
    -25
      server/src/api/post.rs
  39. +17
    -20
      server/src/api/site.rs
  40. +51
    -44
      server/src/api/user.rs
  41. +9
    -3
      server/src/apub/activities.rs
  42. +8
    -10
      server/src/apub/comment.rs
  43. +41
    -43
      server/src/apub/community.rs
  44. +6
    -6
      server/src/apub/community_inbox.rs
  45. +2
    -4
      server/src/apub/extensions/group_extensions.rs
  46. +0
    -18
      server/src/apub/extensions/signatures.rs
  47. +16
    -23
      server/src/apub/fetcher.rs
  48. +20
    -40
      server/src/apub/mod.rs
  49. +7
    -9
      server/src/apub/post.rs
  50. +7
    -7
      server/src/apub/private_message.rs
  51. +15
    -14
      server/src/apub/shared_inbox.rs
  52. +16
    -8
      server/src/apub/user.rs
  53. +10
    -10
      server/src/apub/user_inbox.rs
  54. +17
    -16
      server/src/code_migrations.rs
  55. +43
    -288
      server/src/lib.rs
  56. +9
    -7
      server/src/main.rs
  57. +3
    -2
      server/src/rate_limit/mod.rs
  58. +10
    -12
      server/src/routes/federation.rs
  59. +13
    -18
      server/src/routes/feeds.rs
  60. +1
    -1
      server/src/routes/index.rs
  61. +3
    -9
      server/src/routes/nodeinfo.rs
  62. +3
    -21
      server/src/routes/webfinger.rs
  63. +1
    -1
      ui/package.json

+ 8
- 1
docker/dev/Dockerfile View File

@ -17,13 +17,20 @@ WORKDIR /app
RUN sudo chown -R rust:rust .
RUN USER=root cargo new server
WORKDIR /app/server
RUN mkdir -p lemmy_db/src/ lemmy_utils/src/
COPY server/Cargo.toml server/Cargo.lock ./
COPY server/lemmy_db/Cargo.toml ./lemmy_db/
COPY server/lemmy_utils/Cargo.toml ./lemmy_utils/
RUN sudo chown -R rust:rust .
RUN mkdir -p ./src/bin \
&& echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs
&& echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \
&& cp ./src/bin/main.rs ./lemmy_db/src/main.rs \
&& cp ./src/bin/main.rs ./lemmy_utils/src/main.rs
RUN cargo build
RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server*
COPY server/src ./src/
COPY server/lemmy_db ./lemmy_db/
COPY server/lemmy_utils ./lemmy_utils/
COPY server/migrations ./migrations/
# Build for debug


+ 8
- 2
docker/prod/Dockerfile View File

@ -10,13 +10,19 @@ WORKDIR /app
RUN sudo chown -R rust:rust .
RUN USER=root cargo new server
WORKDIR /app/server
RUN mkdir -p lemmy_db/src/ lemmy_utils/src/
COPY --chown=rust:rust server/Cargo.toml server/Cargo.lock ./
#RUN sudo chown -R rust:rust .
COPY --chown=rust:rust server/lemmy_db/Cargo.toml ./lemmy_db/
COPY --chown=rust:rust server/lemmy_utils/Cargo.toml ./lemmy_utils/
RUN mkdir -p ./src/bin \
&& echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs
&& echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \
&& cp ./src/bin/main.rs ./lemmy_db/src/main.rs \
&& cp ./src/bin/main.rs ./lemmy_utils/src/main.rs
RUN cargo build --release
RUN rm -f ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/deps/lemmy_server*
COPY --chown=rust:rust server/src ./src/
COPY --chown=rust:rust server/lemmy_db ./lemmy_db/
COPY --chown=rust:rust server/lemmy_utils ./lemmy_utils/
COPY --chown=rust:rust server/migrations ./migrations/
# build for release


+ 1
- 1
docs/src/contributing_tests.md View File

@ -9,7 +9,7 @@ following commands in the `server` subfolder:
psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
diesel migration run
RUST_TEST_THREADS=1 cargo test
RUST_TEST_THREADS=1 cargo test --workspace
```
### Federation


+ 1
- 1
install.sh View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
set -e
# Set the database variable to the default first.


+ 37
- 12
server/Cargo.lock View File

@ -1399,12 +1399,6 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "htmlescape"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
[[package]]
name = "http"
version = "0.2.1"
@ -1572,6 +1566,21 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lemmy_db"
version = "0.1.0"
dependencies = [
"bcrypt",
"chrono",
"diesel",
"log",
"serde 1.0.114",
"serde_json",
"sha2",
"strum",
"strum_macros",
]
[[package]]
name = "lemmy_server"
version = "0.0.1"
@ -1589,27 +1598,23 @@ dependencies = [
"base64 0.12.3",
"bcrypt",
"chrono",
"comrak",
"config",
"diesel",
"diesel_migrations",
"dotenv",
"env_logger",
"failure",
"futures",
"htmlescape",
"http",
"http-signature-normalization-actix",
"itertools",
"jsonwebtoken",
"lazy_static",
"lettre",
"lettre_email",
"lemmy_db",
"lemmy_utils",
"log",
"openssl",
"percent-encoding",
"rand 0.7.3",
"regex",
"rss",
"serde 1.0.114",
"serde_json",
@ -1621,6 +1626,26 @@ dependencies = [
"uuid 0.8.1",
]
[[package]]
name = "lemmy_utils"
version = "0.1.0"
dependencies = [
"chrono",
"comrak",
"config",
"itertools",
"lazy_static",
"lettre",
"lettre_email",
"log",
"openssl",
"rand 0.7.3",
"regex",
"serde 1.0.114",
"serde_json",
"url",
]
[[package]]
name = "lettre"
version = "0.9.3"


+ 9
- 8
server/Cargo.toml View File

@ -1,14 +1,21 @@
[package]
name = "lemmy_server"
version = "0.0.1"
authors = ["Dessalines <tyhou13@gmx.com>"]
edition = "2018"
[profile.release]
lto = true
[workspace]
members = [
"lemmy_utils",
"lemmy_db"
]
[dependencies]
diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] }
lemmy_utils = { path = "./lemmy_utils" }
lemmy_db = { path = "./lemmy_db" }
diesel = "1.4.4"
diesel_migrations = "1.4.0"
dotenv = "0.15.0"
activitystreams = "0.6.2"
@ -31,16 +38,10 @@ rand = "0.7.3"
strum = "0.18.0"
strum_macros = "0.18.0"
jsonwebtoken = "7.0.1"
regex = "1.3.5"
lazy_static = "1.3.0"
lettre = "0.9.3"
lettre_email = "0.9.4"
rss = "1.9.0"
htmlescape = "0.3.1"
url = { version = "2.1.1", features = ["serde"] }
config = {version = "0.10.1", default-features = false, features = ["hjson"] }
percent-encoding = "2.1.0"
comrak = "0.7"
openssl = "0.10"
http = "0.2.1"
http-signature-normalization-actix = { version = "0.4.0-alpha.0", default-features = false, features = ["sha-2"] }


+ 2
- 1
server/db-init.sh View File

@ -1,4 +1,5 @@
#!/bin/sh
#!/bin/bash
set -e
# Default configurations
username=lemmy


+ 1
- 1
server/diesel.toml View File

@ -2,4 +2,4 @@
# see diesel.rs/guides/configuring-diesel-cli
[print_schema]
file = "src/schema.rs"
file = "lemmy_db/src/schema.rs"

+ 15
- 0
server/lemmy_db/Cargo.toml View File

@ -0,0 +1,15 @@
[package]
name = "lemmy_db"
version = "0.1.0"
edition = "2018"
[dependencies]
diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] }
chrono = { version = "0.4.7", features = ["serde"] }
serde = { version = "1.0.105", features = ["derive"] }
serde_json = { version = "1.0.52", features = ["preserve_order"]}
strum = "0.18.0"
strum_macros = "0.18.0"
log = "0.4.0"
sha2 = "0.9"
bcrypt = "0.8.0"

server/src/db/activity.rs → server/lemmy_db/src/activity.rs View File


server/src/db/category.rs → server/lemmy_db/src/category.rs View File


server/src/db/comment.rs → server/lemmy_db/src/comment.rs View File


server/src/db/comment_view.rs → server/lemmy_db/src/comment_view.rs View File


server/src/db/community.rs → server/lemmy_db/src/community.rs View File


server/src/db/community_view.rs → server/lemmy_db/src/community_view.rs View File


server/src/db/mod.rs → server/lemmy_db/src/lib.rs View File


server/src/db/moderator.rs → server/lemmy_db/src/moderator.rs View File


server/src/db/moderator_views.rs → server/lemmy_db/src/moderator_views.rs View File


server/src/db/password_reset_request.rs → server/lemmy_db/src/password_reset_request.rs View File


server/src/db/post.rs → server/lemmy_db/src/post.rs View File


server/src/db/post_view.rs → server/lemmy_db/src/post_view.rs View File


server/src/db/private_message.rs → server/lemmy_db/src/private_message.rs View File


server/src/db/private_message_view.rs → server/lemmy_db/src/private_message_view.rs View File


server/src/schema.rs → server/lemmy_db/src/schema.rs View File


server/src/db/site.rs → server/lemmy_db/src/site.rs View File


server/src/db/site_view.rs → server/lemmy_db/src/site_view.rs View File


server/src/db/user.rs → server/lemmy_db/src/user.rs View File


server/src/db/user_mention.rs → server/lemmy_db/src/user_mention.rs View File


server/src/db/user_mention_view.rs → server/lemmy_db/src/user_mention_view.rs View File


server/src/db/user_view.rs → server/lemmy_db/src/user_view.rs View File


+ 22
- 0
server/lemmy_utils/Cargo.toml View File

@ -0,0 +1,22 @@
[package]
name = "lemmy_utils"
version = "0.1.0"
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
regex = "1.3.5"
config = { version = "0.10.1", default-features = false, features = ["hjson"] }
chrono = { version = "0.4.7", features = ["serde"] }
lettre = "0.9.3"
lettre_email = "0.9.4"
log = "0.4.0"
itertools = "0.9.0"
rand = "0.7.3"
serde = { version = "1.0.105", features = ["derive"] }
serde_json = { version = "1.0.52", features = ["preserve_order"]}
comrak = "0.7"
lazy_static = "1.3.0"
openssl = "0.10"
url = { version = "2.1.1", features = ["serde"] }

+ 324
- 0
server/lemmy_utils/src/lib.rs View File

@ -0,0 +1,324 @@
#[macro_use]
pub extern crate lazy_static;
pub extern crate comrak;
pub extern crate lettre;
pub extern crate lettre_email;
pub extern crate openssl;
pub extern crate rand;
pub extern crate regex;
pub extern crate serde_json;
pub extern crate url;
pub mod settings;
use crate::settings::Settings;
use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, Utc};
use itertools::Itertools;
use lettre::{
smtp::{
authentication::{Credentials, Mechanism},
extension::ClientId,
ConnectionReuseParameters,
},
ClientSecurity,
SmtpClient,
Transport,
};
use lettre_email::Email;
use openssl::{pkey::PKey, rsa::Rsa};
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use regex::{Regex, RegexBuilder};
use std::io::{Error, ErrorKind};
use url::Url;
pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime<Utc> {
DateTime::<Utc>::from_utc(ndt, Utc)
}
pub fn naive_from_unix(time: i64) -> NaiveDateTime {
NaiveDateTime::from_timestamp(time, 0)
}
pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
let now = Local::now();
DateTime::<FixedOffset>::from_utc(datetime, *now.offset())
}
pub fn is_email_regex(test: &str) -> bool {
EMAIL_REGEX.is_match(test)
}
pub fn remove_slurs(test: &str) -> String {
SLUR_REGEX.replace_all(test, "*removed*").to_string()
}
pub fn slur_check(test: &str) -> Result<(), Vec<&str>> {
let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect();
// Unique
matches.sort_unstable();
matches.dedup();
if matches.is_empty() {
Ok(())
} else {
Err(matches)
}
}
pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String {
let start = "No slurs - ";
let combined = &slurs.join(", ");
[start, combined].concat()
}
pub fn generate_random_string() -> String {
thread_rng().sample_iter(&Alphanumeric).take(30).collect()
}
pub fn send_email(
subject: &str,
to_email: &str,
to_username: &str,
html: &str,
) -> Result<(), String> {
let email_config = Settings::get().email.ok_or("no_email_setup")?;
let email = Email::builder()
.to((to_email, to_username))
.from(email_config.smtp_from_address.to_owned())
.subject(subject)
.html(html)
.build()
.unwrap();
let mailer = if email_config.use_tls {
SmtpClient::new_simple(&email_config.smtp_server).unwrap()
} else {
SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap()
}
.hello_name(ClientId::Domain(Settings::get().hostname))
.smtp_utf8(true)
.authentication_mechanism(Mechanism::Plain)
.connection_reuse(ConnectionReuseParameters::ReuseUnlimited);
let mailer = if let (Some(login), Some(password)) =
(&email_config.smtp_login, &email_config.smtp_password)
{
mailer.credentials(Credentials::new(login.to_owned(), password.to_owned()))
} else {
mailer
};
let mut transport = mailer.transport();
let result = transport.send(email.into());
transport.close();
match result {
Ok(_) => Ok(()),
Err(e) => Err(e.to_string()),
}
}
pub fn markdown_to_html(text: &str) -> String {
comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
}
// TODO nothing is done with community / group webfingers yet, so just ignore those for now
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct MentionData {
pub name: String,
pub domain: String,
}
impl MentionData {
pub fn is_local(&self) -> bool {
Settings::get().hostname.eq(&self.domain)
}
pub fn full_name(&self) -> String {
format!("@{}@{}", &self.name, &self.domain)
}
}
pub fn scrape_text_for_mentions(text: &str) -> Vec<MentionData> {
let mut out: Vec<MentionData> = Vec::new();
for caps in MENTIONS_REGEX.captures_iter(text) {
out.push(MentionData {
name: caps["name"].to_string(),
domain: caps["domain"].to_string(),
});
}
out.into_iter().unique().collect()
}
pub fn is_valid_username(name: &str) -> bool {
VALID_USERNAME_REGEX.is_match(name)
}
pub fn is_valid_community_name(name: &str) -> bool {
VALID_COMMUNITY_NAME_REGEX.is_match(name)
}
#[cfg(test)]
mod tests {
use crate::{
is_email_regex,
is_valid_community_name,
is_valid_username,
remove_slurs,
scrape_text_for_mentions,
slur_check,
slurs_vec_to_str,
};
#[test]
fn test_mentions_regex() {
let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)";
let mentions = scrape_text_for_mentions(text);
assert_eq!(mentions[0].name, "tedu".to_string());
assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string());
assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string());
}
#[test]
fn test_email() {
assert!(is_email_regex("gush@gmail.com"));
assert!(!is_email_regex("nada_neutho"));
}
#[test]
fn test_valid_register_username() {
assert!(is_valid_username("Hello_98"));
assert!(is_valid_username("ten"));
assert!(!is_valid_username("Hello-98"));
assert!(!is_valid_username("a"));
assert!(!is_valid_username(""));
}
#[test]
fn test_valid_community_name() {
assert!(is_valid_community_name("example"));
assert!(is_valid_community_name("example_community"));
assert!(!is_valid_community_name("Example"));
assert!(!is_valid_community_name("Ex"));
assert!(!is_valid_community_name(""));
}
#[test]
fn test_slur_filter() {
let test =
"coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
let slur_free = "No slurs here";
assert_eq!(
remove_slurs(&test),
"*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
.to_string()
);
let has_slurs_vec = vec![
"Niggerz",
"coons",
"dindu",
"ladyboy",
"retardeds",
"tranny",
];
let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny";
assert_eq!(slur_check(test), Err(has_slurs_vec));
assert_eq!(slur_check(slur_free), Ok(()));
if let Err(slur_vec) = slur_check(test) {
assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str);
}
}
// These helped with testing
// #[test]
// fn test_send_email() {
// let result = send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
// assert!(result.is_ok());
// }
}
lazy_static! {
static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap();
static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap();
static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap();
// TODO keep this old one, it didn't work with port well tho
// static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap();
static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._:-]+)").unwrap();
static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap();
static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap();
pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!(
"^group:([a-z0-9_]{{3, 20}})@{}$",
Settings::get().hostname
))
.unwrap();
pub static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!(
"^acct:([a-z0-9_]{{3, 20}})@{}$",
Settings::get().hostname
))
.unwrap();
pub static ref CACHE_CONTROL_REGEX: Regex =
Regex::new("^((text|image)/.+|application/javascript)$").unwrap();
}
pub struct Keypair {
pub private_key: String,
pub public_key: String,
}
/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
pub fn generate_actor_keypair() -> Result<Keypair, Error> {
let rsa = Rsa::generate(2048)?;
let pkey = PKey::from_rsa(rsa)?;
let public_key = pkey.public_key_to_pem()?;
let private_key = pkey.private_key_to_pem_pkcs8()?;
let key_to_string = |key| match String::from_utf8(key) {
Ok(s) => Ok(s),
Err(e) => Err(Error::new(
ErrorKind::Other,
format!("Failed converting key to string: {}", e),
)),
};
Ok(Keypair {
private_key: key_to_string(private_key)?,
public_key: key_to_string(public_key)?,
})
}
pub enum EndpointType {
Community,
User,
Post,
Comment,
PrivateMessage,
}
pub fn get_apub_protocol_string() -> &'static str {
if Settings::get().federation.tls_enabled {
"https"
} else {
"http"
}
}
/// Generates the ActivityPub ID for a given object type and ID.
pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
let point = match endpoint_type {
EndpointType::Community => "c",
EndpointType::User => "u",
EndpointType::Post => "post",
EndpointType::Comment => "comment",
EndpointType::PrivateMessage => "private_message",
};
Url::parse(&format!(
"{}://{}/{}/{}",
get_apub_protocol_string(),
Settings::get().hostname,
point,
name
))
.unwrap()
}

server/src/settings.rs → server/lemmy_utils/src/settings.rs View File


+ 73
- 0
server/src/api/claims.rs View File

@ -0,0 +1,73 @@
use diesel::{result::Error, PgConnection};
use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation};
use lemmy_db::{user::User_, Crud};
use lemmy_utils::{is_email_regex, settings::Settings};
use serde::{Deserialize, Serialize};
type Jwt = String;
#[derive(Debug, Serialize, Deserialize)]
pub struct Claims {
pub id: i32,
pub username: String,
pub iss: String,
pub show_nsfw: bool,
pub theme: String,
pub default_sort_type: i16,
pub default_listing_type: i16,
pub lang: String,
pub avatar: Option<String>,
pub show_avatars: bool,
}
impl Claims {
pub fn decode(jwt: &str) -> Result<TokenData<Claims>, jsonwebtoken::errors::Error> {
let v = Validation {
validate_exp: false,
..Validation::default()
};
decode::<Claims>(
&jwt,
&DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()),
&v,
)
}
pub fn jwt(user: User_, hostname: String) -> Jwt {
let my_claims = Claims {
id: user.id,
username: user.name.to_owned(),
iss: hostname,
show_nsfw: user.show_nsfw,
theme: user.theme.to_owned(),
default_sort_type: user.default_sort_type,
default_listing_type: user.default_listing_type,
lang: user.lang.to_owned(),
avatar: user.avatar.to_owned(),
show_avatars: user.show_avatars.to_owned(),
};
encode(
&Header::default(),
&my_claims,
&EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()),
)
.unwrap()
}
// TODO: move these into user?
pub fn find_by_email_or_username(
conn: &PgConnection,
username_or_email: &str,
) -> Result<User_, Error> {
if is_email_regex(username_or_email) {
User_::find_by_email(conn, username_or_email)
} else {
User_::find_by_username(conn, username_or_email)
}
}
pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result<User_, Error> {
let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims;
User_::read(&conn, claims.id)
}
}

+ 29
- 23
server/src/api/comment.rs View File

@ -1,28 +1,7 @@
use crate::{
api::{APIError, Oper, Perform},
api::{claims::Claims, APIError, Oper, Perform},
apub::{ApubLikeableType, ApubObjectType},
blocking,
db::{
comment::*,
comment_view::*,
community_view::*,
moderator::*,
post::*,
site_view::*,
user::*,
user_mention::*,
user_view::*,
Crud,
Likeable,
ListingType,
Saveable,
SortType,
},
naive_now,
remove_slurs,
scrape_text_for_mentions,
send_email,
settings::Settings,
websocket::{
server::{JoinCommunityRoom, SendComment},
UserOperation,
@ -30,6 +9,31 @@ use crate::{
},
DbPool,
LemmyError,
};
use lemmy_db::{
comment::*,
comment_view::*,
community_view::*,
moderator::*,
naive_now,
post::*,
site_view::*,
user::*,
user_mention::*,
user_view::*,
Crud,
Likeable,
ListingType,
Saveable,
SortType,
};
use lemmy_utils::{
make_apub_endpoint,
remove_slurs,
scrape_text_for_mentions,
send_email,
settings::Settings,
EndpointType,
MentionData,
};
use log::error;
@ -155,7 +159,9 @@ impl Perform for Oper<CreateComment> {
let inserted_comment_id = inserted_comment.id;
let updated_comment: Comment = match blocking(pool, move |conn| {
Comment::update_ap_id(&conn, inserted_comment_id)
let apub_id =
make_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string()).to_string();
Comment::update_ap_id(&conn, inserted_comment_id, apub_id)
})
.await?
{


+ 12
- 14
server/src/api/community.rs View File

@ -1,26 +1,24 @@
use super::*;
use crate::{
api::{APIError, Oper, Perform},
apub::{
extensions::signatures::generate_actor_keypair,
make_apub_endpoint,
ActorType,
EndpointType,
},
api::{claims::Claims, APIError, Oper, Perform},
apub::ActorType,
blocking,
db::{Bannable, Crud, Followable, Joinable, SortType},
is_valid_community_name,
naive_from_unix,
naive_now,
slur_check,
slurs_vec_to_str,
websocket::{
server::{JoinCommunityRoom, SendCommunityRoomMessage},
UserOperation,
WebsocketInfo,
},
DbPool,
LemmyError,
};
use lemmy_db::{naive_now, Bannable, Crud, Followable, Joinable, SortType};
use lemmy_utils::{
generate_actor_keypair,
is_valid_community_name,
make_apub_endpoint,
naive_from_unix,
slur_check,
slurs_vec_to_str,
EndpointType,
};
use serde::{Deserialize, Serialize};
use std::str::FromStr;


+ 3
- 6
server/src/api/mod.rs View File

@ -1,11 +1,8 @@
use crate::{
db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*},
websocket::WebsocketInfo,
DbPool,
LemmyError,
};
use crate::{websocket::WebsocketInfo, DbPool, LemmyError};
use actix_web::client::Client;
use lemmy_db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*};
pub mod claims;
pub mod comment;
pub mod community;
pub mod post;


+ 29
- 25
server/src/api/post.rs View File

@ -1,27 +1,8 @@
use crate::{
api::{APIError, Oper, Perform},
api::{claims::Claims, APIError, Oper, Perform},
apub::{ApubLikeableType, ApubObjectType},
blocking,
db::{
comment_view::*,
community_view::*,
moderator::*,
post::*,
post_view::*,
site::*,
site_view::*,
user::*,
user_view::*,
Crud,
Likeable,
ListingType,
Saveable,
SortType,
},
fetch_iframely_and_pictrs_data,
naive_now,
slur_check,
slurs_vec_to_str,
websocket::{
server::{JoinCommunityRoom, JoinPostRoom, SendPost},
UserOperation,
@ -30,6 +11,24 @@ use crate::{
DbPool,
LemmyError,
};
use lemmy_db::{
comment_view::*,
community_view::*,
moderator::*,
naive_now,
post::*,
post_view::*,
site::*,
site_view::*,
user::*,
user_view::*,
Crud,
Likeable,
ListingType,
Saveable,
SortType,
};
use lemmy_utils::{make_apub_endpoint, slur_check, slurs_vec_to_str, EndpointType};
use serde::{Deserialize, Serialize};
use std::str::FromStr;
@ -191,11 +190,16 @@ impl Perform for Oper<CreatePost> {
};
let inserted_post_id = inserted_post.id;
let updated_post =
match blocking(pool, move |conn| Post::update_ap_id(conn, inserted_post_id)).await? {
Ok(post) => post,
Err(_e) => return Err(APIError::err("couldnt_create_post").into()),
};
let updated_post = match blocking(pool, move |conn| {
let apub_id =
make_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string()).to_string();
Post::update_ap_id(conn, inserted_post_id, apub_id)
})
.await?
{
Ok(post) => post,
Err(_e) => return Err(APIError::err("couldnt_create_post").into()),
};
updated_post.send_create(&user, &self.client, pool).await?;


+ 17
- 20
server/src/api/site.rs View File

@ -1,31 +1,28 @@
use super::user::Register;
use crate::{
api::{APIError, Oper, Perform},
api::{claims::Claims, APIError, Oper, Perform},
apub::fetcher::search_by_apub_id,
blocking,
db::{
category::*,
comment_view::*,
community_view::*,
moderator::*,
moderator_views::*,
post_view::*,
site::*,
site_view::*,
user::*,
user_view::*,
Crud,
SearchType,
SortType,
},
naive_now,
settings::Settings,
slur_check,
slurs_vec_to_str,
websocket::{server::SendAllMessage, UserOperation, WebsocketInfo},
DbPool,
LemmyError,
};
use lemmy_db::{
category::*,
comment_view::*,
community_view::*,
moderator::*,
moderator_views::*,
naive_now,
post_view::*,
site::*,
site_view::*,
user_view::*,
Crud,
SearchType,
SortType,
};
use lemmy_utils::{settings::Settings, slur_check, slurs_vec_to_str};
use log::{debug, info};
use serde::{Deserialize, Serialize};
use std::str::FromStr;


+ 51
- 44
server/src/api/user.rs View File

@ -1,53 +1,53 @@
use crate::{
api::{APIError, Oper, Perform},
apub::{
extensions::signatures::generate_actor_keypair,
make_apub_endpoint,
ApubObjectType,
EndpointType,
},
api::{claims::Claims, APIError, Oper, Perform},
apub::ApubObjectType,
blocking,
db::{
comment::*,
comment_view::*,
community::*,
community_view::*,
moderator::*,
password_reset_request::*,
post::*,
post_view::*,
private_message::*,
private_message_view::*,
site::*,
site_view::*,
user::*,
user_mention::*,
user_mention_view::*,
user_view::*,
Crud,
Followable,
Joinable,
ListingType,
SortType,
websocket::{
server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage},
UserOperation,
WebsocketInfo,
},
DbPool,
LemmyError,
};
use bcrypt::verify;
use lemmy_db::{
comment::*,
comment_view::*,
community::*,
community_view::*,
moderator::*,
naive_now,
password_reset_request::*,
post::*,
post_view::*,
private_message::*,
private_message_view::*,
site::*,
site_view::*,
user::*,
user_mention::*,
user_mention_view::*,
user_view::*,
Crud,
Followable,
Joinable,
ListingType,
SortType,
};
use lemmy_utils::{
generate_actor_keypair,
generate_random_string,
is_valid_username,
make_apub_endpoint,
naive_from_unix,
naive_now,
remove_slurs,
send_email,
settings::Settings,
slur_check,
slurs_vec_to_str,
websocket::{
server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage},
UserOperation,
WebsocketInfo,
},
DbPool,
LemmyError,
EndpointType,
};
use bcrypt::verify;
use log::error;
use serde::{Deserialize, Serialize};
use std::str::FromStr;
@ -264,7 +264,7 @@ impl Perform for Oper<Login> {
// Fetch that username / email
let username_or_email = data.username_or_email.clone();
let user = match blocking(pool, move |conn| {
User_::find_by_email_or_username(conn, &username_or_email)
Claims::find_by_email_or_username(conn, &username_or_email)
})
.await?
{
@ -279,7 +279,9 @@ impl Perform for Oper<Login> {
}
// Return the jwt
Ok(LoginResponse { jwt: user.jwt() })
Ok(LoginResponse {
jwt: Claims::jwt(user, Settings::get().hostname),
})
}
}
@ -421,7 +423,7 @@ impl Perform for Oper<Register> {
// Return the jwt
Ok(LoginResponse {
jwt: inserted_user.jwt(),
jwt: Claims::jwt(inserted_user, Settings::get().hostname),
})
}
}
@ -532,7 +534,7 @@ impl Perform for Oper<SaveUserSettings> {
// Return the jwt
Ok(LoginResponse {
jwt: updated_user.jwt(),
jwt: Claims::jwt(updated_user, Settings::get().hostname),
})
}
}
@ -1155,7 +1157,7 @@ impl Perform for Oper<PasswordChange> {
// Return the jwt
Ok(LoginResponse {
jwt: updated_user.jwt(),
jwt: Claims::jwt(updated_user, Settings::get().hostname),
})
}
}
@ -1213,7 +1215,12 @@ impl Perform for Oper<CreatePrivateMessage> {
let inserted_private_message_id = inserted_private_message.id;
let updated_private_message = match blocking(pool, move |conn| {
PrivateMessage::update_ap_id(&conn, inserted_private_message_id)
let apub_id = make_apub_endpoint(
EndpointType::PrivateMessage,
&inserted_private_message_id.to_string(),
)
.to_string();
PrivateMessage::update_ap_id(&conn, inserted_private_message_id, apub_id)
})
.await?
{


+ 9
- 3
server/src/apub/activities.rs View File

@ -1,12 +1,18 @@
use crate::{
apub::{extensions::signatures::sign, is_apub_id_valid, ActorType},
db::{activity::insert_activity, community::Community, user::User_},
apub::{
community::do_announce,
extensions::signatures::sign,
insert_activity,
is_apub_id_valid,
ActorType,
},
request::retry_custom,
DbPool,
LemmyError,
};
use activitystreams::{context, object::properties::ObjectProperties, public, Activity, Base};
use actix_web::client::Client;
use lemmy_db::{community::Community, user::User_};
use log::debug;
use serde::Serialize;
use std::fmt::Debug;
@ -43,7 +49,7 @@ where
// if this is a local community, we need to do an announce from the community instead
if community.local {
Community::do_announce(activity, &community, creator, client, pool).await?;
do_announce(activity, &community, creator, client, pool).await?;
} else {
send_activity(client, &activity, creator, to).await?;
}


+ 8
- 10
server/src/apub/comment.rs View File

@ -17,19 +17,9 @@ use crate::{
ToApub,
},
blocking,
convert_datetime,
db::{
comment::{Comment, CommentForm},
community::Community,
post::Post,
user::User_,
Crud,
},
routes::DbPoolParam,
scrape_text_for_mentions,
DbPool,
LemmyError,
MentionData,
};
use activitystreams::{
activity::{Create, Delete, Dislike, Like, Remove, Undo, Update},
@ -40,6 +30,14 @@ use activitystreams::{
use activitystreams_new::object::Tombstone;
use actix_web::{body::Body, client::Client, web::Path, HttpResponse};
use itertools::Itertools;
use lemmy_db::{
comment::{Comment, CommentForm},
community::Community,
post::Post,
user::User_,
Crud,
};
use lemmy_utils::{convert_datetime, scrape_text_for_mentions, MentionData};
use log::debug;
use serde::Deserialize;


+ 41
- 43
server/src/apub/community.rs View File

@ -7,20 +7,13 @@ use crate::{
extensions::group_extensions::GroupExtension,
fetcher::get_or_fetch_and_upsert_remote_user,
get_shared_inbox,
insert_activity,
ActorType,
FromApub,
GroupExt,
ToApub,
},
blocking,
convert_datetime,
db::{
activity::insert_activity,
community::{Community, CommunityForm},
community_view::{CommunityFollowerView, CommunityModeratorView},
user::User_,
},
naive_now,
routes::DbPoolParam,
DbPool,
LemmyError,
@ -44,6 +37,13 @@ use activitystreams_new::{
};
use actix_web::{body::Body, client::Client, web, HttpResponse};
use itertools::Itertools;
use lemmy_db::{
community::{Community, CommunityForm},
community_view::{CommunityFollowerView, CommunityModeratorView},
naive_now,
user::User_,
};
use lemmy_utils::convert_datetime;
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, str::FromStr};
@ -462,39 +462,37 @@ pub async fn get_apub_community_followers(
Ok(create_apub_response(&collection))
}
impl Community {
pub async fn do_announce<A>(
activity: A,
community: &Community,
sender: &dyn ActorType,
client: &Client,
pool: &DbPool,
) -> Result<HttpResponse, LemmyError>
where
A: Activity + Base + Serialize + Debug,
{
let mut announce = Announce::default();
populate_object_props(
&mut announce.object_props,
vec![community.get_followers_url()],
&format!("{}/announce/{}", community.actor_id, uuid::Uuid::new_v4()),
)?;
announce
.announce_props
.set_actor_xsd_any_uri(community.actor_id.to_owned())?
.set_object_base_box(BaseBox::from_concrete(activity)?)?;
insert_activity(community.creator_id, announce.clone(), true, pool).await?;
// dont send to the instance where the activity originally came from, because that would result
// in a database error (same data inserted twice)
let mut to = community.get_follower_inboxes(pool).await?;
// this seems to be the "easiest" stable alternative for remove_item()
to.retain(|x| *x != sender.get_shared_inbox_url());
send_activity(client, &announce, community, to).await?;
Ok(HttpResponse::Ok().finish())
}
pub async fn do_announce<A>(
activity: A,
community: &Community,
sender: &dyn ActorType,
client: &Client,
pool: &DbPool,
) -> Result<HttpResponse, LemmyError>
where
A: Activity + Base + Serialize + Debug,
{
let mut announce = Announce::default();
populate_object_props(
&mut announce.object_props,
vec![community.get_followers_url()],
&format!("{}/announce/{}", community.actor_id, uuid::Uuid::new_v4()),
)?;
announce
.announce_props
.set_actor_xsd_any_uri(community.actor_id.to_owned())?
.set_object_base_box(BaseBox::from_concrete(activity)?)?;
insert_activity(community.creator_id, announce.clone(), true, pool).await?;
// dont send to the instance where the activity originally came from, because that would result
// in a database error (same data inserted twice)
let mut to = community.get_follower_inboxes(pool).await?;
// this seems to be the "easiest" stable alternative for remove_item()
to.retain(|x| *x != sender.get_shared_inbox_url());
send_activity(client, &announce, community, to).await?;
Ok(HttpResponse::Ok().finish())
}

+ 6
- 6
server/src/apub/community_inbox.rs View File

@ -2,21 +2,21 @@ use crate::{
apub::{
extensions::signatures::verify,
fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user},
insert_activity,
ActorType,
},
blocking,
db::{
activity::insert_activity,
community::{Community, CommunityFollower, CommunityFollowerForm},
user::User_,
Followable,
},
routes::{ChatServerParam, DbPoolParam},
LemmyError,
};
use activitystreams::activity::Undo;
use activitystreams_new::activity::Follow;
use actix_web::{client::Client, web, HttpRequest, HttpResponse};
use lemmy_db::{
community::{Community, CommunityFollower, CommunityFollowerForm},
user::User_,
Followable,
};
use log::debug;
use serde::Deserialize;
use std::fmt::Debug;


+ 2
- 4
server/src/apub/extensions/group_extensions.rs View File

@ -1,9 +1,7 @@
use crate::{
db::{category::Category, Crud},
LemmyError,
};
use crate::LemmyError;
use activitystreams::{ext::Extension, Actor};
use diesel::PgConnection;
use lemmy_db::{category::Category, Crud};
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Default, Deserialize, Serialize)]


+ 0
- 18
server/src/apub/extensions/signatures.rs View File

@ -9,7 +9,6 @@ use log::debug;
use openssl::{
hash::MessageDigest,
pkey::PKey,
rsa::Rsa,
sign::{Signer, Verifier},
};
use serde::{Deserialize, Serialize};
@ -19,23 +18,6 @@ lazy_static! {
static ref HTTP_SIG_CONFIG: Config = Config::new();
}
pub struct Keypair {
pub private_key: String,
pub public_key: String,
}
/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
pub fn generate_actor_keypair() -> Result<Keypair, LemmyError> {
let rsa = Rsa::generate(2048)?;
let pkey = PKey::from_rsa(rsa)?;
let public_key = pkey.public_key_to_pem()?;
let private_key = pkey.private_key_to_pem_pkcs8()?;
Ok(Keypair {
private_key: String::from_utf8(private_key)?,
public_key: String::from_utf8(public_key)?,
})
}
/// Signs request headers with the given keypair.
pub async fn sign(
request: ClientRequest,


+ 16
- 23
server/src/apub/fetcher.rs View File

@ -1,29 +1,7 @@
use crate::{
api::site::SearchResponse,
apub::{
get_apub_protocol_string,
is_apub_id_valid,
FromApub,
GroupExt,
PageExt,
PersonExt,
APUB_JSON_CONTENT_TYPE,
},
apub::{is_apub_id_valid, FromApub, GroupExt, PageExt, PersonExt, APUB_JSON_CONTENT_TYPE},
blocking,
db::{
comment::{Comment, CommentForm},
comment_view::CommentView,
community::{Community, CommunityForm, CommunityModerator, CommunityModeratorForm},
community_view::CommunityView,
post::{Post, PostForm},
post_view::PostView,
user::{UserForm, User_},
user_view::UserView,
Crud,
Joinable,
SearchType,
},
naive_now,
request::{retry, RecvError},
routes::nodeinfo::{NodeInfo, NodeInfoWellKnown},
DbPool,
@ -34,6 +12,21 @@ use activitystreams_new::{base::BaseExt, prelude::*, primitives::XsdAnyUri};
use actix_web::client::Client;
use chrono::NaiveDateTime;
use diesel::{result::Error::NotFound, PgConnection};
use lemmy_db::{
comment::{Comment, CommentForm},
comment_view::CommentView,
community::{Community, CommunityForm, CommunityModerator, CommunityModeratorForm},
community_view::CommunityView,
naive_now,
post::{Post, PostForm},
post_view::PostView,
user::{UserForm, User_},
user_view::UserView,
Crud,
Joinable,
SearchType,
};
use lemmy_utils::get_apub_protocol_string;
use log::debug;
use serde::Deserialize;
use std::{fmt::Debug, time::Duration};


+ 20
- 40
server/src/apub/mod.rs View File

@ -16,14 +16,11 @@ use crate::{
page_extension::PageExtension,
signatures::{PublicKey, PublicKeyExtension},
},
convert_datetime,
db::user::User_,
blocking,
request::{retry, RecvError},
routes::webfinger::WebFingerResponse,
DbPool,
LemmyError,
MentionData,
Settings,
};
use activitystreams::object::Page;
use activitystreams_ext::{Ext1, Ext2};
@ -35,6 +32,9 @@ use activitystreams_new::{
};
use actix_web::{body::Body, client::Client, HttpResponse};
use chrono::NaiveDateTime;
use failure::_core::fmt::Debug;
use lemmy_db::{activity::do_insert_activity, user::User_};
use lemmy_utils::{convert_datetime, get_apub_protocol_string, settings::Settings, MentionData};
use log::debug;
use serde::Serialize;
use url::Url;
@ -45,14 +45,6 @@ type PageExt = Ext1<Page, PageExtension>;
pub static APUB_JSON_CONTENT_TYPE: &str = "application/activity+json";
pub enum EndpointType {
Community,
User,
Post,