Browse Source

Proxy pictrs requests through Lemmy (fixes #371) (#77)

fix check_only value for image rate limit

Fix image rate limit

Add rate limit for image uploads

Proxy pictrs requests through Lemmy (fixes #371)

Co-authored-by: Felix Ableitner <me@nutomic.com>
Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/77
remove_make_mod_or_admin_for_federated_users
nutomic 1 year ago
parent
commit
c34fffc2c4
  1. 12
      ansible/templates/nginx.conf
  2. 3
      docker/dev/docker-compose.yml
  3. 36
      docker/federation/nginx.conf
  4. 9
      docker/lemmy.hjson
  5. 6
      server/config/defaults.hjson
  6. 3
      server/lemmy_utils/src/settings.rs
  7. 5
      server/src/main.rs
  8. 13
      server/src/rate_limit/mod.rs
  9. 1
      server/src/rate_limit/rate_limiter.rs
  10. 146
      server/src/routes/images.rs
  11. 1
      server/src/routes/mod.rs

12
ansible/templates/nginx.conf

@ -74,18 +74,6 @@ server {
return 301 /pictrs/image/$1;
}
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://0.0.0.0:8537/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ {
proxy_pass http://0.0.0.0:8061/;
proxy_set_header X-Real-IP $remote_addr;

3
docker/dev/docker-compose.yml

@ -21,7 +21,8 @@ services:
postgres:
image: postgres:12-alpine
ports:
- "127.0.0.1:5432:5432"
# use a different port so it doesnt conflict with postgres running on the host
- "127.0.0.1:5433:5432"
environment:
- POSTGRES_USER=lemmy
- POSTGRES_PASSWORD=password

36
docker/federation/nginx.conf

@ -26,18 +26,6 @@ http {
proxy_set_header Connection "upgrade";
}
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://pictrs:8080/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ {
proxy_pass http://iframely:80/;
proxy_set_header X-Real-IP $remote_addr;
@ -69,18 +57,6 @@ http {
proxy_set_header Connection "upgrade";
}
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://pictrs:8080/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ {
proxy_pass http://iframely:80/;
proxy_set_header X-Real-IP $remote_addr;
@ -112,18 +88,6 @@ http {
proxy_set_header Connection "upgrade";
}
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://pictrs:8080/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ {
proxy_pass http://iframely:80/;
proxy_set_header X-Real-IP $remote_addr;

9
docker/lemmy.hjson

@ -2,6 +2,15 @@
# for more info about the config, check out the documentation
# https://dev.lemmy.ml/docs/administration_configuration.html
setup: {
# username for the admin user
admin_username: "lemmy"
# password for the admin user
admin_password: "lemmy"
# name of the site (can be changed later)
site_name: "lemmy-test"
}
# the domain name of your instance (eg "dev.lemmy.ml")
hostname: "my_domain"
# address where lemmy should listen for incoming requests

6
server/config/defaults.hjson

@ -35,6 +35,8 @@
jwt_secret: "changeme"
# The location of the frontend
front_end_dir: "../ui/dist"
# address where pictrs is available
pictrs_url: "http://pictrs:8080"
# rate limits for various user actions, by user ip
rate_limit: {
# maximum number of messages created in interval
@ -49,6 +51,10 @@
register: 3
# interval length for registration limit
register_per_second: 3600
# maximum number of image uploads in interval
image: 6
# interval length for image uploads
image_per_second: 3600
}
# settings related to activitypub federation
federation: {

3
server/lemmy_utils/src/settings.rs

@ -14,6 +14,7 @@ pub struct Settings {
pub port: u16,
pub jwt_secret: String,
pub front_end_dir: String,
pub pictrs_url: String,
pub rate_limit: RateLimitConfig,
pub email: Option<EmailConfig>,
pub federation: Federation,
@ -36,6 +37,8 @@ pub struct RateLimitConfig {
pub post_per_second: i32,
pub register: i32,
pub register_per_second: i32,
pub image: i32,
pub image_per_second: i32,
}
#[derive(Debug, Deserialize, Clone)]

5
server/src/main.rs

@ -27,7 +27,7 @@ use lemmy_server::{
blocking,
code_migrations::run_advanced_migrations,
rate_limit::{rate_limiter::RateLimiter, RateLimit},
routes::{api, federation, feeds, index, nodeinfo, webfinger},
routes::*,
websocket::server::*,
LemmyError,
};
@ -91,9 +91,10 @@ async fn main() -> Result<(), LemmyError> {
.data(server.clone())
.data(Client::default())
// The routes
.configure(move |cfg| api::config(cfg, &rate_limiter))
.configure(|cfg| api::config(cfg, &rate_limiter))
.configure(federation::config)
.configure(feeds::config)
.configure(|cfg| images::config(cfg, &rate_limiter))
.configure(index::config)
.configure(nodeinfo::config)
.configure(webfinger::config)

13
server/src/rate_limit/mod.rs

@ -45,6 +45,10 @@ impl RateLimit {
self.kind(RateLimitType::Register)
}
pub fn image(&self) -> RateLimited {
self.kind(RateLimitType::Image)
}
fn kind(&self, type_: RateLimitType) -> RateLimited {
RateLimited {
rate_limiter: self.rate_limiter.clone(),
@ -101,6 +105,15 @@ impl RateLimited {
true,
)?;
}
RateLimitType::Image => {
limiter.check_rate_limit_full(
self.type_,
&ip_addr,
rate_limit.image,
rate_limit.image_per_second,
false,
)?;
}
};
}

1
server/src/rate_limit/rate_limiter.rs

@ -15,6 +15,7 @@ pub enum RateLimitType {
Message,
Register,
Post,
Image,
}
/// Rate limiting based on rate type and IP addr

146
server/src/routes/images.rs

@ -0,0 +1,146 @@
use crate::rate_limit::RateLimit;
use actix::clock::Duration;
use actix_web::{body::BodyStream, http::StatusCode, *};
use awc::Client;
use lemmy_utils::settings::Settings;
use serde::{Deserialize, Serialize};
const THUMBNAIL_SIZES: &[u64] = &[256];
pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) {
let client = Client::build()
.header("User-Agent", "pict-rs-frontend, v0.1.0")
.timeout(Duration::from_secs(30))
.finish();
cfg
.data(client)
.service(
web::resource("/pictrs/image")
.wrap(rate_limit.image())
.route(web::post().to(upload)),
)
.service(web::resource("/pictrs/image/{filename}").route(web::get().to(full_res)))
.service(
web::resource("/pictrs/image/thumbnail{size}/{filename}").route(web::get().to(thumbnail)),
)
.service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete)));
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Image {
file: String,
delete_token: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Images {
msg: String,
files: Option<Vec<Image>>,
}
async fn upload(
req: HttpRequest,
body: web::Payload,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
// TODO: check auth and rate limit here
let mut res = client
.request_from(format!("{}/image", Settings::get().pictrs_url), req.head())
.if_some(req.head().peer_addr, |addr, req| {
req.header("X-Forwarded-For", addr.to_string())
})
.send_stream(body)
.await?;
let images = res.json::<Images>().await?;
Ok(HttpResponse::build(res.status()).json(images))
}
async fn full_res(
filename: web::Path<String>,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let url = format!(
"{}/image/{}",
Settings::get().pictrs_url,
&filename.into_inner()
);
image(url, req, client).await
}
async fn thumbnail(
parts: web::Path<(u64, String)>,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let (size, file) = parts.into_inner();
if THUMBNAIL_SIZES.contains(&size) {
let url = format!(
"{}/image/thumbnail{}/{}",
Settings::get().pictrs_url,
size,
&file
);
return image(url, req, client).await;
}
Ok(HttpResponse::NotFound().finish())
}
async fn image(
url: String,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let res = client
.request_from(url, req.head())
.if_some(req.head().peer_addr, |addr, req| {
req.header("X-Forwarded-For", addr.to_string())
})
.no_decompress()
.send()
.await?;
if res.status() == StatusCode::NOT_FOUND {
return Ok(HttpResponse::NotFound().finish());
}
let mut client_res = HttpResponse::build(res.status());
for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") {
client_res.header(name.clone(), value.clone());
}
Ok(client_res.body(BodyStream::new(res)))
}
async fn delete(
components: web::Path<(String, String)>,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let (token, file) = components.into_inner();
let url = format!(
"{}/image/delete/{}/{}",
Settings::get().pictrs_url,
&token,
&file
);
let res = client
.request_from(url, req.head())
.if_some(req.head().peer_addr, |addr, req| {
req.header("X-Forwarded-For", addr.to_string())
})
.no_decompress()
.send()
.await?;
Ok(HttpResponse::build(res.status()).body(BodyStream::new(res)))
}

1
server/src/routes/mod.rs

@ -1,6 +1,7 @@
pub mod api;
pub mod federation;
pub mod feeds;
pub mod images;
pub mod index;
pub mod nodeinfo;
pub mod webfinger;

Loading…
Cancel
Save