From 369e9bb42fc907f2e3f92e7e44dc52d6940dc046 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 13 Jul 2019 14:49:39 +0300 Subject: [PATCH 1/3] [#1041] Rate-limited status actions (per user and per user+status). --- CHANGELOG.md | 3 +- config/config.exs | 4 +- lib/pleroma/plugs/rate_limiter.ex | 67 ++++++++++++---- .../mastodon_api/mastodon_api_controller.ex | 22 ++++- test/plugs/rate_limiter_test.exs | 80 ++++++++++++++++--- 5 files changed, 149 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 405b7680c..c6c1ba1f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) -Configuration: `federation_incoming_replies_max_depth` option +- Configuration: `federation_incoming_replies_max_depth` option - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header - Mastodon API, extension: Ability to reset avatar, profile banner, and background @@ -32,6 +32,7 @@ Configuration: `federation_incoming_replies_max_depth` option - Added synchronization of following/followers counters for external users - Configuration: `enabled` option for `Pleroma.Emails.Mailer`, defaulting to `false`. - Mastodon API: Add support for categories for custom emojis by reusing the group feature. +- Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/config/config.exs b/config/config.exs index 99b500993..3d48a5584 100644 --- a/config/config.exs +++ b/config/config.exs @@ -519,7 +519,9 @@ config :pleroma, :rate_limit, search: [{1000, 10}, {1000, 30}], - app_account_creation: {1_800_000, 25} + app_account_creation: {1_800_000, 25}, + statuses_actions: {10_000, 15}, + status_id_action: {60_000, 3} # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/lib/pleroma/plugs/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter.ex index c5e0957e8..31388f574 100644 --- a/lib/pleroma/plugs/rate_limiter.ex +++ b/lib/pleroma/plugs/rate_limiter.ex @@ -31,12 +31,28 @@ defmodule Pleroma.Plugs.RateLimiter do ## Usage + AllowedSyntax: + + plug(Pleroma.Plugs.RateLimiter, :limiter_name) + plug(Pleroma.Plugs.RateLimiter, {:limiter_name, options}) + + Allowed options: + + * `bucket_name` overrides bucket name (e.g. to have a separate limit for a set of actions) + * `params` appends values of specified request params (e.g. ["id"]) to bucket name + Inside a controller: plug(Pleroma.Plugs.RateLimiter, :one when action == :one) plug(Pleroma.Plugs.RateLimiter, :two when action in [:two, :three]) - or inside a router pipiline: + plug( + Pleroma.Plugs.RateLimiter, + {:status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]} + when action in ~w(fav_status unfav_status)a + ) + + or inside a router pipeline: pipeline :api do ... @@ -49,33 +65,56 @@ defmodule Pleroma.Plugs.RateLimiter do alias Pleroma.User - def init(limiter_name) do + def init(limiter_name) when is_atom(limiter_name) do + init({limiter_name, []}) + end + + def init({limiter_name, opts}) do case Pleroma.Config.get([:rate_limit, limiter_name]) do nil -> nil - config -> {limiter_name, config} + config -> {limiter_name, config, opts} end end - # do not limit if there is no limiter configuration + # Do not limit if there is no limiter configuration def call(conn, nil), do: conn - def call(conn, opts) do - case check_rate(conn, opts) do - {:ok, _count} -> conn - {:error, _count} -> render_throttled_error(conn) + def call(conn, settings) do + case check_rate(conn, settings) do + {:ok, _count} -> + conn + + {:error, _count} -> + render_throttled_error(conn) end end - defp check_rate(%{assigns: %{user: %User{id: user_id}}}, {limiter_name, [_, {scale, limit}]}) do - ExRated.check_rate("#{limiter_name}:#{user_id}", scale, limit) + defp bucket_name(conn, limiter_name, opts) do + bucket_name = opts[:bucket_name] || limiter_name + + if params_names = opts[:params] do + params_values = for p <- Enum.sort(params_names), do: conn.params[p] + Enum.join([bucket_name] ++ params_values, ":") + else + bucket_name + end end - defp check_rate(conn, {limiter_name, [{scale, limit} | _]}) do - ExRated.check_rate("#{limiter_name}:#{ip(conn)}", scale, limit) + defp check_rate( + %{assigns: %{user: %User{id: user_id}}} = conn, + {limiter_name, [_, {scale, limit}], opts} + ) do + bucket_name = bucket_name(conn, limiter_name, opts) + ExRated.check_rate("#{bucket_name}:#{user_id}", scale, limit) end - defp check_rate(conn, {limiter_name, {scale, limit}}) do - check_rate(conn, {limiter_name, [{scale, limit}]}) + defp check_rate(conn, {limiter_name, [{scale, limit} | _], opts}) do + bucket_name = bucket_name(conn, limiter_name, opts) + ExRated.check_rate("#{bucket_name}:#{ip(conn)}", scale, limit) + end + + defp check_rate(conn, {limiter_name, {scale, limit}, opts}) do + check_rate(conn, {limiter_name, [{scale, limit}, {scale, limit}], opts}) end def ip(%{remote_ip: remote_ip}) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 8c2033c3a..76648b9f7 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Pagination + alias Pleroma.Plugs.RateLimiter alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.Stats @@ -46,8 +47,25 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger - plug(Pleroma.Plugs.RateLimiter, :app_account_creation when action == :account_register) - plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) + @rate_limited_status_crud_actions ~w(post_status delete_status)a + @rate_limited_status_reactions ~w(reblog_status unreblog_status fav_status unfav_status)a + @rate_limited_status_actions @rate_limited_status_crud_actions ++ @rate_limited_status_reactions + + plug( + RateLimiter, + {:status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]} + when action in ~w(reblog_status unreblog_status)a + ) + + plug( + RateLimiter, + {:status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]} + when action in ~w(fav_status unfav_status)a + ) + + plug(RateLimiter, :statuses_actions when action in @rate_limited_status_actions) + plug(RateLimiter, :app_account_creation when action == :account_register) + plug(RateLimiter, :search when action in [:search, :search2, :account_search]) @local_mastodon_name "Mastodon-Local" diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs index f8251b5c7..395095079 100644 --- a/test/plugs/rate_limiter_test.exs +++ b/test/plugs/rate_limiter_test.exs @@ -10,12 +10,13 @@ defmodule Pleroma.Plugs.RateLimiterTest do import Pleroma.Factory - @limiter_name :testing + # Note: each example must work with separate buckets in order to prevent concurrency issues test "init/1" do - Pleroma.Config.put([:rate_limit, @limiter_name], {1, 1}) + limiter_name = :test_init + Pleroma.Config.put([:rate_limit, limiter_name], {1, 1}) - assert {@limiter_name, {1, 1}} == RateLimiter.init(@limiter_name) + assert {limiter_name, {1, 1}, []} == RateLimiter.init(limiter_name) assert nil == RateLimiter.init(:foo) end @@ -24,14 +25,15 @@ test "ip/1" do end test "it restricts by opts" do + limiter_name = :test_opts scale = 1000 limit = 5 - Pleroma.Config.put([:rate_limit, @limiter_name], {scale, limit}) + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) - opts = RateLimiter.init(@limiter_name) + opts = RateLimiter.init(limiter_name) conn = conn(:get, "/") - bucket_name = "#{@limiter_name}:#{RateLimiter.ip(conn)}" + bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" conn = RateLimiter.call(conn, opts) assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) @@ -65,18 +67,78 @@ test "it restricts by opts" do refute conn.halted end + test "`bucket_name` option overrides default bucket name" do + limiter_name = :test_bucket_name + scale = 1000 + limit = 5 + + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + base_bucket_name = "#{limiter_name}:group1" + opts = RateLimiter.init({limiter_name, bucket_name: base_bucket_name}) + + conn = conn(:get, "/") + default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" + customized_bucket_name = "#{base_bucket_name}:#{RateLimiter.ip(conn)}" + + RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(customized_bucket_name, scale, limit) + assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) + end + + test "`params` option appends specified params' values to bucket name" do + limiter_name = :test_params + scale = 1000 + limit = 5 + + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + opts = RateLimiter.init({limiter_name, params: ["id"]}) + id = "1" + + conn = conn(:get, "/?id=#{id}") + conn = Plug.Conn.fetch_query_params(conn) + + default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" + parametrized_bucket_name = "#{limiter_name}:#{id}:#{RateLimiter.ip(conn)}" + + RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(parametrized_bucket_name, scale, limit) + assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) + end + + test "it supports combination of options modifying bucket name" do + limiter_name = :test_options_combo + scale = 1000 + limit = 5 + + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + base_bucket_name = "#{limiter_name}:group1" + opts = RateLimiter.init({limiter_name, bucket_name: base_bucket_name, params: ["id"]}) + id = "100" + + conn = conn(:get, "/?id=#{id}") + conn = Plug.Conn.fetch_query_params(conn) + + default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" + parametrized_bucket_name = "#{base_bucket_name}:#{id}:#{RateLimiter.ip(conn)}" + + RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(parametrized_bucket_name, scale, limit) + assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) + end + test "optional limits for authenticated users" do + limiter_name = :test_authenticated Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) scale = 1000 limit = 5 - Pleroma.Config.put([:rate_limit, @limiter_name], [{1, 10}, {scale, limit}]) + Pleroma.Config.put([:rate_limit, limiter_name], [{1, 10}, {scale, limit}]) - opts = RateLimiter.init(@limiter_name) + opts = RateLimiter.init(limiter_name) user = insert(:user) conn = conn(:get, "/") |> assign(:user, user) - bucket_name = "#{@limiter_name}:#{user.id}" + bucket_name = "#{limiter_name}:#{user.id}" conn = RateLimiter.call(conn, opts) assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) From b74d11e20ab214d533f746bee81fde589d319f64 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 13 Jul 2019 15:13:26 +0300 Subject: [PATCH 2/3] [#1041] Added documentation on existing rate limiters. --- docs/config.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/config.md b/docs/config.md index 140789d87..3ec13cff2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -640,3 +640,10 @@ A keyword list of rate limiters where a key is a limiter name and value is the l It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. + +Supported rate limiters: + +* `:search` for the search requests (account & status search etc.) +* `:app_account_creation` for registering user accounts from the same IP address +* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses +* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user From d72876c57dd8f519f63f7bb14abcfaceedf41410 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 13 Jul 2019 15:21:50 +0300 Subject: [PATCH 3/3] [#1041] Minor refactoring. --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 76648b9f7..8a7b75025 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -47,9 +47,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger - @rate_limited_status_crud_actions ~w(post_status delete_status)a - @rate_limited_status_reactions ~w(reblog_status unreblog_status fav_status unfav_status)a - @rate_limited_status_actions @rate_limited_status_crud_actions ++ @rate_limited_status_reactions + @rate_limited_status_actions ~w(reblog_status unreblog_status fav_status unfav_status + post_status delete_status)a plug( RateLimiter,