From bcd16676a78065a1b9549189ed3d594b4c62584c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Dec 2019 14:27:10 +0300 Subject: [PATCH 1/4] Publisher: check out a connection for inserting publish_one jobs Related to #1474, federation of one post on my istance creates in best-case 360 jobs, so if they for some reason take a while to insert, it will exhaust the connection pool. This fixes it by checking out one dedicated connection for inserting them. --- lib/pleroma/web/activity_pub/publisher.ex | 49 +++++++++++++---------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 4ea37fc7b..4073d3d63 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier @@ -188,31 +189,35 @@ def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity) recipients = recipients(actor, activity) - recipients - |> Enum.filter(&User.ap_enabled?/1) - |> Enum.map(fn %{source_data: data} -> data["inbox"] end) - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - |> Instances.filter_reachable() - |> Enum.each(fn {inbox, unreachable_since} -> - %User{ap_id: ap_id} = - Enum.find(recipients, fn %{source_data: data} -> data["inbox"] == inbox end) + inboxes = + recipients + |> Enum.filter(&User.ap_enabled?/1) + |> Enum.map(fn %{source_data: data} -> data["inbox"] end) + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() - # Get all the recipients on the same host and add them to cc. Otherwise, a remote - # instance would only accept a first message for the first recipient and ignore the rest. - cc = get_cc_ap_ids(ap_id, recipients) + Repo.checkout(fn -> + Enum.each(inboxes, fn {inbox, unreachable_since} -> + %User{ap_id: ap_id} = + Enum.find(recipients, fn %{source_data: data} -> data["inbox"] == inbox end) - json = - data - |> Map.put("cc", cc) - |> Jason.encode!() + # Get all the recipients on the same host and add them to cc. Otherwise, a remote + # instance would only accept a first message for the first recipient and ignore the rest. + cc = get_cc_ap_ids(ap_id, recipients) - Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ - inbox: inbox, - json: json, - actor_id: actor.id, - id: activity.data["id"], - unreachable_since: unreachable_since - }) + json = + data + |> Map.put("cc", cc) + |> Jason.encode!() + + Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ + inbox: inbox, + json: json, + actor_id: actor.id, + id: activity.data["id"], + unreachable_since: unreachable_since + }) + end) end) end From 82c62c5028f517ded121b46b9806053989defdc2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Dec 2019 15:32:33 +0700 Subject: [PATCH 2/4] Add default endpoint to Pleroma.Captcha.Kocaptcha --- config/test.exs | 2 -- lib/pleroma/captcha/kocaptcha.ex | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/test.exs b/config/test.exs index b48b89c8f..d52ced612 100644 --- a/config/test.exs +++ b/config/test.exs @@ -95,8 +95,6 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" - if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index 4e1a07c59..a3128decb 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -5,11 +5,13 @@ defmodule Pleroma.Captcha.Kocaptcha do import Pleroma.Web.Gettext alias Pleroma.Captcha.Service + @behaviour Service + @default_endpoint "https://captcha.kotobank.ch" @impl Service def new do - endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) + endpoint = Pleroma.Config.get([__MODULE__, :endpoint], @default_endpoint) case Tesla.get(endpoint <> "/new") do {:error, _} -> From 8d242f52d4bf44a012603fb2cc6fe4dd9b282ef8 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Dec 2019 19:33:51 +0700 Subject: [PATCH 3/4] Revert "Add default endpoint to Pleroma.Captcha.Kocaptcha" This reverts commit 82c62c5028f517ded121b46b9806053989defdc2. --- config/test.exs | 2 ++ lib/pleroma/captcha/kocaptcha.ex | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/test.exs b/config/test.exs index d52ced612..b48b89c8f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -95,6 +95,8 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock +config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index a3128decb..4e1a07c59 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -5,13 +5,11 @@ defmodule Pleroma.Captcha.Kocaptcha do import Pleroma.Web.Gettext alias Pleroma.Captcha.Service - @behaviour Service - @default_endpoint "https://captcha.kotobank.ch" @impl Service def new do - endpoint = Pleroma.Config.get([__MODULE__, :endpoint], @default_endpoint) + endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) case Tesla.get(endpoint <> "/new") do {:error, _} -> From fc484e5ce67117389ce9e2cce22ab1944a759242 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Dec 2019 19:35:10 +0700 Subject: [PATCH 4/4] Add Kocaptcha endpoint to config.exs --- config/config.exs | 2 ++ config/test.exs | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 370ddd855..47098858b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -70,6 +70,8 @@ seconds_valid: 60, method: Pleroma.Captcha.Native +config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" + config :pleroma, :hackney_pools, federation: [ max_connections: 50, diff --git a/config/test.exs b/config/test.exs index b48b89c8f..d52ced612 100644 --- a/config/test.exs +++ b/config/test.exs @@ -95,8 +95,6 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" - if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else