From e93cc561cd42ff4ca7f3c95cdbf8dfa7fb9f4a74 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 25 Jan 2020 18:42:04 +0300 Subject: [PATCH] restarting pleroma from outside application --- lib/pleroma/config/loader.ex | 6 +- lib/pleroma/config/transfer_task.ex | 95 +++++++++++++++---- .../web/admin_api/admin_api_controller.ex | 33 +++++-- .../web/admin_api/views/config_view.ex | 10 +- lib/pleroma/web/router.ex | 1 + mix.exs | 10 +- mix.lock | 1 + test/config/transfer_task_test.exs | 67 +++++++++++++ .../admin_api/admin_api_controller_test.exs | 43 ++++++++- 9 files changed, 231 insertions(+), 35 deletions(-) diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex index 68b247381..b8787cb49 100644 --- a/lib/pleroma/config/loader.ex +++ b/lib/pleroma/config/loader.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Config.Loader do - @paths ["config/config.exs", "config/#{Mix.env()}.exs"] - @reject_keys [ Pleroma.Repo, Pleroma.Web.Endpoint, @@ -35,8 +33,8 @@ defp do_merge(conf1, conf2), do: Mix.Config.merge(conf1, conf2) def load_and_merge do all_paths = if Pleroma.Config.get(:release), - do: @paths ++ ["config/releases.exs"], - else: @paths + do: ["config/config.exs", "config/releases.exs"], + else: ["config/config.exs"] all_paths |> Enum.map(&load(&1)) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index d54f38ee4..6c5ba1f95 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -10,6 +10,30 @@ defmodule Pleroma.Config.TransferTask do require Logger + @type env() :: :test | :benchmark | :dev | :prod + + @reboot_time_keys [ + {:pleroma, :hackney_pools}, + {:pleroma, :chat}, + {:pleroma, Oban}, + {:pleroma, :rate_limit}, + {:pleroma, :markup}, + {:plerome, :streamer} + ] + + @reboot_time_subkeys [ + {:pleroma, Pleroma.Captcha, [:seconds_valid]}, + {:pleroma, Pleroma.Upload, [:proxy_remote]}, + {:pleroma, :instance, [:upload_limit]}, + {:pleroma, :email_notifications, [:digest]}, + {:pleroma, :oauth2, [:clean_expired_tokens]}, + {:pleroma, Pleroma.ActivityExpiration, [:enabled]}, + {:pleroma, Pleroma.ScheduledActivity, [:enabled]}, + {:pleroma, :gopher, [:enabled]} + ] + + @reject [nil, :prometheus] + def start_link(_) do load_and_update_env() if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) @@ -17,21 +41,34 @@ def start_link(_) do end @spec load_and_update_env([ConfigDB.t()]) :: :ok | false - def load_and_update_env(deleted \\ []) do + def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do with true <- Pleroma.Config.get(:configurable_from_database), true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"), started_applications <- Application.started_applications() do # We need to restart applications for loaded settings take effect + in_db = Repo.all(ConfigDB) with_deleted = in_db ++ deleted - with_deleted - |> Enum.map(&merge_and_update(&1)) - |> Enum.uniq() - # TODO: some problem with prometheus after restart! - |> Enum.reject(&(&1 in [:pleroma, nil, :prometheus])) - |> Enum.each(&restart(started_applications, &1)) + reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject] + + applications = + with_deleted + |> Enum.map(&merge_and_update(&1)) + |> Enum.uniq() + # TODO: some problem with prometheus after restart! + |> Enum.reject(&(&1 in reject_for_restart)) + + # to be ensured that pleroma will be restarted last + applications = + if :pleroma in applications do + List.delete(applications, :pleroma) ++ [:pleroma] + else + applications + end + + Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env))) :ok end @@ -43,12 +80,25 @@ defp merge_and_update(setting) do group = ConfigDB.from_string(setting.group) default = Pleroma.Config.Holder.config(group, key) - merged_value = merge_value(setting, default, group, key) + value = ConfigDB.from_binary(setting.value) + + merged_value = + if Ecto.get_meta(setting, :state) == :deleted do + default + else + if can_be_merged?(default, value) do + ConfigDB.merge_group(group, key, default, value) + else + value + end + end :ok = update_env(group, key, merged_value) if group != :logger do - group + if group != :pleroma or pleroma_need_restart?(group, key, value) do + group + end else # change logger configuration in runtime, without restart if Keyword.keyword?(merged_value) and @@ -76,22 +126,31 @@ defp merge_and_update(setting) do end end - defp merge_value(%{__meta__: %{state: :deleted}}, default, _group, _key), do: default + @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean() + def pleroma_need_restart?(group, key, value) do + group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value) + end - defp merge_value(setting, default, group, key) do - value = ConfigDB.from_binary(setting.value) + defp group_and_key_need_reboot?(group, key) do + Enum.any?(@reboot_time_keys, fn {g, k} -> g == group and k == key end) + end - if can_be_merged?(default, value) do - ConfigDB.merge_group(group, key, default, value) - else - value - end + defp group_and_subkey_need_reboot?(group, key, value) do + Keyword.keyword?(value) and + Enum.any?(@reboot_time_subkeys, fn {g, k, subkeys} -> + g == group and k == key and + Enum.any?(Keyword.keys(value), &(&1 in subkeys)) + end) end defp update_env(group, key, nil), do: Application.delete_env(group, key) defp update_env(group, key, value), do: Application.put_env(group, key, value) - defp restart(started_applications, app) do + defp restart(_, :pleroma, :test), do: Logger.warn("pleroma restarted") + + defp restart(_, :pleroma, _), do: send(Restarter.Pleroma, :after_boot) + + defp restart(started_applications, app, _) do with {^app, _, _} <- List.keyfind(started_applications, app, 0), :ok <- Application.stop(app) do :ok = Application.start(app) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2314d3274..6f0449418 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -890,17 +890,36 @@ def config_update(conn, %{"configs" => configs}) do Ecto.get_meta(config, :state) == :deleted end) - Pleroma.Config.TransferTask.load_and_update_env(deleted) + Pleroma.Config.TransferTask.load_and_update_env(deleted, false) - Mix.Tasks.Pleroma.Config.run([ - "migrate_from_db", - "--env", - to_string(Pleroma.Config.get(:env)) - ]) + need_reboot? = + Enum.any?(updated, fn config -> + group = ConfigDB.from_string(config.group) + key = ConfigDB.from_string(config.key) + value = ConfigDB.from_binary(config.value) + Pleroma.Config.TransferTask.pleroma_need_restart?(group, key, value) + end) + + response = %{configs: updated} + + response = + if need_reboot?, do: Map.put(response, :need_reboot, need_reboot?), else: response conn |> put_view(ConfigView) - |> render("index.json", %{configs: updated}) + |> render("index.json", response) + end + end + + def restart(conn, _params) do + with :ok <- configurable_from_database(conn) do + if Pleroma.Config.get(:env) == :test do + Logger.warn("pleroma restarted") + else + send(Restarter.Pleroma, {:restart, 50}) + end + + json(conn, %{}) end end diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex index 23d97e847..bbb53efcd 100644 --- a/lib/pleroma/web/admin_api/views/config_view.ex +++ b/lib/pleroma/web/admin_api/views/config_view.ex @@ -5,10 +5,16 @@ defmodule Pleroma.Web.AdminAPI.ConfigView do use Pleroma.Web, :view - def render("index.json", %{configs: configs}) do - %{ + def render("index.json", %{configs: configs} = params) do + map = %{ configs: render_many(configs, __MODULE__, "show.json", as: :config) } + + if params[:need_reboot] do + Map.put(map, :need_reboot, true) + else + map + end end def render("show.json", %{config: config}) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ef6e5a565..43fee8a0f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -197,6 +197,7 @@ defmodule Pleroma.Web.Router do post("/config", AdminAPIController, :config_update) get("/config/descriptions", AdminAPIController, :config_descriptions) get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) + get("/restart", AdminAPIController, :restart) get("/moderation_log", AdminAPIController, :list_log) diff --git a/mix.exs b/mix.exs index 0aa7c862f..1d0b59e56 100644 --- a/mix.exs +++ b/mix.exs @@ -8,7 +8,7 @@ def project do elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), - elixirc_options: [warnings_as_errors: true], + elixirc_options: [warnings_as_errors: warnings_as_errors(Mix.env())], xref: [exclude: [:eldap]], start_permanent: Mix.env() == :prod, aliases: aliases(), @@ -73,6 +73,11 @@ defp elixirc_paths(:benchmark), do: ["lib", "benchmarks"] defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] + defp warnings_as_errors(:prod), do: false + # Uncomment this if you need testing configurable_from_database logic + # defp warnings_as_errors(:dev), do: false + defp warnings_as_errors(_), do: true + # Specifies OAuth dependencies. defp oauth_deps do oauth_strategy_packages = @@ -166,7 +171,8 @@ defp deps do {:captcha, git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, - {:mox, "~> 0.5", only: :test} + {:mox, "~> 0.5", only: :test}, + {:restarter, git: "https://git.pleroma.social/alex.s/restarter"} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index c1fe223c0..538337cf3 100644 --- a/mix.lock +++ b/mix.lock @@ -94,6 +94,7 @@ "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, + "restarter": {:git, "https://git.pleroma.social/alex.s/restarter", "1932655b80a1409405d897911c06ebee4ac8c2d8", []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index 53e8703fd..0d328f0c3 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.Config.TransferTaskTest do use Pleroma.DataCase + import ExUnit.CaptureLog + alias Pleroma.Config.TransferTask alias Pleroma.ConfigDB @@ -105,4 +107,69 @@ test "transfer config values with full subkey update" do Application.put_env(:pleroma, :assets, assets) end) end + + describe "pleroma restart" do + test "don't restart if no reboot time settings were changed" do + emoji = Application.get_env(:pleroma, :emoji) + on_exit(fn -> Application.put_env(:pleroma, :emoji, emoji) end) + + ConfigDB.create(%{ + group: ":pleroma", + key: ":emoji", + value: [groups: [a: 1, b: 2]] + }) + + assert capture_log(fn -> TransferTask.start_link([]) end) =~ "" + end + + test "restart pleroma on reboot time key" do + chat = Application.get_env(:pleroma, :chat) + on_exit(fn -> Application.put_env(:pleroma, :chat, chat) end) + + ConfigDB.create(%{ + group: ":pleroma", + key: ":chat", + value: [enabled: false] + }) + + assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + end + + test "restart pleroma on reboot time subkey" do + captcha = Application.get_env(:pleroma, Pleroma.Captcha) + on_exit(fn -> Application.put_env(:pleroma, Pleroma.Captcha, captcha) end) + + ConfigDB.create(%{ + group: ":pleroma", + key: "Pleroma.Captcha", + value: [seconds_valid: 60] + }) + + assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + end + + test "don't restart pleroma on reboot time key and subkey if there is false flag" do + chat = Application.get_env(:pleroma, :chat) + captcha = Application.get_env(:pleroma, Pleroma.Captcha) + + on_exit(fn -> + Application.put_env(:pleroma, :chat, chat) + Application.put_env(:pleroma, Pleroma.Captcha, captcha) + end) + + ConfigDB.create(%{ + group: ":pleroma", + key: ":chat", + value: [enabled: false] + }) + + ConfigDB.create(%{ + group: ":pleroma", + key: "Pleroma.Captcha", + value: [seconds_valid: 60] + }) + + assert capture_log(fn -> TransferTask.load_and_update_env([], false) end) =~ "" + end + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 5c767219a..81e346fb8 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2043,7 +2043,6 @@ test "POST /api/pleroma/admin/config error", %{conn: conn} do Application.delete_env(:pleroma, Pleroma.Captcha.NotReal) Application.put_env(:pleroma, :http, http) Application.put_env(:tesla, :adapter, Tesla.Mock) - :ok = File.rm("config/test.exported_from_db.secret.exs") end) end @@ -2170,7 +2169,7 @@ test "create new config setting in db", %{conn: conn} do assert Application.get_env(:idna, :key5) == {"string", Pleroma.Captcha.NotReal, []} end - test "save config setting without key", %{conn: conn} do + test "save configs setting without explicit key", %{conn: conn} do level = Application.get_env(:quack, :level) meta = Application.get_env(:quack, :meta) webhook_url = Application.get_env(:quack, :webhook_url) @@ -2256,6 +2255,34 @@ test "saving config with partial update", %{conn: conn} do } end + test "saving config which need pleroma reboot", %{conn: conn} do + chat = Pleroma.Config.get(:chat) + on_exit(fn -> Pleroma.Config.put(:chat, chat) end) + + conn = + post( + conn, + "/api/pleroma/admin/config", + %{ + configs: [ + %{group: ":pleroma", key: ":chat", value: [%{"tuple" => [":enabled", true]}]} + ] + } + ) + + assert json_response(conn, 200) == %{ + "configs" => [ + %{ + "db" => [":enabled"], + "group" => ":pleroma", + "key" => ":chat", + "value" => [%{"tuple" => [":enabled", true]}] + } + ], + "need_reboot" => true + } + end + test "saving config with nested merge", %{conn: conn} do config = insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: [k1: 1, k2: 2])) @@ -3001,6 +3028,18 @@ test "returns error if configuration from database is off", %{conn: conn} do end end + describe "GET /api/pleroma/admin/restart" do + clear_config(:configurable_from_database) do + Pleroma.Config.put(:configurable_from_database, true) + end + + test "pleroma restarts", %{conn: conn} do + ExUnit.CaptureLog.capture_log(fn -> + assert conn |> get("/api/pleroma/admin/restart") |> json_response(200) == %{} + end) =~ "pleroma restarted" + end + end + describe "GET /api/pleroma/admin/users/:nickname/statuses" do setup do user = insert(:user)