From 257e059e61b89752bcde9544cb5ae645b167c96b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 19 Aug 2020 15:31:33 +0400 Subject: [PATCH 01/38] Add account export --- lib/pleroma/export.ex | 118 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 lib/pleroma/export.ex diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex new file mode 100644 index 000000000..82a4b7ace --- /dev/null +++ b/lib/pleroma/export.ex @@ -0,0 +1,118 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Export do + alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.UserView + + import Ecto.Query + + def run(user) do + with {:ok, dir} <- create_dir(), + :ok <- actor(dir, user), + :ok <- statuses(dir, user), + :ok <- likes(dir, user), + :ok <- bookmarks(dir, user) do + IO.inspect({"DONE", dir}) + else + err -> IO.inspect({"export error", err}) + end + end + + def actor(dir, user) do + with {:ok, json} <- + UserView.render("user.json", %{user: user}) + |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) + |> Jason.encode() do + File.write(dir <> "/actor.json", json) + end + end + + defp create_dir do + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + dir = Path.join(System.tmp_dir!(), "archive-" <> datetime) + + with :ok <- File.mkdir(dir), do: {:ok, dir} + end + + defp write_header(file, name) do + IO.write( + file, + """ + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "#{name}.json", + "type": "OrderedCollection", + "orderedItems": [ + """ + ) + end + + defp write(query, dir, name, fun) do + path = dir <> "/#{name}.json" + + with {:ok, file} <- File.open(path, [:write, :utf8]), + :ok <- write_header(file, name) do + counter = :counters.new(1, []) + + query + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn items -> + Enum.each(items, fn i -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + :counters.add(counter, 1, 1) + end + end) + end) + |> Stream.run() + + total = :counters.get(counter, 1) + + with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do + File.close(file) + end + end + end + + def bookmarks(dir, %{id: user_id} = _user) do + Bookmark + |> where(user_id: ^user_id) + |> join(:inner, [b], activity in assoc(b, :activity)) + |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) + |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) + end + + def likes(dir, user) do + user.ap_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Like") + |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) + |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) + end + + def statuses(dir, user) do + opts = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] + |> Enum.concat() + |> ActivityPub.fetch_activities_query(opts) + |> write(dir, "outbox", fn a -> + with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do + activity |> Map.delete("@context") |> Jason.encode() + end + end) + end +end From 9d564ffc2988f145bc9cf26477eea93b1bf01cb0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 24 Aug 2020 20:59:57 +0400 Subject: [PATCH 02/38] Zip exported files --- lib/pleroma/export.ex | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index 82a4b7ace..f0f1ef093 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -12,15 +12,17 @@ defmodule Pleroma.Export do import Ecto.Query + @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def run(user) do - with {:ok, dir} <- create_dir(), - :ok <- actor(dir, user), - :ok <- statuses(dir, user), - :ok <- likes(dir, user), - :ok <- bookmarks(dir, user) do - IO.inspect({"DONE", dir}) - else - err -> IO.inspect({"export error", err}) + with {:ok, path} <- create_dir(user), + :ok <- actor(path, user), + :ok <- statuses(path, user), + :ok <- likes(path, user), + :ok <- bookmarks(path, user), + {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), + {:ok, _} <- File.rm_rf(path) do + {:ok, zip_path} end end @@ -33,9 +35,9 @@ def actor(dir, user) do end end - defp create_dir do + defp create_dir(user) do datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-" <> datetime) + dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") with :ok <- File.mkdir(dir), do: {:ok, dir} end From c01a81804835fb92c145b90e3a264c5d4cf9c886 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 25 Aug 2020 18:51:09 +0400 Subject: [PATCH 03/38] Add tests --- lib/pleroma/export.ex | 8 +-- test/export_test.exs | 111 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 test/export_test.exs diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index f0f1ef093..45b8ce749 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -26,7 +26,7 @@ def run(user) do end end - def actor(dir, user) do + defp actor(dir, user) do with {:ok, json} <- UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) @@ -82,7 +82,7 @@ defp write(query, dir, name, fun) do end end - def bookmarks(dir, %{id: user_id} = _user) do + defp bookmarks(dir, %{id: user_id} = _user) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) @@ -90,7 +90,7 @@ def bookmarks(dir, %{id: user_id} = _user) do |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) end - def likes(dir, user) do + defp likes(dir, user) do user.ap_id |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") @@ -98,7 +98,7 @@ def likes(dir, user) do |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) end - def statuses(dir, user) do + defp statuses(dir, user) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) diff --git a/test/export_test.exs b/test/export_test.exs new file mode 100644 index 000000000..5afd58ccc --- /dev/null +++ b/test/export_test.exs @@ -0,0 +1,111 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ExportTest do + use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + alias Pleroma.Bookmark + + test "it exports user data" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, %{object: %{data: %{"id" => id1}}} = status1} = + CommonAPI.post(user, %{status: "status1"}) + + {:ok, %{object: %{data: %{"id" => id2}}} = status2} = + CommonAPI.post(user, %{status: "status2"}) + + {:ok, %{object: %{data: %{"id" => id3}}} = status3} = + CommonAPI.post(user, %{status: "status3"}) + + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, zipfile} = :zip.zip_open(path, [:memory]) + assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) + + assert %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + %{"@language" => "und"} + ], + "bookmarks" => "bookmarks.json", + "followers" => "http://cofe.io/users/cofe/followers", + "following" => "http://cofe.io/users/cofe/following", + "id" => "http://cofe.io/users/cofe", + "inbox" => "http://cofe.io/users/cofe/inbox", + "likes" => "likes.json", + "name" => "Cofe", + "outbox" => "http://cofe.io/users/cofe/outbox", + "preferredUsername" => "cofe", + "publicKey" => %{ + "id" => "http://cofe.io/users/cofe#main-key", + "owner" => "http://cofe.io/users/cofe" + }, + "type" => "Person", + "url" => "http://cofe.io/users/cofe" + } = Jason.decode!(json) + + assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "outbox.json", + "orderedItems" => [ + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status1", + "type" => "Note" + }, + "type" => "Create" + }, + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status2" + } + }, + %{ + "actor" => "http://cofe.io/users/cofe", + "object" => %{ + "content" => "status3" + } + } + ], + "totalItems" => 3, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "likes.json", + "orderedItems" => [^id1, ^id2], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "bookmarks.json", + "orderedItems" => [^id2, ^id3], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + :zip.zip_close(zipfile) + File.rm!(path) + end +end From c82f9129592553718be4bd4712a2b1848dd0a447 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 25 Aug 2020 19:16:01 +0400 Subject: [PATCH 04/38] Fix credo warning --- test/export_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/export_test.exs b/test/export_test.exs index 5afd58ccc..01ca8e7e8 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.ExportTest do use Pleroma.DataCase import Pleroma.Factory - alias Pleroma.Web.CommonAPI alias Pleroma.Bookmark + alias Pleroma.Web.CommonAPI test "it exports user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) From be42ab70dc9538df54ac6f30ee123666223b7287 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 31 Aug 2020 20:31:21 +0400 Subject: [PATCH 05/38] Add backup upload --- lib/pleroma/export.ex | 20 +++++++++++++++++++- test/export_test.exs | 17 ++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index 45b8ce749..b84eccd78 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -22,7 +22,25 @@ def run(user) do :ok <- bookmarks(path, user), {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), {:ok, _} <- File.rm_rf(path) do - {:ok, zip_path} + {:ok, :binary.list_to_bin(zip_path)} + end + end + + def upload(zip_path) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + file_name = zip_path |> String.split("/") |> List.last() + id = Ecto.UUID.generate() + + upload = %Pleroma.Upload{ + id: id, + name: file_name, + tempfile: zip_path, + content_type: "application/zip", + path: id <> "/" <> file_name + } + + with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do + {:ok, upload} end end diff --git a/test/export_test.exs b/test/export_test.exs index 01ca8e7e8..fae269974 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -28,7 +28,7 @@ test "it exports user data" do Bookmark.create(user.id, status3.id) assert {:ok, path} = Pleroma.Export.run(user) - assert {:ok, zipfile} = :zip.zip_open(path, [:memory]) + assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) assert %{ @@ -108,4 +108,19 @@ test "it exports user data" do :zip.zip_close(zipfile) File.rm!(path) end + + test "it uploads an exported backup archive" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) + {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) + {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end end From 75e07ba206b94155c5210151a49e29a11bce6e50 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 31 Aug 2020 23:07:14 +0400 Subject: [PATCH 06/38] Fix tests --- lib/pleroma/export.ex | 3 ++- test/export_test.exs | 47 +++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index b84eccd78..8b1bfefe2 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -39,7 +39,8 @@ def upload(zip_path) do path: id <> "/" <> file_name } - with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do + with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), + :ok <- File.rm(zip_path) do {:ok, upload} end end diff --git a/test/export_test.exs b/test/export_test.exs index fae269974..d7e8f558c 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.ExportTest do use Pleroma.DataCase import Pleroma.Factory + import Mock alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI @@ -109,18 +110,42 @@ test "it exports user data" do File.rm!(path) end - test "it uploads an exported backup archive" do - user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + describe "it uploads an exported backup archive" do + setup do + clear_config(Pleroma.Uploaders.S3, + bucket: "test_bucket", + public_endpoint: "https://s3.amazonaws.com" + ) - {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) - {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) - {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) - CommonAPI.favorite(user, status1.id) - CommonAPI.favorite(user, status2.id) - Bookmark.create(user.id, status2.id) - Bookmark.create(user.id, status3.id) + clear_config([Pleroma.Upload, :uploader]) - assert {:ok, path} = Pleroma.Export.run(user) - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) + {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) + {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + + [path: path] + end + + test "S3", %{path: path} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) + + with_mock ExAws, request: fn _ -> {:ok, :ok} end do + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end + end + + test "Local", %{path: path} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end end end From 4f3a6337454807f4145bbc1830c3d55dd883d46d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Sep 2020 20:21:33 +0400 Subject: [PATCH 07/38] Add `backups` table --- lib/pleroma/{export.ex => backup.ex} | 110 ++++++++++++++---- .../20200831192323_create_backups.exs | 17 +++ test/{export_test.exs => backup_test.exs} | 48 ++++++-- 3 files changed, 141 insertions(+), 34 deletions(-) rename lib/pleroma/{export.ex => backup.ex} (60%) create mode 100644 priv/repo/migrations/20200831192323_create_backups.exs rename test/{export_test.exs => backup_test.exs} (75%) diff --git a/lib/pleroma/export.ex b/lib/pleroma/backup.ex similarity index 60% rename from lib/pleroma/export.ex rename to lib/pleroma/backup.ex index 8b1bfefe2..4580d8f92 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/backup.ex @@ -2,41 +2,110 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Export do +defmodule Pleroma.Backup do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView - import Ecto.Query + schema "backups" do + field(:content_type, :string) + field(:file_name, :string) + field(:file_size, :integer, default: 0) + field(:processed, :boolean, default: false) + + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + + timestamps() + end + + def create(user) do + with :ok <- validate_limit(user), + {:ok, backup} <- user |> new() |> Repo.insert() do + {:ok, backup} + end + end + + def new(user) do + rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" + + %__MODULE__{ + user_id: user.id, + content_type: "application/zip", + file_name: name + } + end + + defp validate_limit(user) do + case get_last(user.id) do + %__MODULE__{inserted_at: inserted_at} -> + days = 7 + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) + + if diff > days do + :ok + else + {:error, "Last export was less than #{days} days ago"} + end + + nil -> + :ok + end + end + + def get_last(user_id) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> limit(1) + |> Repo.one() + end + + def process(%__MODULE__{} = backup) do + with {:ok, zip_file} <- zip(backup), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- upload(backup, zip_file) do + backup + |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> Repo.update() + end + end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def zip(%__MODULE__{} = backup) do + backup = Repo.preload(backup, :user) + name = String.trim_trailing(backup.file_name, ".zip") + dir = Path.join(System.tmp_dir!(), name) - def run(user) do - with {:ok, path} <- create_dir(user), - :ok <- actor(path, user), - :ok <- statuses(path, user), - :ok <- likes(path, user), - :ok <- bookmarks(path, user), - {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), - {:ok, _} <- File.rm_rf(path) do + with :ok <- File.mkdir(dir), + :ok <- actor(dir, backup.user), + :ok <- statuses(dir, backup.user), + :ok <- likes(dir, backup.user), + :ok <- bookmarks(dir, backup.user), + {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), + {:ok, _} <- File.rm_rf(dir) do {:ok, :binary.list_to_bin(zip_path)} end end - def upload(zip_path) do + def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - file_name = zip_path |> String.split("/") |> List.last() - id = Ecto.UUID.generate() upload = %Pleroma.Upload{ - id: id, - name: file_name, + name: backup.file_name, tempfile: zip_path, - content_type: "application/zip", - path: id <> "/" <> file_name + content_type: backup.content_type, + path: "backups/" <> backup.file_name } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), @@ -54,13 +123,6 @@ defp actor(dir, user) do end end - defp create_dir(user) do - datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") - - with :ok <- File.mkdir(dir), do: {:ok, dir} - end - defp write_header(file, name) do IO.write( file, diff --git a/priv/repo/migrations/20200831192323_create_backups.exs b/priv/repo/migrations/20200831192323_create_backups.exs new file mode 100644 index 000000000..3ac5889e2 --- /dev/null +++ b/priv/repo/migrations/20200831192323_create_backups.exs @@ -0,0 +1,17 @@ +defmodule Pleroma.Repo.Migrations.CreateBackups do + use Ecto.Migration + + def change do + create_if_not_exists table(:backups) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:file_name, :string, null: false) + add(:content_type, :string, null: false) + add(:processed, :boolean, null: false, default: false) + add(:file_size, :bigint) + + timestamps() + end + + create_if_not_exists(index(:backups, [:user_id])) + end +end diff --git a/test/export_test.exs b/test/backup_test.exs similarity index 75% rename from test/export_test.exs rename to test/backup_test.exs index d7e8f558c..27f5cb7f7 100644 --- a/test/export_test.exs +++ b/test/backup_test.exs @@ -2,15 +2,41 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ExportTest do +defmodule Pleroma.BackupTest do use Pleroma.DataCase import Pleroma.Factory import Mock + alias Pleroma.Backup alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI - test "it exports user data" do + test "it creates a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, backup} = Backup.create(user) + + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + end + + test "it return an error if the export limit is over" do + %{id: user_id} = user = insert(:user) + limit_days = 7 + + assert {:ok, backup} = Backup.create(user) + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + + assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} + end + + test "it process a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, %{id: backup_id} = backup} = Backup.create(user) + assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert backup.file_size > 0 + assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + end + + test "it creates a zip archive with user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) {:ok, %{object: %{data: %{"id" => id1}}} = status1} = @@ -28,7 +54,8 @@ test "it exports user data" do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -110,7 +137,7 @@ test "it exports user data" do File.rm!(path) end - describe "it uploads an exported backup archive" do + describe "it uploads a backup archive" do setup do clear_config(Pleroma.Uploaders.S3, bucket: "test_bucket", @@ -129,23 +156,24 @@ test "it exports user data" do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) - [path: path] + [path: path, backup: backup] end - test "S3", %{path: path} do + test "S3", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) with_mock ExAws, request: fn _ -> {:ok, :ok} end do - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end - test "Local", %{path: path} do + test "Local", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end end From a0ad9bd734e9af0ce912c32c7480a60ff87a4368 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Sep 2020 21:45:22 +0400 Subject: [PATCH 08/38] Add BackupWorker --- config/config.exs | 1 + config/description.exs | 6 ++++++ lib/pleroma/backup.ex | 11 ++++++++++- lib/pleroma/workers/backup_worker.ex | 17 +++++++++++++++++ test/backup_test.exs | 20 ++++++++++++++------ 5 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/workers/backup_worker.ex diff --git a/config/config.exs b/config/config.exs index 2e6b0796a..1f10167e5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -551,6 +551,7 @@ queues: [ activity_expiration: 10, token_expiration: 5, + backup: 1, federator_incoming: 50, federator_outgoing: 50, ingestion_queue: 50, diff --git a/config/description.exs b/config/description.exs index 6fa78a5d1..13e44afe8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2288,6 +2288,12 @@ description: "Activity expiration queue", suggestions: [10] }, + %{ + key: :backup, + type: :integer, + description: "Backup queue", + suggestions: [1] + }, %{ key: :attachments_cleanup, type: :integer, diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 4580d8f92..9b5d2625f 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Backup do def create(user) do with :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - {:ok, backup} + Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id}) end end @@ -71,6 +71,15 @@ def get_last(user_id) do |> Repo.one() end + def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> where([b], b.id != ^latest_id) + |> Repo.delete_all() + end + + def get(id), do: Repo.get(__MODULE__, id) + def process(%__MODULE__{} = backup) do with {:ok, zip_file} <- zip(backup), {:ok, %{size: size}} <- File.stat(zip_file), diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex new file mode 100644 index 000000000..c982ffa3a --- /dev/null +++ b/lib/pleroma/workers/backup_worker.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.BackupWorker do + alias Pleroma.Backup + + use Pleroma.Workers.WorkerHelper, queue: "backup" + + @impl Oban.Worker + def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do + with {:ok, %Backup{} = backup} <- + backup_id |> Backup.get() |> Backup.process() do + {:ok, backup} + end + end +end diff --git a/test/backup_test.exs b/test/backup_test.exs index 27f5cb7f7..5b1f76dd9 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -3,35 +3,43 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.BackupTest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase + import Pleroma.Factory import Mock alias Pleroma.Backup alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.BackupWorker - test "it creates a backup record" do + setup do: clear_config([Pleroma.Upload, :uploader]) + + test "it creates a backup record and an Oban job" do %{id: user_id} = user = insert(:user) - assert {:ok, backup} = Backup.create(user) + assert {:ok, %Oban.Job{args: args}} = Backup.create(user) + assert_enqueued(worker: BackupWorker, args: args) + backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup end test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) limit_days = 7 - - assert {:ok, backup} = Backup.create(user) + assert {:ok, %Oban.Job{args: args}} = Backup.create(user) + backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} end test "it process a backup record" do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) %{id: user_id} = user = insert(:user) - assert {:ok, %{id: backup_id} = backup} = Backup.create(user) - assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user) + assert {:ok, backup} = BackupWorker.perform(job) assert backup.file_size > 0 assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup end From 3ad7492f9dd1c76cdbc64ad2246f8e9c8c5c4ae6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 18:30:39 +0400 Subject: [PATCH 09/38] Add config for Pleroma.Backup --- config/config.exs | 4 ++++ config/description.exs | 20 ++++++++++++++++++++ docs/configuration/cheatsheet.md | 5 +++++ lib/pleroma/backup.ex | 2 +- test/backup_test.exs | 2 +- 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1f10167e5..09023e2c3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -818,6 +818,10 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator +config :pleroma, Pleroma.Backup, + purge_after_days: 30, + limit_days: 7 + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index 13e44afe8..4942e196d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3712,5 +3712,25 @@ ] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Backup, + type: :group, + description: "Account Backup", + children: [ + %{ + key: :purge_after_days, + type: :integer, + description: "Remove backup achives after N days", + suggestions: [30] + }, + %{ + key: :limit_days, + type: :integer, + description: "Limit user to export not more often than once per N days", + suggestions: [7] + } + ] } ] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 42e5fe808..cc4081f14 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1083,6 +1083,11 @@ Control favicons for instances. * `enabled`: Allow/disallow displaying and getting instances favicons +## Account Backup + +* `:purge_after_days` an integer, remove backup achives after N days. +* `:limit_days` an integer, limit user to export not more often than once per N days. + ## Frontend management Frontends in Pleroma are swappable - you can specify which one to use here. diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 9b5d2625f..e384b6b00 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -49,7 +49,7 @@ def new(user) do defp validate_limit(user) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> - days = 7 + days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) if diff > days do diff --git a/test/backup_test.exs b/test/backup_test.exs index 5b1f76dd9..f343b0361 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -27,7 +27,7 @@ test "it creates a backup record and an Oban job" do test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) - limit_days = 7 + limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) assert {:ok, %Oban.Job{args: args}} = Backup.create(user) backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup From 739cb1463ba07513f047b2ac8f7e22a16c89ef4e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 21:48:52 +0400 Subject: [PATCH 10/38] Add backups deletion --- lib/pleroma/backup.ex | 14 +++++++-- lib/pleroma/workers/backup_worker.ex | 37 ++++++++++++++++++++-- test/backup_test.exs | 47 +++++++++++++++++++++++++--- test/support/oban_helpers.ex | 3 ++ 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index e384b6b00..bd50fd910 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Backup do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView + alias Pleroma.Workers.BackupWorker schema "backups" do field(:content_type, :string) @@ -30,7 +31,7 @@ defmodule Pleroma.Backup do def create(user) do with :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id}) + BackupWorker.process(backup) end end @@ -46,6 +47,14 @@ def new(user) do } end + def delete(backup) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + with :ok <- uploader.delete_file("backups/" <> backup.file_name) do + Repo.delete(backup) + end + end + defp validate_limit(user) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> @@ -75,7 +84,8 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do __MODULE__ |> where(user_id: ^user_id) |> where([b], b.id != ^latest_id) - |> Repo.delete_all() + |> Repo.all() + |> Enum.each(&BackupWorker.delete/1) end def get(id), do: Repo.get(__MODULE__, id) diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index c982ffa3a..f40020794 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -3,15 +3,46 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do + use Oban.Worker, queue: :backup, max_attempts: 1 + + alias Oban.Job alias Pleroma.Backup - use Pleroma.Workers.WorkerHelper, queue: "backup" + def process(backup) do + %{"op" => "process", "backup_id" => backup.id} + |> new() + |> Oban.insert() + end + + def schedule_deletion(backup) do + days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days]) + time = 60 * 60 * 24 * days + scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) + + %{"op" => "delete", "backup_id" => backup.id} + |> new(scheduled_at: scheduled_at) + |> Oban.insert() + end + + def delete(backup) do + %{"op" => "delete", "backup_id" => backup.id} + |> new() + |> Oban.insert() + end - @impl Oban.Worker def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do with {:ok, %Backup{} = backup} <- - backup_id |> Backup.get() |> Backup.process() do + backup_id |> Backup.get() |> Backup.process(), + {:ok, _job} <- schedule_deletion(backup), + :ok <- Backup.remove_outdated(backup) do {:ok, backup} end end + + def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do + case Backup.get(backup_id) do + %Backup{} = backup -> Backup.delete(backup) + nil -> :ok + end + end end diff --git a/test/backup_test.exs b/test/backup_test.exs index f343b0361..59aebe360 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -13,8 +13,12 @@ defmodule Pleroma.BackupTest do alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker + alias Pleroma.Tests.ObanHelpers - setup do: clear_config([Pleroma.Upload, :uploader]) + setup do + clear_config([Pleroma.Upload, :uploader]) + clear_config([Pleroma.Backup, :limit_days]) + end test "it creates a backup record and an Oban job" do %{id: user_id} = user = insert(:user) @@ -38,10 +42,34 @@ test "it return an error if the export limit is over" do test "it process a backup record" do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) %{id: user_id} = user = insert(:user) - assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user) - assert {:ok, backup} = BackupWorker.perform(job) + + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user) + assert {:ok, backup} = perform_job(BackupWorker, args) assert backup.file_size > 0 assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + + delete_job_args = %{"op" => "delete", "backup_id" => backup_id} + + assert_enqueued(worker: BackupWorker, args: delete_job_args) + assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) + refute Backup.get(backup_id) + end + + test "it removes outdated backups after creating a fresh one" do + Pleroma.Config.put([Pleroma.Backup, :limit_days], -1) + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + user = insert(:user) + + assert {:ok, job1} = Backup.create(user) + + assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1) + assert {:ok, job2} = Backup.create(user) + assert Pleroma.Repo.aggregate(Backup, :count) == 2 + assert {:ok, backup2} = ObanHelpers.perform(job2) + + ObanHelpers.perform_all() + + assert [^backup2] = Pleroma.Repo.all(Backup) end test "it creates a zip archive with user data" do @@ -145,7 +173,7 @@ test "it creates a zip archive with user data" do File.rm!(path) end - describe "it uploads a backup archive" do + describe "it uploads and deletes a backup archive" do setup do clear_config(Pleroma.Uploaders.S3, bucket: "test_bucket", @@ -173,8 +201,16 @@ test "it creates a zip archive with user data" do test "S3", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) - with_mock ExAws, request: fn _ -> {:ok, :ok} end do + with_mock ExAws, + request: fn + %{http_method: :put} -> {:ok, :ok} + %{http_method: :delete} -> {:ok, %{status_code: 204}} + end do assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + assert {:ok, _backup} = Backup.delete(backup) + end + + with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do end end @@ -182,6 +218,7 @@ test "Local", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + assert {:ok, _backup} = Backup.delete(backup) end end end diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index 9f90a821c..2468f66dc 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do Oban test helpers. """ + require Ecto.Query + alias Pleroma.Repo def wipe_all do @@ -15,6 +17,7 @@ def wipe_all do def perform_all do Oban.Job + |> Ecto.Query.where(state: "available") |> Repo.all() |> perform() end From abdffc6b8c2eec8f81ffe89f943f11d1f90d7074 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 22:00:26 +0400 Subject: [PATCH 11/38] Fix Credo warning --- test/backup_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backup_test.exs b/test/backup_test.exs index 59aebe360..5fc519eab 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -11,9 +11,9 @@ defmodule Pleroma.BackupTest do alias Pleroma.Backup alias Pleroma.Bookmark + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker - alias Pleroma.Tests.ObanHelpers setup do clear_config([Pleroma.Upload, :uploader]) From 2c73bfe1227065fa203b0b78c9eb12cf86ab3948 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 9 Sep 2020 01:04:00 +0400 Subject: [PATCH 12/38] Add API endpoints for Backups --- lib/pleroma/backup.ex | 7 ++ .../operations/pleroma_backup_operation.ex | 79 +++++++++++++++++ .../controllers/backup_controller.ex | 27 ++++++ .../web/pleroma_api/views/backup_view.ex | 24 ++++++ lib/pleroma/web/router.ex | 3 + .../controllers/backup_controller_test.exs | 84 +++++++++++++++++++ 6 files changed, 224 insertions(+) create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex create mode 100644 lib/pleroma/web/pleroma_api/views/backup_view.ex create mode 100644 test/web/pleroma_api/controllers/backup_controller_test.exs diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index bd50fd910..348e537a8 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -80,6 +80,13 @@ def get_last(user_id) do |> Repo.one() end + def list(%User{id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> Repo.all() + end + def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do __MODULE__ |> where(user_id: ^user_id) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex new file mode 100644 index 000000000..f877ca31b --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -0,0 +1,79 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Backups"], + summary: "List backups", + security: [%{"oAuth" => ["read:account"]}], + operationId: "PleromaAPI.BackupController.index", + responses: %{ + 200 => + Operation.response( + "An array of backups", + "application/json", + %Schema{ + type: :array, + items: backup() + } + ), + 400 => Operation.response("Bad Request", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Backups"], + summary: "Create a backup", + security: [%{"oAuth" => ["read:account"]}], + operationId: "PleromaAPI.BackupController.create", + responses: %{ + 200 => + Operation.response( + "An array of backups", + "application/json", + %Schema{ + type: :array, + items: backup() + } + ), + 400 => Operation.response("Bad Request", "application/json", ApiError) + } + } + end + + defp backup do + %Schema{ + title: "Backup", + description: "Response schema for a backup", + type: :object, + properties: %{ + inserted_at: %Schema{type: :string, format: :"date-time"}, + content_type: %Schema{type: :string}, + file_name: %Schema{type: :string}, + file_size: %Schema{type: :integer}, + processed: %Schema{type: :boolean} + }, + example: %{ + "content_type" => "application/zip", + "file_name" => + "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip", + "file_size" => 1024, + "inserted_at" => "2020-09-08T19:58:20", + "processed" => true + } + } + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex new file mode 100644 index 000000000..e52c77ff2 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupController do + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) + plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation + + def index(%{assigns: %{user: user}} = conn, _params) do + backups = Pleroma.Backup.list(user) + render(conn, "index.json", backups: backups) + end + + def create(%{assigns: %{user: user}} = conn, _params) do + with {:ok, _} <- Pleroma.Backup.create(user) do + backups = Pleroma.Backup.list(user) + render(conn, "index.json", backups: backups) + end + end +end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex new file mode 100644 index 000000000..02b94ce4f --- /dev/null +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupView do + use Pleroma.Web, :view + + alias Pleroma.Backup + alias Pleroma.Web.CommonAPI.Utils + + def render("show.json", %{backup: %Backup{} = backup}) do + %{ + content_type: backup.content_type, + file_name: backup.file_name, + file_size: backup.file_size, + processed: backup.processed, + inserted_at: Utils.to_masto_date(backup.inserted_at) + } + end + + def render("index.json", %{backups: backups}) do + render_many(backups, __MODULE__, "show.json") + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e22b31b4c..a1a5a1cb5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -293,6 +293,9 @@ defmodule Pleroma.Web.Router do get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup) post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm) delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable) + + get("/backups", BackupController, :index) + post("/backups", BackupController, :create) end scope "/oauth", Pleroma.Web.OAuth do diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs new file mode 100644 index 000000000..1ad1b63c4 --- /dev/null +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -0,0 +1,84 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Backup + + setup do + clear_config([Pleroma.Upload, :uploader]) + clear_config([Backup, :limit_days]) + oauth_access(["read:accounts"]) + end + + test "GET /api/pleroma/backups", %{user: user, conn: conn} do + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user) + + backup = Backup.get(backup_id) + + response = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + assert [ + %{ + "content_type" => "application/zip", + "file_name" => file_name, + "file_size" => 0, + "processed" => false, + "inserted_at" => _ + } + ] = response + + assert file_name == backup.file_name + + Pleroma.Tests.ObanHelpers.perform_all() + + assert [ + %{ + "file_name" => ^file_name, + "processed" => true + } + ] = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + end + + test "POST /api/pleroma/backups", %{user: _user, conn: conn} do + assert [ + %{ + "content_type" => "application/zip", + "file_name" => file_name, + "file_size" => 0, + "processed" => false, + "inserted_at" => _ + } + ] = + conn + |> post("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + Pleroma.Tests.ObanHelpers.perform_all() + + assert [ + %{ + "file_name" => ^file_name, + "processed" => true + } + ] = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + days = Pleroma.Config.get([Backup, :limit_days]) + + assert %{"error" => "Last export was less than #{days} days ago"} == + conn + |> post("/api/pleroma/backups") + |> json_response_and_validate_schema(400) + end +end From 86ce4afd9338d81f741fa57f962509a6f0f50aff Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 9 Sep 2020 20:02:20 +0400 Subject: [PATCH 13/38] Improve backup urls --- .../api_spec/operations/pleroma_backup_operation.ex | 6 +++--- lib/pleroma/web/pleroma_api/views/backup_view.ex | 6 +++++- .../controllers/backup_controller_test.exs | 11 ++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex index f877ca31b..6993794db 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -69,9 +69,9 @@ defp backup do example: %{ "content_type" => "application/zip", "file_name" => - "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip", - "file_size" => 1024, - "inserted_at" => "2020-09-08T19:58:20", + "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip", + "file_size" => 4105, + "inserted_at" => "2020-09-08T16:42:07.000Z", "processed" => true } } diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index 02b94ce4f..bf40a001e 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do def render("show.json", %{backup: %Backup{} = backup}) do %{ content_type: backup.content_type, - file_name: backup.file_name, + url: download_url(backup), file_size: backup.file_size, processed: backup.processed, inserted_at: Utils.to_masto_date(backup.inserted_at) @@ -21,4 +21,8 @@ def render("show.json", %{backup: %Backup{} = backup}) do def render("index.json", %{backups: backups}) do render_many(backups, __MODULE__, "show.json") end + + def download_url(%Backup{file_name: file_name}) do + Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name + end end diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs index 1ad1b63c4..5d2f1206e 100644 --- a/test/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Backup + alias Pleroma.Web.PleromaAPI.BackupView setup do clear_config([Pleroma.Upload, :uploader]) @@ -26,20 +27,20 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do assert [ %{ "content_type" => "application/zip", - "file_name" => file_name, + "url" => url, "file_size" => 0, "processed" => false, "inserted_at" => _ } ] = response - assert file_name == backup.file_name + assert url == BackupView.download_url(backup) Pleroma.Tests.ObanHelpers.perform_all() assert [ %{ - "file_name" => ^file_name, + "url" => ^url, "processed" => true } ] = @@ -52,7 +53,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do assert [ %{ "content_type" => "application/zip", - "file_name" => file_name, + "url" => url, "file_size" => 0, "processed" => false, "inserted_at" => _ @@ -66,7 +67,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do assert [ %{ - "file_name" => ^file_name, + "url" => ^url, "processed" => true } ] = From cd13613db3f675b6a9171dea56fc5b03e43ae6b0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 10 Sep 2020 20:53:06 +0400 Subject: [PATCH 14/38] Fix query --- lib/pleroma/backup.ex | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 348e537a8..ce54a413a 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Backup do import Ecto.Changeset import Ecto.Query + require Pleroma.Constants + alias Pleroma.Activity alias Pleroma.Bookmark alias Pleroma.Repo @@ -158,6 +160,7 @@ defp write_header(file, name) do "id": "#{name}.json", "type": "OrderedCollection", "orderedItems": [ + """ ) end @@ -209,13 +212,13 @@ defp statuses(dir, user) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) - |> Map.put(:blocking_user, user) - |> Map.put(:muting_user, user) - |> Map.put(:reply_filtering_user, user) - |> Map.put(:announce_filtering_user, user) - |> Map.put(:user, user) + |> Map.put(:actor_id, user.ap_id) - [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] + [ + [Pleroma.Constants.as_public(), user.ap_id], + User.following(user), + Pleroma.List.memberships(user) + ] |> Enum.concat() |> ActivityPub.fetch_activities_query(opts) |> write(dir, "outbox", fn a -> From 386199063b9be9fc30ad403f6afb03bf6ca47298 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 10 Sep 2020 21:09:20 +0400 Subject: [PATCH 15/38] Document `/api/pleroma/backups` API endpoint --- docs/API/pleroma_api.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 3fd141bd2..aeb266159 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]} ] ``` + +## `POST /api/pleroma/backups` +### Create a user backup archive + +* Method: `POST` +* Authentication: not required +* Params: none +* Response: JSON +* Example response: + +```json +[{ + "content_type": "application/zip", + "file_size": 0, + "inserted_at": "2020-09-10T16:18:03.000Z", + "processed": false, + "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip" +}] +``` + +## `GET /api/pleroma/backups` +### Lists user backups + +* Method: `GET` +* Authentication: not required +* Params: none +* Response: JSON +* Example response: + +```json +[{ + "content_type": "application/zip", + "file_size": 55457, + "inserted_at": "2020-09-10T16:18:03.000Z", + "processed": true, + "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip" +}] +``` From 27bc121ec00a7b088030d6fb36c7e731f5b072b6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 15 Sep 2020 18:07:28 +0400 Subject: [PATCH 16/38] Require email --- docs/configuration/cheatsheet.md | 3 +++ lib/pleroma/backup.ex | 19 ++++++++++++++++--- test/backup_test.exs | 16 ++++++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index cc4081f14..8da8a7bd6 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1085,6 +1085,9 @@ Control favicons for instances. ## Account Backup +!!! note + Requires enabled email + * `:purge_after_days` an integer, remove backup achives after N days. * `:limit_days` an integer, limit user to export not more often than once per N days. diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index ce54a413a..3b85dd1c1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -31,7 +31,9 @@ defmodule Pleroma.Backup do end def create(user) do - with :ok <- validate_limit(user), + with :ok <- validate_email_enabled(), + :ok <- validate_user_email(user), + :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do BackupWorker.process(backup) end @@ -74,6 +76,17 @@ defp validate_limit(user) do end end + defp validate_email_enabled do + if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do + :ok + else + {:error, "Backups require enabled email"} + end + end + + defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"} + defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok + def get_last(user_id) do __MODULE__ |> where(user_id: ^user_id) @@ -100,7 +113,7 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do def get(id), do: Repo.get(__MODULE__, id) def process(%__MODULE__{} = backup) do - with {:ok, zip_file} <- zip(backup), + with {:ok, zip_file} <- export(backup), {:ok, %{size: size}} <- File.stat(zip_file), {:ok, _upload} <- upload(backup, zip_file) do backup @@ -110,7 +123,7 @@ def process(%__MODULE__{} = backup) do end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] - def zip(%__MODULE__{} = backup) do + def export(%__MODULE__{} = backup) do backup = Repo.preload(backup, :user) name = String.trim_trailing(backup.file_name, ".zip") dir = Path.join(System.tmp_dir!(), name) diff --git a/test/backup_test.exs b/test/backup_test.exs index 5fc519eab..318c8c419 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -18,6 +18,18 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) clear_config([Pleroma.Backup, :limit_days]) + clear_config([Pleroma.Emails.Mailer, :enabled]) + end + + test "it requries enabled email" do + Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false) + user = insert(:user) + assert {:error, "Backups require enabled email"} == Backup.create(user) + end + + test "it requries user's email" do + user = insert(:user, %{email: nil}) + assert {:error, "Email is required"} == Backup.create(user) end test "it creates a backup record and an Oban job" do @@ -91,7 +103,7 @@ test "it creates a zip archive with user data" do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.zip(backup) + assert {:ok, path} = Backup.export(backup) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -193,7 +205,7 @@ test "it creates a zip archive with user data" do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.zip(backup) + assert {:ok, path} = Backup.export(backup) [path: path, backup: backup] end From e52dd62e14a956a28a706124464f3ac4b985080d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 16 Sep 2020 23:21:13 +0400 Subject: [PATCH 17/38] Add configurable temporary directory --- config/config.exs | 3 ++- docs/configuration/cheatsheet.md | 6 ++++++ lib/pleroma/backup.ex | 7 ++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 09023e2c3..0e12d6e15 100644 --- a/config/config.exs +++ b/config/config.exs @@ -820,7 +820,8 @@ config :pleroma, Pleroma.Backup, purge_after_days: 30, - limit_days: 7 + limit_days: 7, + dir: nil # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8da8a7bd6..9271964f1 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1090,6 +1090,12 @@ Control favicons for instances. * `:purge_after_days` an integer, remove backup achives after N days. * `:limit_days` an integer, limit user to export not more often than once per N days. +* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order: + 1. the directory named by the TMPDIR environment variable + 2. the directory named by the TEMP environment variable + 3. the directory named by the TMP environment variable + 4. C:\TMP on Windows or /tmp on Unix-like operating systems + 5. as a last resort, the current working directory ## Frontend management diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 3b85dd1c1..450dd5b84 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -126,7 +126,7 @@ def process(%__MODULE__{} = backup) do def export(%__MODULE__{} = backup) do backup = Repo.preload(backup, :user) name = String.trim_trailing(backup.file_name, ".zip") - dir = Path.join(System.tmp_dir!(), name) + dir = dir(name) with :ok <- File.mkdir(dir), :ok <- actor(dir, backup.user), @@ -139,6 +139,11 @@ def export(%__MODULE__{} = backup) do end end + def dir(name) do + dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!() + Path.join(dir, name) + end + def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) From 7fdd81d000d857cbcd5bf442f68c91b1c5b1cebb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 17 Sep 2020 18:42:24 +0400 Subject: [PATCH 18/38] Add "Your backup is ready" email --- lib/pleroma/emails/user_email.ex | 16 ++++++++++++++++ lib/pleroma/workers/backup_worker.ex | 6 +++++- test/backup_test.exs | 5 ++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 1d8c72ae9..f943dda0d 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -189,4 +189,20 @@ def unsubscribe_url(user, notifications_type) do Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end + + def backup_is_ready_email(backup) do + %{user: user} = Pleroma.Repo.preload(backup, :user) + download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) + + html_body = """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + + new() + |> to(recipient(user)) + |> from(sender()) + |> subject("Your account archive is ready") + |> html_body(html_body) + end end diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index f40020794..405d55269 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -34,7 +34,11 @@ def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do with {:ok, %Backup{} = backup} <- backup_id |> Backup.get() |> Backup.process(), {:ok, _job} <- schedule_deletion(backup), - :ok <- Backup.remove_outdated(backup) do + :ok <- Backup.remove_outdated(backup), + {:ok, _} <- + backup + |> Pleroma.Emails.UserEmail.backup_is_ready_email() + |> Pleroma.Emails.Mailer.deliver() do {:ok, backup} end end diff --git a/test/backup_test.exs b/test/backup_test.exs index 318c8c419..0ea40e6fd 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -6,8 +6,9 @@ defmodule Pleroma.BackupTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase - import Pleroma.Factory import Mock + import Pleroma.Factory + import Swoosh.TestAssertions alias Pleroma.Backup alias Pleroma.Bookmark @@ -65,6 +66,8 @@ test "it process a backup record" do assert_enqueued(worker: BackupWorker, args: delete_job_args) assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) refute Backup.get(backup_id) + + assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup)) end test "it removes outdated backups after creating a fresh one" do From 7c22c9afb410668d87dcd4a90651d62d9a1e9e4d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:18:34 +0400 Subject: [PATCH 19/38] Allow admins request user backups --- lib/pleroma/backup.ex | 4 ++-- lib/pleroma/emails/user_email.ex | 20 +++++++++++++----- .../controllers/admin_api_controller.ex | 12 ++++++++++- lib/pleroma/web/router.ex | 2 ++ lib/pleroma/workers/backup_worker.ex | 10 +++++---- .../controllers/admin_api_controller_test.exs | 21 +++++++++++++++++++ 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 450dd5b84..d589f12f1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,12 +30,12 @@ defmodule Pleroma.Backup do timestamps() end - def create(user) do + def create(user, admin_user_id \\ nil) do with :ok <- validate_email_enabled(), :ok <- validate_user_email(user), :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup) + BackupWorker.process(backup, admin_user_id) end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index f943dda0d..5745794ec 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -190,14 +190,24 @@ def unsubscribe_url(user, notifications_type) do Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end - def backup_is_ready_email(backup) do + def backup_is_ready_email(backup, admin_user_id \\ nil) do %{user: user} = Pleroma.Repo.preload(backup, :user) download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) - html_body = """ -

You requested a full backup of your Pleroma account. It's ready for download:

-

- """ + html_body = + if is_nil(admin_user_id) do + """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + else + admin = Pleroma.Repo.get(User, admin_user_id) + + """ +

Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + end new() |> to(recipient(user)) diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index d5713c3dd..f7d2fe5b1 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -23,12 +23,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.Endpoint alias Pleroma.Web.Router + require Logger + @users_page_size 50 plug( OAuthScopesPlug, %{scopes: ["read:accounts"], admin: true} - when action in [:list_users, :user_show, :right_get, :show_user_credentials] + when action in [:list_users, :user_show, :right_get, :show_user_credentials, :create_backup] ) plug( @@ -681,6 +683,14 @@ def stats(conn, params) do json(conn, %{"status_visibility" => counters}) end + def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_by_nickname(nickname), + {:ok, _} <- Pleroma.Backup.create(user, admin.id) do + Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}") + json(conn, "") + end + end + defp page_params(params) do {get_page(params["page"]), get_page_size(params["page_size"])} end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a1a5a1cb5..e539eeeeb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -129,6 +129,8 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:admin_api) + post("/backups", AdminAPIController, :create_backup) + post("/users/follow", AdminAPIController, :user_follow) post("/users/unfollow", AdminAPIController, :user_unfollow) diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 405d55269..65754b6a2 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Workers.BackupWorker do alias Oban.Job alias Pleroma.Backup - def process(backup) do - %{"op" => "process", "backup_id" => backup.id} + def process(backup, admin_user_id \\ nil) do + %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} |> new() |> Oban.insert() end @@ -30,14 +30,16 @@ def delete(backup) do |> Oban.insert() end - def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do + def perform(%Job{ + args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id} + }) do with {:ok, %Backup{} = backup} <- backup_id |> Backup.get() |> Backup.process(), {:ok, _job} <- schedule_deletion(backup), :ok <- Backup.remove_outdated(backup), {:ok, _} <- backup - |> Pleroma.Emails.UserEmail.backup_is_ready_email() + |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id) |> Pleroma.Emails.Mailer.deliver() do {:ok, backup} end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index cba6b43d3..4d331779e 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2024,6 +2024,27 @@ test "by instance", %{conn: conn} do response["status_visibility"] end end + + describe "/api/pleroma/backups" do + test "it creates a backup", %{conn: conn} do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + user = insert(:user) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert [backup] = Repo.all(Pleroma.Backup) + + ObanHelpers.perform_all() + + assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) + end + end end # Needed for testing From 563801716a0aa54e30f680b4e985d4b8c79578fb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:01:46 +0400 Subject: [PATCH 20/38] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc1750d1..04b49d80a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Account backup ### Changed From e50314d9d342dbf9a03ca484654b07717592d4bd Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:33:12 +0400 Subject: [PATCH 21/38] Fix export --- lib/pleroma/backup.ex | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index d589f12f1..242773bdb 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -191,16 +191,13 @@ defp write(query, dir, name, fun) do counter = :counters.new(1, []) query - |> Pleroma.RepoStreamer.chunk_stream(100) - |> Stream.each(fn items -> - Enum.each(items, fn i -> - with {:ok, str} <- fun.(i), - :ok <- IO.write(file, str <> ",\n") do - :counters.add(counter, 1, 1) - end - end) + |> Pleroma.Repo.chunk_stream(100) + |> Enum.each(fn i -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + :counters.add(counter, 1, 1) + end end) - |> Stream.run() total = :counters.get(counter, 1) From a9efd441e242f1d8ac608b866d0cfafe4833243a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 19:57:09 +0400 Subject: [PATCH 22/38] Use `Pleroma.Repo.chunk_stream/2` instead of `Pleroma.RepoStreamer.chunk_stream/2` --- lib/pleroma/backup.ex | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 242773bdb..f5f39431d 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -188,18 +188,17 @@ defp write(query, dir, name, fun) do with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do - counter = :counters.new(1, []) - - query - |> Pleroma.Repo.chunk_stream(100) - |> Enum.each(fn i -> - with {:ok, str} <- fun.(i), - :ok <- IO.write(file, str <> ",\n") do - :counters.add(counter, 1, 1) - end - end) - - total = :counters.get(counter, 1) + total = + query + |> Pleroma.Repo.chunk_stream(100) + |> Enum.reduce(0, fn i, acc -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + acc + 1 + else + _ -> acc + end + end) with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) From 17562bf4147ab03e171b1f1d365a512f2e5b3202 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 20:43:27 +0400 Subject: [PATCH 23/38] Move API endpoints to `/api/v1/pleroma/backups` --- docs/API/pleroma_api.md | 4 ++-- lib/pleroma/web/router.ex | 6 +++--- .../controllers/backup_controller_test.exs | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index aeb266159..fa3a9a449 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -616,7 +616,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to ] ``` -## `POST /api/pleroma/backups` +## `POST /api/v1/pleroma/backups` ### Create a user backup archive * Method: `POST` @@ -635,7 +635,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to }] ``` -## `GET /api/pleroma/backups` +## `GET /api/v1/pleroma/backups` ### Lists user backups * Method: `GET` diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e539eeeeb..ad7e315c7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -295,9 +295,6 @@ defmodule Pleroma.Web.Router do get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup) post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm) delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable) - - get("/backups", BackupController, :index) - post("/backups", BackupController, :create) end scope "/oauth", Pleroma.Web.OAuth do @@ -358,6 +355,9 @@ defmodule Pleroma.Web.Router do put("/mascot", MascotController, :update) post("/scrobble", ScrobbleController, :create) + + get("/backups", BackupController, :index) + post("/backups", BackupController, :create) end scope [] do diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs index 5d2f1206e..b2ac74c7d 100644 --- a/test/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -14,14 +14,14 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do oauth_access(["read:accounts"]) end - test "GET /api/pleroma/backups", %{user: user, conn: conn} do + test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user) backup = Backup.get(backup_id) response = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) assert [ @@ -45,11 +45,11 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do } ] = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) end - test "POST /api/pleroma/backups", %{user: _user, conn: conn} do + test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do assert [ %{ "content_type" => "application/zip", @@ -60,7 +60,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do } ] = conn - |> post("/api/pleroma/backups") + |> post("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) Pleroma.Tests.ObanHelpers.perform_all() @@ -72,14 +72,14 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do } ] = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) days = Pleroma.Config.get([Backup, :limit_days]) assert %{"error" => "Last export was less than #{days} days ago"} == conn - |> post("/api/pleroma/backups") + |> post("/api/v1/pleroma/backups") |> json_response_and_validate_schema(400) end end From e4792ce76af3094d378a3a201ca429ae38203696 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 21:06:16 +0400 Subject: [PATCH 24/38] Do not limit admins --- lib/pleroma/backup.ex | 10 ++++---- .../controllers/admin_api_controller_test.exs | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index f5f39431d..e2673db80 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,12 +30,12 @@ defmodule Pleroma.Backup do timestamps() end - def create(user, admin_user_id \\ nil) do + def create(user, admin_id \\ nil) do with :ok <- validate_email_enabled(), :ok <- validate_user_email(user), - :ok <- validate_limit(user), + :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup, admin_user_id) + BackupWorker.process(backup, admin_id) end end @@ -59,7 +59,9 @@ def delete(backup) do end end - defp validate_limit(user) do + defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok + + defp validate_limit(user, nil) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index 4d331779e..4b3abce0d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2044,6 +2044,30 @@ test "it creates a backup", %{conn: conn} do assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) end + + test "it doesn't limit admins", %{conn: conn} do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + user = insert(:user) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert [_backup] = Repo.all(Pleroma.Backup) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert Repo.aggregate(Pleroma.Backup, :count) == 2 + end end end From 8baee855d90530def46dc62b81e6a0cb0c315914 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 21 Sep 2020 21:47:36 +0400 Subject: [PATCH 25/38] Fix emails --- lib/pleroma/emails/user_email.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 5745794ec..806a61fd2 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -198,14 +198,14 @@ def backup_is_ready_email(backup, admin_user_id \\ nil) do if is_nil(admin_user_id) do """

You requested a full backup of your Pleroma account. It's ready for download:

-

+

#{download_url}

""" else admin = Pleroma.Repo.get(User, admin_user_id) """

Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

-

+

#{download_url}

""" end From f1e4333dd7976e8cbef44a3bcfe5c96bef177c6f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 23 Sep 2020 20:23:11 +0400 Subject: [PATCH 26/38] Fix test --- test/backup_test.exs | 7 ++++++- .../admin_api/controllers/admin_api_controller_test.exs | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/backup_test.exs b/test/backup_test.exs index 0ea40e6fd..23c08b680 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -67,7 +67,12 @@ test "it process a backup record" do assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) refute Backup.get(backup_id) - assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup)) + email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup) + + assert_email_sent( + to: {user.name, user.email}, + html_body: email.html_body + ) end test "it removes outdated backups after creating a fresh one" do diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index 4b3abce0d..a6dc4f62d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2042,7 +2042,10 @@ test "it creates a backup", %{conn: conn} do ObanHelpers.perform_all() - assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) + email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id) + + assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup") + assert_email_sent(to: {user.name, user.email}, html_body: email.html_body) end test "it doesn't limit admins", %{conn: conn} do From 6d5f02a1da81ed7693c5ae364a25bc0b54ee1a38 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 20:34:44 +0400 Subject: [PATCH 27/38] Fix API documentation --- docs/API/pleroma_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index fa3a9a449..7a0a80dad 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -620,7 +620,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to ### Create a user backup archive * Method: `POST` -* Authentication: not required +* Authentication: required * Params: none * Response: JSON * Example response: From d7a5291b4fa3b7568674c0f7643fe287fcd21eff Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:24:35 +0400 Subject: [PATCH 28/38] Use `Jason.encode/1` for likes and bookmarks --- lib/pleroma/backup.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index e2673db80..b43dc94d6 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -194,7 +194,8 @@ defp write(query, dir, name, fun) do query |> Pleroma.Repo.chunk_stream(100) |> Enum.reduce(0, fn i, acc -> - with {:ok, str} <- fun.(i), + with {:ok, data} <- fun.(i), + {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do acc + 1 else @@ -213,7 +214,7 @@ defp bookmarks(dir, %{id: user_id} = _user) do |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) + |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) end defp likes(dir, user) do @@ -221,7 +222,7 @@ defp likes(dir, user) do |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) + |> write(dir, "likes", fn a -> {:ok, a.object} end) end defp statuses(dir, user) do @@ -239,7 +240,7 @@ defp statuses(dir, user) do |> ActivityPub.fetch_activities_query(opts) |> write(dir, "outbox", fn a -> with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do - activity |> Map.delete("@context") |> Jason.encode() + {:ok, Map.delete(activity, "@context")} end end) end From 9af9f02f4b3c4eac859a69ab9b2f546a91110287 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:45:03 +0400 Subject: [PATCH 29/38] Use Gettext for error messages --- lib/pleroma/backup.ex | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index b43dc94d6..0ebaf02e5 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Backup do import Ecto.Changeset import Ecto.Query + import Pleroma.Web.Gettext require Pleroma.Constants @@ -70,7 +71,14 @@ defp validate_limit(user, nil) do if diff > days do :ok else - {:error, "Last export was less than #{days} days ago"} + {:error, + dngettext( + "errors", + "Last export was less than a day ago", + "Last export was less than %{days} days ago", + days, + days: days + )} end nil -> @@ -82,11 +90,14 @@ defp validate_email_enabled do if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do :ok else - {:error, "Backups require enabled email"} + {:error, dgettext("errors", "Backups require enabled email")} end end - defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"} + defp validate_user_email(%User{email: nil}) do + {:error, dgettext("errors", "Email is required")} + end + defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok def get_last(user_id) do From 08972dd135c200073f5de0c8731b886cc2e72eeb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:50:31 +0400 Subject: [PATCH 30/38] Use Path.join/2 --- lib/pleroma/backup.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 0ebaf02e5..cee51d7c1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -55,7 +55,7 @@ def new(user) do def delete(backup) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - with :ok <- uploader.delete_file("backups/" <> backup.file_name) do + with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do Repo.delete(backup) end end @@ -164,7 +164,7 @@ def upload(%__MODULE__{} = backup, zip_path) do name: backup.file_name, tempfile: zip_path, content_type: backup.content_type, - path: "backups/" <> backup.file_name + path: Path.join("backups", backup.file_name) } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), @@ -178,7 +178,7 @@ defp actor(dir, user) do UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) |> Jason.encode() do - File.write(dir <> "/actor.json", json) + File.write(Path.join(dir, "actor.json"), json) end end @@ -197,7 +197,7 @@ defp write_header(file, name) do end defp write(query, dir, name, fun) do - path = dir <> "/#{name}.json" + path = Path.join(dir, "#{name}.json") with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do From 8545d533ddee2978e9bf7f3284cc7dcb822a77e6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:53:04 +0400 Subject: [PATCH 31/38] Use to_string/1 instead of :binary.list_to_bin/1 --- lib/pleroma/backup.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index cee51d7c1..629e879a7 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -148,7 +148,7 @@ def export(%__MODULE__{} = backup) do :ok <- bookmarks(dir, backup.user), {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do - {:ok, :binary.list_to_bin(zip_path)} + {:ok, to_string(zip_path)} end end From bc3db724030707e9903d161a70b10fe217a83212 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 23:16:56 +0400 Subject: [PATCH 32/38] Use ModerationLog instead of Logger --- lib/pleroma/moderation_log.ex | 10 ++++++++ .../controllers/admin_api_controller.ex | 3 ++- .../controllers/admin_api_controller_test.exs | 23 +++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 47036a6f6..be1e81467 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -651,6 +651,16 @@ def get_log_entry_message(%ModerationLog{ "@#{actor_nickname} deleted chat message ##{subject_id}" end + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "create_backup", + "subject" => %{"nickname" => user_nickname} + } + }) do + "@#{actor_nickname} requested account backup for @#{user_nickname}" + end + defp nicknames_to_string(nicknames) do nicknames |> Enum.map(&"@#{&1}") diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index f7d2fe5b1..8b5310d80 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -686,7 +686,8 @@ def stats(conn, params) do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), {:ok, _} <- Pleroma.Backup.create(user, admin.id) do - Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}") + ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) + json(conn, "") end end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index a6dc4f62d..34d48c2c1 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2027,9 +2027,9 @@ test "by instance", %{conn: conn} do describe "/api/pleroma/backups" do test "it creates a backup", %{conn: conn} do - admin = insert(:user, is_admin: true) + admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true) token = insert(:oauth_admin_token, user: admin) - user = insert(:user) + user = %{id: user_id, nickname: user_nickname} = insert(:user) assert "" == conn @@ -2046,6 +2046,25 @@ test "it creates a backup", %{conn: conn} do assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup") assert_email_sent(to: {user.name, user.email}, html_body: email.html_body) + + log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}" + + assert [ + %{ + data: %{ + "action" => "create_backup", + "actor" => %{ + "id" => ^admin_id, + "nickname" => ^admin_nickname + }, + "message" => ^log_message, + "subject" => %{ + "id" => ^user_id, + "nickname" => ^user_nickname + } + } + } + ] = Pleroma.ModerationLog |> Repo.all() end test "it doesn't limit admins", %{conn: conn} do From 98f32cf8204113c6d019653c22e446e558147248 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 19 Oct 2020 15:30:32 +0400 Subject: [PATCH 33/38] Fix tests --- lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +- test/backup_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index e52c77ff2..8e3d081f3 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller - alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.Plugs.OAuthScopesPlug action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) diff --git a/test/backup_test.exs b/test/backup_test.exs index 23c08b680..078e03621 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -19,7 +19,7 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) clear_config([Pleroma.Backup, :limit_days]) - clear_config([Pleroma.Emails.Mailer, :enabled]) + clear_config([Pleroma.Emails.Mailer, :enabled], true) end test "it requries enabled email" do From c1976d5b19fbceaecf1f52711fe35e1c7d5312aa Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 19 Oct 2020 18:14:39 +0400 Subject: [PATCH 34/38] Fix credo warnings --- test/{ => pleroma}/backup_test.exs | 0 .../web/pleroma_api/controllers/backup_controller_test.exs | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/{ => pleroma}/backup_test.exs (100%) rename test/{ => pleroma}/web/pleroma_api/controllers/backup_controller_test.exs (100%) diff --git a/test/backup_test.exs b/test/pleroma/backup_test.exs similarity index 100% rename from test/backup_test.exs rename to test/pleroma/backup_test.exs diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs similarity index 100% rename from test/web/pleroma_api/controllers/backup_controller_test.exs rename to test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs From ad605e3e16ba3f6ee3df7a0a3e6705036fef369f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 20 Oct 2020 17:16:58 +0400 Subject: [PATCH 35/38] Rename `Pleroma.Backup` to `Pleroma.User.Backup` --- config/config.exs | 2 +- config/description.exs | 2 +- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/{ => user}/backup.ex | 4 ++-- .../web/admin_api/controllers/admin_api_controller.ex | 2 +- .../web/pleroma_api/controllers/backup_controller.ex | 7 ++++--- lib/pleroma/web/pleroma_api/views/backup_view.ex | 2 +- lib/pleroma/workers/backup_worker.ex | 4 ++-- test/pleroma/{ => user}/backup_test.exs | 10 +++++----- .../controllers/admin_api_controller_test.exs | 6 +++--- .../pleroma_api/controllers/backup_controller_test.exs | 2 +- 11 files changed, 22 insertions(+), 21 deletions(-) rename lib/pleroma/{ => user}/backup.ex (98%) rename test/pleroma/{ => user}/backup_test.exs (97%) diff --git a/config/config.exs b/config/config.exs index 63e386250..c758c818c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -831,7 +831,7 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator -config :pleroma, Pleroma.Backup, +config :pleroma, Pleroma.User.Backup, purge_after_days: 30, limit_days: 7, dir: nil diff --git a/config/description.exs b/config/description.exs index 88f2a6133..9f23b6d3d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3731,7 +3731,7 @@ }, %{ group: :pleroma, - key: Pleroma.Backup, + key: Pleroma.User.Backup, type: :group, description: "Account Backup", children: [ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index aafc43f3d..b40a2aebf 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1077,7 +1077,7 @@ Control favicons for instances. * `enabled`: Allow/disallow displaying and getting instances favicons -## Account Backup +## Pleroma.User.Backup !!! note Requires enabled email diff --git a/lib/pleroma/backup.ex b/lib/pleroma/user/backup.ex similarity index 98% rename from lib/pleroma/backup.ex rename to lib/pleroma/user/backup.ex index 629e879a7..a9041fd94 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Backup do +defmodule Pleroma.User.Backup do use Ecto.Schema import Ecto.Changeset @@ -65,7 +65,7 @@ defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok defp validate_limit(user, nil) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> - days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) + days = Pleroma.Config.get([__MODULE__, :limit_days]) diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) if diff > days do diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index a4f0d7d34..0a27c5861 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -685,7 +685,7 @@ def stats(conn, params) do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), - {:ok, _} <- Pleroma.Backup.create(user, admin.id) do + {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) json(conn, "") diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index 8e3d081f3..bd7b36880 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.User.Backup action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) @@ -14,13 +15,13 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation def index(%{assigns: %{user: user}} = conn, _params) do - backups = Pleroma.Backup.list(user) + backups = Backup.list(user) render(conn, "index.json", backups: backups) end def create(%{assigns: %{user: user}} = conn, _params) do - with {:ok, _} <- Pleroma.Backup.create(user) do - backups = Pleroma.Backup.list(user) + with {:ok, _} <- Backup.create(user) do + backups = Backup.list(user) render(conn, "index.json", backups: backups) end end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index bf40a001e..af75876aa 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do use Pleroma.Web, :view - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Web.CommonAPI.Utils def render("show.json", %{backup: %Backup{} = backup}) do diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 65754b6a2..5b4985983 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.BackupWorker do use Oban.Worker, queue: :backup, max_attempts: 1 alias Oban.Job - alias Pleroma.Backup + alias Pleroma.User.Backup def process(backup, admin_user_id \\ nil) do %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} @@ -15,7 +15,7 @@ def process(backup, admin_user_id \\ nil) do end def schedule_deletion(backup) do - days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days]) + days = Pleroma.Config.get([Backup, :purge_after_days]) time = 60 * 60 * 24 * days scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) diff --git a/test/pleroma/backup_test.exs b/test/pleroma/user/backup_test.exs similarity index 97% rename from test/pleroma/backup_test.exs rename to test/pleroma/user/backup_test.exs index 078e03621..5ad587833 100644 --- a/test/pleroma/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.BackupTest do +defmodule Pleroma.User.BackupTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase @@ -10,7 +10,7 @@ defmodule Pleroma.BackupTest do import Pleroma.Factory import Swoosh.TestAssertions - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Bookmark alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI @@ -18,7 +18,7 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) - clear_config([Pleroma.Backup, :limit_days]) + clear_config([Backup, :limit_days]) clear_config([Pleroma.Emails.Mailer, :enabled], true) end @@ -44,7 +44,7 @@ test "it creates a backup record and an Oban job" do test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) - limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) + limit_days = Pleroma.Config.get([Backup, :limit_days]) assert {:ok, %Oban.Job{args: args}} = Backup.create(user) backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup @@ -76,7 +76,7 @@ test "it process a backup record" do end test "it removes outdated backups after creating a fresh one" do - Pleroma.Config.put([Pleroma.Backup, :limit_days], -1) + Pleroma.Config.put([Backup, :limit_days], -1) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) user = insert(:user) diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index 34d48c2c1..5efe8ef71 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -2038,7 +2038,7 @@ test "it creates a backup", %{conn: conn} do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert [backup] = Repo.all(Pleroma.Backup) + assert [backup] = Repo.all(Pleroma.User.Backup) ObanHelpers.perform_all() @@ -2079,7 +2079,7 @@ test "it doesn't limit admins", %{conn: conn} do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert [_backup] = Repo.all(Pleroma.Backup) + assert [_backup] = Repo.all(Pleroma.User.Backup) assert "" == conn @@ -2088,7 +2088,7 @@ test "it doesn't limit admins", %{conn: conn} do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert Repo.aggregate(Pleroma.Backup, :count) == 2 + assert Repo.aggregate(Pleroma.User.Backup, :count) == 2 end end end diff --git a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs index b2ac74c7d..f1941f6dd 100644 --- a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do use Pleroma.Web.ConnCase - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Web.PleromaAPI.BackupView setup do From 034ac43f3a91178694d3c621c52ce68207ec4f69 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 20 Oct 2020 17:47:04 +0400 Subject: [PATCH 36/38] Fix credo warnings --- lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +- test/pleroma/user/backup_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index bd7b36880..dd0a2e22f 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller - alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.User.Backup + alias Pleroma.Web.Plugs.OAuthScopesPlug action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 5ad587833..513798911 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -10,9 +10,9 @@ defmodule Pleroma.User.BackupTest do import Pleroma.Factory import Swoosh.TestAssertions - alias Pleroma.User.Backup alias Pleroma.Bookmark alias Pleroma.Tests.ObanHelpers + alias Pleroma.User.Backup alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker From 4f90077767b416f3469fe7c8acfaa6932c579ec2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 28 Oct 2020 15:32:44 +0400 Subject: [PATCH 37/38] Fix warning --- test/pleroma/user/backup_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 513798911..f68e4a029 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -82,7 +82,7 @@ test "it removes outdated backups after creating a fresh one" do assert {:ok, job1} = Backup.create(user) - assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1) + assert {:ok, %Backup{}} = ObanHelpers.perform(job1) assert {:ok, job2} = Backup.create(user) assert Pleroma.Repo.aggregate(Backup, :count) == 2 assert {:ok, backup2} = ObanHelpers.perform(job2) From d1698267a27bd5084916f5f6f36d66b1ff2ffc5f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 31 Oct 2020 00:26:11 +0400 Subject: [PATCH 38/38] Fix credo warning --- lib/pleroma/web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9592d0f38..efe67ad7a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -148,7 +148,7 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:admin_api) - + put("/users/disable_mfa", AdminAPIController, :disable_mfa) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users)