From 42d034505a1904fae83856ab96f528fe874e70be Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 17 Mar 2019 15:37:55 +0100 Subject: [PATCH 01/79] Add test for conversation API beforehand --- .../mastodon_api_controller_test.exs | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 059d5237d..1560ec79c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -248,6 +248,57 @@ test "direct timeline", %{conn: conn} do assert status["url"] != direct.data["id"] end + test "Conversations", %{conn: conn} do + user_one = insert(:user) + user_two = insert(:user) + + {:ok, user_two} = User.follow(user_two, user_one) + + {:ok, direct} = + CommonAPI.post(user_one, %{ + "status" => "Hi @#{user_two.nickname}!", + "visibility" => "direct" + }) + + {:ok, _follower_only} = + CommonAPI.post(user_one, %{ + "status" => "Hi @#{user_two.nickname}!", + "visibility" => "private" + }) + + res_conn = + conn + |> assign(:user, user) + |> get("/api/v1/conversations") + + assert response = json_response(res_conn, 200) + + assert %{ + "id" => res_id, + "accounts" => res_accounts, + "last_status" => res_last_status, + "unread" => unread + } = reponse + + assert unread == false + + # Apparently undocumented API endpoint + res_conn = + conn + |> assign(:user, user) + |> get("/api/v1/conversations/#{res_id}/read") + + assert response == json_response(res_conn, 200) + + # (vanilla) Mastodon frontend behaviour + res_conn = + conn + |> assign(:user, user) + |> get("/api/v1/statuses/#{res_last_status.id}/context") + + assert %{ancestors: [], descendants: []} == json_response(res_conn, 200) + end + test "doesn't include DMs from blocked users", %{conn: conn} do blocker = insert(:user) blocked = insert(:user) From f6fab01ba7a08fe0e5147f82d9e3dd294922dc93 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 17 Mar 2019 17:06:28 +0100 Subject: [PATCH 02/79] Web.Router: Add routes for Conversation mastoAPI --- lib/pleroma/web/router.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index befd382ba..6d000b7f5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -243,6 +243,9 @@ defmodule Pleroma.Web.Router do get("/suggestions", MastodonAPIController, :suggestions) + get("/conversations", MastodonAPIController, :conversations) + get("/conversations/:id/read", MastodonAPIController, :get_conversation) + get("/endorsements", MastodonAPIController, :empty_array) get("/pleroma/flavour", MastodonAPIController, :get_flavour) From a2e03d4f3cfc7ccae98ae19db253aec8f1d3e9d0 Mon Sep 17 00:00:00 2001 From: Zachary Dunn Date: Mon, 18 Mar 2019 13:56:59 +0000 Subject: [PATCH 03/79] Initial attempt at updating return type --- lib/pleroma/user.ex | 33 ++++++++++++------- lib/pleroma/web/activity_pub/relay.ex | 4 +-- .../web/activity_pub/transmogrifier.ex | 24 +++++++------- .../mastodon_api/mastodon_api_controller.ex | 2 +- lib/pleroma/web/twitter_api/twitter_api.ex | 2 +- 5 files changed, 37 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 1ce9882f6..6e8103c1c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -411,7 +411,7 @@ def follow_import(%User{} = follower, followed_identifiers) Enum.map( followed_identifiers, fn followed_identifier -> - with %User{} = followed <- get_or_fetch(followed_identifier), + with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier), {:ok, follower} <- maybe_direct_follow(follower, followed), {:ok, _} <- ActivityPub.follow(follower, followed) do followed @@ -492,7 +492,14 @@ def get_cached_by_id(id) do def get_cached_by_nickname(nickname) do key = "nickname:#{nickname}" - Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end) + Cachex.fetch!(:user_cache, key, fn -> + user_result = get_or_fetch_by_nickname(nickname) + + case user_result do + {:ok, user} -> {:commit, user} + {:error, error} -> {:ignore, error} + end + end) end def get_cached_by_nickname_or_id(nickname_or_id) do @@ -529,7 +536,7 @@ def fetch_by_nickname(nickname) do def get_or_fetch_by_nickname(nickname) do with %User{} = user <- get_by_nickname(nickname) do - user + {:ok, user} else _e -> with [_nick, _domain] <- String.split(nickname, "@"), @@ -538,9 +545,9 @@ def get_or_fetch_by_nickname(nickname) do {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user]) end - user + {:ok, user} else - _e -> nil + _e -> {:error, "Error"} end end end @@ -939,7 +946,7 @@ def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_i Enum.map( blocked_identifiers, fn blocked_identifier -> - with %User{} = blocked <- get_or_fetch(blocked_identifier), + with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier), {:ok, blocker} <- block(blocker, blocked), {:ok, _} <- ActivityPub.block(blocker, blocked) do blocked @@ -1157,17 +1164,19 @@ def get_or_fetch_by_ap_id(ap_id) do user = get_by_ap_id(ap_id) if !is_nil(user) and !User.needs_update?(user) do - user + {:ok, user} else user = fetch_by_ap_id(ap_id) - if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do - with %User{} = user do + with %User{} = user do + if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user]) end - end - user + {:ok, user} + else + _ -> {:error, "Could not fetch by AP id"} + end end end @@ -1209,7 +1218,7 @@ def public_key_from_info(%{magic_key: magic_key}) do end def get_public_key_for_ap_id(ap_id) do - with %User{} = user <- get_or_fetch_by_ap_id(ap_id), + with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id), {:ok, public_key} <- public_key_from_info(user.info) do {:ok, public_key} else diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 01fef71b9..2bab9b03c 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -15,7 +15,7 @@ def get_actor do def follow(target_instance) do with %User{} = local_user <- get_actor(), - %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance), + {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance), {:ok, activity} <- ActivityPub.follow(local_user, target_user) do Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}") {:ok, activity} @@ -28,7 +28,7 @@ def follow(target_instance) do def unfollow(target_instance) do with %User{} = local_user <- get_actor(), - %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance), + {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance), {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}") {:ok, activity} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 1247e4b61..a90b934e5 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -373,7 +373,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj |> fix_addressing with nil <- Activity.get_create_by_object_ap_id(object["id"]), - %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do + {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do object = fix_object(data["object"]) params = %{ @@ -402,7 +402,7 @@ def handle_incoming( %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), - %User{} = follower <- User.get_or_fetch_by_ap_id(follower), + {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), {:ok, activity} <- ActivityPub.follow(follower, followed, id, false) do if not User.locked?(followed) do ActivityPub.accept(%{ @@ -425,7 +425,7 @@ def handle_incoming( %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data ) do with actor <- get_actor(data), - %User{} = followed <- User.get_or_fetch_by_ap_id(actor), + {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"), %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]), @@ -451,7 +451,7 @@ def handle_incoming( %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data ) do with actor <- get_actor(data), - %User{} = followed <- User.get_or_fetch_by_ap_id(actor), + {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"), %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]), @@ -475,7 +475,7 @@ def handle_incoming( %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data ) do with actor <- get_actor(data), - %User{} = actor <- User.get_or_fetch_by_ap_id(actor), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do {:ok, activity} @@ -488,7 +488,7 @@ def handle_incoming( %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data ) do with actor <- get_actor(data), - %User{} = actor <- User.get_or_fetch_by_ap_id(actor), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), public <- Visibility.is_public?(data), {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do @@ -543,7 +543,7 @@ def handle_incoming( object_id = Utils.get_ap_id(object_id) with actor <- get_actor(data), - %User{} = actor <- User.get_or_fetch_by_ap_id(actor), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), :ok <- contain_origin(actor.ap_id, object.data), {:ok, activity} <- ActivityPub.delete(object, false) do @@ -562,7 +562,7 @@ def handle_incoming( } = data ) do with actor <- get_actor(data), - %User{} = actor <- User.get_or_fetch_by_ap_id(actor), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do {:ok, activity} @@ -580,7 +580,7 @@ def handle_incoming( } = _data ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), - %User{} = follower <- User.get_or_fetch_by_ap_id(follower), + {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), {:ok, activity} <- ActivityPub.unfollow(follower, followed, id, false) do User.unfollow(follower, followed) {:ok, activity} @@ -599,7 +599,7 @@ def handle_incoming( ) do with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), - %User{} = blocker <- User.get_or_fetch_by_ap_id(blocker), + {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker), {:ok, activity} <- ActivityPub.unblock(blocker, blocked, id, false) do User.unblock(blocker, blocked) {:ok, activity} @@ -613,7 +613,7 @@ def handle_incoming( ) do with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), - %User{} = blocker = User.get_or_fetch_by_ap_id(blocker), + {:ok, %User{} = blocker = User.get_or_fetch_by_ap_id(blocker), {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do User.unfollow(blocker, blocked) User.block(blocker, blocked) @@ -632,7 +632,7 @@ def handle_incoming( } = data ) do with actor <- get_actor(data), - %User{} = actor <- User.get_or_fetch_by_ap_id(actor), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do {:ok, activity} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e578f707e..cbf2a7ce1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1450,7 +1450,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do x, "id", case User.get_or_fetch(x["acct"]) do - %{id: id} -> id + {:ok, %User{} = %{id: id}} -> id _ -> 0 end ) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index d57100491..feadf8670 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -312,7 +312,7 @@ def conversation_id_to_context(id) do end def get_external_profile(for_user, uri) do - with %User{} = user <- User.get_or_fetch(uri) do + with {:ok, %User{} = user} <- User.get_or_fetch(uri) do {:ok, UserView.render("show.json", %{user: user, for: for_user})} else _e -> From 5ba14c664b46faf633692cafa4290c0e6521d03c Mon Sep 17 00:00:00 2001 From: Zachary Dunn Date: Mon, 18 Mar 2019 14:05:10 +0000 Subject: [PATCH 04/79] Fix missing end brace --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a90b934e5..546134f46 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -613,7 +613,7 @@ def handle_incoming( ) do with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), - {:ok, %User{} = blocker = User.get_or_fetch_by_ap_id(blocker), + {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker), {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do User.unfollow(blocker, blocked) User.block(blocker, blocked) From af4338da0c26d992a5189fe940aa08ba69222e3b Mon Sep 17 00:00:00 2001 From: Zachary Dunn Date: Mon, 18 Mar 2019 14:52:24 +0000 Subject: [PATCH 05/79] Use better error message --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 6e8103c1c..dcc03ae95 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -547,7 +547,7 @@ def get_or_fetch_by_nickname(nickname) do {:ok, user} else - _e -> {:error, "Error"} + e -> {:error, e} end end end From e572786dad231a954c87ce54bb63e9be7fb396e9 Mon Sep 17 00:00:00 2001 From: Zachary Dunn Date: Mon, 18 Mar 2019 14:53:30 +0000 Subject: [PATCH 06/79] Run --- lib/pleroma/user.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index dcc03ae95..a29f836be 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -492,6 +492,7 @@ def get_cached_by_id(id) do def get_cached_by_nickname(nickname) do key = "nickname:#{nickname}" + Cachex.fetch!(:user_cache, key, fn -> user_result = get_or_fetch_by_nickname(nickname) From 97b35e00b049c8f908484163b5ffdbcb55db7867 Mon Sep 17 00:00:00 2001 From: Zachary Dunn Date: Mon, 18 Mar 2019 14:59:52 +0000 Subject: [PATCH 07/79] Fix with expression always matching --- lib/pleroma/user.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a29f836be..5e8dfc669 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1167,9 +1167,7 @@ def get_or_fetch_by_ap_id(ap_id) do if !is_nil(user) and !User.needs_update?(user) do {:ok, user} else - user = fetch_by_ap_id(ap_id) - - with %User{} = user do + with %User{} = user <- fetch_by_ap_id(ap_id) do if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user]) end From c810fb81a489dc3faeb1f21092c89a3131188ac1 Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 31 Mar 2019 14:55:09 +0200 Subject: [PATCH 08/79] Basic SSH daemon. --- lib/pleroma/bbs/bbs.ex | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 lib/pleroma/bbs/bbs.ex diff --git a/lib/pleroma/bbs/bbs.ex b/lib/pleroma/bbs/bbs.ex new file mode 100644 index 000000000..486ab9183 --- /dev/null +++ b/lib/pleroma/bbs/bbs.ex @@ -0,0 +1,34 @@ +defmodule Pleroma.BBS do + def start_daemon do + :ok = :ssh.start() + + options = [ + system_dir: 'ssh_keys', + auth_method_kb_interactive_data: fn (_, user, _) -> { + 'Welcome to Pleroma BBS', + 'Hello #{user}', + 'Password: ', + false } + end, + auth_methods: 'keyboard-interactive,password', + pwdfun: fn(user, password) -> true end, + shell: &start_prompt/1 + ] + :ssh.daemon(13121, options) + end + + def start_prompt(user) do + spawn(__MODULE__, :prompt, [user]) + end + + def prompt(user) do + IO.puts("Hey #{user}.\n") + IO.puts("Here's your timeline:\n") + + user = Pleroma.User.get_cached_by_nickname(to_string(user)) + Pleroma.Web.TwitterAPI.TwitterAPI.fetch_friend_statuses(user) + |> Enum.each(fn (status) -> + IO.puts("#{status["user"]["name"]} (#{status["user"]["screen_name"]}): #{status["text"]}") + end) + end +end From 17ab9fa45bf8a359746d30f4a91cb57a9c94f206 Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 31 Mar 2019 18:01:16 +0200 Subject: [PATCH 09/79] BBS: Some more functionality. --- lib/pleroma/bbs/authenticator.ex | 16 ++++ lib/pleroma/bbs/bbs.ex | 34 -------- lib/pleroma/bbs/handler.ex | 130 +++++++++++++++++++++++++++++++ mix.exs | 5 +- mix.lock | 1 + 5 files changed, 150 insertions(+), 36 deletions(-) create mode 100644 lib/pleroma/bbs/authenticator.ex delete mode 100644 lib/pleroma/bbs/bbs.ex create mode 100644 lib/pleroma/bbs/handler.ex diff --git a/lib/pleroma/bbs/authenticator.ex b/lib/pleroma/bbs/authenticator.ex new file mode 100644 index 000000000..a2c153720 --- /dev/null +++ b/lib/pleroma/bbs/authenticator.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.BBS.Authenticator do + use Sshd.PasswordAuthenticator + alias Comeonin.Pbkdf2 + alias Pleroma.User + + def authenticate(username, password) do + username = to_string(username) + password = to_string(password) + + with %User{} = user <- User.get_by_nickname(username) do + Pbkdf2.checkpw(password, user.password_hash) + else + _e -> false + end + end +end diff --git a/lib/pleroma/bbs/bbs.ex b/lib/pleroma/bbs/bbs.ex deleted file mode 100644 index 486ab9183..000000000 --- a/lib/pleroma/bbs/bbs.ex +++ /dev/null @@ -1,34 +0,0 @@ -defmodule Pleroma.BBS do - def start_daemon do - :ok = :ssh.start() - - options = [ - system_dir: 'ssh_keys', - auth_method_kb_interactive_data: fn (_, user, _) -> { - 'Welcome to Pleroma BBS', - 'Hello #{user}', - 'Password: ', - false } - end, - auth_methods: 'keyboard-interactive,password', - pwdfun: fn(user, password) -> true end, - shell: &start_prompt/1 - ] - :ssh.daemon(13121, options) - end - - def start_prompt(user) do - spawn(__MODULE__, :prompt, [user]) - end - - def prompt(user) do - IO.puts("Hey #{user}.\n") - IO.puts("Here's your timeline:\n") - - user = Pleroma.User.get_cached_by_nickname(to_string(user)) - Pleroma.Web.TwitterAPI.TwitterAPI.fetch_friend_statuses(user) - |> Enum.each(fn (status) -> - IO.puts("#{status["user"]["name"]} (#{status["user"]["screen_name"]}): #{status["text"]}") - end) - end -end diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex new file mode 100644 index 000000000..010929ed2 --- /dev/null +++ b/lib/pleroma/bbs/handler.ex @@ -0,0 +1,130 @@ +defmodule Pleroma.BBS.Handler do + @moduledoc """ + An example implementation of `Sshd.ShellHandler`, implementing a very simple + Read-Eval-Loop, that does nothing. + """ + use Sshd.ShellHandler + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.ActivityPub.ActivityPub + + def on_shell(username, _pubkey, _ip, _port) do + :ok = IO.puts "Welcome to #{Pleroma.Config.get([:instance, :name])}!" + user = Pleroma.User.get_by_nickname(to_string(username)) + Logger.debug("#{inspect user}") + loop(run_state([user: user])) + end + + def on_connect(username, ip, port, method) do + Logger.debug fn -> + """ + Incoming SSH shell #{inspect self()} requested for #{username} from #{inspect ip}:#{inspect port} using #{inspect method} + """ + end + end + + def on_disconnect(username, ip, port) do + Logger.debug fn -> + "Disconnecting SSH shell for #{username} from #{inspect ip}:#{inspect port}" + end + end + + defp loop(state) do + self_pid = self() + counter = state.counter + prefix = state.prefix + user = state.user + + input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end) + wait_input state, input + end + + def puts_activity(activity) do + status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity}) + IO.puts("#{status.id} by #{status.account.display_name} (#{status.account.acct}):") + IO.puts(HtmlSanitizeEx.strip_tags(status.content)) + IO.puts("") + end + + def handle_command(state, "help") do + IO.puts("Available commands:") + IO.puts("help - This help") + IO.puts("home - Show the home timeline") + IO.puts("p - Post the given text") + IO.puts("quit - Quit") + + state + end + + def handle_command(%{user: user} = state, "p " <> text) do + text = String.trim(text) + + with {:ok, _activity} <- CommonAPI.post(user, %{"status" => text}) do + IO.puts("Posted!") + else + _e -> IO.puts("Could not post...") + end + state + end + + def handle_command(state, "home") do + user = state.user + params = + %{} + |> Map.put("type", ["Create", "Announce"]) + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> Map.put("user", user) + + activities = + [user.ap_id | user.following] + |> ActivityPub.fetch_activities(params) + |> ActivityPub.contain_timeline(user) + |> Enum.reverse() + + Enum.each(activities, fn (activity) -> + puts_activity(activity) + end) + + state + end + + def handle_command(_state, command) do + IO.puts("Unknown command '#{command}'") + end + + defp wait_input(state, input) do + receive do + {:input, ^input, "quit\n"} -> + IO.puts "Exiting..." + + {:input, ^input, code} when is_binary(code) -> + code = String.trim(code) + + handle_command(state, code) + + loop(%{state | counter: state.counter + 1}) + + {:error, :interrupted} -> + IO.puts "Caught Ctrl+C..." + loop(%{state | counter: state.counter + 1}) + + {:input, ^input, msg} -> + :ok = Logger.warn "received unknown message: #{inspect msg}" + loop(%{state | counter: state.counter + 1}) + end + end + + defp run_state(opts) do + %{prefix: "pleroma", counter: 1, user: opts[:user]} + end + + defp io_get(pid, prefix, counter, username) do + prompt = prompt(prefix, counter, username) + send pid, {:input, self(), IO.gets(:stdio, prompt)} + end + + defp prompt(prefix, counter, username) do + prompt = "#{username}@#{prefix}:#{counter}>" + prompt <> " " + end +end diff --git a/mix.exs b/mix.exs index 333f21a91..58b0db9a1 100644 --- a/mix.exs +++ b/mix.exs @@ -41,7 +41,7 @@ def project do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin], + extra_applications: [:logger, :runtime_tools, :comeonin, :esshd], included_applications: [:ex_syslogger] ] end @@ -94,7 +94,8 @@ defp deps do {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"}, - {:pleroma_job_queue, "~> 0.2.0"} + {:pleroma_job_queue, "~> 0.2.0"}, + {:esshd, "~> 0.1.0"} ] end diff --git a/mix.lock b/mix.lock index f401258e9..a8b06b433 100644 --- a/mix.lock +++ b/mix.lock @@ -18,6 +18,7 @@ "earmark": {:hex, :earmark, "1.3.0", "17f0c38eaafb4800f746b457313af4b2442a8c2405b49c645768680f900be603", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, From 3fc4ea45df240c84135727a239d6bf20fd016a3f Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 31 Mar 2019 18:14:51 +0200 Subject: [PATCH 10/79] BBS: Add documentation. --- .gitignore | 1 + docs/config.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.gitignore b/.gitignore index 04c61ede7..e8052d817 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ /test/tmp/ /doc /instance +/priv/ssh_keys # Prevent committing custom emojis /priv/static/emoji/custom/* diff --git a/docs/config.md b/docs/config.md index 3624e295b..4fb4f530b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -356,3 +356,21 @@ Pleroma account will be created with the same name as the LDAP user name. * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator * `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +## BBS / SSH access + +To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: + +```exs +app_dir = File.cwd! +priv_dir = Path.join([app_dir, "priv/ssh_keys"]) + +config :esshd, + enabled: true, + priv_dir: priv_dir, + handler: "Pleroma.BBS.Handler", + port: 10_022, + password_authenticator: "Pleroma.BBS.Authenticator" +``` + +Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` From 10fdc080a0048a4776abb4bd1b5aa22d8c65e2da Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 31 Mar 2019 20:35:10 +0200 Subject: [PATCH 11/79] BBS: Tests and formatting. --- config/config.exs | 3 +++ lib/pleroma/bbs/handler.ex | 40 +++++++++++++++++++++----------------- test/bbs/handler_test.exs | 30 ++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 18 deletions(-) create mode 100644 test/bbs/handler_test.exs diff --git a/config/config.exs b/config/config.exs index 0df38d75a..33724346e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -378,6 +378,9 @@ base: System.get_env("LDAP_BASE") || "dc=example,dc=com", uid: System.get_env("LDAP_UID") || "cn" +config :esshd, + enabled: false + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 010929ed2..d999dcf76 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -8,34 +8,36 @@ defmodule Pleroma.BBS.Handler do alias Pleroma.Web.ActivityPub.ActivityPub def on_shell(username, _pubkey, _ip, _port) do - :ok = IO.puts "Welcome to #{Pleroma.Config.get([:instance, :name])}!" + :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!") user = Pleroma.User.get_by_nickname(to_string(username)) - Logger.debug("#{inspect user}") - loop(run_state([user: user])) + Logger.debug("#{inspect(user)}") + loop(run_state(user: user)) end def on_connect(username, ip, port, method) do - Logger.debug fn -> + Logger.debug(fn -> """ - Incoming SSH shell #{inspect self()} requested for #{username} from #{inspect ip}:#{inspect port} using #{inspect method} + Incoming SSH shell #{inspect(self())} requested for #{username} from #{inspect(ip)}:#{ + inspect(port) + } using #{inspect(method)} """ - end + end) end def on_disconnect(username, ip, port) do - Logger.debug fn -> - "Disconnecting SSH shell for #{username} from #{inspect ip}:#{inspect port}" - end + Logger.debug(fn -> + "Disconnecting SSH shell for #{username} from #{inspect(ip)}:#{inspect(port)}" + end) end defp loop(state) do self_pid = self() - counter = state.counter - prefix = state.prefix - user = state.user + counter = state.counter + prefix = state.prefix + user = state.user input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end) - wait_input state, input + wait_input(state, input) end def puts_activity(activity) do @@ -63,11 +65,13 @@ def handle_command(%{user: user} = state, "p " <> text) do else _e -> IO.puts("Could not post...") end + state end def handle_command(state, "home") do user = state.user + params = %{} |> Map.put("type", ["Create", "Announce"]) @@ -81,7 +85,7 @@ def handle_command(state, "home") do |> ActivityPub.contain_timeline(user) |> Enum.reverse() - Enum.each(activities, fn (activity) -> + Enum.each(activities, fn activity -> puts_activity(activity) end) @@ -95,7 +99,7 @@ def handle_command(_state, command) do defp wait_input(state, input) do receive do {:input, ^input, "quit\n"} -> - IO.puts "Exiting..." + IO.puts("Exiting...") {:input, ^input, code} when is_binary(code) -> code = String.trim(code) @@ -105,11 +109,11 @@ defp wait_input(state, input) do loop(%{state | counter: state.counter + 1}) {:error, :interrupted} -> - IO.puts "Caught Ctrl+C..." + IO.puts("Caught Ctrl+C...") loop(%{state | counter: state.counter + 1}) {:input, ^input, msg} -> - :ok = Logger.warn "received unknown message: #{inspect msg}" + :ok = Logger.warn("received unknown message: #{inspect(msg)}") loop(%{state | counter: state.counter + 1}) end end @@ -120,7 +124,7 @@ defp run_state(opts) do defp io_get(pid, prefix, counter, username) do prompt = prompt(prefix, counter, username) - send pid, {:input, self(), IO.gets(:stdio, prompt)} + send(pid, {:input, self(), IO.gets(:stdio, prompt)}) end defp prompt(prefix, counter, username) do diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs new file mode 100644 index 000000000..ee5f194bb --- /dev/null +++ b/test/bbs/handler_test.exs @@ -0,0 +1,30 @@ +defmodule Pleroma.BBS.HandlerTest do + use Pleroma.DataCase + alias Pleroma.BBS.Handler + alias Pleroma.Web.CommonAPI + alias Pleroma.User + + import ExUnit.CaptureIO + import Pleroma.Factory + + test "getting the home timeline" do + user = insert(:user) + followed = insert(:user) + + {:ok, user} = User.follow(user, followed) + + {:ok, _first} = CommonAPI.post(user, %{"status" => "hey"}) + {:ok, _second} = CommonAPI.post(followed, %{"status" => "hello"}) + + output = + capture_io(fn -> + Handler.handle_command(%{user: user}, "home") + end) + + assert output =~ user.nickname + assert output =~ followed.nickname + + assert output =~ "hey" + assert output =~ "hello" + end +end From e3bf6655ba412268e4f5fee645609c9738e453ef Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 31 Mar 2019 21:14:21 +0200 Subject: [PATCH 12/79] Add replying. --- lib/pleroma/bbs/handler.ex | 21 ++++++++++++++-- test/bbs/handler_test.exs | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index d999dcf76..7749eb3af 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -6,6 +6,7 @@ defmodule Pleroma.BBS.Handler do use Sshd.ShellHandler alias Pleroma.Web.CommonAPI alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Activity def on_shell(username, _pubkey, _ip, _port) do :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!") @@ -42,7 +43,7 @@ defp loop(state) do def puts_activity(activity) do status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity}) - IO.puts("#{status.id} by #{status.account.display_name} (#{status.account.acct}):") + IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})") IO.puts(HtmlSanitizeEx.strip_tags(status.content)) IO.puts("") end @@ -52,11 +53,27 @@ def handle_command(state, "help") do IO.puts("help - This help") IO.puts("home - Show the home timeline") IO.puts("p - Post the given text") + IO.puts("r - Reply to the post with the given id") IO.puts("quit - Quit") state end + def handle_command(%{user: user} = state, "r " <> text) do + text = String.trim(text) + [activity_id, rest] = String.split(text, " ", parts: 2) + + with %Activity{} <- Activity.get_by_id(activity_id), + {:ok, _activity} <- + CommonAPI.post(user, %{"status" => rest, "in_reply_to_status_id" => activity_id}) do + IO.puts("Replied!") + else + _e -> IO.puts("Could not reply...") + end + + state + end + def handle_command(%{user: user} = state, "p " <> text) do text = String.trim(text) @@ -104,7 +121,7 @@ defp wait_input(state, input) do {:input, ^input, code} when is_binary(code) -> code = String.trim(code) - handle_command(state, code) + state = handle_command(state, code) loop(%{state | counter: state.counter + 1}) diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs index ee5f194bb..a22c6d64d 100644 --- a/test/bbs/handler_test.exs +++ b/test/bbs/handler_test.exs @@ -3,9 +3,12 @@ defmodule Pleroma.BBS.HandlerTest do alias Pleroma.BBS.Handler alias Pleroma.Web.CommonAPI alias Pleroma.User + alias Pleroma.Repo + alias Pleroma.Activity import ExUnit.CaptureIO import Pleroma.Factory + import Ecto.Query test "getting the home timeline" do user = insert(:user) @@ -27,4 +30,51 @@ test "getting the home timeline" do assert output =~ "hey" assert output =~ "hello" end + + test "posting" do + user = insert(:user) + + output = + capture_io(fn -> + Handler.handle_command(%{user: user}, "p this is a test post") + end) + + assert output =~ "Posted" + + activity = + Repo.one( + from(a in Activity, + where: fragment("?->>'type' = ?", a.data, "Create") + ) + ) + + assert activity.actor == user.ap_id + assert activity.data["object"]["content"] == "this is a test post" + end + + test "replying" do + user = insert(:user) + another_user = insert(:user) + + {:ok, activity} = CommonAPI.post(another_user, %{"status" => "this is a test post"}) + + output = + capture_io(fn -> + Handler.handle_command(%{user: user}, "r #{activity.id} this is a reply") + end) + + assert output =~ "Replied" + + reply = + Repo.one( + from(a in Activity, + where: fragment("?->>'type' = ?", a.data, "Create"), + where: a.actor == ^user.ap_id + ) + ) + + assert reply.actor == user.ap_id + assert reply.data["object"]["content"] == "this is a reply" + assert reply.data["object"]["inReplyTo"] == activity.data["object"]["id"] + end end From 629ad1766ce5da434bf095f6baa81a460334e1b2 Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 31 Mar 2019 21:53:17 +0200 Subject: [PATCH 13/79] BBS: Some fixes. --- lib/pleroma/bbs/handler.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 7749eb3af..1ebba77d2 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -91,7 +91,7 @@ def handle_command(state, "home") do params = %{} - |> Map.put("type", ["Create", "Announce"]) + |> Map.put("type", ["Create"]) |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) @@ -100,7 +100,6 @@ def handle_command(state, "home") do [user.ap_id | user.following] |> ActivityPub.fetch_activities(params) |> ActivityPub.contain_timeline(user) - |> Enum.reverse() Enum.each(activities, fn activity -> puts_activity(activity) @@ -109,8 +108,9 @@ def handle_command(state, "home") do state end - def handle_command(_state, command) do + def handle_command(state, command) do IO.puts("Unknown command '#{command}'") + state end defp wait_input(state, input) do From b5cecebbc14c80d08f1a9f541722218c48dc514f Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 09:32:17 +0200 Subject: [PATCH 14/79] Conversations: Fix specs. --- test/web/mastodon_api/mastodon_api_controller_test.exs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 9e19fb48e..519ad8f4d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -320,7 +320,7 @@ test "Conversations", %{conn: conn} do res_conn = conn - |> assign(:user, user) + |> assign(:user, user_one) |> get("/api/v1/conversations") assert response = json_response(res_conn, 200) @@ -330,22 +330,22 @@ test "Conversations", %{conn: conn} do "accounts" => res_accounts, "last_status" => res_last_status, "unread" => unread - } = reponse + } = response assert unread == false # Apparently undocumented API endpoint res_conn = conn - |> assign(:user, user) - |> get("/api/v1/conversations/#{res_id}/read") + |> assign(:user, user_one) + |> post("/api/v1/conversations/#{res_id}/read") assert response == json_response(res_conn, 200) # (vanilla) Mastodon frontend behaviour res_conn = conn - |> assign(:user, user) + |> assign(:user, user_one) |> get("/api/v1/statuses/#{res_last_status.id}/context") assert %{ancestors: [], descendants: []} == json_response(res_conn, 200) From d1da6b155ab758ae4eb8fa154997a0a2a179897c Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 09:34:53 +0200 Subject: [PATCH 15/79] Conversation: Add Conversations and Participations. --- lib/conversation.ex | 30 ++++++++++++++++++ lib/conversation/participation.ex | 31 +++++++++++++++++++ .../20190408123347_create_conversations.exs | 26 ++++++++++++++++ test/conversation/participation_test.exs | 22 +++++++++++++ test/conversation_test.exs | 12 +++++++ test/support/factory.ex | 6 ++++ 6 files changed, 127 insertions(+) create mode 100644 lib/conversation.ex create mode 100644 lib/conversation/participation.ex create mode 100644 priv/repo/migrations/20190408123347_create_conversations.exs create mode 100644 test/conversation/participation_test.exs create mode 100644 test/conversation_test.exs diff --git a/lib/conversation.ex b/lib/conversation.ex new file mode 100644 index 000000000..cfb78d925 --- /dev/null +++ b/lib/conversation.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Conversation do + alias Pleroma.Repo + alias Pleroma.Conversation.Participation + use Ecto.Schema + import Ecto.Changeset + + schema "conversations" do + field(:ap_id, :string) + has_many(:participations, Participation) + + timestamps() + end + + def creation_cng(struct, params) do + struct + |> cast(params, [:ap_id]) + |> validate_required([:ap_id]) + |> unique_constraint(:ap_id) + end + + def create_for_ap_id(ap_id) do + %__MODULE__{} + |> creation_cng(%{ap_id: ap_id}) + |> Repo.insert() + end +end diff --git a/lib/conversation/participation.ex b/lib/conversation/participation.ex new file mode 100644 index 000000000..244d37c46 --- /dev/null +++ b/lib/conversation/participation.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Conversation.Participation do + use Ecto.Schema + alias Pleroma.User + alias Pleroma.Conversation + alias Pleroma.Repo + import Ecto.Changeset + + schema "conversation_participations" do + belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:conversation, Conversation) + field(:read, :boolean, default: false) + + timestamps() + end + + def creation_cng(struct, params) do + struct + |> cast(params, [:user_id, :conversation_id]) + |> validate_required([:user_id, :conversation_id]) + end + + def create_for_user_and_conversation(user, conversation) do + %__MODULE__{} + |> creation_cng(%{user_id: user.id, conversation_id: conversation.id}) + |> Repo.insert() + end +end diff --git a/priv/repo/migrations/20190408123347_create_conversations.exs b/priv/repo/migrations/20190408123347_create_conversations.exs new file mode 100644 index 000000000..68bf766bc --- /dev/null +++ b/priv/repo/migrations/20190408123347_create_conversations.exs @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.CreateConversations do + use Ecto.Migration + + def change do + create table(:conversations) do + add(:ap_id, :string, null: false) + timestamps() + end + + create table(:conversation_participations) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:conversation_id, references(:conversations, on_delete: :delete_all)) + add(:read, :boolean, default: false) + + timestamps() + end + + create index(:conversation_participations, [:user_id]) + create index(:conversation_participations, [:conversation_id]) + create unique_index(:conversations, [:ap_id]) + end +end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs new file mode 100644 index 000000000..8dc15a802 --- /dev/null +++ b/test/conversation/participation_test.exs @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Conversation.ParticipationTest do + use Pleroma.DataCase + + import Pleroma.Factory + + alias Pleroma.Conversation.Participation + + test "it creates a participation for a conversation and a user" do + user = insert(:user) + conversation = insert(:conversation) + + {:ok, %Participation{} = participation} = + Participation.create_for_user_and_conversation(user, conversation) + + assert participation.user_id == user.id + assert participation.conversation_id == conversation.id + end +end diff --git a/test/conversation_test.exs b/test/conversation_test.exs new file mode 100644 index 000000000..8fb55d51c --- /dev/null +++ b/test/conversation_test.exs @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ConversationTest do + use Pleroma.DataCase + alias Pleroma.Conversation + + test "it creates a conversation for given ap_id" do + assert {:ok, %Conversation{}} = Conversation.create_for_ap_id("https://some_ap_id") + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index ea59912cf..af38be46c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -5,6 +5,12 @@ defmodule Pleroma.Factory do use ExMachina.Ecto, repo: Pleroma.Repo + def conversation_factory do + %Pleroma.Conversation{ + ap_id: sequence(:ap_id, &"https://some_conversation/#{&1}") + } + end + def user_factory do user = %Pleroma.User{ name: sequence(:name, &"Test テスト User #{&1}"), From 64c1c3a4071f3f99a59f38e2dcde499bda3969cf Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 15:12:01 +0200 Subject: [PATCH 16/79] Participations: Add marking as read and unread. --- lib/conversation/participation.ex | 18 ++++++++++++++++++ test/conversation/participation_test.exs | 14 ++++++++++++++ test/support/factory.ex | 11 +++++++++++ 3 files changed, 43 insertions(+) diff --git a/lib/conversation/participation.ex b/lib/conversation/participation.ex index 244d37c46..ab59a529e 100644 --- a/lib/conversation/participation.ex +++ b/lib/conversation/participation.ex @@ -28,4 +28,22 @@ def create_for_user_and_conversation(user, conversation) do |> creation_cng(%{user_id: user.id, conversation_id: conversation.id}) |> Repo.insert() end + + def read_cng(struct, params) do + struct + |> cast(params, [:read]) + |> validate_required([:read]) + end + + def mark_as_read(participation) do + participation + |> read_cng(%{read: true}) + |> Repo.update() + end + + def mark_as_unread(participation) do + participation + |> read_cng(%{read: false}) + |> Repo.update() + end end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 8dc15a802..eae1873ca 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -19,4 +19,18 @@ test "it creates a participation for a conversation and a user" do assert participation.user_id == user.id assert participation.conversation_id == conversation.id end + + test "it marks a participation as read" do + participation = insert(:participation, %{read: false}) + {:ok, participation} = Participation.mark_as_read(participation) + + assert participation.read + end + + test "it marks a participation as unread" do + participation = insert(:participation, %{read: true}) + {:ok, participation} = Participation.mark_as_unread(participation) + + refute participation.read + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index af38be46c..2a2954ad6 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -5,6 +5,17 @@ defmodule Pleroma.Factory do use ExMachina.Ecto, repo: Pleroma.Repo + def participation_factory do + conversation = insert(:conversation) + user = insert(:user) + + %Pleroma.Conversation.Participation{ + conversation: conversation, + user: user, + read: false + } + end + def conversation_factory do %Pleroma.Conversation{ ap_id: sequence(:ap_id, &"https://some_conversation/#{&1}") From 280172f6f6d74872349e3b4e6f1feaa9c95b3900 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 16:33:45 +0200 Subject: [PATCH 17/79] Conversations: Create or bump on inserting a dm. --- lib/conversation.ex | 42 +++++++++- lib/conversation/participation.ex | 6 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 + .../20190408123347_create_conversations.exs | 2 +- test/conversation/participation_test.exs | 22 ++++- test/conversation_test.exs | 84 ++++++++++++++++++- 6 files changed, 152 insertions(+), 6 deletions(-) diff --git a/lib/conversation.ex b/lib/conversation.ex index cfb78d925..3d53e91b7 100644 --- a/lib/conversation.ex +++ b/lib/conversation.ex @@ -5,10 +5,12 @@ defmodule Pleroma.Conversation do alias Pleroma.Repo alias Pleroma.Conversation.Participation + alias Pleroma.User use Ecto.Schema import Ecto.Changeset schema "conversations" do + # This is the context ap id. field(:ap_id, :string) has_many(:participations, Participation) @@ -25,6 +27,44 @@ def creation_cng(struct, params) do def create_for_ap_id(ap_id) do %__MODULE__{} |> creation_cng(%{ap_id: ap_id}) - |> Repo.insert() + |> Repo.insert( + on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]], + returning: true, + conflict_target: :ap_id + ) + end + + def get_for_ap_id(ap_id) do + Repo.get_by(__MODULE__, ap_id: ap_id) + end + + @doc """ + This will + 1. Create a conversation if there isn't one already + 2. Create a participation for all the people involved who don't have one already + 3. Bump all relevant participations to 'unread' + """ + def create_or_bump_for(activity) do + with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), + "Create" <- activity.data["type"], + "Note" <- activity.data["object"]["type"], + ap_id when is_binary(ap_id) <- activity.data["object"]["context"] do + {:ok, conversation} = create_for_ap_id(ap_id) + + local_users = User.get_users_from_set(activity.recipients, true) + + participations = + Enum.map(local_users, fn user -> + {:ok, participation} = + Participation.create_for_user_and_conversation(user, conversation) + + participation + end) + + %{ + conversation + | participations: participations + } + end end end diff --git a/lib/conversation/participation.ex b/lib/conversation/participation.ex index ab59a529e..a58d0ca0d 100644 --- a/lib/conversation/participation.ex +++ b/lib/conversation/participation.ex @@ -26,7 +26,11 @@ def creation_cng(struct, params) do def create_for_user_and_conversation(user, conversation) do %__MODULE__{} |> creation_cng(%{user_id: user.id, conversation_id: conversation.id}) - |> Repo.insert() + |> Repo.insert( + on_conflict: [set: [read: false, updated_at: NaiveDateTime.utc_now()]], + returning: true, + conflict_target: [:user_id, :conversation_id] + ) end def read_cng(struct, params) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f217e7bac..880d19a5e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity + alias Pleroma.Conversation alias Pleroma.Instances alias Pleroma.Notification alias Pleroma.Object @@ -143,6 +144,7 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do end) Notification.create_notifications(activity) + Conversation.create_or_bump_for(activity) stream_out(activity) {:ok, activity} else diff --git a/priv/repo/migrations/20190408123347_create_conversations.exs b/priv/repo/migrations/20190408123347_create_conversations.exs index 68bf766bc..0e0af30ae 100644 --- a/priv/repo/migrations/20190408123347_create_conversations.exs +++ b/priv/repo/migrations/20190408123347_create_conversations.exs @@ -19,8 +19,8 @@ def change do timestamps() end - create index(:conversation_participations, [:user_id]) create index(:conversation_participations, [:conversation_id]) + create unique_index(:conversation_participations, [:user_id, :conversation_id]) create unique_index(:conversations, [:ap_id]) end end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index eae1873ca..4e7d9dc92 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -4,9 +4,7 @@ defmodule Pleroma.Conversation.ParticipationTest do use Pleroma.DataCase - import Pleroma.Factory - alias Pleroma.Conversation.Participation test "it creates a participation for a conversation and a user" do @@ -18,6 +16,26 @@ test "it creates a participation for a conversation and a user" do assert participation.user_id == user.id assert participation.conversation_id == conversation.id + + :timer.sleep(1000) + # Creating again returns the same participation + {:ok, %Participation{} = participation_two} = + Participation.create_for_user_and_conversation(user, conversation) + + assert participation.id == participation_two.id + refute participation.updated_at == participation_two.updated_at + end + + test "recreating an existing participations sets it to unread" do + participation = insert(:participation, %{read: true}) + + {:ok, participation} = + Participation.create_for_user_and_conversation( + participation.user, + participation.conversation + ) + + refute participation.read end test "it marks a participation as read" do diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 8fb55d51c..1c9d485ff 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -5,8 +5,90 @@ defmodule Pleroma.ConversationTest do use Pleroma.DataCase alias Pleroma.Conversation + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory test "it creates a conversation for given ap_id" do - assert {:ok, %Conversation{}} = Conversation.create_for_ap_id("https://some_ap_id") + assert {:ok, %Conversation{} = conversation} = + Conversation.create_for_ap_id("https://some_ap_id") + + # Inserting again returns the same + assert {:ok, conversation_two} = Conversation.create_for_ap_id("https://some_ap_id") + assert conversation_two.id == conversation.id + end + + test "public posts don't create conversations" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey"}) + + context = activity.data["object"]["context"] + + conversation = Conversation.get_for_ap_id(context) + + refute conversation + end + + test "it creates or updates a conversation and participations for a given DM" do + har = insert(:user) + jafnhar = insert(:user) + tridi = insert(:user) + + {:ok, activity} = + CommonAPI.post(har, %{"status" => "Hey @#{jafnhar.nickname}", "visibility" => "direct"}) + + context = activity.data["object"]["context"] + + conversation = + Conversation.get_for_ap_id(context) + |> Repo.preload(:participations) + + assert conversation + [har_participation, jafnhar_participation] = conversation.participations + + assert har_participation.user_id == har.id + assert jafnhar_participation.user_id == jafnhar.id + + {:ok, activity} = + CommonAPI.post(jafnhar, %{ + "status" => "Hey @#{har.nickname}", + "visibility" => "direct", + "in_reply_to_status_id" => activity.id + }) + + context = activity.data["object"]["context"] + + conversation_two = + Conversation.get_for_ap_id(context) + |> Repo.preload(:participations) + + assert conversation_two.id == conversation.id + + [har_participation_two, jafnhar_participation_two] = conversation_two.participations + + assert har_participation_two.user_id == har.id + assert jafnhar_participation_two.user_id == jafnhar.id + + {:ok, activity} = + CommonAPI.post(tridi, %{ + "status" => "Hey @#{har.nickname}", + "visibility" => "direct", + "in_reply_to_status_id" => activity.id + }) + + context = activity.data["object"]["context"] + + conversation_three = + Conversation.get_for_ap_id(context) + |> Repo.preload(:participations) + + assert conversation_three.id == conversation.id + + [har_participation_three, jafnhar_participation_three, tridi_participation] = + conversation_three.participations + + assert har_participation_three.user_id == har.id + assert jafnhar_participation_three.user_id == jafnhar.id + assert tridi_participation.user_id == tridi.id end end From 20d9b9076051d2dea60919ad85aaf88154629dc4 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 17:05:33 +0200 Subject: [PATCH 18/79] Participation: Get for a user. --- lib/conversation.ex | 4 ++-- lib/conversation/participation.ex | 9 +++++++ test/conversation/participation_test.exs | 30 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/conversation.ex b/lib/conversation.ex index 3d53e91b7..6eedc72b6 100644 --- a/lib/conversation.ex +++ b/lib/conversation.ex @@ -51,10 +51,10 @@ def create_or_bump_for(activity) do ap_id when is_binary(ap_id) <- activity.data["object"]["context"] do {:ok, conversation} = create_for_ap_id(ap_id) - local_users = User.get_users_from_set(activity.recipients, true) + users = User.get_users_from_set(activity.recipients) participations = - Enum.map(local_users, fn user -> + Enum.map(users, fn user -> {:ok, participation} = Participation.create_for_user_and_conversation(user, conversation) diff --git a/lib/conversation/participation.ex b/lib/conversation/participation.ex index a58d0ca0d..23e6409f1 100644 --- a/lib/conversation/participation.ex +++ b/lib/conversation/participation.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Conversation.Participation do alias Pleroma.Conversation alias Pleroma.Repo import Ecto.Changeset + import Ecto.Query schema "conversation_participations" do belongs_to(:user, User, type: Pleroma.FlakeId) @@ -50,4 +51,12 @@ def mark_as_unread(participation) do |> read_cng(%{read: false}) |> Repo.update() end + + def for_user(user, params \\ %{}) do + from(p in __MODULE__, + where: p.user_id == ^user.id, + order_by: [desc: p.updated_at] + ) + |> Pleroma.Pagination.fetch_paginated(params) + end end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 4e7d9dc92..c52b4ed88 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Conversation.ParticipationTest do use Pleroma.DataCase import Pleroma.Factory alias Pleroma.Conversation.Participation + alias Pleroma.Web.CommonAPI test "it creates a participation for a conversation and a user" do user = insert(:user) @@ -51,4 +52,33 @@ test "it marks a participation as unread" do refute participation.read end + + test "gets all the participations for a user, ordered by updated at descending" do + user = insert(:user) + {:ok, activity_one} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"}) + :timer.sleep(1000) + {:ok, activity_two} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"}) + :timer.sleep(1000) + + {:ok, activity_three} = + CommonAPI.post(user, %{ + "status" => "x", + "visibility" => "direct", + "in_reply_to_status_id" => activity_one.id + }) + + assert [participation_one, participation_two] = + Participation.for_user(user) + |> Repo.preload(:conversation) + + assert participation_one.conversation.ap_id == activity_three.data["object"]["context"] + assert participation_two.conversation.ap_id == activity_two.data["object"]["context"] + + # Pagination + assert [participation_one] = + Participation.for_user(user, %{limit: 1}) + |> Repo.preload(:conversation) + + assert participation_one.conversation.ap_id == activity_three.data["object"]["context"] + end end From cf353514feff50c2ccb9a8079ce5e695eb7f8cb6 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 17:28:02 +0200 Subject: [PATCH 19/79] Participations: Add last activity. --- lib/conversation/participation.ex | 27 ++++++++++++++++++++++++ test/conversation/participation_test.exs | 7 ++++++ 2 files changed, 34 insertions(+) diff --git a/lib/conversation/participation.ex b/lib/conversation/participation.ex index 23e6409f1..4183af8a7 100644 --- a/lib/conversation/participation.ex +++ b/lib/conversation/participation.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Conversation.Participation do alias Pleroma.User alias Pleroma.Conversation alias Pleroma.Repo + alias Pleroma.Web.ActivityPub.ActivityPub import Ecto.Changeset import Ecto.Query @@ -14,6 +15,7 @@ defmodule Pleroma.Conversation.Participation do belongs_to(:user, User, type: Pleroma.FlakeId) belongs_to(:conversation, Conversation) field(:read, :boolean, default: false) + field(:last_activity_id, Pleroma.FlakeId, virtual: true) timestamps() end @@ -59,4 +61,29 @@ def for_user(user, params \\ %{}) do ) |> Pleroma.Pagination.fetch_paginated(params) end + + def for_user_with_last_activity_id(user, params \\ %{}) do + for_user(user, params) + |> Repo.preload(:conversation) + |> Enum.map(fn participation -> + # TODO: Don't load all those activities, just get the most recent + # Involves splitting up the query. + activities = + ActivityPub.fetch_activities_for_context(participation.conversation.ap_id, %{ + "user" => user, + "blocking_user" => user + }) + + activity_id = + case activities do + [activity | _] -> activity.id + _ -> nil + end + + %{ + participation + | last_activity_id: activity_id + } + end) + end end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index c52b4ed88..5791fa0db 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -80,5 +80,12 @@ test "gets all the participations for a user, ordered by updated at descending" |> Repo.preload(:conversation) assert participation_one.conversation.ap_id == activity_three.data["object"]["context"] + + # With last_activity_id + assert [participation_one] = + Participation.for_user_with_last_activity_id(user, %{limit: 1}) + |> Repo.preload(:conversation) + + assert participation_one.last_activity_id == activity_three.id end end From de57094fca505e7dbf1a84fef5fb31fae68f2709 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 17:30:25 +0200 Subject: [PATCH 20/79] Participations: Add sort index. --- .../20190410152859_add_participation_updated_at_index.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs diff --git a/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs new file mode 100644 index 000000000..1ce688c52 --- /dev/null +++ b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddParticipationUpdatedAtIndex do + use Ecto.Migration + + def change do + create index(:conversation_participations, ["updated_at desc"]) + end +end From c352a0aba601ae444bf5b479ab3c643728a8b35e Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 17:48:31 +0200 Subject: [PATCH 21/79] Conversations: Make tests run. --- .../mastodon_api/mastodon_api_controller.ex | 36 +++++++++++++++++++ lib/pleroma/web/router.ex | 2 +- .../mastodon_api_controller_test.exs | 24 +++++++------ 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 5462ce8be..57f73dacd 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Ecto.Changeset alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.Conversation.Participation alias Pleroma.Filter alias Pleroma.Notification alias Pleroma.Object @@ -1584,6 +1585,41 @@ def reports(%{assigns: %{user: user}} = conn, params) do end end + def conversations(%{assigns: %{user: user}} = conn, params) do + participations = Participation.for_user_with_last_activity_id(user, params) + + conversations = + Enum.map(participations, fn participation -> + %{ + id: participation.id, + # TODO: Add this. + accounts: [], + unread: !participation.read, + last_status: participation.last_activity_id + } + end) + + conn + |> add_link_headers(:conversations, participations) + |> json(conversations) + end + + def conversation_read(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do + with %Participation{} = participation <- + Repo.get_by(Participation, id: participation_id, user_id: user.id), + {:ok, participation} <- Participation.mark_as_read(participation) do + conn + |> json(%{ + id: participation.id, + # TODO: Add this. + accounts: [], + unread: !participation.read, + # TODO: Add this. + last_status: nil + }) + end + end + def try_render(conn, target, params) when is_binary(target) do res = render(conn, target, params) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 0af743b80..dc5119c50 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -273,7 +273,7 @@ defmodule Pleroma.Web.Router do get("/suggestions", MastodonAPIController, :suggestions) get("/conversations", MastodonAPIController, :conversations) - get("/conversations/:id/read", MastodonAPIController, :get_conversation) + post("/conversations/:id/read", MastodonAPIController, :conversation_read) get("/endorsements", MastodonAPIController, :empty_array) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 519ad8f4d..d1d22edde 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -325,14 +325,17 @@ test "Conversations", %{conn: conn} do assert response = json_response(res_conn, 200) - assert %{ - "id" => res_id, - "accounts" => res_accounts, - "last_status" => res_last_status, - "unread" => unread - } = response + assert [ + %{ + "id" => res_id, + "accounts" => res_accounts, + "last_status" => res_last_status, + "unread" => unread + } + ] = response - assert unread == false + assert unread == true + assert res_last_status == direct.id # Apparently undocumented API endpoint res_conn = @@ -340,15 +343,16 @@ test "Conversations", %{conn: conn} do |> assign(:user, user_one) |> post("/api/v1/conversations/#{res_id}/read") - assert response == json_response(res_conn, 200) + assert response = json_response(res_conn, 200) + assert response["unread"] == false # (vanilla) Mastodon frontend behaviour res_conn = conn |> assign(:user, user_one) - |> get("/api/v1/statuses/#{res_last_status.id}/context") + |> get("/api/v1/statuses/#{res_last_status}/context") - assert %{ancestors: [], descendants: []} == json_response(res_conn, 200) + assert %{"ancestors" => [], "descendants" => []} == json_response(res_conn, 200) end test "doesn't include DMs from blocked users", %{conn: conn} do From d115d2a27e2e7a9df466fc4393416f804cb7e8e2 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 10 Apr 2019 18:17:22 +0200 Subject: [PATCH 22/79] Conversations: Tidying up. --- lib/{ => pleroma}/conversation.ex | 2 +- lib/{ => pleroma}/conversation/participation.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename lib/{ => pleroma}/conversation.ex (100%) rename lib/{ => pleroma}/conversation/participation.ex (100%) diff --git a/lib/conversation.ex b/lib/pleroma/conversation.ex similarity index 100% rename from lib/conversation.ex rename to lib/pleroma/conversation.ex index 6eedc72b6..a77a7cd6e 100644 --- a/lib/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -3,8 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Conversation do - alias Pleroma.Repo alias Pleroma.Conversation.Participation + alias Pleroma.Repo alias Pleroma.User use Ecto.Schema import Ecto.Changeset diff --git a/lib/conversation/participation.ex b/lib/pleroma/conversation/participation.ex similarity index 100% rename from lib/conversation/participation.ex rename to lib/pleroma/conversation/participation.ex index 4183af8a7..1a2ceafeb 100644 --- a/lib/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -4,9 +4,9 @@ defmodule Pleroma.Conversation.Participation do use Ecto.Schema - alias Pleroma.User alias Pleroma.Conversation alias Pleroma.Repo + alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub import Ecto.Changeset import Ecto.Query From 36ec8d969405fc8e58a43100554c358fbeb6b4fc Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 11 Apr 2019 13:20:46 +0200 Subject: [PATCH 23/79] ActivityPub: Fix specs. --- test/web/activity_pub/activity_pub_test.exs | 48 ++++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 17fec05b1..c3911500e 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -126,9 +126,15 @@ test "drops activities beyond a certain limit" do end test "doesn't drop activities with content being null" do + user = insert(:user) + data = %{ - "ok" => true, + "actor" => user.ap_id, + "to" => [], "object" => %{ + "actor" => user.ap_id, + "to" => [], + "type" => "Note", "content" => nil } } @@ -144,8 +150,17 @@ test "returns the activity if one with the same id is already in" do end test "inserts a given map into the activity database, giving it an id if it has none." do + user = insert(:user) + data = %{ - "ok" => true + "actor" => user.ap_id, + "to" => [], + "object" => %{ + "actor" => user.ap_id, + "to" => [], + "type" => "Note", + "content" => "hey" + } } {:ok, %Activity{} = activity} = ActivityPub.insert(data) @@ -155,9 +170,16 @@ test "inserts a given map into the activity database, giving it an id if it has given_id = "bla" data = %{ - "ok" => true, "id" => given_id, - "context" => "blabla" + "actor" => user.ap_id, + "to" => [], + "context" => "blabla", + "object" => %{ + "actor" => user.ap_id, + "to" => [], + "type" => "Note", + "content" => "hey" + } } {:ok, %Activity{} = activity} = ActivityPub.insert(data) @@ -168,10 +190,16 @@ test "inserts a given map into the activity database, giving it an id if it has end test "adds a context when none is there" do + user = insert(:user) + data = %{ - "id" => "some_id", + "actor" => user.ap_id, + "to" => [], "object" => %{ - "id" => "object_id" + "actor" => user.ap_id, + "to" => [], + "type" => "Note", + "content" => "hey" } } @@ -184,10 +212,16 @@ test "adds a context when none is there" do end test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do + user = insert(:user) + data = %{ + "actor" => user.ap_id, + "to" => [], "object" => %{ + "actor" => user.ap_id, + "to" => [], "type" => "Note", - "ok" => true + "content" => "hey" } } From 6f880b162736861189526ef0602f54bae6ff6153 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 11 Apr 2019 13:31:20 +0200 Subject: [PATCH 24/79] Conversation: Fix tests. --- test/conversation_test.exs | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 1c9d485ff..4e3e86c8d 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -44,10 +44,12 @@ test "it creates or updates a conversation and participations for a given DM" do |> Repo.preload(:participations) assert conversation - [har_participation, jafnhar_participation] = conversation.participations - assert har_participation.user_id == har.id - assert jafnhar_participation.user_id == jafnhar.id + assert Enum.find(conversation.participations, fn %{user_id: user_id} -> har.id == user_id end) + + assert Enum.find(conversation.participations, fn %{user_id: user_id} -> + jafnhar.id == user_id + end) {:ok, activity} = CommonAPI.post(jafnhar, %{ @@ -64,10 +66,13 @@ test "it creates or updates a conversation and participations for a given DM" do assert conversation_two.id == conversation.id - [har_participation_two, jafnhar_participation_two] = conversation_two.participations + assert Enum.find(conversation_two.participations, fn %{user_id: user_id} -> + har.id == user_id + end) - assert har_participation_two.user_id == har.id - assert jafnhar_participation_two.user_id == jafnhar.id + assert Enum.find(conversation_two.participations, fn %{user_id: user_id} -> + jafnhar.id == user_id + end) {:ok, activity} = CommonAPI.post(tridi, %{ @@ -84,11 +89,16 @@ test "it creates or updates a conversation and participations for a given DM" do assert conversation_three.id == conversation.id - [har_participation_three, jafnhar_participation_three, tridi_participation] = - conversation_three.participations + assert Enum.find(conversation_three.participations, fn %{user_id: user_id} -> + har.id == user_id + end) - assert har_participation_three.user_id == har.id - assert jafnhar_participation_three.user_id == jafnhar.id - assert tridi_participation.user_id == tridi.id + assert Enum.find(conversation_three.participations, fn %{user_id: user_id} -> + jafnhar.id == user_id + end) + + assert Enum.find(conversation_three.participations, fn %{user_id: user_id} -> + tridi.id == user_id + end) end end From b421dd3dd23fd2df6086bcad1d46a4262f259459 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 12 Apr 2019 07:32:46 +0200 Subject: [PATCH 25/79] installation/download-mastofe-build.sh: Add mastofe CI-artifacts download [ci skip] --- installation/download-mastofe-build.sh | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 installation/download-mastofe-build.sh diff --git a/installation/download-mastofe-build.sh b/installation/download-mastofe-build.sh new file mode 100755 index 000000000..ab31a7ee7 --- /dev/null +++ b/installation/download-mastofe-build.sh @@ -0,0 +1,42 @@ +#!/bin/sh +project_id="74" +project_branch="rebase/glitch-soc" +static_dir="instance/static" +# For bundling: +# project_branch="pleroma" +# static_dir="priv/static" + +if [[ ! -d "${static_dir}" ]] +then + echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?" + exit 1 +fi + +last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)" + +echo "branch:${project_branch}" +echo "Last-Modified:${last_modified}" + +artifact="mastofe.zip" + +if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]] +then + if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]] + then + echo "MastoFE is up-to-date, exiting…" + exit 0 + fi +fi + +curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit + +# TODO: Update the emoji as well +rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit +unzip -q "${artifact}" || exit + +cp public/assets/sw.js "${static_dir}/sw.js" || exit +cp -r public/packs "${static_dir}/packs" || exit + +echo "${last_modified}" > mastofe.timestamp +rm -fr public +rm -i "${artifact}" From 378b964d8eba2f216fa6f670c2de42ecee0c6277 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 12 Apr 2019 07:39:49 +0200 Subject: [PATCH 26/79] installation/download-mastofe-build.sh: Add copyright header [ci skip] --- installation/download-mastofe-build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installation/download-mastofe-build.sh b/installation/download-mastofe-build.sh index ab31a7ee7..7e293867d 100755 --- a/installation/download-mastofe-build.sh +++ b/installation/download-mastofe-build.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only project_id="74" project_branch="rebase/glitch-soc" static_dir="instance/static" From c1ebb38d3adc1d222be832405ec0d7497b61f94a Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 15 Apr 2019 21:45:25 +0200 Subject: [PATCH 27/79] Conversation: Also create participations for remote users. Needed to get the participating user list. --- lib/pleroma/conversation.ex | 2 +- test/conversation_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index a77a7cd6e..5a2a3fc6d 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -51,7 +51,7 @@ def create_or_bump_for(activity) do ap_id when is_binary(ap_id) <- activity.data["object"]["context"] do {:ok, conversation} = create_for_ap_id(ap_id) - users = User.get_users_from_set(activity.recipients) + users = User.get_users_from_set(activity.recipients, false) participations = Enum.map(users, fn user -> diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 4e3e86c8d..150d55631 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -31,7 +31,7 @@ test "public posts don't create conversations" do test "it creates or updates a conversation and participations for a given DM" do har = insert(:user) - jafnhar = insert(:user) + jafnhar = insert(:user, local: false) tridi = insert(:user) {:ok, activity} = From 0da985182f26927de0d2d9733816600afb89e79e Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 15 Apr 2019 21:58:58 +0200 Subject: [PATCH 28/79] Conversation: Return full status object, id is a string. --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 8 ++++++-- test/web/mastodon_api/mastodon_api_controller_test.exs | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 57f73dacd..3ffb767b9 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1590,12 +1590,16 @@ def conversations(%{assigns: %{user: user}} = conn, params) do conversations = Enum.map(participations, fn participation -> + activity = Activity.get_by_id_with_object(participation.last_activity_id) + + last_status = StatusView.render("status.json", %{activity: activity, for: user}) + %{ - id: participation.id, + id: participation.id |> to_string(), # TODO: Add this. accounts: [], unread: !participation.read, - last_status: participation.last_activity_id + last_status: last_status } end) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index d1d22edde..bd13f870c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -334,8 +334,9 @@ test "Conversations", %{conn: conn} do } ] = response + assert is_binary(res_id) assert unread == true - assert res_last_status == direct.id + assert res_last_status["id"] == direct.id # Apparently undocumented API endpoint res_conn = @@ -350,7 +351,7 @@ test "Conversations", %{conn: conn} do res_conn = conn |> assign(:user, user_one) - |> get("/api/v1/statuses/#{res_last_status}/context") + |> get("/api/v1/statuses/#{res_last_status["id"]}/context") assert %{"ancestors" => [], "descendants" => []} == json_response(res_conn, 200) end From 76999c73a790232ff0c30fff7a51589fb44651a3 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 15 Apr 2019 22:28:42 +0200 Subject: [PATCH 29/79] Conversation: Add accounts to output. --- lib/pleroma/conversation.ex | 1 + lib/pleroma/conversation/participation.ex | 1 + .../web/mastodon_api/mastodon_api_controller.ex | 9 +++++++-- test/conversation_test.exs | 14 +++++++++++++- .../mastodon_api/mastodon_api_controller_test.exs | 1 + 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 5a2a3fc6d..d9c84cb1b 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Conversation do # This is the context ap id. field(:ap_id, :string) has_many(:participations, Participation) + has_many(:users, through: [:participations, :user]) timestamps() end diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 1a2ceafeb..f200c1df5 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -60,6 +60,7 @@ def for_user(user, params \\ %{}) do order_by: [desc: p.updated_at] ) |> Pleroma.Pagination.fetch_paginated(params) + |> Repo.preload(conversation: [:users]) end def for_user_with_last_activity_id(user, params \\ %{}) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 3ffb767b9..c7166ff28 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1594,10 +1594,15 @@ def conversations(%{assigns: %{user: user}} = conn, params) do last_status = StatusView.render("status.json", %{activity: activity, for: user}) + accounts = + AccountView.render("accounts.json", %{ + users: participation.conversation.users, + as: :user + }) + %{ id: participation.id |> to_string(), - # TODO: Add this. - accounts: [], + accounts: accounts, unread: !participation.read, last_status: last_status } diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 150d55631..239dda04f 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -85,7 +85,7 @@ test "it creates or updates a conversation and participations for a given DM" do conversation_three = Conversation.get_for_ap_id(context) - |> Repo.preload(:participations) + |> Repo.preload([:participations, :users]) assert conversation_three.id == conversation.id @@ -100,5 +100,17 @@ test "it creates or updates a conversation and participations for a given DM" do assert Enum.find(conversation_three.participations, fn %{user_id: user_id} -> tridi.id == user_id end) + + assert Enum.find(conversation_three.users, fn %{id: user_id} -> + har.id == user_id + end) + + assert Enum.find(conversation_three.users, fn %{id: user_id} -> + jafnhar.id == user_id + end) + + assert Enum.find(conversation_three.users, fn %{id: user_id} -> + tridi.id == user_id + end) end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index bd13f870c..4fa5254f3 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -334,6 +334,7 @@ test "Conversations", %{conn: conn} do } ] = response + assert length(res_accounts) == 2 assert is_binary(res_id) assert unread == true assert res_last_status["id"] == direct.id From 24073f829f18d1ebd2cb23dd815c775b39145e42 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 21 Apr 2019 00:40:41 +0700 Subject: [PATCH 30/79] Refactor query to return only 1 message instead of 20 --- lib/pleroma/conversation/participation.ex | 13 +---- lib/pleroma/web/activity_pub/activity_pub.ex | 53 ++++++++++++-------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index f200c1df5..61021fb18 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -65,22 +65,13 @@ def for_user(user, params \\ %{}) do def for_user_with_last_activity_id(user, params \\ %{}) do for_user(user, params) - |> Repo.preload(:conversation) |> Enum.map(fn participation -> - # TODO: Don't load all those activities, just get the most recent - # Involves splitting up the query. - activities = - ActivityPub.fetch_activities_for_context(participation.conversation.ap_id, %{ + activity_id = + ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ "user" => user, "blocking_user" => user }) - activity_id = - case activities do - [activity | _] -> activity.id - _ -> nil - end - %{ participation | last_activity_id: activity_id diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 880d19a5e..577e6a59e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -459,35 +459,44 @@ def flag( end end - def fetch_activities_for_context(context, opts \\ %{}) do + defp fetch_activities_for_context_query(context, opts) do public = ["https://www.w3.org/ns/activitystreams#Public"] recipients = if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public - query = from(activity in Activity) - - query = - query - |> restrict_blocked(opts) - |> restrict_recipients(recipients, opts["user"]) - - query = - from( - activity in query, - where: - fragment( - "?->>'type' = ? and ?->>'context' = ?", - activity.data, - "Create", - activity.data, - ^context - ), - order_by: [desc: :id] + from(activity in Activity) + |> restrict_blocked(opts) + |> restrict_recipients(recipients, opts["user"]) + |> where( + [activity], + fragment( + "?->>'type' = ? and ?->>'context' = ?", + activity.data, + "Create", + activity.data, + ^context ) - |> Activity.with_preloaded_object() + ) + |> order_by([activity], desc: activity.id) + end - Repo.all(query) + @spec fetch_activities_for_context(String.t(), keyword() | map()) :: [Activity.t()] + def fetch_activities_for_context(context, opts \\ %{}) do + context + |> fetch_activities_for_context_query(opts) + |> Activity.with_preloaded_object() + |> Repo.all() + end + + @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) :: + Pleroma.FlakeId.t() | nil + def fetch_latest_activity_id_for_context(context, opts \\ %{}) do + context + |> fetch_activities_for_context_query(opts) + |> limit(1) + |> select([a], a.id) + |> Repo.one() end def fetch_public_activities(opts \\ %{}) do From 2662bea4e0d945edf7a24a44edf3ed39b2e64de9 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 21 Apr 2019 20:26:13 +0700 Subject: [PATCH 31/79] Add accounts and last_status to conversation read response --- .../mastodon_api/mastodon_api_controller.ex | 24 +++++++++++++++---- .../mastodon_api_controller_test.exs | 2 ++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index c7166ff28..86cacb0b0 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1617,14 +1617,30 @@ def conversation_read(%{assigns: %{user: user}} = conn, %{"id" => participation_ with %Participation{} = participation <- Repo.get_by(Participation, id: participation_id, user_id: user.id), {:ok, participation} <- Participation.mark_as_read(participation) do + participation = Repo.preload(participation, conversation: :users) + + accounts = + AccountView.render("accounts.json", %{ + users: participation.conversation.users, + as: :user + }) + + last_activity_id = + ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ + "user" => user, + "blocking_user" => user + }) + + activity = Activity.get_by_id_with_object(last_activity_id) + + last_status = StatusView.render("status.json", %{activity: activity, for: user}) + conn |> json(%{ id: participation.id, - # TODO: Add this. - accounts: [], + accounts: accounts, unread: !participation.read, - # TODO: Add this. - last_status: nil + last_status: last_status }) end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 4fa5254f3..cf77dff78 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -346,6 +346,8 @@ test "Conversations", %{conn: conn} do |> post("/api/v1/conversations/#{res_id}/read") assert response = json_response(res_conn, 200) + assert length(response["accounts"]) == 2 + assert response["last_status"]["id"] == direct.id assert response["unread"] == false # (vanilla) Mastodon frontend behaviour From e56afefef9c0f256561c6ecab79e5520fdeb6d5e Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 21 Apr 2019 23:14:27 +0700 Subject: [PATCH 32/79] Refactor conversation function in MastodonAPIController to use a View --- .../mastodon_api/mastodon_api_controller.ex | 44 +++---------------- .../mastodon_api/views/conversation_view.ex | 38 ++++++++++++++++ 2 files changed, 43 insertions(+), 39 deletions(-) create mode 100644 lib/pleroma/web/mastodon_api/views/conversation_view.ex diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 86cacb0b0..d5b6a943f 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -22,6 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.AppView + alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Web.MastodonAPI.FilterView alias Pleroma.Web.MastodonAPI.ListView alias Pleroma.Web.MastodonAPI.MastodonAPI @@ -1590,22 +1591,7 @@ def conversations(%{assigns: %{user: user}} = conn, params) do conversations = Enum.map(participations, fn participation -> - activity = Activity.get_by_id_with_object(participation.last_activity_id) - - last_status = StatusView.render("status.json", %{activity: activity, for: user}) - - accounts = - AccountView.render("accounts.json", %{ - users: participation.conversation.users, - as: :user - }) - - %{ - id: participation.id |> to_string(), - accounts: accounts, - unread: !participation.read, - last_status: last_status - } + ConversationView.render("participation.json", %{participation: participation, user: user}) end) conn @@ -1617,31 +1603,11 @@ def conversation_read(%{assigns: %{user: user}} = conn, %{"id" => participation_ with %Participation{} = participation <- Repo.get_by(Participation, id: participation_id, user_id: user.id), {:ok, participation} <- Participation.mark_as_read(participation) do - participation = Repo.preload(participation, conversation: :users) - - accounts = - AccountView.render("accounts.json", %{ - users: participation.conversation.users, - as: :user - }) - - last_activity_id = - ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ - "user" => user, - "blocking_user" => user - }) - - activity = Activity.get_by_id_with_object(last_activity_id) - - last_status = StatusView.render("status.json", %{activity: activity, for: user}) + participation_view = + ConversationView.render("participation.json", %{participation: participation, user: user}) conn - |> json(%{ - id: participation.id, - accounts: accounts, - unread: !participation.read, - last_status: last_status - }) + |> json(participation_view) end end diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex new file mode 100644 index 000000000..d841a840c --- /dev/null +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -0,0 +1,38 @@ +defmodule Pleroma.Web.MastodonAPI.ConversationView do + use Pleroma.Web, :view + + alias Pleroma.Activity + alias Pleroma.Repo + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.MastodonAPI.AccountView + + def render("participation.json", %{participation: participation, user: user}) do + participation = Repo.preload(participation, conversation: :users) + + last_activity_id = + with nil <- participation.last_activity_id do + ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ + "user" => user, + "blocking_user" => user + }) + end + + activity = Activity.get_by_id_with_object(last_activity_id) + + last_status = StatusView.render("status.json", %{activity: activity, for: user}) + + accounts = + AccountView.render("accounts.json", %{ + users: participation.conversation.users, + as: :user + }) + + %{ + id: participation.id |> to_string(), + accounts: accounts, + unread: !participation.read, + last_status: last_status + } + end +end From eeb093631cd50583feb4864a016d0dc1e4c58f5e Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 21 Apr 2019 23:19:36 +0700 Subject: [PATCH 33/79] Fix Credo warning --- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index d841a840c..eb61baa03 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -3,9 +3,9 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Activity alias Pleroma.Repo + alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.MastodonAPI.AccountView def render("participation.json", %{participation: participation, user: user}) do participation = Repo.preload(participation, conversation: :users) From 4908e0eeee2ecb58b204198c20720d52548b6f4a Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 21 Apr 2019 23:24:33 +0700 Subject: [PATCH 34/79] Fix Credo warning --- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index eb61baa03..8e8f7cf31 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -3,8 +3,8 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Activity alias Pleroma.Repo - alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView def render("participation.json", %{participation: participation, user: user}) do From 963d5774af7efb57fa306b3ac164049f8958a72c Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Wed, 24 Apr 2019 07:06:17 +0545 Subject: [PATCH 35/79] fix the status notification with special char --- lib/pleroma/web/twitter_api/views/activity_view.ex | 2 +- test/web/twitter_api/views/activity_view_test.exs | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index c64152da8..1007a2a48 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -289,7 +289,7 @@ def render( "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user, for: opts[:for]}), "statusnet_html" => html, - "text" => text, + "text" => HtmlEntities.decode(text), "is_local" => activity.local, "is_post_verb" => true, "created_at" => created_at, diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index d84ab7420..85815ba7e 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -371,4 +371,14 @@ test "a peertube video" do assert length(result["attachments"]) == 1 assert result["summary"] == "Friday Night" end + + test "special characters are not escaped in text field for status created" do + text = "<3 is on the way" + + {:ok, activity} = CommonAPI.post(insert(:user), %{"status" => text}) + + result = ActivityView.render("activity.json", activity: activity) + + assert result["text"] == text + end end From ce4825c1dc8a2b7ec2712170f45cde0ae14b46cf Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 30 Apr 2019 20:21:28 +0300 Subject: [PATCH 36/79] Do not normalize objects in stream_out unless the activity type is Create Saves quite a bit of time with delete activities because they would always query the db --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 604ffae7b..483a2153f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -168,7 +168,6 @@ def stream_out(activity) do public = "https://www.w3.org/ns/activitystreams#Public" if activity.data["type"] in ["Create", "Announce", "Delete"] do - object = Object.normalize(activity) Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) @@ -180,6 +179,8 @@ def stream_out(activity) do end if activity.data["type"] in ["Create"] do + object = Object.normalize(activity) + object.data |> Map.get("tag", []) |> Enum.filter(fn tag -> is_bitstring(tag) end) From 85fa2fbce4ee315a15b517fae4bc9b5474d1db5a Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Wed, 1 May 2019 01:37:17 +0545 Subject: [PATCH 37/79] add scrubber for html special char --- lib/pleroma/html.ex | 27 +++++++++++++++---- .../web/twitter_api/views/activity_view.ex | 2 +- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index cf6c0ee0a..eb33d12d9 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -28,12 +28,18 @@ def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber) def filter_tags(html), do: filter_tags(html, nil) def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags) - def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do + def get_cached_scrubbed_html_for_activity( + content, + scrubbers, + activity, + key \\ "", + callback \\ fn x -> x end + ) do key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}" Cachex.fetch!(:scrubber_cache, key, fn _key -> object = Pleroma.Object.normalize(activity) - ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false) + ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback) end) end @@ -42,16 +48,27 @@ def get_cached_stripped_html_for_activity(content, activity, key) do content, HtmlSanitizeEx.Scrubber.StripTags, activity, - key + key, + &HtmlEntities.decode/1 ) end def ensure_scrubbed_html( content, scrubbers, - false = _fake + fake, + callback ) do - {:commit, filter_tags(content, scrubbers)} + content = + content + |> filter_tags(scrubbers) + |> callback.() + + if fake do + {:ignore, content} + else + {:commit, content} + end end def ensure_scrubbed_html( diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 1007a2a48..c64152da8 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -289,7 +289,7 @@ def render( "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user, for: opts[:for]}), "statusnet_html" => html, - "text" => HtmlEntities.decode(text), + "text" => text, "is_local" => activity.local, "is_post_verb" => true, "created_at" => created_at, From 2982061dfa7533893eb4a3cca2977e19bd962953 Mon Sep 17 00:00:00 2001 From: xse Date: Wed, 1 May 2019 01:26:14 +0200 Subject: [PATCH 38/79] Fix syntax highlighting + clarify :frontend_configuration --- docs/config.md | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/config.md b/docs/config.md index 7e31e6fb7..ad55d44a7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -37,7 +37,7 @@ This filter replaces the filename (not the path) of an upload. For complete obfu An example for Sendgrid adapter: -```exs +```elixir config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendgrid, api_key: "YOUR_API_KEY" @@ -45,7 +45,7 @@ config :pleroma, Pleroma.Emails.Mailer, An example for SMTP adapter: -```exs +```elixir config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.SMTP, relay: "smtp.gmail.com", @@ -109,7 +109,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: -``` +```elixir config :logger, backends: [{ExSyslogger, :ex_syslogger}] @@ -118,7 +118,7 @@ config :logger, :ex_syslogger, ``` Another example, keeping console output and adding the pid to syslog output: -``` +```elixir config :logger, backends: [:console, {ExSyslogger, :ex_syslogger}] @@ -130,7 +130,7 @@ config :logger, :ex_syslogger, See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) An example of logging info to local syslog, but warn to a Slack channel: -``` +```elixir config :logger, backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], level: :info @@ -156,14 +156,30 @@ Frontends can access these settings at `/api/pleroma/frontend_configurations` To add your own configuration for PleromaFE, use it like this: -`config :pleroma, :frontend_configurations, pleroma_fe: %{redirectRootNoLogin: "/main/all", ...}` +```elixir +config :pleroma, :frontend_configurations, + pleroma_fe: %{ + theme: "pleroma-dark", + # ... see /priv/static/static/config.json for the available keys. +}, + masto_fe: %{ + showInstanceSpecificPanel: true + } +``` -These settings need to be complete, they will override the defaults. See `priv/static/static/config.json` for the available keys. +These settings **need to be complete**, they will override the defaults. + +NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. ## :fe __THIS IS DEPRECATED__ -If you are using this method, please change it to the `frontend_configurations` method. Please set this option to false in your config like this: `config :pleroma, :fe, false`. +If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. +Please **set this option to false** in your config like this: + +```elixir +config :pleroma, :fe, false +``` This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false. @@ -274,7 +290,7 @@ their ActivityPub ID. An example: -```exs +```elixir config :pleroma, :mrf_user_allowlist, "example.org": ["https://example.org/users/admin"] ``` @@ -303,7 +319,7 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: -```exs +```elixir config :pleroma, :admin_token, "somerandomtoken" ``` @@ -387,7 +403,7 @@ Configuration for the `auto_linker` library: Example: -```exs +```elixir config :auto_linker, opts: [ scheme: true, @@ -460,7 +476,7 @@ Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you ha Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: -``` +```elixir # Twitter config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), From 04fd7cf81729c5ccec0b1f857acb96fc562fbdb0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 1 May 2019 15:18:12 +0700 Subject: [PATCH 39/79] Update `auto_linker` --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index efaa06a1c..9ded9931c 100644 --- a/mix.exs +++ b/mix.exs @@ -102,7 +102,7 @@ defp deps do {:ueberauth, "~> 0.4"}, {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", - ref: "90613b4bae875a3610c275b7056b61ffdd53210d"}, + ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"}, {:pleroma_job_queue, "~> 0.2.0"}, {:telemetry, "~> 0.3"}, {:prometheus_ex, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index 979d599b4..08221eadc 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,6 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, - "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "90613b4bae875a3610c275b7056b61ffdd53210d", [ref: "90613b4bae875a3610c275b7056b61ffdd53210d"]}, + "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, From f11e7037c21d611cddd7f2eab64ebfc39630a078 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 1 May 2019 16:09:53 +0700 Subject: [PATCH 40/79] test fixes --- lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/user.ex | 18 +++++++------- .../activity_pub/activity_pub_controller.ex | 2 +- .../web/activity_pub/transmogrifier.ex | 2 +- .../mastodon_api/mastodon_api_controller.ex | 2 +- test/formatter_test.exs | 2 +- test/user_test.exs | 24 +++++++++---------- test/web/activity_pub/transmogrifier_test.exs | 2 +- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index b396ff0de..9e2523b18 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -126,7 +126,7 @@ def run(["new", nickname, email | rest]) do proceed? = assume_yes? or Mix.shell().yes?("Continue?") - unless not proceed? do + if proceed? do Common.start_pleroma() params = %{ diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 755d8f773..1c62f238e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -513,7 +513,7 @@ def get_cached_by_nickname(nickname) do case user_result do {:ok, user} -> {:commit, user} - {:error, error} -> {:ignore, error} + {:error, _error} -> {:ignore, nil} end end) end @@ -563,7 +563,7 @@ def get_or_fetch_by_nickname(nickname) do {:ok, user} else - e -> {:error, e} + _e -> {:error, "not found " <> nickname} end end end @@ -1210,11 +1210,11 @@ def fetch_by_ap_id(ap_id) do case ap_try do {:ok, user} -> - user + {:ok, user} _ -> case OStatus.make_user(ap_id) do - {:ok, user} -> user + {:ok, user} -> {:ok, user} _ -> {:error, "Could not fetch by AP id"} end end @@ -1229,15 +1229,15 @@ def get_or_fetch_by_ap_id(ap_id) do # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled) should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled]) + resp = fetch_by_ap_id(ap_id) + if should_fetch_initial do - with {:ok, %User{} = user} = fetch_by_ap_id(ap_id) do + with {:ok, %User{} = user} = resp do {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user]) end - - {:ok, user} - else - _ -> {:error, "Could not fetch by AP id"} end + + resp end end diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 0b80566bf..c967ab7a9 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -155,7 +155,7 @@ def outbox(conn, %{"nickname" => nickname} = params) do def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do with %User{} = recipient <- User.get_cached_by_nickname(nickname), - %User{} = actor <- User.get_or_fetch_by_ap_id(params["actor"]), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), true <- Utils.recipient_in_message(recipient, actor, params), params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do Federator.incoming_ap_doc(params) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 80317171d..b774c2afa 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -126,7 +126,7 @@ def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collec def fix_implicit_addressing(object, _), do: object def fix_addressing(object) do - %User{} = user = User.get_or_fetch_by_ap_id(object["actor"]) + {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"]) followers_collection = User.ap_followers(user) object diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 727e1c310..ed585098a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1653,7 +1653,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do x, "id", case User.get_or_fetch(x["acct"]) do - {:ok, %User{} = %{id: id}} -> id + {:ok, %User{id: id}} -> id _ -> 0 end ) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 97eb2f583..fdaf29742 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -147,7 +147,7 @@ test "gives a replacement for user links, using local nicknames in user links te end test "gives a replacement for user links when the user is using Osada" do - mike = User.get_or_fetch("mike@osada.macgirvin.com") + {:ok, mike} = User.get_or_fetch("mike@osada.macgirvin.com") text = "@mike@osada.macgirvin.com test" diff --git a/test/user_test.exs b/test/user_test.exs index 7be47e5fb..67266cb7a 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -362,7 +362,7 @@ test "it creates confirmed user if :confirmed option is given" do describe "get_or_fetch/1" do test "gets an existing user by nickname" do user = insert(:user) - fetched_user = User.get_or_fetch(user.nickname) + {:ok, fetched_user} = User.get_or_fetch(user.nickname) assert user == fetched_user end @@ -379,7 +379,7 @@ test "gets an existing user by ap_id" do info: %{} ) - fetched_user = User.get_or_fetch(ap_id) + {:ok, fetched_user} = User.get_or_fetch(ap_id) freshed_user = refresh_record(user) assert freshed_user == fetched_user end @@ -388,14 +388,14 @@ test "gets an existing user by ap_id" do describe "fetching a user from nickname or trying to build one" do test "gets an existing user" do user = insert(:user) - fetched_user = User.get_or_fetch_by_nickname(user.nickname) + {:ok, fetched_user} = User.get_or_fetch_by_nickname(user.nickname) assert user == fetched_user end test "gets an existing user, case insensitive" do user = insert(:user, nickname: "nick") - fetched_user = User.get_or_fetch_by_nickname("NICK") + {:ok, fetched_user} = User.get_or_fetch_by_nickname("NICK") assert user == fetched_user end @@ -403,7 +403,7 @@ test "gets an existing user, case insensitive" do test "gets an existing user by fully qualified nickname" do user = insert(:user) - fetched_user = + {:ok, fetched_user} = User.get_or_fetch_by_nickname(user.nickname <> "@" <> Pleroma.Web.Endpoint.host()) assert user == fetched_user @@ -413,24 +413,24 @@ test "gets an existing user by fully qualified nickname, case insensitive" do user = insert(:user, nickname: "nick") casing_altered_fqn = String.upcase(user.nickname <> "@" <> Pleroma.Web.Endpoint.host()) - fetched_user = User.get_or_fetch_by_nickname(casing_altered_fqn) + {:ok, fetched_user} = User.get_or_fetch_by_nickname(casing_altered_fqn) assert user == fetched_user end test "fetches an external user via ostatus if no user exists" do - fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la") + {:ok, fetched_user} = User.get_or_fetch_by_nickname("shp@social.heldscal.la") assert fetched_user.nickname == "shp@social.heldscal.la" end test "returns nil if no user could be fetched" do - fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la") - assert fetched_user == nil + {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la") + assert fetched_user == "not found nonexistant@social.heldscal.la" end test "returns nil for nonexistant local user" do - fetched_user = User.get_or_fetch_by_nickname("nonexistant") - assert fetched_user == nil + {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant") + assert fetched_user == "not found nonexistant" end test "updates an existing user, if stale" do @@ -448,7 +448,7 @@ test "updates an existing user, if stale" do assert orig_user.last_refreshed_at == a_week_ago - user = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin") + {:ok, user} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin") assert user.info.source_data["endpoints"] refute user.last_refreshed_at == orig_user.last_refreshed_at diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 78429c7c6..c24b50f8c 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -219,7 +219,7 @@ test "it rejects incoming follow requests from blocked users when deny_follow_bl Pleroma.Config.put([:user, :deny_follow_blocked], true) user = insert(:user) - target = User.get_or_fetch("http://mastodon.example.org/users/admin") + {:ok, target} = User.get_or_fetch("http://mastodon.example.org/users/admin") {:ok, user} = User.block(user, target) From 51e26f14f7fc342c23fe1fe643a1cf444ef9392b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 1 May 2019 13:52:44 +0300 Subject: [PATCH 41/79] Remove redundant ensure_scrubbed_html It is never used as handling for fake and non-fake activities was merged into one function above it --- lib/pleroma/html.ex | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index eb33d12d9..726c370ad 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -71,14 +71,6 @@ def ensure_scrubbed_html( end end - def ensure_scrubbed_html( - content, - scrubbers, - true = _fake - ) do - {:ignore, filter_tags(content, scrubbers)} - end - defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do generate_scrubber_signature([scrubber]) end From c854bff8f528b4f1560707ac3aa74f4a37044f52 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 1 May 2019 13:28:04 +0000 Subject: [PATCH 42/79] Refactored Pleroma.Web.Auth.Authenticator --- lib/pleroma/web/auth/authenticator.ex | 26 ++++++++++++ lib/pleroma/web/auth/ldap_authenticator.ex | 41 ++++++++---------- lib/pleroma/web/auth/pleroma_authenticator.ex | 15 +++---- test/media_proxy_test.exs | 14 +++---- test/web/auth/authenticator_test.exs | 42 +++++++++++++++++++ 5 files changed, 97 insertions(+), 41 deletions(-) create mode 100644 test/web/auth/authenticator_test.exs diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index b02f595dc..d4e0ffa80 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -42,4 +42,30 @@ def oauth_consumer_template do implementation().oauth_consumer_template() || Pleroma.Config.get([:auth, :oauth_consumer_template], "consumer.html") end + + @doc "Gets user by nickname or email for auth." + @spec fetch_user(String.t()) :: User.t() | nil + def fetch_user(name) do + User.get_by_nickname_or_email(name) + end + + # Gets name and password from conn + # + @spec fetch_credentials(Plug.Conn.t() | map()) :: + {:ok, {name :: any, password :: any}} | {:error, :invalid_credentials} + def fetch_credentials(%Plug.Conn{params: params} = _), + do: fetch_credentials(params) + + def fetch_credentials(params) do + case params do + %{"authorization" => %{"name" => name, "password" => password}} -> + {:ok, {name, password}} + + %{"grant_type" => "password", "username" => name, "password" => password} -> + {:ok, {name, password}} + + _ -> + {:error, :invalid_credentials} + end + end end diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 363c99597..177c05636 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -7,6 +7,9 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do require Logger + import Pleroma.Web.Auth.Authenticator, + only: [fetch_credentials: 1, fetch_user: 1] + @behaviour Pleroma.Web.Auth.Authenticator @base Pleroma.Web.Auth.PleromaAuthenticator @@ -20,30 +23,20 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do defdelegate oauth_consumer_template, to: @base def get_user(%Plug.Conn{} = conn) do - if Pleroma.Config.get([:ldap, :enabled]) do - {name, password} = - case conn.params do - %{"authorization" => %{"name" => name, "password" => password}} -> - {name, password} - - %{"grant_type" => "password", "username" => name, "password" => password} -> - {name, password} - end - - case ldap_user(name, password) do - %User{} = user -> - {:ok, user} - - {:error, {:ldap_connection_error, _}} -> - # When LDAP is unavailable, try default authenticator - @base.get_user(conn) - - error -> - error - end + with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])}, + {:ok, {name, password}} <- fetch_credentials(conn), + %User{} = user <- ldap_user(name, password) do + {:ok, user} else - # Fall back to default authenticator - @base.get_user(conn) + {:error, {:ldap_connection_error, _}} -> + # When LDAP is unavailable, try default authenticator + @base.get_user(conn) + + {:ldap, _} -> + @base.get_user(conn) + + error -> + error end end @@ -94,7 +87,7 @@ defp bind_user(connection, ldap, name, password) do case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do :ok -> - case User.get_by_nickname_or_email(name) do + case fetch_user(name) do %User{} = user -> user diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index d647f1e05..dd79cdcf7 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -8,19 +8,14 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do alias Pleroma.Repo alias Pleroma.User + import Pleroma.Web.Auth.Authenticator, + only: [fetch_credentials: 1, fetch_user: 1] + @behaviour Pleroma.Web.Auth.Authenticator def get_user(%Plug.Conn{} = conn) do - {name, password} = - case conn.params do - %{"authorization" => %{"name" => name, "password" => password}} -> - {name, password} - - %{"grant_type" => "password", "username" => name, "password" => password} -> - {name, password} - end - - with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)}, + with {:ok, {name, password}} <- fetch_credentials(conn), + {_, %User{} = user} <- {:user, fetch_user(name)}, {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do {:ok, user} else diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index a4331478e..0a02039a6 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -7,15 +7,15 @@ defmodule Pleroma.MediaProxyTest do import Pleroma.Web.MediaProxy alias Pleroma.Web.MediaProxy.MediaProxyController + setup do + enabled = Pleroma.Config.get([:media_proxy, :enabled]) + on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end) + :ok + end + describe "when enabled" do setup do - enabled = Pleroma.Config.get([:media_proxy, :enabled]) - - unless enabled do - Pleroma.Config.put([:media_proxy, :enabled], true) - on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end) - end - + Pleroma.Config.put([:media_proxy, :enabled], true) :ok end diff --git a/test/web/auth/authenticator_test.exs b/test/web/auth/authenticator_test.exs new file mode 100644 index 000000000..fea5c8209 --- /dev/null +++ b/test/web/auth/authenticator_test.exs @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Auth.AuthenticatorTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Web.Auth.Authenticator + import Pleroma.Factory + + describe "fetch_user/1" do + test "returns user by name" do + user = insert(:user) + assert Authenticator.fetch_user(user.nickname) == user + end + + test "returns user by email" do + user = insert(:user) + assert Authenticator.fetch_user(user.email) == user + end + + test "returns nil" do + assert Authenticator.fetch_user("email") == nil + end + end + + describe "fetch_credentials/1" do + test "returns name and password from authorization params" do + params = %{"authorization" => %{"name" => "test", "password" => "test-pass"}} + assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}} + end + + test "returns name and password with grant_type 'password'" do + params = %{"grant_type" => "password", "username" => "test", "password" => "test-pass"} + assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}} + end + + test "returns error" do + assert Authenticator.fetch_credentials(%{}) == {:error, :invalid_credentials} + end + end +end From 8af55728e47f9f62d237704cd5a33fba5f946fa2 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Mon, 29 Apr 2019 03:44:04 +0700 Subject: [PATCH 43/79] Fix tests --- lib/pleroma/conversation.ex | 5 +++-- test/conversation/participation_test.exs | 20 +++++++++----------- test/conversation_test.exs | 12 ++++++++---- test/web/activity_pub/activity_pub_test.exs | 5 +++-- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index d9c84cb1b..e6a4ccc85 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -47,9 +47,10 @@ def get_for_ap_id(ap_id) do """ def create_or_bump_for(activity) do with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), + object <- Pleroma.Object.normalize(activity), "Create" <- activity.data["type"], - "Note" <- activity.data["object"]["type"], - ap_id when is_binary(ap_id) <- activity.data["object"]["context"] do + "Note" <- object.data["type"], + ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do {:ok, conversation} = create_for_ap_id(ap_id) users = User.get_users_from_set(activity.recipients, false) diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 5791fa0db..568953b07 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -67,24 +67,22 @@ test "gets all the participations for a user, ordered by updated at descending" "in_reply_to_status_id" => activity_one.id }) - assert [participation_one, participation_two] = - Participation.for_user(user) - |> Repo.preload(:conversation) + assert [participation_one, participation_two] = Participation.for_user(user) - assert participation_one.conversation.ap_id == activity_three.data["object"]["context"] - assert participation_two.conversation.ap_id == activity_two.data["object"]["context"] + object2 = Pleroma.Object.normalize(activity_two) + object3 = Pleroma.Object.normalize(activity_three) + + assert participation_one.conversation.ap_id == object3.data["context"] + assert participation_two.conversation.ap_id == object2.data["context"] # Pagination - assert [participation_one] = - Participation.for_user(user, %{limit: 1}) - |> Repo.preload(:conversation) + assert [participation_one] = Participation.for_user(user, %{"limit" => 1}) - assert participation_one.conversation.ap_id == activity_three.data["object"]["context"] + assert participation_one.conversation.ap_id == object3.data["context"] # With last_activity_id assert [participation_one] = - Participation.for_user_with_last_activity_id(user, %{limit: 1}) - |> Repo.preload(:conversation) + Participation.for_user_with_last_activity_id(user, %{"limit" => 1}) assert participation_one.last_activity_id == activity_three.id end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 239dda04f..763183d6b 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -22,7 +22,8 @@ test "public posts don't create conversations" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey"}) - context = activity.data["object"]["context"] + object = Pleroma.Object.normalize(activity) + context = object.data["context"] conversation = Conversation.get_for_ap_id(context) @@ -37,7 +38,8 @@ test "it creates or updates a conversation and participations for a given DM" do {:ok, activity} = CommonAPI.post(har, %{"status" => "Hey @#{jafnhar.nickname}", "visibility" => "direct"}) - context = activity.data["object"]["context"] + object = Pleroma.Object.normalize(activity) + context = object.data["context"] conversation = Conversation.get_for_ap_id(context) @@ -58,7 +60,8 @@ test "it creates or updates a conversation and participations for a given DM" do "in_reply_to_status_id" => activity.id }) - context = activity.data["object"]["context"] + object = Pleroma.Object.normalize(activity) + context = object.data["context"] conversation_two = Conversation.get_for_ap_id(context) @@ -81,7 +84,8 @@ test "it creates or updates a conversation and participations for a given DM" do "in_reply_to_status_id" => activity.id }) - context = activity.data["object"]["context"] + object = Pleroma.Object.normalize(activity) + context = object.data["context"] conversation_three = Conversation.get_for_ap_id(context) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 15276ba7b..047270a2a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -208,11 +208,12 @@ test "adds a context when none is there" do } {:ok, %Activity{} = activity} = ActivityPub.insert(data) + object = Pleroma.Object.normalize(activity) assert is_binary(activity.data["context"]) - assert is_binary(activity.data["object"]["context"]) + assert is_binary(object.data["context"]) assert activity.data["context_id"] - assert activity.data["object"]["context_id"] + assert object.data["context_id"] end test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do From 533d8cd5816343ccfb6e26495124416e9808554c Mon Sep 17 00:00:00 2001 From: AkiraFukushima Date: Thu, 2 May 2019 21:04:00 +0900 Subject: [PATCH 44/79] Parse access_token from body parameters and URL parameters --- lib/pleroma/plugs/oauth_plug.ex | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/pleroma/plugs/oauth_plug.ex b/lib/pleroma/plugs/oauth_plug.ex index 5888d596a..9d43732eb 100644 --- a/lib/pleroma/plugs/oauth_plug.ex +++ b/lib/pleroma/plugs/oauth_plug.ex @@ -16,6 +16,16 @@ def init(options), do: options def call(%{assigns: %{user: %User{}}} = conn, _), do: conn + def call(%{params: %{"access_token" => access_token}} = conn, _) do + with {:ok, user, token_record} <- fetch_user_and_token(access_token) do + conn + |> assign(:token, token_record) + |> assign(:user, user) + else + _ -> conn + end + end + def call(conn, _) do with {:ok, token_str} <- fetch_token_str(conn), {:ok, user, token_record} <- fetch_user_and_token(token_str) do From dff6afc7c88f20cb719a4189d463605589869e8e Mon Sep 17 00:00:00 2001 From: AkiraFukushima Date: Mon, 29 Apr 2019 23:53:48 +0900 Subject: [PATCH 45/79] fix: Add mix deps.get before unit-testing --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c07f1a5d3..dc99b81ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,7 @@ unit-testing: - name: postgres:9.6.2 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: + - mix deps.get - mix ecto.create - mix ecto.migrate - mix test --trace --preload-modules @@ -77,4 +78,4 @@ docs-deploy: - echo "${SSH_HOST_KEY}" > ~/.ssh/known_hosts - eval $(ssh-agent -s) - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}" + - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}" From a53a6c9d64f2c32ca3b53a4317980b3e7c0b37a5 Mon Sep 17 00:00:00 2001 From: AkiraFukushima Date: Thu, 2 May 2019 22:25:21 +0900 Subject: [PATCH 46/79] Add oauth plug tests for url and body parameters --- test/plugs/oauth_plug_test.exs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/plugs/oauth_plug_test.exs b/test/plugs/oauth_plug_test.exs index 17fdba916..5a2ed11cc 100644 --- a/test/plugs/oauth_plug_test.exs +++ b/test/plugs/oauth_plug_test.exs @@ -38,6 +38,26 @@ test "with valid token(downcase), it assigns the user", %{conn: conn} = opts do assert conn.assigns[:user] == opts[:user] end + test "with valid token(downcase) in url parameters, it assings the user", opts do + conn = + :get + |> build_conn("/?access_token=#{opts[:token]}") + |> put_req_header("content-type", "application/json") + |> fetch_query_params() + |> OAuthPlug.call(%{}) + + assert conn.assigns[:user] == opts[:user] + end + + test "with valid token(downcase) in body parameters, it assigns the user", opts do + conn = + :post + |> build_conn("/api/v1/statuses", access_token: opts[:token], status: "test") + |> OAuthPlug.call(%{}) + + assert conn.assigns[:user] == opts[:user] + end + test "with invalid token, it not assigns the user", %{conn: conn} do conn = conn From 38b79461dfe6d14eb95799013b2fdd502e73245c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 2 May 2019 22:19:14 +0300 Subject: [PATCH 47/79] Fix embeded relationships in Mastodon API Currently some endpoints render accounts without for user resulting in embedded relationship being empty. It causes bugs in followers/following tab in pleroma-fe but I fixed it for other endpoints as well just in case --- .../web/mastodon_api/mastodon_api_controller.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index ed585098a..78dae1c64 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -704,7 +704,7 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do end end - def favourited_by(conn, %{"id" => id}) do + def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), %Object{data: %{"likes" => likes}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^likes) @@ -712,13 +712,13 @@ def favourited_by(conn, %{"id" => id}) do conn |> put_view(AccountView) - |> render(AccountView, "accounts.json", %{users: users, as: :user}) + |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user}) else _ -> json(conn, []) end end - def reblogged_by(conn, %{"id" => id}) do + def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^announces) @@ -726,7 +726,7 @@ def reblogged_by(conn, %{"id" => id}) do conn |> put_view(AccountView) - |> render("accounts.json", %{users: users, as: :user}) + |> render("accounts.json", %{for: user, users: users, as: :user}) else _ -> json(conn, []) end @@ -783,7 +783,7 @@ def followers(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do conn |> add_link_headers(:followers, followers, user) |> put_view(AccountView) - |> render("accounts.json", %{users: followers, as: :user}) + |> render("accounts.json", %{for: for_user, users: followers, as: :user}) end end @@ -800,7 +800,7 @@ def following(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do conn |> add_link_headers(:following, followers, user) |> put_view(AccountView) - |> render("accounts.json", %{users: followers, as: :user}) + |> render("accounts.json", %{for: for_user, users: followers, as: :user}) end end @@ -808,7 +808,7 @@ def follow_requests(%{assigns: %{user: followed}} = conn, _params) do with {:ok, follow_requests} <- User.get_follow_requests(followed) do conn |> put_view(AccountView) - |> render("accounts.json", %{users: follow_requests, as: :user}) + |> render("accounts.json", %{for: followed, users: follow_requests, as: :user}) end end @@ -1235,7 +1235,7 @@ def list_accounts(%{assigns: %{user: user}} = conn, %{"id" => id}) do {:ok, users} = Pleroma.List.get_following(list) do conn |> put_view(AccountView) - |> render("accounts.json", %{users: users, as: :user}) + |> render("accounts.json", %{for: user, users: users, as: :user}) end end From 81d1aa424d65b364ec8f2aee45247e7d95e3f255 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 May 2019 13:39:14 +0200 Subject: [PATCH 48/79] Streamer: Stream out Conversations/Participations. --- lib/pleroma/conversation.ex | 11 +++++--- lib/pleroma/web/activity_pub/activity_pub.ex | 22 ++++++++++++++- lib/pleroma/web/streamer.ex | 29 ++++++++++++++++++++ mix.exs | 2 +- mix.lock | 2 +- test/conversation_test.exs | 17 ++++++++++++ test/web/activity_pub/activity_pub_test.exs | 22 +++++++++++++++ 7 files changed, 98 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index e6a4ccc85..6e26c5fd4 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -63,10 +63,13 @@ def create_or_bump_for(activity) do participation end) - %{ - conversation - | participations: participations - } + {:ok, + %{ + conversation + | participations: participations + }} + else + e -> {:error, e} end end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 28754e864..6c737d0a4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -142,8 +142,14 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do end) Notification.create_notifications(activity) - Conversation.create_or_bump_for(activity) + + participations = + activity + |> Conversation.create_or_bump_for() + |> get_participations() + stream_out(activity) + stream_out_participations(participations) {:ok, activity} else %Activity{} = activity -> @@ -166,6 +172,19 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do end end + defp get_participations({:ok, %{participations: participations}}), do: participations + defp get_participations(_), do: [] + + def stream_out_participations(participations) do + participations = + participations + |> Repo.preload(:user) + + Enum.each(participations, fn participation -> + Pleroma.Web.Streamer.stream("participation", participation) + end) + end + def stream_out(activity) do public = "https://www.w3.org/ns/activitystreams#Public" @@ -197,6 +216,7 @@ def stream_out(activity) do end end else + # TODO: Write test, replace with visibility test if !Enum.member?(activity.data["cc"] || [], public) && !Enum.member?( activity.data["to"], diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 72eaf2084..b8f6663a1 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Streamer do use GenServer require Logger + alias Pleroma.Conversation.Participation alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.Object @@ -71,6 +72,15 @@ def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do {:noreply, topics} end + def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do + user_topic = "direct:#{participation.user_id}" + Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") + + push_to_socket(topics, user_topic, participation) + + {:noreply, topics} + end + def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do # filter the recipient list if the activity is not public, see #270. recipient_lists = @@ -192,6 +202,19 @@ defp represent_update(%Activity{} = activity) do |> Jason.encode!() end + def represent_conversation(%Participation{} = participation) do + %{ + event: "conversation", + payload: + Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ + participation: participation, + user: participation.user + }) + |> Jason.encode!() + } + |> Jason.encode!() + end + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do Enum.each(topics[topic] || [], fn socket -> # Get the current user so we have up-to-date blocks etc. @@ -214,6 +237,12 @@ def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = ite end) end + def push_to_socket(topics, topic, %Participation{} = participation) do + Enum.each(topics[topic] || [], fn socket -> + send(socket.transport_pid, {:text, represent_conversation(participation)}) + end) + end + def push_to_socket(topics, topic, %Activity{ data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} }) do diff --git a/mix.exs b/mix.exs index 9ded9931c..ad8f1123a 100644 --- a/mix.exs +++ b/mix.exs @@ -87,7 +87,7 @@ defp deps do {:bbcode, "~> 0.1"}, {:ex_machina, "~> 2.3", only: :test}, {:credo, "~> 0.9.3", only: [:dev, :test]}, - {:mock, "~> 0.3.1", only: :test}, + {:mock, "~> 0.3.3", only: :test}, {:crypt, git: "https://github.com/msantos/crypt", ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"}, {:cors_plug, "~> 1.5"}, diff --git a/mix.lock b/mix.lock index 08221eadc..bb298a68b 100644 --- a/mix.lock +++ b/mix.lock @@ -43,7 +43,7 @@ "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, "mochiweb": {:hex, :mochiweb, "2.15.0", "e1daac474df07651e5d17cc1e642c4069c7850dc4508d3db7263a0651330aacc", [:rebar3], [], "hexpm"}, - "mock": {:hex, :mock, "0.3.1", "994f00150f79a0ea50dc9d86134cd9ebd0d177ad60bd04d1e46336cdfdb98ff9", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, + "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 763183d6b..f3300e7d1 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -117,4 +117,21 @@ test "it creates or updates a conversation and participations for a given DM" do tridi.id == user_id end) end + + test "create_or_bump_for returns the conversation with participations" do + har = insert(:user) + jafnhar = insert(:user, local: false) + + {:ok, activity} = + CommonAPI.post(har, %{"status" => "Hey @#{jafnhar.nickname}", "visibility" => "direct"}) + + {:ok, conversation} = Conversation.create_or_bump_for(activity) + + assert length(conversation.participations) == 2 + + {:ok, activity} = + CommonAPI.post(har, %{"status" => "Hey @#{jafnhar.nickname}", "visibility" => "public"}) + + assert {:error, _} = Conversation.create_or_bump_for(activity) + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 047270a2a..1e056b7ee 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -22,6 +22,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do :ok end + describe "streaming out participations" do + test "it streams them out" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + {:ok, conversation} = Pleroma.Conversation.create_or_bump_for(activity) + + participations = + conversation.participations + |> Repo.preload(:user) + + with_mock Pleroma.Web.Streamer, + stream: fn _, _ -> nil end do + ActivityPub.stream_out_participations(conversation.participations) + + Enum.each(participations, fn participation -> + assert called(Pleroma.Web.Streamer.stream("participation", participation)) + end) + end + end + end + describe "fetching restricted by visibility" do test "it restricts by the appropriate visibility" do user = insert(:user) From a0c755cc4ac3af7ddb7e8fe91fcfc1bae9750e9b Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 May 2019 13:40:43 +0200 Subject: [PATCH 49/79] MastodonApi: Bump api level. --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index aa3f46482..754946624 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -159,7 +159,7 @@ def user(%{assigns: %{user: for_user}} = conn, %{"id" => nickname_or_id}) do end end - @mastodon_api_level "2.5.0" + @mastodon_api_level "2.6.5" def masto_instance(conn, _params) do instance = Config.get(:instance) From acb04306b6954aac8d66a74438d0e213d93a9046 Mon Sep 17 00:00:00 2001 From: feld Date: Fri, 3 May 2019 11:45:04 +0000 Subject: [PATCH 50/79] Standardize construction of websocket URL This follows up on the change made in d747bd98 --- lib/pleroma/plugs/http_security_plug.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index f701aaaa5..a476f1d49 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -35,7 +35,7 @@ defp headers do defp csp_string do scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() - websocket_url = String.replace(static_url, "http", "ws") + websocket_url = Pleroma.Web.Endpoint.websocket_url() connect_src = "connect-src 'self' #{static_url} #{websocket_url}" diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index ed585098a..201a21f50 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1295,8 +1295,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do initial_state = %{ meta: %{ - streaming_api_base_url: - String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"), + streaming_api_base_url: Pleroma.Web.Endpoint.websocket_url(), access_token: token, locale: "en", domain: Pleroma.Web.Endpoint.host(), From c42ded13a2caa02f3f5aa00accce69b183546f9e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 May 2019 13:53:17 +0200 Subject: [PATCH 51/79] Credo fixes. --- lib/pleroma/web/streamer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index b8f6663a1..133decfc4 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.Streamer do use GenServer require Logger - alias Pleroma.Conversation.Participation alias Pleroma.Activity + alias Pleroma.Conversation.Participation alias Pleroma.Notification alias Pleroma.Object alias Pleroma.User From 85b5c60694e07d3bfb1f885d5fda14be6b7bade9 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 7 Feb 2019 16:41:20 +0100 Subject: [PATCH 52/79] Pleroma.Formatter: width/height to class=emoji --- lib/pleroma/formatter.ex | 4 +--- lib/pleroma/html.ex | 2 ++ test/formatter_test.exs | 4 ++-- test/web/twitter_api/views/activity_view_test.exs | 2 +- test/web/twitter_api/views/user_view_test.exs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index dab8910c1..0ec6bcee0 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -113,9 +113,7 @@ def emojify(text, emoji, strip \\ false) do html = if not strip do - "#{emoji}" + "#{emoji}" else "" end diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 726c370ad..d1da746de 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -151,6 +151,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do Meta.allow_tag_with_these_attributes("img", [ "width", "height", + "class", "title", "alt" ]) @@ -221,6 +222,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("img", [ "width", "height", + "class", "title", "alt" ]) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index fdaf29742..06f4f6e50 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -248,7 +248,7 @@ test "it adds cool emoji" do text = "I love :firefox:" expected_result = - "I love \"firefox\"" + "I love \"firefox\"" assert Formatter.emojify(text) == expected_result end @@ -263,7 +263,7 @@ test "it does not add XSS emoji" do } expected_result = - "I love \"\"" + "I love \"\"" assert Formatter.emojify(text, custom_emoji) == expected_result end diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index 85815ba7e..1aa533b48 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -100,7 +100,7 @@ test "a create activity with a summary containing emoji" do expected = ":firefox: meow" expected_html = - "\"firefox\" meow" + "\"firefox\" meow" assert result["summary"] == expected assert result["summary_html"] == expected_html diff --git a/test/web/twitter_api/views/user_view_test.exs b/test/web/twitter_api/views/user_view_test.exs index c99dbddeb..74526673c 100644 --- a/test/web/twitter_api/views/user_view_test.exs +++ b/test/web/twitter_api/views/user_view_test.exs @@ -32,7 +32,7 @@ test "A user with an avatar object", %{user: user} do test "A user with emoji in username" do expected = - "\"karjalanpiirakka\" man" + "\"karjalanpiirakka\" man" user = insert(:user, %{ From d70af32127bda1431bacad58e7c6516a2e652fcd Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 11 Feb 2019 23:47:32 +0100 Subject: [PATCH 53/79] Pleroma.User: remove emojify on parse_bio --- lib/pleroma/user.ex | 21 ++++++++----------- .../mastodon_api/mastodon_api_controller.ex | 2 +- test/user_test.exs | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 1c62f238e..1741ce684 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -11,7 +11,6 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Pleroma.Activity alias Pleroma.Bookmark - alias Pleroma.Formatter alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Registration @@ -1331,18 +1330,15 @@ def wait_and_refresh(timeout, %User{} = a, %User{} = b) do end end - def parse_bio(bio, user \\ %User{info: %{source_data: %{}}}) - def parse_bio(nil, _user), do: "" - def parse_bio(bio, _user) when bio == "", do: bio + def parse_bio(bio) when is_binary(bio) and bio != "" do + bio + |> CommonUtils.format_input("text/plain", mentions_format: :full) + |> elem(0) + end - def parse_bio(bio, user) do - emoji = - (user.info.source_data["tag"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) - |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} -> - {String.trim(name, ":"), url} - end) + def parse_bio(_), do: "" + def parse_bio(bio, user) when is_binary(bio) and bio != "" do # TODO: get profile URLs other than user.ap_id profile_urls = [user.ap_id] @@ -1352,9 +1348,10 @@ def parse_bio(bio, user) do rel: &RelMe.maybe_put_rel_me(&1, profile_urls) ) |> elem(0) - |> Formatter.emojify(emoji) end + def parse_bio(_, _), do: "" + def tag(user_identifiers, tags) when is_list(user_identifiers) do Repo.transaction(fn -> for user_identifier <- user_identifiers, do: tag(user_identifier, tags) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 201a21f50..564dbb829 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -86,7 +86,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do user_params = %{} |> add_if_present(params, "display_name", :name) - |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value)} end) + |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end) |> add_if_present(params, "avatar", :avatar, fn value -> with %Plug.Upload{} <- value, {:ok, object} <- ActivityPub.upload(value, type: :avatar) do diff --git a/test/user_test.exs b/test/user_test.exs index 67266cb7a..6d21b56f7 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1103,7 +1103,7 @@ test "preserves hosts in user links text" do expected_text = "A.k.a. " <> "@nick@domain.com" + }'>@nick@domain.com" assert expected_text == User.parse_bio(bio, user) end From 2f76a40d028c45e99425b061298a1b05e4b59923 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 12 Feb 2019 14:59:34 +0100 Subject: [PATCH 54/79] formatter.ex: Add get_emoji_map/1 --- lib/pleroma/formatter.ex | 11 ++++++++ lib/pleroma/user/info.ex | 1 + .../web/activity_pub/transmogrifier.ex | 14 ++++++++-- .../web/activity_pub/views/user_view.ex | 7 ++++- lib/pleroma/web/common_api/common_api.ex | 9 ++---- .../web/twitter_api/twitter_api_controller.ex | 12 +++++++- .../twitter_api_controller_test.exs | 28 +++++++++++++++++++ 7 files changed, 72 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 0ec6bcee0..3d7c36d21 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -128,12 +128,23 @@ def demojify(text) do def demojify(text, nil), do: text + @doc "Outputs a list of the emoji-shortcodes in a text" def get_emoji(text) when is_binary(text) do Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end) end def get_emoji(_), do: [] + @doc "Outputs a list of the emoji-Maps in a text" + def get_emoji_map(text) when is_binary(text) do + get_emoji(text) + |> Enum.reduce(%{}, fn {name, file, _group}, acc -> + Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") + end) + end + + def get_emoji_map(_), do: [] + def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index a3658d57f..1b81619ce 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -41,6 +41,7 @@ defmodule Pleroma.User.Info do field(:hide_favorites, :boolean, default: true) field(:pinned_activities, {:array, :string}, default: []) field(:flavour, :string, default: nil) + field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b774c2afa..508f3532f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -856,10 +856,16 @@ def add_mention_tags(object) do |> Map.put("tag", tags ++ mentions) end + def add_emoji_tags(%User{info: %{"emoji" => _emoji} = user_info} = object) do + user_info = add_emoji_tags(user_info) + + object + |> Map.put(:info, user_info) + end + # TODO: we should probably send mtime instead of unix epoch time for updated - def add_emoji_tags(object) do + def add_emoji_tags(%{"emoji" => emoji} = object) do tags = object["tag"] || [] - emoji = object["emoji"] || [] out = emoji @@ -877,6 +883,10 @@ def add_emoji_tags(object) do |> Map.put("tag", tags ++ out) end + def add_emoji_tags(object) do + object + end + def set_conversation(object) do Map.put(object, "conversation", object["context"]) end diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 5926a3294..1254fdf6c 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -69,6 +69,11 @@ def render("user.json", %{user: user}) do endpoints = render("endpoints.json", %{user: user}) + user_tags = + user + |> Transmogrifier.add_emoji_tags() + |> Map.get("tag", []) + %{ "id" => user.ap_id, "type" => "Person", @@ -87,7 +92,7 @@ def render("user.json", %{user: user}) do "publicKeyPem" => public_key }, "endpoints" => endpoints, - "tag" => user.info.source_data["tag"] || [] + "tag" => (user.info.source_data["tag"] || []) ++ user_tags } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index ecd183110..b53869c75 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -151,8 +151,8 @@ def post(user, %{"status" => status} = data) do ), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), - cw <- data["spoiler_text"], - full_payload <- String.trim(status <> (data["spoiler_text"] || "")), + cw <- data["spoiler_text"] || "", + full_payload <- String.trim(status <> cw), length when length in 1..limit <- String.length(full_payload), object <- make_note_data( @@ -170,10 +170,7 @@ def post(user, %{"status" => status} = data) do Map.put( object, "emoji", - (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"])) - |> Enum.reduce(%{}, fn {name, file, _}, acc -> - Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") - end) + Formatter.get_emoji_map(full_payload) ) do res = ActivityPub.create( diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 79ed9dad2..261cc4462 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do alias Ecto.Changeset alias Pleroma.Activity + alias Pleroma.Formatter alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo @@ -653,7 +654,16 @@ defp build_info_cng(user, params) do defp parse_profile_bio(user, params) do if bio = params["description"] do - Map.put(params, "bio", User.parse_bio(bio, user)) + user_info = + user.info + |> Map.put( + "emojis", + Formatter.get_emoji_map(params["description"]) + ) + + params + |> Map.put("bio", User.parse_bio(bio, user)) + |> Map.put("info", user_info) else params end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 43ad71a16..90718cfb4 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1611,6 +1611,34 @@ test "it unlocks an account", %{conn: conn} do assert json_response(conn, 200) == UserView.render("user.json", %{user: user, for: user}) end + + # Broken before the change to class="emoji" and non- in the DB + @tag :skip + test "it formats emojos", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> post("/api/account/update_profile.json", %{ + "bio" => "I love our :moominmamma:​" + }) + + assert response = json_response(conn, 200) + + assert %{ + "description" => "I love our :moominmamma:", + "description_html" => + ~s{I love our moominmamma Date: Fri, 22 Feb 2019 22:35:47 +0100 Subject: [PATCH 55/79] Web.TwitterAPI.UserView: Also view local user emojis --- lib/pleroma/web/twitter_api/views/user_view.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index ea015b8f0..6857055c9 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -67,6 +67,8 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do {String.trim(name, ":"), url} end) + emoji = Enum.dedup(emoji ++ user.info.emoji) + # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] fields = From 46bbf9e1cff4e00f2fbd95dcc21c038d7d686b86 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 23 Feb 2019 00:09:11 +0100 Subject: [PATCH 56/79] TwitterAPI: profile update with emoji_map --- lib/pleroma/web/twitter_api/twitter_api_controller.ex | 10 ++++++++-- lib/pleroma/web/twitter_api/views/user_view.ex | 7 ++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 261cc4462..ef7b6fe65 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -654,11 +654,17 @@ defp build_info_cng(user, params) do defp parse_profile_bio(user, params) do if bio = params["description"] do + emojis_text = (params["description"] || "") <> " " <> (params["name"] || "") + + emojis = + ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text)) + |> Enum.dedup() + user_info = user.info |> Map.put( - "emojis", - Formatter.get_emoji_map(params["description"]) + "emoji", + emojis ) params diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 6857055c9..f0a4ddbd3 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -69,6 +69,11 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do emoji = Enum.dedup(emoji ++ user.info.emoji) + description_html = + (user.bio || "") + |> HTML.filter_tags(User.html_filter_policy(for_user)) + |> Formatter.emojify(emoji) + # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] fields = @@ -80,7 +85,7 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do %{ "created_at" => user.inserted_at |> Utils.format_naive_asctime(), "description" => HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), - "description_html" => HTML.filter_tags(user.bio, User.html_filter_policy(for_user)), + "description_html" => description_html, "favourites_count" => 0, "followers_count" => user_info[:follower_count], "following" => following, From b5ad1715b2d4a2a5bdaefa2d56bde71120d23acb Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 23 Feb 2019 00:57:42 +0100 Subject: [PATCH 57/79] MastoAPI: profile update with emoji_map --- .../mastodon_api/mastodon_api_controller.ex | 8 ++++++ .../mastodon_api_controller_test.exs | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 564dbb829..0840c2c5a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Bookmark alias Pleroma.Config alias Pleroma.Filter + alias Pleroma.Formatter alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Object.Fetcher @@ -96,6 +97,12 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do end end) + emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "") + + user_info_emojis = + ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text)) + |> Enum.dedup() + info_params = [:no_rich_text, :locked, :hide_followers, :hide_follows, :hide_favorites, :show_role] |> Enum.reduce(%{}, fn key, acc -> @@ -112,6 +119,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do _ -> :error end end) + |> Map.put(:emoji, user_info_emojis) info_cng = User.Info.profile_update(user.info, info_params) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index c2a12d3c7..610aa486e 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2351,6 +2351,33 @@ test "requires 'write' permission", %{conn: conn} do end end end + + test "updates profile emojos", %{conn: conn} do + user = insert(:user) + + note = "*sips :blank:*" + name = "I am :firefox:" + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{ + "note" => note, + "display_name" => name + }) + + assert json_response(conn, 200) + + conn = + conn + |> get("/api/v1/accounts/#{user.id}") + + assert user = json_response(conn, 200) + + assert user["note"] == note + assert user["display_name"] == name + assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"] + end end test "get instance information", %{conn: conn} do From 0e37fddd5abad5db0378c8186513fbba0fbd6586 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 May 2019 19:15:55 +0200 Subject: [PATCH 58/79] Search: Add fts index on objects table. --- CHANGELOG.md | 1 + lib/mix/tasks/benchmark.ex | 25 +++++++++++++++++++ mix.exs | 3 ++- mix.lock | 2 ++ ...0190501125843_add_fts_index_to_objects.exs | 8 ++++++ 5 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 lib/mix/tasks/benchmark.ex create mode 100644 priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 67b9649e1..038a001de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Added support max_id & since_id for bookmark timeline endpoints. ### Fixed +- Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. - Followers counter not being updated when a follower is blocked - Deactivated users being able to request an access token - Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak diff --git a/lib/mix/tasks/benchmark.ex b/lib/mix/tasks/benchmark.ex new file mode 100644 index 000000000..0fbb4dbb1 --- /dev/null +++ b/lib/mix/tasks/benchmark.ex @@ -0,0 +1,25 @@ +defmodule Mix.Tasks.Pleroma.Benchmark do + use Mix.Task + alias Mix.Tasks.Pleroma.Common + + def run(["search"]) do + Common.start_pleroma() + + Benchee.run(%{ + "search" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe") + end + }) + end + + def run(["tag"]) do + Common.start_pleroma() + + Benchee.run(%{ + "tag" => fn -> + %{"type" => "Create", "tag" => "cofe"} + |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() + end + }) + end +end diff --git a/mix.exs b/mix.exs index 9ded9931c..3b368e57e 100644 --- a/mix.exs +++ b/mix.exs @@ -111,7 +111,8 @@ defp deps do {:prometheus_ecto, "~> 1.4"}, {:prometheus_process_collector, "~> 1.4"}, {:recon, github: "ferd/recon", tag: "2.4.0"}, - {:quack, "~> 0.1.1"} + {:quack, "~> 0.1.1"}, + {:benchee, "~> 1.0", only: :dev} ] ++ oauth_deps end diff --git a/mix.lock b/mix.lock index 08221eadc..e97f4ec38 100644 --- a/mix.lock +++ b/mix.lock @@ -3,6 +3,7 @@ "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, + "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, "cachex": {:hex, :cachex, "3.0.2", "1351caa4e26e29f7d7ec1d29b53d6013f0447630bbf382b4fb5d5bad0209f203", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"}, "calendar": {:hex, :calendar, "0.17.4", "22c5e8d98a4db9494396e5727108dffb820ee0d18fed4b0aa8ab76e4f5bc32f1", [:mix], [{:tzdata, "~> 0.5.8 or ~> 0.1.201603", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, @@ -17,6 +18,7 @@ "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"}, + "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs new file mode 100644 index 000000000..9b274695e --- /dev/null +++ b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.AddFTSIndexToObjects do + use Ecto.Migration + + def change do + drop_if_exists index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], using: :gin, name: :activities_fts) + create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + end +end From a7709cc26726350a0c90e0e4117cab4a8a07bb35 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 May 2019 19:42:49 +0200 Subject: [PATCH 59/79] Mix: Also have benchee in test. --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 3b368e57e..ef338c7c0 100644 --- a/mix.exs +++ b/mix.exs @@ -112,7 +112,7 @@ defp deps do {:prometheus_process_collector, "~> 1.4"}, {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"}, - {:benchee, "~> 1.0", only: :dev} + {:benchee, "~> 1.0", only: [:dev, :test]} ] ++ oauth_deps end From a23206a4ae0abe02036c40f9142ee1e6a46f5fae Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 May 2019 20:28:06 +0200 Subject: [PATCH 60/79] Just have Benchee in all environments. --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index ef338c7c0..c553b835b 100644 --- a/mix.exs +++ b/mix.exs @@ -112,7 +112,7 @@ defp deps do {:prometheus_process_collector, "~> 1.4"}, {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"}, - {:benchee, "~> 1.0", only: [:dev, :test]} + {:benchee, "~> 1.0"} ] ++ oauth_deps end From 4c76f49e60287e6618f87a2bae3e0d4e8c444895 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 4 May 2019 15:06:18 +0200 Subject: [PATCH 61/79] BBS: small fixes. --- lib/pleroma/bbs/handler.ex | 6 +----- mix.exs | 2 +- test/bbs/handler_test.exs | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 1ebba77d2..75ba35dc2 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -1,12 +1,8 @@ defmodule Pleroma.BBS.Handler do - @moduledoc """ - An example implementation of `Sshd.ShellHandler`, implementing a very simple - Read-Eval-Loop, that does nothing. - """ use Sshd.ShellHandler + alias Pleroma.Activity alias Pleroma.Web.CommonAPI alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Activity def on_shell(username, _pubkey, _ip, _port) do :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!") diff --git a/mix.exs b/mix.exs index 38e83e679..47f7c2903 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defp deps do {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"}, {:benchee, "~> 1.0"}, - {:esshd, "~> 0.1.0"} + {:esshd, "~> 0.1.0"} ] ++ oauth_deps end diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs index a22c6d64d..1386a2d16 100644 --- a/test/bbs/handler_test.exs +++ b/test/bbs/handler_test.exs @@ -1,10 +1,10 @@ defmodule Pleroma.BBS.HandlerTest do use Pleroma.DataCase + alias Pleroma.Activity alias Pleroma.BBS.Handler alias Pleroma.Web.CommonAPI - alias Pleroma.User alias Pleroma.Repo - alias Pleroma.Activity + alias Pleroma.User import ExUnit.CaptureIO import Pleroma.Factory From c9d1cb2dcefb850357b953a7a2d308de39b35e07 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 4 May 2019 15:08:07 +0200 Subject: [PATCH 62/79] BBS: Use cached user fetcher. --- lib/pleroma/bbs/handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 75ba35dc2..14e6a6807 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -6,7 +6,7 @@ defmodule Pleroma.BBS.Handler do def on_shell(username, _pubkey, _ip, _port) do :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!") - user = Pleroma.User.get_by_nickname(to_string(username)) + user = Pleroma.User.get_cached_by_nickname(to_string(username)) Logger.debug("#{inspect(user)}") loop(run_state(user: user)) end From c58fd4c038da8305d8840c38f525ceb9f13a644d Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 4 May 2019 15:36:48 +0200 Subject: [PATCH 63/79] BBS: Fix tests. --- test/bbs/handler_test.exs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs index 1386a2d16..148df6ddd 100644 --- a/test/bbs/handler_test.exs +++ b/test/bbs/handler_test.exs @@ -3,6 +3,7 @@ defmodule Pleroma.BBS.HandlerTest do alias Pleroma.Activity alias Pleroma.BBS.Handler alias Pleroma.Web.CommonAPI + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -49,7 +50,8 @@ test "posting" do ) assert activity.actor == user.ap_id - assert activity.data["object"]["content"] == "this is a test post" + object = Object.normalize(activity) + assert object.data["content"] == "this is a test post" end test "replying" do @@ -74,7 +76,8 @@ test "replying" do ) assert reply.actor == user.ap_id - assert reply.data["object"]["content"] == "this is a reply" - assert reply.data["object"]["inReplyTo"] == activity.data["object"]["id"] + object = Object.normalize(reply) + assert object.data["content"] == "this is a reply" + assert object.data["inReplyTo"] == activity.data["object"] end end From eb0fb73ddbed109ca4dcd758b60a25ff0dafc883 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 4 May 2019 15:47:50 +0200 Subject: [PATCH 64/79] BBS: Credo fixes. --- lib/pleroma/bbs/handler.ex | 2 +- test/bbs/handler_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 14e6a6807..106fe5d18 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -1,8 +1,8 @@ defmodule Pleroma.BBS.Handler do use Sshd.ShellHandler alias Pleroma.Activity - alias Pleroma.Web.CommonAPI alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.CommonAPI def on_shell(username, _pubkey, _ip, _port) do :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!") diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs index 148df6ddd..7d5d68d11 100644 --- a/test/bbs/handler_test.exs +++ b/test/bbs/handler_test.exs @@ -2,10 +2,10 @@ defmodule Pleroma.BBS.HandlerTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.BBS.Handler - alias Pleroma.Web.CommonAPI alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.CommonAPI import ExUnit.CaptureIO import Pleroma.Factory From af62ace9545e60a0c93498ec3054697e781e058a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 6 May 2019 04:28:04 +0200 Subject: [PATCH 65/79] Add short documentation on every MRF Policy --- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 1 + lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex | 1 + lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex | 1 + lib/pleroma/web/activity_pub/mrf/noop_policy.ex | 1 + lib/pleroma/web/activity_pub/mrf/normalize_markup.ex | 1 + lib/pleroma/web/activity_pub/mrf/reject_non_public.ex | 1 + lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 1 + lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 1 + lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 1 + 12 files changed, 15 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 34665a3a6..87fa514c3 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do alias Pleroma.User + @moduledoc "Prevent followbots from following with a bit of heuristic" + @behaviour Pleroma.Web.ActivityPub.MRF # XXX: this should become User.normalize_by_ap_id() or similar, really. diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index a93ccf386..b8d38aae6 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do require Logger + @moduledoc "Drop and log everything received" @behaviour Pleroma.Web.ActivityPub.MRF @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 895376c9d..15d8514be 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do alias Pleroma.Object + @moduledoc "Ensure a re: is prepended on replies to a post with a Subject" @behaviour Pleroma.Web.ActivityPub.MRF @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 6736f3cb9..a699f6a7e 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -4,6 +4,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do alias Pleroma.User + @moduledoc "Block messages with too much mentions (configurable)" + @behaviour Pleroma.Web.ActivityPub.MRF defp delist_message(message, threshold) when threshold > 0 do diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index e8dfba672..d5c341433 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do + @moduledoc "Reject or Word-Replace messages with a keyword or regex" + @behaviour Pleroma.Web.ActivityPub.MRF defp string_matches?(string, _) when not is_binary(string) do false diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index 081456046..f30fee0d5 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do + @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)" @behaviour Pleroma.Web.ActivityPub.MRF @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex index 40f37bdb1..c47cb3298 100644 --- a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do + @moduledoc "Does nothing (lets the messages go through unmodified)" @behaviour Pleroma.Web.ActivityPub.MRF @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 3d13cdb32..9c87c6963 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do + @moduledoc "Scrub configured hypertext markup" alias Pleroma.HTML @behaviour Pleroma.Web.ActivityPub.MRF diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 4197be847..ea3df1b4d 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do alias Pleroma.User + @moduledoc "Rejects non-public (followers-only, direct) activities" @behaviour Pleroma.Web.ActivityPub.MRF @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 798ba9687..2f105700b 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do alias Pleroma.User + @moduledoc "Filter activities depending on their origin instance" @behaviour Pleroma.Web.ActivityPub.MRF defp check_accept(%{host: actor_host} = _actor_info, object) do diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index b242e44e6..5ed1ee77c 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do alias Pleroma.User @behaviour Pleroma.Web.ActivityPub.MRF + @moduledoc "Apply policies based on user tags" defp get_tags(%User{tags: tags}) when is_list(tags), do: tags defp get_tags(_), do: [] diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index a3b1f8aa0..f5078d818 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do alias Pleroma.Config + @moduledoc "Accept-list of users from specified instances" @behaviour Pleroma.Web.ActivityPub.MRF defp filter_by_list(object, []), do: {:ok, object} From e41a2f98d5f061d263cf731b76a20851931e6e33 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 6 May 2019 03:43:11 +0200 Subject: [PATCH 66/79] mrf/tag_policy.ex: Add some documentation mrf_tag:disable-remote-subscription exact way of working is quite unclear to me. Is it the requester that is denied if they have a tag, or is it the requestee if they have one? --- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index 5ed1ee77c..b52be30e7 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -5,7 +5,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do alias Pleroma.User @behaviour Pleroma.Web.ActivityPub.MRF - @moduledoc "Apply policies based on user tags" + @moduledoc """ + Apply policies based on user tags + + This policy applies policies on a user activities depending on their tags + on your instance. + + - `mrf_tag:media-force-nsfw`: Mark as sensitive on presence of attachments + - `mrf_tag:media-strip`: Remove attachments + - `mrf_tag:force-unlisted`: Mark as unlisted (removes from the federated timeline) + - `mrf_tag:sandbox`: Remove from public (local and federated) timelines + - `mrf_tag:disable-remote-subscription`: Reject non-local follow requests + - `mrf_tag:disable-any-subscription`: Reject any follow requests + """ defp get_tags(%User{tags: tags}) when is_list(tags), do: tags defp get_tags(_), do: [] From 69a5074893d050a9a3ce6d245f1aa05c385ea4fb Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 6 May 2019 04:47:04 +0200 Subject: [PATCH 67/79] Remove H1 in @moduledoc --- lib/pleroma/object/containment.ex | 2 +- lib/pleroma/upload.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 25bd911fb..70325d6be 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Object.Containment do @moduledoc """ - # Object Containment + help on getting the origin of contained objects This module contains some useful functions for containing objects to specific origins and determining those origins. They previously lived in the diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index f72334930..c47d65241 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Upload do @moduledoc """ - # Upload + Manage user uploads Options: * `:type`: presets for activity type (defaults to Document) and size limits from app configuration From ce6ca0fefe7feb1c31447287aec117d40233652a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 6 May 2019 16:45:22 +0000 Subject: [PATCH 68/79] Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into feature/845-improve-status-deletion --- CHANGELOG.md | 1 + config/config.exs | 6 ++-- docs/config.md | 1 + lib/mix/tasks/pleroma/user.ex | 4 +-- lib/pleroma/activity.ex | 7 +++++ lib/pleroma/user.ex | 30 +++++++++++-------- lib/pleroma/user_invite_token.ex | 2 +- .../controllers/util_controller.ex | 2 +- test/user_test.exs | 8 +++-- 9 files changed, 40 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 038a001de..0d44f6786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Deactivated users being able to request an access token - Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak - proper Twitter Card generation instead of a dummy +- Deletions failing for users with a large number of posts - NodeInfo: Include admins in `staffAccounts` - ActivityPub: Crashing when requesting empty local user's outbox - Federation: Handling of objects without `summary` property diff --git a/config/config.exs b/config/config.exs index 1a9738cff..12dbe9941 100644 --- a/config/config.exs +++ b/config/config.exs @@ -232,7 +232,8 @@ welcome_message: nil, max_report_comment_size: 1000, safe_dm_mentions: false, - healthcheck: false + healthcheck: false, + repo_batch_size: 500 config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because @@ -416,7 +417,8 @@ web_push: 50, mailer: 10, transmogrifier: 20, - scheduled_activities: 10 + scheduled_activities: 10, + background: 5 config :pleroma, :fetch_initial_posts, enabled: false, diff --git a/docs/config.md b/docs/config.md index ad55d44a7..731eff99a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``. +* `repo_batch_size`: Repo batch size. The number of loaded rows from the database to the memory for processing chunks. E.g. deleting user statuses. ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 9e2523b18..6a83a8c0d 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -163,7 +163,7 @@ def run(["rm", nickname]) do Common.start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.delete(user) + User.perform(:delete, user) Mix.shell().info("User #{nickname} deleted.") else _ -> @@ -380,7 +380,7 @@ def run(["delete_activities", nickname]) do Common.start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.delete_user_activities(user) + {:ok, _} = User.delete_user_activities(user) Mix.shell().info("User #{nickname} statuses deleted.") else _ -> diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4a2ded518..73e63bb14 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -14,6 +14,8 @@ defmodule Pleroma.Activity do import Ecto.Query @type t :: %__MODULE__{} + @type actor :: String.t() + @primary_key {:id, Pleroma.FlakeId, autogenerate: true} # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19 @@ -260,4 +262,9 @@ def all_by_actor_and_id(actor, status_ids) do |> where([s], s.actor == ^actor) |> Repo.all() end + + @spec query_by_actor(actor()) :: Ecto.Query.t() + def query_by_actor(actor) do + from(a in Activity, where: a.actor == ^actor) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 1741ce684..fd2ce81ad 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1164,7 +1164,12 @@ def update_notification_settings(%User{} = user, settings \\ %{}) do |> update_and_set_cache() end - def delete(%User{} = user) do + @spec delete(User.t()) :: :ok + def delete(%User{} = user), + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) + + @spec perform(atom(), User.t()) :: {:ok, User.t()} + def perform(:delete, %User{} = user) do {:ok, user} = User.deactivate(user) # Remove all relationships @@ -1180,22 +1185,23 @@ def delete(%User{} = user) do end def delete_user_activities(%User{ap_id: ap_id} = user) do - Activity - |> where(actor: ^ap_id) - |> Activity.with_preloaded_object() - |> Repo.all() - |> Enum.each(fn - %{data: %{"type" => "Create"}} = activity -> - activity |> Object.normalize() |> ActivityPub.delete() + stream = + ap_id + |> Activity.query_by_actor() + |> Activity.with_preloaded_object() + |> Repo.stream() - # TODO: Do something with likes, follows, repeats. - _ -> - "Doing nothing" - end) + Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity) {:ok, user} end + defp delete_activity(%{data: %{"type" => "Create"}} = activity) do + Object.normalize(activity) |> ActivityPub.delete() + end + + defp delete_activity(_activity), do: "Doing nothing" + def html_filter_policy(%User{info: %{no_rich_text: true}}) do Pleroma.HTML.Scrubber.TwitterText end diff --git a/lib/pleroma/user_invite_token.ex b/lib/pleroma/user_invite_token.ex index 86f0a5486..fadc89891 100644 --- a/lib/pleroma/user_invite_token.ex +++ b/lib/pleroma/user_invite_token.ex @@ -24,7 +24,7 @@ defmodule Pleroma.UserInviteToken do timestamps() end - @spec create_invite(map()) :: UserInviteToken.t() + @spec create_invite(map()) :: {:ok, UserInviteToken.t()} def create_invite(params \\ %{}) do %UserInviteToken{} |> cast(params, [:max_use, :expires_at]) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 1122e6c5d..c03f8ab3a 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -352,7 +352,7 @@ def change_password(%{assigns: %{user: user}} = conn, params) do def delete_account(%{assigns: %{user: user}} = conn, params) do case CommonAPI.Utils.confirm_current_password(user, params["password"]) do {:ok, user} -> - Task.start(fn -> User.delete(user) end) + User.delete(user) json(conn, %{status: "success"}) {:error, msg} -> diff --git a/test/user_test.exs b/test/user_test.exs index 6d21b56f7..adc77a264 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -829,10 +829,12 @@ test ".delete_user_activities deletes all create activities" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"}) - {:ok, _} = User.delete_user_activities(user) - # TODO: Remove favorites, repeats, delete activities. - refute Activity.get_by_id(activity.id) + Ecto.Adapters.SQL.Sandbox.unboxed_run(Repo, fn -> + {:ok, _} = User.delete_user_activities(user) + # TODO: Remove favorites, repeats, delete activities. + refute Activity.get_by_id(activity.id) + end) end test ".delete deactivates a user, all follow relationships and all create activities" do From 1040caf096347b638b9fda5b23fcccde87b32ede Mon Sep 17 00:00:00 2001 From: Maksim Date: Mon, 6 May 2019 17:51:03 +0000 Subject: [PATCH 69/79] fix format Modified-by: Maksim Pechnikov --- CHANGELOG.md | 1 + config/config.exs | 4 + docs/api/differences_in_mastoapi_responses.md | 9 +- docs/config.md | 9 +- lib/pleroma/repo.ex | 28 +++ lib/pleroma/web/oauth/app.ex | 1 + lib/pleroma/web/oauth/authorization.ex | 8 + lib/pleroma/web/oauth/oauth_controller.ex | 157 +++++++------- lib/pleroma/web/oauth/token.ex | 77 +++++-- .../web/oauth/token/strategy/refresh_token.ex | 54 +++++ .../web/oauth/token/strategy/revoke.ex | 22 ++ lib/pleroma/web/oauth/token/utils.ex | 30 +++ ...33552_add_refresh_token_index_to_token.exs | 7 + test/repo_test.exs | 44 ++++ test/web/oauth/oauth_controller_test.exs | 196 ++++++++++++++++++ 15 files changed, 561 insertions(+), 86 deletions(-) create mode 100644 lib/pleroma/web/oauth/token/strategy/refresh_token.ex create mode 100644 lib/pleroma/web/oauth/token/strategy/revoke.ex create mode 100644 lib/pleroma/web/oauth/token/utils.ex create mode 100644 priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs create mode 100644 test/repo_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d44f6786..210aae2e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: [Reports](https://docs.joinmastodon.org/api/rest/reports/) - ActivityPub C2S: OAuth endpoints - Metadata RelMe provider +- OAuth: added support for refresh tokens - Emoji packs and emoji pack manager ### Changed diff --git a/config/config.exs b/config/config.exs index 7792e9a87..946ba9adf 100644 --- a/config/config.exs +++ b/config/config.exs @@ -473,6 +473,10 @@ total_user_limit: 300, enabled: true +config :pleroma, :oauth2, + token_expires_in: 600, + issue_new_refresh_token: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 1350ace43..d3ba41b6a 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -1,6 +1,6 @@ # Differences in Mastodon API responses from vanilla Mastodon -A Pleroma instance can be identified by " (compatible; Pleroma )" present in `version` field in response from `/api/v1/instance` +A Pleroma instance can be identified by " (compatible; Pleroma )" present in `version` field in response from `/api/v1/instance` ## Flake IDs @@ -80,3 +80,10 @@ Additional parameters can be added to the JSON body/Form data: - `hide_favorites` - if true, user's favorites timeline will be hidden - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API - `default_scope` - the scope returned under `privacy` key in Source subentity + +## Authentication + +*Pleroma supports refreshing tokens. + +`POST /oauth/token` +Post here request with grant_type=refresh_token to obtain new access token. Returns an access token. diff --git a/docs/config.md b/docs/config.md index bbdc22688..d999952e1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -474,7 +474,7 @@ Authentication / authorization settings. * `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. * `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable. -# OAuth consumer mode +## OAuth consumer mode OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). @@ -527,6 +527,13 @@ config :ueberauth, Ueberauth, ] ``` +## OAuth 2.0 provider - :oauth2 + +Configure OAuth 2 provider capabilities: + +* `token_expires_in` - The lifetime in seconds of the access token. +* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. + ## :emoji * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index aa5d427ae..f57e088bc 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -19,4 +19,32 @@ defmodule Instrumenter do def init(_, opts) do {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))} end + + @doc "find resource based on prepared query" + @spec find_resource(Ecto.Query.t()) :: {:ok, struct()} | {:error, :not_found} + def find_resource(%Ecto.Query{} = query) do + case __MODULE__.one(query) do + nil -> {:error, :not_found} + resource -> {:ok, resource} + end + end + + def find_resource(_query), do: {:error, :not_found} + + @doc """ + Gets association from cache or loads if need + + ## Examples + + iex> Repo.get_assoc(token, :user) + %User{} + + """ + @spec get_assoc(struct(), atom()) :: {:ok, struct()} | {:error, :not_found} + def get_assoc(resource, association) do + case __MODULE__.preload(resource, association) do + %{^association => assoc} when not is_nil(assoc) -> {:ok, assoc} + _ -> {:error, :not_found} + end + end end diff --git a/lib/pleroma/web/oauth/app.ex b/lib/pleroma/web/oauth/app.ex index 3476da484..bccc2ac96 100644 --- a/lib/pleroma/web/oauth/app.ex +++ b/lib/pleroma/web/oauth/app.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.App do use Ecto.Schema import Ecto.Changeset + @type t :: %__MODULE__{} schema "apps" do field(:client_name, :string) field(:redirect_uris, :string) diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index 3461f9983..ca3901cc4 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.OAuth.Authorization do import Ecto.Changeset import Ecto.Query + @type t :: %__MODULE__{} schema "oauth_authorizations" do field(:token, :string) field(:scopes, {:array, :string}, default: []) @@ -63,4 +64,11 @@ def delete_user_authorizations(%User{id: user_id}) do ) |> Repo.delete_all() end + + @doc "gets auth for app by token" + @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} + def get_by_token(%App{id: app_id} = _app, token) do + from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token) + |> Repo.find_resource() + end end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 688eaca11..e3c01217d 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -13,11 +13,15 @@ defmodule Pleroma.Web.OAuth.OAuthController do alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken + alias Pleroma.Web.OAuth.Token.Strategy.Revoke, as: RevokeToken import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2] if Pleroma.Config.oauth_consumer_enabled?(), do: plug(Ueberauth) + @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600) + plug(:fetch_session) plug(:fetch_flash) @@ -138,25 +142,33 @@ defp handle_create_authorization_error(conn, error, %{"authorization" => _}) do Authenticator.handle_error(conn, error) end + @doc "Renew access_token with refresh_token" + def token_exchange( + conn, + %{"grant_type" => "refresh_token", "refresh_token" => token} = params + ) do + with %App{} = app <- get_app_from_request(conn, params), + {:ok, %{user: user} = token} <- Token.get_by_refresh_token(app, token), + {:ok, token} <- RefreshToken.grant(token) do + response_attrs = %{created_at: Token.Utils.format_created_at(token)} + + json(conn, response_token(user, token, response_attrs)) + else + _error -> + put_status(conn, 400) + |> json(%{error: "Invalid credentials"}) + end + end + def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do with %App{} = app <- get_app_from_request(conn, params), - fixed_token = fix_padding(params["code"]), - %Authorization{} = auth <- - Repo.get_by(Authorization, token: fixed_token, app_id: app.id), + fixed_token = Token.Utils.fix_padding(params["code"]), + {:ok, auth} <- Authorization.get_by_token(app, fixed_token), %User{} = user <- User.get_cached_by_id(auth.user_id), - {:ok, token} <- Token.exchange_token(app, auth), - {:ok, inserted_at} <- DateTime.from_naive(token.inserted_at, "Etc/UTC") do - response = %{ - token_type: "Bearer", - access_token: token.token, - refresh_token: token.refresh_token, - created_at: DateTime.to_unix(inserted_at), - expires_in: 60 * 10, - scope: Enum.join(token.scopes, " "), - me: user.ap_id - } + {:ok, token} <- Token.exchange_token(app, auth) do + response_attrs = %{created_at: Token.Utils.format_created_at(token)} - json(conn, response) + json(conn, response_token(user, token, response_attrs)) else _error -> put_status(conn, 400) @@ -177,16 +189,7 @@ def token_exchange( true <- Enum.any?(scopes), {:ok, auth} <- Authorization.create_authorization(app, user, scopes), {:ok, token} <- Token.exchange_token(app, auth) do - response = %{ - token_type: "Bearer", - access_token: token.token, - refresh_token: token.refresh_token, - expires_in: 60 * 10, - scope: Enum.join(token.scopes, " "), - me: user.ap_id - } - - json(conn, response) + json(conn, response_token(user, token)) else {:auth_active, false} -> # Per https://github.com/tootsuite/mastodon/blob/ @@ -218,10 +221,12 @@ def token_exchange( token_exchange(conn, params) end - def token_revoke(conn, %{"token" => token} = params) do + # Bad request + def token_exchange(conn, params), do: bad_request(conn, params) + + def token_revoke(conn, %{"token" => _token} = params) do with %App{} = app <- get_app_from_request(conn, params), - %Token{} = token <- Repo.get_by(Token, token: token, app_id: app.id), - {:ok, %Token{}} <- Repo.delete(token) do + {:ok, _token} <- RevokeToken.revoke(app, params) do json(conn, %{}) else _error -> @@ -230,6 +235,15 @@ def token_revoke(conn, %{"token" => token} = params) do end end + def token_revoke(conn, params), do: bad_request(conn, params) + + # Response for bad request + defp bad_request(conn, _) do + conn + |> put_status(500) + |> json(%{error: "Bad request"}) + end + @doc "Prepares OAuth request to provider for Ueberauth" def prepare_request(conn, %{"provider" => provider, "authorization" => auth_attrs}) do scope = @@ -278,25 +292,22 @@ def callback(conn, params) do params = callback_params(params) with {:ok, registration} <- Authenticator.get_registration(conn) do - user = Repo.preload(registration, :user).user auth_attrs = Map.take(params, ~w(client_id redirect_uri scope scopes state)) - if user do - create_authorization( - conn, - %{"authorization" => auth_attrs}, - user: user - ) - else - registration_params = - Map.merge(auth_attrs, %{ - "nickname" => Registration.nickname(registration), - "email" => Registration.email(registration) - }) + case Repo.get_assoc(registration, :user) do + {:ok, user} -> + create_authorization(conn, %{"authorization" => auth_attrs}, user: user) - conn - |> put_session(:registration_id, registration.id) - |> registration_details(%{"authorization" => registration_params}) + _ -> + registration_params = + Map.merge(auth_attrs, %{ + "nickname" => Registration.nickname(registration), + "email" => Registration.email(registration) + }) + + conn + |> put_session(:registration_id, registration.id) + |> registration_details(%{"authorization" => registration_params}) end else _ -> @@ -399,36 +410,30 @@ defp do_create_authorization( end end - # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be - # decoding it. Investigate sometime. - defp fix_padding(token) do - token - |> URI.decode() - |> Base.url_decode64!(padding: false) - |> Base.url_encode64(padding: false) + defp get_app_from_request(conn, params) do + conn + |> fetch_client_credentials(params) + |> fetch_client end - defp get_app_from_request(conn, params) do - # Per RFC 6749, HTTP Basic is preferred to body params - {client_id, client_secret} = - with ["Basic " <> encoded] <- get_req_header(conn, "authorization"), - {:ok, decoded} <- Base.decode64(encoded), - [id, secret] <- - String.split(decoded, ":") - |> Enum.map(fn s -> URI.decode_www_form(s) end) do - {id, secret} - else - _ -> {params["client_id"], params["client_secret"]} - end + defp fetch_client({id, secret}) when is_binary(id) and is_binary(secret) do + Repo.get_by(App, client_id: id, client_secret: secret) + end - if client_id && client_secret do - Repo.get_by( - App, - client_id: client_id, - client_secret: client_secret - ) + defp fetch_client({_id, _secret}), do: nil + + defp fetch_client_credentials(conn, params) do + # Per RFC 6749, HTTP Basic is preferred to body params + with ["Basic " <> encoded] <- get_req_header(conn, "authorization"), + {:ok, decoded} <- Base.decode64(encoded), + [id, secret] <- + Enum.map( + String.split(decoded, ":"), + fn s -> URI.decode_www_form(s) end + ) do + {id, secret} else - nil + _ -> {params["client_id"], params["client_secret"]} end end @@ -441,4 +446,16 @@ defp get_session_registration_id(conn), do: get_session(conn, :registration_id) defp put_session_registration_id(conn, registration_id), do: put_session(conn, :registration_id, registration_id) + + defp response_token(%User{} = user, token, opts \\ %{}) do + %{ + token_type: "Bearer", + access_token: token.token, + refresh_token: token.refresh_token, + expires_in: @expires_in, + scope: Enum.join(token.scopes, " "), + me: user.ap_id + } + |> Map.merge(opts) + end end diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 399140003..4e5d1d118 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.Token do use Ecto.Schema import Ecto.Query + import Ecto.Changeset alias Pleroma.Repo alias Pleroma.User @@ -13,6 +14,9 @@ defmodule Pleroma.Web.OAuth.Token do alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token + @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600) + @type t :: %__MODULE__{} + schema "oauth_tokens" do field(:token, :string) field(:refresh_token, :string) @@ -24,28 +28,67 @@ defmodule Pleroma.Web.OAuth.Token do timestamps() end + @doc "Gets token for app by access token" + @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} + def get_by_token(%App{id: app_id} = _app, token) do + from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token) + |> Repo.find_resource() + end + + @doc "Gets token for app by refresh token" + @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} + def get_by_refresh_token(%App{id: app_id} = _app, token) do + from(t in __MODULE__, + where: t.app_id == ^app_id and t.refresh_token == ^token, + preload: [:user] + ) + |> Repo.find_resource() + end + def exchange_token(app, auth) do with {:ok, auth} <- Authorization.use_token(auth), true <- auth.app_id == app.id do - create_token(app, User.get_cached_by_id(auth.user_id), auth.scopes) + create_token( + app, + User.get_cached_by_id(auth.user_id), + %{scopes: auth.scopes} + ) end end - def create_token(%App{} = app, %User{} = user, scopes \\ nil) do - scopes = scopes || app.scopes - token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) - refresh_token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + defp put_token(changeset) do + changeset + |> change(%{token: Token.Utils.generate_token()}) + |> validate_required([:token]) + |> unique_constraint(:token) + end - token = %Token{ - token: token, - refresh_token: refresh_token, - scopes: scopes, - user_id: user.id, - app_id: app.id, - valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10) - } + defp put_refresh_token(changeset, attrs) do + refresh_token = Map.get(attrs, :refresh_token, Token.Utils.generate_token()) - Repo.insert(token) + changeset + |> change(%{refresh_token: refresh_token}) + |> validate_required([:refresh_token]) + |> unique_constraint(:refresh_token) + end + + defp put_valid_until(changeset, attrs) do + expires_in = + Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in)) + + changeset + |> change(%{valid_until: expires_in}) + |> validate_required([:valid_until]) + end + + def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do + %__MODULE__{user_id: user.id, app_id: app.id} + |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes]) + |> validate_required([:scopes, :user_id, :app_id]) + |> put_valid_until(attrs) + |> put_token + |> put_refresh_token(attrs) + |> Repo.insert() end def delete_user_tokens(%User{id: user_id}) do @@ -73,4 +116,10 @@ def get_user_tokens(%User{id: user_id}) do |> Repo.all() |> Repo.preload(:app) end + + def is_expired?(%__MODULE__{valid_until: valid_until}) do + NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 + end + + def is_expired?(_), do: false end diff --git a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex new file mode 100644 index 000000000..7df0be14e --- /dev/null +++ b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex @@ -0,0 +1,54 @@ +defmodule Pleroma.Web.OAuth.Token.Strategy.RefreshToken do + @moduledoc """ + Functions for dealing with refresh token strategy. + """ + + alias Pleroma.Config + alias Pleroma.Repo + alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token.Strategy.Revoke + + @doc """ + Will grant access token by refresh token. + """ + @spec grant(Token.t()) :: {:ok, Token.t()} | {:error, any()} + def grant(token) do + access_token = Repo.preload(token, [:user, :app]) + + result = + Repo.transaction(fn -> + token_params = %{ + app: access_token.app, + user: access_token.user, + scopes: access_token.scopes + } + + access_token + |> revoke_access_token() + |> create_access_token(token_params) + end) + + case result do + {:ok, {:error, reason}} -> {:error, reason} + {:ok, {:ok, token}} -> {:ok, token} + {:error, reason} -> {:error, reason} + end + end + + defp revoke_access_token(token) do + Revoke.revoke(token) + end + + defp create_access_token({:error, error}, _), do: {:error, error} + + defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do + Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token)) + end + + defp add_refresh_token(params, token) do + case Config.get([:oauth2, :issue_new_refresh_token], false) do + true -> Map.put(params, :refresh_token, token) + false -> params + end + end +end diff --git a/lib/pleroma/web/oauth/token/strategy/revoke.ex b/lib/pleroma/web/oauth/token/strategy/revoke.ex new file mode 100644 index 000000000..dea63ca54 --- /dev/null +++ b/lib/pleroma/web/oauth/token/strategy/revoke.ex @@ -0,0 +1,22 @@ +defmodule Pleroma.Web.OAuth.Token.Strategy.Revoke do + @moduledoc """ + Functions for dealing with revocation. + """ + + alias Pleroma.Repo + alias Pleroma.Web.OAuth.App + alias Pleroma.Web.OAuth.Token + + @doc "Finds and revokes access token for app and by token" + @spec revoke(App.t(), map()) :: {:ok, Token.t()} | {:error, :not_found | Ecto.Changeset.t()} + def revoke(%App{} = app, %{"token" => token} = _attrs) do + with {:ok, token} <- Token.get_by_token(app, token), + do: revoke(token) + end + + @doc "Revokes access token" + @spec revoke(Token.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} + def revoke(%Token{} = token) do + Repo.delete(token) + end +end diff --git a/lib/pleroma/web/oauth/token/utils.ex b/lib/pleroma/web/oauth/token/utils.ex new file mode 100644 index 000000000..a81560a1c --- /dev/null +++ b/lib/pleroma/web/oauth/token/utils.ex @@ -0,0 +1,30 @@ +defmodule Pleroma.Web.OAuth.Token.Utils do + @moduledoc """ + Auxiliary functions for dealing with tokens. + """ + + @doc "convert token inserted_at to unix timestamp" + def format_created_at(%{inserted_at: inserted_at} = _token) do + inserted_at + |> DateTime.from_naive!("Etc/UTC") + |> DateTime.to_unix() + end + + @doc false + @spec generate_token(keyword()) :: binary() + def generate_token(opts \\ []) do + opts + |> Keyword.get(:size, 32) + |> :crypto.strong_rand_bytes() + |> Base.url_encode64(padding: false) + end + + # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be + # decoding it. Investigate sometime. + def fix_padding(token) do + token + |> URI.decode() + |> Base.url_decode64!(padding: false) + |> Base.url_encode64(padding: false) + end +end diff --git a/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs new file mode 100644 index 000000000..449f2a3d4 --- /dev/null +++ b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddRefreshTokenIndexToToken do + use Ecto.Migration + + def change do + create(unique_index(:oauth_tokens, [:refresh_token])) + end +end diff --git a/test/repo_test.exs b/test/repo_test.exs new file mode 100644 index 000000000..5382289c7 --- /dev/null +++ b/test/repo_test.exs @@ -0,0 +1,44 @@ +defmodule Pleroma.RepoTest do + use Pleroma.DataCase + import Pleroma.Factory + + describe "find_resource/1" do + test "returns user" do + user = insert(:user) + query = from(t in Pleroma.User, where: t.id == ^user.id) + assert Repo.find_resource(query) == {:ok, user} + end + + test "returns not_found" do + query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw") + assert Repo.find_resource(query) == {:error, :not_found} + end + end + + describe "get_assoc/2" do + test "get assoc from preloaded data" do + user = %Pleroma.User{name: "Agent Smith"} + token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user} + assert Repo.get_assoc(token, :user) == {:ok, user} + end + + test "get one-to-one assoc from repo" do + user = insert(:user, name: "Jimi Hendrix") + token = refresh_record(insert(:oauth_token, user: user)) + + assert Repo.get_assoc(token, :user) == {:ok, user} + end + + test "get one-to-many assoc from repo" do + user = insert(:user) + notification = refresh_record(insert(:notification, user: user)) + + assert Repo.get_assoc(user, :notifications) == {:ok, [notification]} + end + + test "return error if has not assoc " do + token = insert(:oauth_token, user: nil) + assert Repo.get_assoc(token, :user) == {:error, :not_found} + end + end +end diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index 6e96537ec..cb6836983 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -12,6 +12,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token + @oauth_config_path [:oauth2, :issue_new_refresh_token] @session_opts [ store: :cookie, key: "_test", @@ -714,4 +715,199 @@ test "rejects an invalid authorization code" do refute Map.has_key?(resp, "access_token") end end + + describe "POST /oauth/token - refresh token" do + setup do + oauth_token_config = Pleroma.Config.get(@oauth_config_path) + + on_exit(fn -> + Pleroma.Config.get(@oauth_config_path, oauth_token_config) + end) + end + + test "issues a new access token with keep fresh token" do + Pleroma.Config.put(@oauth_config_path, true) + user = insert(:user) + app = insert(:oauth_app, scopes: ["read", "write"]) + + {:ok, auth} = Authorization.create_authorization(app, user, ["write"]) + {:ok, token} = Token.exchange_token(app, auth) + + response = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "refresh_token", + "refresh_token" => token.refresh_token, + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + |> json_response(200) + + ap_id = user.ap_id + + assert match?( + %{ + "scope" => "write", + "token_type" => "Bearer", + "expires_in" => 600, + "access_token" => _, + "refresh_token" => _, + "me" => ^ap_id + }, + response + ) + + refute Repo.get_by(Token, token: token.token) + new_token = Repo.get_by(Token, token: response["access_token"]) + assert new_token.refresh_token == token.refresh_token + assert new_token.scopes == auth.scopes + assert new_token.user_id == user.id + assert new_token.app_id == app.id + end + + test "issues a new access token with new fresh token" do + Pleroma.Config.put(@oauth_config_path, false) + user = insert(:user) + app = insert(:oauth_app, scopes: ["read", "write"]) + + {:ok, auth} = Authorization.create_authorization(app, user, ["write"]) + {:ok, token} = Token.exchange_token(app, auth) + + response = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "refresh_token", + "refresh_token" => token.refresh_token, + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + |> json_response(200) + + ap_id = user.ap_id + + assert match?( + %{ + "scope" => "write", + "token_type" => "Bearer", + "expires_in" => 600, + "access_token" => _, + "refresh_token" => _, + "me" => ^ap_id + }, + response + ) + + refute Repo.get_by(Token, token: token.token) + new_token = Repo.get_by(Token, token: response["access_token"]) + refute new_token.refresh_token == token.refresh_token + assert new_token.scopes == auth.scopes + assert new_token.user_id == user.id + assert new_token.app_id == app.id + end + + test "returns 400 if we try use access token" do + user = insert(:user) + app = insert(:oauth_app, scopes: ["read", "write"]) + + {:ok, auth} = Authorization.create_authorization(app, user, ["write"]) + {:ok, token} = Token.exchange_token(app, auth) + + response = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "refresh_token", + "refresh_token" => token.token, + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + |> json_response(400) + + assert %{"error" => "Invalid credentials"} == response + end + + test "returns 400 if refresh_token invalid" do + app = insert(:oauth_app, scopes: ["read", "write"]) + + response = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "refresh_token", + "refresh_token" => "token.refresh_token", + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + |> json_response(400) + + assert %{"error" => "Invalid credentials"} == response + end + + test "issues a new token if token expired" do + user = insert(:user) + app = insert(:oauth_app, scopes: ["read", "write"]) + + {:ok, auth} = Authorization.create_authorization(app, user, ["write"]) + {:ok, token} = Token.exchange_token(app, auth) + + change = + Ecto.Changeset.change( + token, + %{valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -86_400 * 30)} + ) + + {:ok, access_token} = Repo.update(change) + + response = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "refresh_token", + "refresh_token" => access_token.refresh_token, + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + |> json_response(200) + + ap_id = user.ap_id + + assert match?( + %{ + "scope" => "write", + "token_type" => "Bearer", + "expires_in" => 600, + "access_token" => _, + "refresh_token" => _, + "me" => ^ap_id + }, + response + ) + + refute Repo.get_by(Token, token: token.token) + token = Repo.get_by(Token, token: response["access_token"]) + assert token + assert token.scopes == auth.scopes + assert token.user_id == user.id + assert token.app_id == app.id + end + end + + describe "POST /oauth/token - bad request" do + test "returns 500" do + response = + build_conn() + |> post("/oauth/token", %{}) + |> json_response(500) + + assert %{"error" => "Bad request"} == response + end + end + + describe "POST /oauth/revoke - bad request" do + test "returns 500" do + response = + build_conn() + |> post("/oauth/revoke", %{}) + |> json_response(500) + + assert %{"error" => "Bad request"} == response + end + end end From c81b7070301ca329353af7417fd55be3786fcb40 Mon Sep 17 00:00:00 2001 From: xse Date: Tue, 7 May 2019 00:32:07 +0000 Subject: [PATCH 70/79] Fix/ex_doc source links --- mix.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index 47f7c2903..c859bed40 100644 --- a/mix.exs +++ b/mix.exs @@ -16,11 +16,11 @@ def project do # Docs name: "Pleroma", - source_url: "https://git.pleroma.social/pleroma/pleroma", - source_url_pattern: - "https://git.pleroma.social/pleroma/pleroma/blob/develop/%{path}#L%{line}", homepage_url: "https://pleroma.social/", + source_url: "https://git.pleroma.social/pleroma/pleroma", docs: [ + source_url_pattern: + "https://git.pleroma.social/pleroma/pleroma/blob/develop/%{path}#L%{line}", logo: "priv/static/static/logo.png", extras: ["README.md", "CHANGELOG.md"] ++ Path.wildcard("docs/**/*.md"), groups_for_extras: [ From f46295faa7002b3afd8cde1c1de302212195ada2 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 7 May 2019 09:07:06 +0700 Subject: [PATCH 71/79] unused config paramater --- config/config.exs | 3 +-- docs/config.md | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 946ba9adf..1e64b79a7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -232,8 +232,7 @@ welcome_message: nil, max_report_comment_size: 1000, safe_dm_mentions: false, - healthcheck: false, - repo_batch_size: 500 + healthcheck: false config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because diff --git a/docs/config.md b/docs/config.md index d999952e1..43ea24d80 100644 --- a/docs/config.md +++ b/docs/config.md @@ -104,7 +104,6 @@ config :pleroma, Pleroma.Emails.Mailer, * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``. -* `repo_batch_size`: Repo batch size. The number of loaded rows from the database to the memory for processing chunks. E.g. deleting user statuses. ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack @@ -170,13 +169,13 @@ config :pleroma, :frontend_configurations, These settings **need to be complete**, they will override the defaults. -NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. +NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. ## :fe __THIS IS DEPRECATED__ If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. -Please **set this option to false** in your config like this: +Please **set this option to false** in your config like this: ```elixir config :pleroma, :fe, false From e71ddf23bac6e9b92f2158ac647e6fb68677b1b0 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 7 May 2019 16:11:10 +0000 Subject: [PATCH 72/79] containment: remove pointless moduledoc line --- lib/pleroma/object/containment.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 70325d6be..2f4687fa2 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -1,7 +1,5 @@ defmodule Pleroma.Object.Containment do @moduledoc """ - help on getting the origin of contained objects - This module contains some useful functions for containing objects to specific origins and determining those origins. They previously lived in the ActivityPub `Transmogrifier` module. From 06947c91471fda6e774c5b6fc04720b87db2b1e6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 3 May 2019 22:15:47 +0300 Subject: [PATCH 73/79] Remove bookmarks assoc --- lib/pleroma/user.ex | 2 -- .../mastodon_api/mastodon_api_controller.ex | 24 +------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fd2ce81ad..b1adaad2f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -10,7 +10,6 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Pleroma.Activity - alias Pleroma.Bookmark alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Registration @@ -54,7 +53,6 @@ defmodule Pleroma.User do field(:search_type, :integer, virtual: true) field(:tags, {:array, :string}, default: []) field(:last_refreshed_at, :naive_datetime_usec) - has_many(:bookmarks, Bookmark) has_many(:notifications, Notification) has_many(:registrations, Registration) embeds_one(:info, Pleroma.User.Info) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index be60e5e3c..2a3d58592 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -295,8 +295,6 @@ def home_timeline(%{assigns: %{user: user}} = conn, params) do |> ActivityPub.contain_timeline(user) |> Enum.reverse() - user = Repo.preload(user, bookmarks: :activity) - conn |> add_link_headers(:home_timeline, activities) |> put_view(StatusView) @@ -315,8 +313,6 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do |> ActivityPub.fetch_public_activities() |> Enum.reverse() - user = Repo.preload(user, bookmarks: :activity) - conn |> add_link_headers(:public_timeline, activities, false, %{"local" => local_only}) |> put_view(StatusView) @@ -324,8 +320,7 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do end def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do - with %User{} = user <- User.get_cached_by_id(params["id"]), - reading_user <- Repo.preload(reading_user, :bookmarks) do + with %User{} = user <- User.get_cached_by_id(params["id"]) do activities = ActivityPub.fetch_user_activities(user, reading_user, params) conn @@ -352,8 +347,6 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do |> ActivityPub.fetch_activities_query(params) |> Pagination.fetch_paginated(params) - user = Repo.preload(user, bookmarks: :activity) - conn |> add_link_headers(:dm_timeline, activities) |> put_view(StatusView) @@ -363,8 +356,6 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do - user = Repo.preload(user, bookmarks: :activity) - conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user}) @@ -514,8 +505,6 @@ def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do with {:ok, announce, _activity} <- CommonAPI.repeat(ap_id_or_id, user), %Activity{} = announce <- Activity.normalize(announce.data) do - user = Repo.preload(user, bookmarks: :activity) - conn |> put_view(StatusView) |> try_render("status.json", %{activity: announce, for: user, as: :activity}) @@ -525,8 +514,6 @@ def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user), %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do - user = Repo.preload(user, bookmarks: :activity) - conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -577,8 +564,6 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), {:ok, _bookmark} <- Bookmark.create(user.id, activity.id) do - user = Repo.preload(user, bookmarks: :activity) - conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -590,8 +575,6 @@ def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), {:ok, _bookmark} <- Bookmark.destroy(user.id, activity.id) do - user = Repo.preload(user, bookmarks: :activity) - conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -1112,8 +1095,6 @@ def favourites(%{assigns: %{user: user}} = conn, params) do ActivityPub.fetch_activities([], params) |> Enum.reverse() - user = Repo.preload(user, bookmarks: :activity) - conn |> add_link_headers(:favourites, activities) |> put_view(StatusView) @@ -1159,7 +1140,6 @@ def user_favourites(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params def bookmarks(%{assigns: %{user: user}} = conn, params) do user = User.get_cached_by_id(user.id) - user = Repo.preload(user, bookmarks: :activity) bookmarks = Bookmark.for_user_query(user.id) @@ -1276,8 +1256,6 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params) |> ActivityPub.fetch_activities_bounded(following, params) |> Enum.reverse() - user = Repo.preload(user, bookmarks: :activity) - conn |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) From f841eb7cdb83afc444dfe260581b6be6e690a717 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 4 May 2019 12:46:42 +0300 Subject: [PATCH 74/79] Preload bookmarks wherever the object is preloaded --- lib/pleroma/activity.ex | 12 ++++++-- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +++++ test/activity_test.exs | 29 ++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 73e63bb14..7845c264a 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Activity do use Ecto.Schema alias Pleroma.Activity + alias Pleroma.Bookmark alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo @@ -36,6 +37,7 @@ defmodule Pleroma.Activity do field(:actor, :string) field(:recipients, {:array, :string}, default: []) has_many(:notifications, Notification, on_delete: :delete_all) + has_many(:bookmarks, Bookmark, on_delete: :delete_all) # Attention: this is a fake relation, don't try to preload it blindly and expect it to work! # The foreign key is embedded in a jsonb field. @@ -71,6 +73,7 @@ def with_preloaded_object(query) do ) ) |> preload([activity, object], object: object) + |> preload(:bookmarks) end def get_by_ap_id(ap_id) do @@ -102,7 +105,8 @@ def get_by_ap_id_with_object(ap_id) do activity.data, activity.data ), - preload: [object: o] + preload: [object: o], + preload: :bookmarks ) ) end @@ -122,7 +126,8 @@ def get_by_id_with_object(id) do activity.data, activity.data ), - preload: [object: o] + preload: [object: o], + preload: :bookmarks ) |> Repo.one() end @@ -200,7 +205,8 @@ def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do activity.data, activity.data ), - preload: [object: o] + preload: [object: o], + preload: :bookmarks ) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6c737d0a4..bd2544470 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -137,6 +137,13 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do activity end + activity = + if activity.data["type"] in ["Create", "Announce"] do + Repo.preload(activity, :bookmarks) + else + activity + end + Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) diff --git a/test/activity_test.exs b/test/activity_test.exs index ad889f544..e2a8baada 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.Object import Pleroma.Factory test "returns an activity by it's AP id" do @@ -28,4 +30,31 @@ test "returns the activity that created an object" do assert activity == found_activity end + + test "preloading object preloads bookmarks" do + user1 = insert(:user) + user2 = insert(:user) + activity = insert(:note_activity) + {:ok, bookmark1} = Bookmark.create(user1.id, activity.id) + {:ok, bookmark2} = Bookmark.create(user2.id, activity.id) + bookmarks = Enum.sort([bookmark1, bookmark2]) + + queried_activity = + Ecto.Query.from(a in Activity, where: a.id == ^activity.id) + |> Activity.with_preloaded_object() + |> Repo.one() + + assert Enum.sort(queried_activity.bookmarks) == bookmarks + + queried_activity = Activity.get_by_ap_id_with_object(activity.data["id"]) + assert Enum.sort(queried_activity.bookmarks) == bookmarks + + queried_activity = Activity.get_by_id_with_object(activity.id) + assert Enum.sort(queried_activity.bookmarks) == bookmarks + + queried_activity = + Activity.get_create_by_object_ap_id_with_object(Object.normalize(activity).data["id"]) + + assert Enum.sort(queried_activity.bookmarks) == bookmarks + end end From 3a7c14645ed726bd6b7deb6489ec0578c4d8cd79 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 4 May 2019 13:42:54 +0300 Subject: [PATCH 75/79] - Actually use preloaded bookmarks in views - Preload bookmarks in bookmark timeline - Rework bookmark preload tests --- lib/pleroma/activity.ex | 13 ++-- lib/pleroma/bookmark.ex | 3 +- .../mastodon_api/mastodon_api_controller.ex | 9 ++- .../web/mastodon_api/views/status_view.ex | 18 +++++- test/activity_test.exs | 60 +++++++++++++------ test/web/mastodon_api/status_view_test.exs | 2 + 6 files changed, 77 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 7845c264a..e432fcb07 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -73,6 +73,11 @@ def with_preloaded_object(query) do ) ) |> preload([activity, object], object: object) + |> with_preloaded_bookmarks() + end + + def with_preloaded_bookmarks(query) do + query |> preload(:bookmarks) end @@ -105,9 +110,9 @@ def get_by_ap_id_with_object(ap_id) do activity.data, activity.data ), - preload: [object: o], - preload: :bookmarks + preload: [object: o] ) + |> with_preloaded_bookmarks() ) end @@ -126,9 +131,9 @@ def get_by_id_with_object(id) do activity.data, activity.data ), - preload: [object: o], - preload: :bookmarks + preload: [object: o] ) + |> with_preloaded_bookmarks() |> Repo.one() end diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index 7f8fd43b6..bbb51c22c 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -38,7 +38,8 @@ def for_user_query(user_id) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) - |> preload([b, a], activity: a) + |> join(:inner, [_b, a], bookmarks in assoc(a, :bookmarks)) + |> preload([b, a, b2], activity: {a, bookmarks: b2}) end def get(user_id, activity_id) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 2a3d58592..451fc85ce 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -563,7 +563,9 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), %User{} = user <- User.get_cached_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, _bookmark} <- Bookmark.create(user.id, activity.id) do + {:ok, bookmark} <- Bookmark.create(user.id, activity.id) do + activity = %{activity | bookmarks: [bookmark | activity.bookmarks]} + conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -575,6 +577,11 @@ def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), {:ok, _bookmark} <- Bookmark.destroy(user.id, activity.id) do + bookmarks = + Enum.filter(activity.bookmarks, fn %{user_id: user_id} -> user_id != user.id end) + + activity = %{activity | bookmarks: bookmarks} + conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 62d064d71..c5a7bcbab 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -80,13 +80,21 @@ def render( user = get_user(activity.data["actor"]) created_at = Utils.to_masto_date(activity.data["published"]) - reblogged_activity = Activity.get_create_by_object_ap_id(object) + reblogged_activity = + Activity.create_by_object_ap_id(object) + |> Activity.with_preloaded_bookmarks() + |> Repo.one() + reblogged = render("status.json", Map.put(opts, :activity, reblogged_activity)) activity_object = Object.normalize(activity) favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || []) - bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], reblogged_activity) + bookmarked = + opts[:for] && Ecto.assoc_loaded?(reblogged_activity.bookmarks) && + Enum.any?(reblogged_activity.bookmarks, fn %{user_id: user_id} -> + user_id == opts[:for].id + end) mentions = activity.recipients @@ -149,7 +157,11 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) - bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], activity) + bookmarked = + opts[:for] && Ecto.assoc_loaded?(activity.bookmarks) && + Enum.any?(activity.bookmarks, fn %{user_id: user_id} -> + user_id == opts[:for].id + end) attachment_data = object.data["attachment"] || [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) diff --git a/test/activity_test.exs b/test/activity_test.exs index e2a8baada..dd149543c 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -31,30 +31,52 @@ test "returns the activity that created an object" do assert activity == found_activity end - test "preloading object preloads bookmarks" do - user1 = insert(:user) - user2 = insert(:user) - activity = insert(:note_activity) - {:ok, bookmark1} = Bookmark.create(user1.id, activity.id) - {:ok, bookmark2} = Bookmark.create(user2.id, activity.id) - bookmarks = Enum.sort([bookmark1, bookmark2]) + describe "preloading bookmarks" do + setup do + user1 = insert(:user) + user2 = insert(:user) + activity = insert(:note_activity) + {:ok, bookmark1} = Bookmark.create(user1.id, activity.id) + {:ok, bookmark2} = Bookmark.create(user2.id, activity.id) + [activity: activity, bookmarks: Enum.sort([bookmark1, bookmark2])] + end - queried_activity = - Ecto.Query.from(a in Activity, where: a.id == ^activity.id) - |> Activity.with_preloaded_object() - |> Repo.one() + test "using with_preloaded_bookmarks", %{activity: activity, bookmarks: bookmarks} do + queried_activity = + Ecto.Query.from(a in Activity, where: a.id == ^activity.id) + |> Activity.with_preloaded_bookmarks() + |> Repo.one() - assert Enum.sort(queried_activity.bookmarks) == bookmarks + assert Enum.sort(queried_activity.bookmarks) == bookmarks + end - queried_activity = Activity.get_by_ap_id_with_object(activity.data["id"]) - assert Enum.sort(queried_activity.bookmarks) == bookmarks + test "using with_preloaded_object", %{activity: activity, bookmarks: bookmarks} do + queried_activity = + Ecto.Query.from(a in Activity, where: a.id == ^activity.id) + |> Activity.with_preloaded_object() + |> Repo.one() - queried_activity = Activity.get_by_id_with_object(activity.id) - assert Enum.sort(queried_activity.bookmarks) == bookmarks + assert Enum.sort(queried_activity.bookmarks) == bookmarks + end - queried_activity = - Activity.get_create_by_object_ap_id_with_object(Object.normalize(activity).data["id"]) + test "using get_by_ap_id_with_object", %{activity: activity, bookmarks: bookmarks} do + queried_activity = Activity.get_by_ap_id_with_object(activity.data["id"]) + assert Enum.sort(queried_activity.bookmarks) == bookmarks + end - assert Enum.sort(queried_activity.bookmarks) == bookmarks + test "using get_by_id_with_object", %{activity: activity, bookmarks: bookmarks} do + queried_activity = Activity.get_by_id_with_object(activity.id) + assert Enum.sort(queried_activity.bookmarks) == bookmarks + end + + test "using get_create_by_object_ap_id_with_object", %{ + activity: activity, + bookmarks: bookmarks + } do + queried_activity = + Activity.get_create_by_object_ap_id_with_object(Object.normalize(activity).data["id"]) + + assert Enum.sort(queried_activity.bookmarks) == bookmarks + end end end diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 5fddc6c58..d7c800e83 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -168,6 +168,8 @@ test "tells if the status is bookmarked" do {:ok, _bookmark} = Bookmark.create(user.id, activity.id) + activity = Activity.get_by_id_with_object(activity.id) + status = StatusView.render("status.json", %{activity: activity, for: user}) assert status.bookmarked == true From be067ec2ab7d3ebcbad7b1306ab19b266dcda664 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 4 May 2019 13:50:39 +0300 Subject: [PATCH 76/79] Use with_preloaded_bookmark in create_by_object_ap_id_with_object --- lib/pleroma/activity.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index e432fcb07..ab41364dc 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -210,9 +210,9 @@ def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do activity.data, activity.data ), - preload: [object: o], - preload: :bookmarks + preload: [object: o] ) + |> with_preloaded_bookmarks() end def create_by_object_ap_id_with_object(_), do: nil From 4c5125dedc429ac021c861c11eb2e41c856ae4e8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 7 May 2019 18:00:50 +0300 Subject: [PATCH 77/79] Remove `bookmarks` assoc and add a fake `bookmark` assoc instead --- lib/pleroma/activity.ex | 29 +++++--- lib/pleroma/bookmark.ex | 3 +- lib/pleroma/web/activity_pub/activity_pub.ex | 15 ++-- .../mastodon_api/mastodon_api_controller.ex | 11 +-- .../web/mastodon_api/views/status_view.ex | 24 +++---- test/activity_test.exs | 68 +++++++++---------- 6 files changed, 71 insertions(+), 79 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index ab41364dc..2b661edc1 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Activity do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.User import Ecto.Changeset import Ecto.Query @@ -36,8 +37,9 @@ defmodule Pleroma.Activity do field(:local, :boolean, default: true) field(:actor, :string) field(:recipients, {:array, :string}, default: []) + # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark + has_one(:bookmark, Bookmark) has_many(:notifications, Notification, on_delete: :delete_all) - has_many(:bookmarks, Bookmark, on_delete: :delete_all) # Attention: this is a fake relation, don't try to preload it blindly and expect it to work! # The foreign key is embedded in a jsonb field. @@ -73,14 +75,18 @@ def with_preloaded_object(query) do ) ) |> preload([activity, object], object: object) - |> with_preloaded_bookmarks() end - def with_preloaded_bookmarks(query) do - query - |> preload(:bookmarks) + def with_preloaded_bookmark(query, %User{} = user) do + from([a] in query, + left_join: b in Bookmark, + on: b.user_id == ^user.id and b.activity_id == a.id, + preload: [bookmark: b] + ) end + def with_preloaded_bookmark(query, _), do: query + def get_by_ap_id(ap_id) do Repo.one( from( @@ -90,6 +96,16 @@ def get_by_ap_id(ap_id) do ) end + def get_bookmark(%Activity{} = activity, %User{} = user) do + if Ecto.assoc_loaded?(activity.bookmark) do + activity.bookmark + else + Bookmark.get(user.id, activity.id) + end + end + + def get_bookmark(_, _), do: nil + def change(struct, params \\ %{}) do struct |> cast(params, [:data]) @@ -112,7 +128,6 @@ def get_by_ap_id_with_object(ap_id) do ), preload: [object: o] ) - |> with_preloaded_bookmarks() ) end @@ -133,7 +148,6 @@ def get_by_id_with_object(id) do ), preload: [object: o] ) - |> with_preloaded_bookmarks() |> Repo.one() end @@ -212,7 +226,6 @@ def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do ), preload: [object: o] ) - |> with_preloaded_bookmarks() end def create_by_object_ap_id_with_object(_), do: nil diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index bbb51c22c..7f8fd43b6 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -38,8 +38,7 @@ def for_user_query(user_id) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) - |> join(:inner, [_b, a], bookmarks in assoc(a, :bookmarks)) - |> preload([b, a, b2], activity: {a, bookmarks: b2}) + |> preload([b, a], activity: a) end def get(user_id, activity_id) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bd2544470..a4053986f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -137,13 +137,6 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do activity end - activity = - if activity.data["type"] in ["Create", "Announce"] do - Repo.preload(activity, :bookmarks) - else - activity - end - Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) @@ -822,11 +815,19 @@ defp maybe_preload_objects(query, _) do |> Activity.with_preloaded_object() end + defp maybe_preload_bookmarks(query, %{"skip_preload" => true}), do: query + + defp maybe_preload_bookmarks(query, opts) do + query + |> Activity.with_preloaded_bookmark(opts["user"]) + end + def fetch_activities_query(recipients, opts \\ %{}) do base_query = from(activity in Activity) base_query |> maybe_preload_objects(opts) + |> maybe_preload_bookmarks(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 451fc85ce..83ad90989 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -563,9 +563,7 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), %User{} = user <- User.get_cached_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, bookmark} <- Bookmark.create(user.id, activity.id) do - activity = %{activity | bookmarks: [bookmark | activity.bookmarks]} - + {:ok, _bookmark} <- Bookmark.create(user.id, activity.id) do conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -577,11 +575,6 @@ def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), {:ok, _bookmark} <- Bookmark.destroy(user.id, activity.id) do - bookmarks = - Enum.filter(activity.bookmarks, fn %{user_id: user_id} -> user_id != user.id end) - - activity = %{activity | bookmarks: bookmarks} - conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -1154,7 +1147,7 @@ def bookmarks(%{assigns: %{user: user}} = conn, params) do activities = bookmarks - |> Enum.map(fn b -> b.activity end) + |> Enum.map(fn b -> Map.put(b.activity, :bookmark, Map.delete(b, :activity)) end) conn |> add_link_headers(:bookmarks, bookmarks) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c5a7bcbab..bd2372944 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -75,26 +75,22 @@ def render("index.json", opts) do def render( "status.json", - %{activity: %{data: %{"type" => "Announce", "object" => object}} = activity} = opts + %{activity: %{data: %{"type" => "Announce", "object" => _object}} = activity} = opts ) do user = get_user(activity.data["actor"]) created_at = Utils.to_masto_date(activity.data["published"]) + activity_object = Object.normalize(activity) reblogged_activity = - Activity.create_by_object_ap_id(object) - |> Activity.with_preloaded_bookmarks() + Activity.create_by_object_ap_id(activity_object.data["id"]) + |> Activity.with_preloaded_bookmark(opts[:for]) |> Repo.one() reblogged = render("status.json", Map.put(opts, :activity, reblogged_activity)) - activity_object = Object.normalize(activity) favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || []) - bookmarked = - opts[:for] && Ecto.assoc_loaded?(reblogged_activity.bookmarks) && - Enum.any?(reblogged_activity.bookmarks, fn %{user_id: user_id} -> - user_id == opts[:for].id - end) + bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil mentions = activity.recipients @@ -104,8 +100,8 @@ def render( %{ id: to_string(activity.id), - uri: object, - url: object, + uri: activity_object.data["id"], + url: activity_object.data["id"], account: AccountView.render("account.json", %{user: user}), in_reply_to_id: nil, in_reply_to_account_id: nil, @@ -157,11 +153,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) - bookmarked = - opts[:for] && Ecto.assoc_loaded?(activity.bookmarks) && - Enum.any?(activity.bookmarks, fn %{user_id: user_id} -> - user_id == opts[:for].id - end) + bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil attachment_data = object.data["attachment"] || [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) diff --git a/test/activity_test.exs b/test/activity_test.exs index dd149543c..7e91d534b 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark - alias Pleroma.Object import Pleroma.Factory test "returns an activity by it's AP id" do @@ -31,52 +30,47 @@ test "returns the activity that created an object" do assert activity == found_activity end - describe "preloading bookmarks" do - setup do - user1 = insert(:user) - user2 = insert(:user) + test "preloading a bookmark" do + user = insert(:user) + user2 = insert(:user) + user3 = insert(:user) + activity = insert(:note_activity) + {:ok, _bookmark} = Bookmark.create(user.id, activity.id) + {:ok, _bookmark2} = Bookmark.create(user2.id, activity.id) + {:ok, bookmark3} = Bookmark.create(user3.id, activity.id) + + queried_activity = + Ecto.Query.from(Pleroma.Activity) + |> Activity.with_preloaded_bookmark(user3) + |> Repo.one() + + assert queried_activity.bookmark == bookmark3 + end + + describe "getting a bookmark" do + test "when association is loaded" do + user = insert(:user) activity = insert(:note_activity) - {:ok, bookmark1} = Bookmark.create(user1.id, activity.id) - {:ok, bookmark2} = Bookmark.create(user2.id, activity.id) - [activity: activity, bookmarks: Enum.sort([bookmark1, bookmark2])] - end + {:ok, bookmark} = Bookmark.create(user.id, activity.id) - test "using with_preloaded_bookmarks", %{activity: activity, bookmarks: bookmarks} do queried_activity = - Ecto.Query.from(a in Activity, where: a.id == ^activity.id) - |> Activity.with_preloaded_bookmarks() + Ecto.Query.from(Pleroma.Activity) + |> Activity.with_preloaded_bookmark(user) |> Repo.one() - assert Enum.sort(queried_activity.bookmarks) == bookmarks + assert Activity.get_bookmark(queried_activity, user) == bookmark end - test "using with_preloaded_object", %{activity: activity, bookmarks: bookmarks} do + test "when association is not loaded" do + user = insert(:user) + activity = insert(:note_activity) + {:ok, bookmark} = Bookmark.create(user.id, activity.id) + queried_activity = - Ecto.Query.from(a in Activity, where: a.id == ^activity.id) - |> Activity.with_preloaded_object() + Ecto.Query.from(Pleroma.Activity) |> Repo.one() - assert Enum.sort(queried_activity.bookmarks) == bookmarks - end - - test "using get_by_ap_id_with_object", %{activity: activity, bookmarks: bookmarks} do - queried_activity = Activity.get_by_ap_id_with_object(activity.data["id"]) - assert Enum.sort(queried_activity.bookmarks) == bookmarks - end - - test "using get_by_id_with_object", %{activity: activity, bookmarks: bookmarks} do - queried_activity = Activity.get_by_id_with_object(activity.id) - assert Enum.sort(queried_activity.bookmarks) == bookmarks - end - - test "using get_create_by_object_ap_id_with_object", %{ - activity: activity, - bookmarks: bookmarks - } do - queried_activity = - Activity.get_create_by_object_ap_id_with_object(Object.normalize(activity).data["id"]) - - assert Enum.sort(queried_activity.bookmarks) == bookmarks + assert Activity.get_bookmark(queried_activity, user) == bookmark end end end From 6020ff3fb696c6b46e948d69a8687182c701a743 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 7 May 2019 19:30:27 +0000 Subject: [PATCH 78/79] activitypub: add optional order constraint to timeline query builder --- lib/pleroma/web/activity_pub/activity_pub.ex | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a4053986f..8f8c23a9b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -822,12 +822,25 @@ defp maybe_preload_bookmarks(query, opts) do |> Activity.with_preloaded_bookmark(opts["user"]) end + defp maybe_order(query, %{order: :desc}) do + query + |> order_by(desc: :id) + end + + defp maybe_order(query, %{order: :asc}) do + query + |> order_by(asc: :id) + end + + defp maybe_order(query, _), do: query + def fetch_activities_query(recipients, opts \\ %{}) do base_query = from(activity in Activity) base_query |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) + |> maybe_order(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) From d64c3b604e4eb5e8dd3f51aff7d1f15024d5fa33 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 7 May 2019 19:30:51 +0000 Subject: [PATCH 79/79] twitterapi: use order constraint to force descending order --- lib/pleroma/web/twitter_api/twitter_api_controller.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index ef7b6fe65..21e6c555a 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -182,6 +182,7 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do |> Map.put("blocking_user", user) |> Map.put("user", user) |> Map.put(:visibility, "direct") + |> Map.put(:order, :desc) activities = ActivityPub.fetch_activities_query([user.ap_id], params)