From fcf2f38d20eed40a53b03374467d9e52b013da07 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 17:37:00 +0200 Subject: [PATCH 1/9] Conversations: Add a function to 'import' old DMs. --- lib/pleroma/conversation.ex | 14 ++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++++ test/conversation_test.exs | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 6e26c5fd4..aa73edd75 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -72,4 +72,18 @@ def create_or_bump_for(activity) do e -> {:error, e} end end + + @doc """ + This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database. + """ + def bump_for_all_activities() do + stream = + Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() + |> Repo.stream() + + Repo.transaction(fn -> + stream + |> Enum.each(&create_or_bump_for/1) + end) + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8f8c23a9b..23cf4e9c4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1061,4 +1061,10 @@ def contain_timeline(timeline, user) do contain_activity(activity, user) end) end + + def fetch_direct_messages_query() do + Activity + |> restrict_type(%{"type" => "Create"}) + |> restrict_visibility(%{visibility: "direct"}) + end end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index f3300e7d1..59368b0e7 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -9,6 +9,24 @@ defmodule Pleroma.ConversationTest do import Pleroma.Factory + test "it goes through old direct conversations" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"visibility" => "direct", "status" => "hey @#{other_user.nickname}"}) + + Repo.delete_all(Conversation) + Repo.delete_all(Conversation.Participation) + + refute Repo.one(Conversation) + + Conversation.bump_for_all_activities() + + assert Repo.one(Conversation) + assert length(Repo.all(Conversation.Participation)) == 2 + end + test "it creates a conversation for given ap_id" do assert {:ok, %Conversation{} = conversation} = Conversation.create_for_ap_id("https://some_ap_id") From 920bd4705526d8dfa8ada516853bbb4e5438cbf1 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 17:40:24 +0200 Subject: [PATCH 2/9] ActivityPub: Remove leftover printf debugging. --- lib/pleroma/web/activity_pub/activity_pub.ex | 42 ++++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 23cf4e9c4..cd8495035 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -533,22 +533,17 @@ def fetch_public_activities(opts \\ %{}) do defp restrict_visibility(query, %{visibility: visibility}) when is_list(visibility) do if Enum.all?(visibility, &(&1 in @valid_visibilities)) do - query = - from( - a in query, - where: - fragment( - "activity_visibility(?, ?, ?) = ANY (?)", - a.actor, - a.recipients, - a.data, - ^visibility - ) - ) - - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - - query + from( + a in query, + where: + fragment( + "activity_visibility(?, ?, ?) = ANY (?)", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) else Logger.error("Could not restrict visibility to #{visibility}") end @@ -556,16 +551,11 @@ defp restrict_visibility(query, %{visibility: visibility}) defp restrict_visibility(query, %{visibility: visibility}) when visibility in @valid_visibilities do - query = - from( - a in query, - where: - fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) - ) - - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - - query + from( + a in query, + where: + fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) + ) end defp restrict_visibility(_query, %{visibility: visibility}) From a4598b5e8bc640ffc1a052438e21f3573ff837ee Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:08:50 +0200 Subject: [PATCH 3/9] Visibility: Make it more resilient. --- lib/pleroma/web/activity_pub/visibility.ex | 9 +++++---- test/web/activity_pub/visibilty_test.exs | 10 ++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 6dee61dd6..e7613a5c8 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -13,11 +13,12 @@ def is_public?(data) do end def is_private?(activity) do - unless is_public?(activity) do - follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address - Enum.any?(activity.data["to"], &(&1 == follower_address)) + with false <- is_public?(activity), + %User{follower_address: follower_address} <- + User.get_cached_by_ap_id(activity.data["actor"]) do + follower_address in activity.data["to"] else - false + _ -> false end end diff --git a/test/web/activity_pub/visibilty_test.exs b/test/web/activity_pub/visibilty_test.exs index 24b96c4aa..ff0e72401 100644 --- a/test/web/activity_pub/visibilty_test.exs +++ b/test/web/activity_pub/visibilty_test.exs @@ -95,4 +95,14 @@ test "visible_for_user?", %{ refute Visibility.visible_for_user?(private, unrelated) refute Visibility.visible_for_user?(direct, unrelated) end + + test "doesn't die when the user doesn't exist", + %{ + direct: direct, + user: user + } do + Repo.delete(user) + Cachex.clear(:user_cache) + refute Visibility.is_private?(direct) + end end From 6d19bb4eae43270099a68f749519ba0f323da01a Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:09:07 +0200 Subject: [PATCH 4/9] Conversations: Add mix task to 'import' old DMs. --- lib/mix/tasks/pleroma/conversations.ex | 23 +++++++++++++++++++++++ lib/pleroma/conversation.ex | 11 +++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 lib/mix/tasks/pleroma/conversations.ex diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex new file mode 100644 index 000000000..125d8851a --- /dev/null +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Conversations do + use Mix.Task + alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation + + @shortdoc "Manages Pleroma users" + @moduledoc """ + Manages Pleroma conversations. + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.conversations bump_all + + """ + def run(["bump_all"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end +end diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index aa73edd75..10c2403e8 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -81,9 +81,12 @@ def bump_for_all_activities() do Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() |> Repo.stream() - Repo.transaction(fn -> - stream - |> Enum.each(&create_or_bump_for/1) - end) + Repo.transaction( + fn -> + stream + |> Enum.each(&create_or_bump_for/1) + end, + timeout: :infinity + ) end end From e6d7f8d223b2604df38f8efa8baf09e2c607c487 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:19:20 +0200 Subject: [PATCH 5/9] Credo fixes. --- lib/pleroma/conversation.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 10c2403e8..0c6ca9f72 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -76,7 +76,7 @@ def create_or_bump_for(activity) do @doc """ This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database. """ - def bump_for_all_activities() do + def bump_for_all_activities do stream = Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() |> Repo.stream() diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cd8495035..8137ac83b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1052,7 +1052,7 @@ def contain_timeline(timeline, user) do end) end - def fetch_direct_messages_query() do + def fetch_direct_messages_query do Activity |> restrict_type(%{"type" => "Create"}) |> restrict_visibility(%{visibility: "direct"}) From a33bec7d58091059d578f6b7537513de11eb0679 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 9 May 2019 16:39:28 +0200 Subject: [PATCH 6/9] Conversations: Import order, import as read. --- lib/pleroma/conversation.ex | 6 +++--- lib/pleroma/conversation/participation.ex | 10 ++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + test/conversation_test.exs | 4 +++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 0c6ca9f72..5f6ab902c 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -45,7 +45,7 @@ def get_for_ap_id(ap_id) do 2. Create a participation for all the people involved who don't have one already 3. Bump all relevant participations to 'unread' """ - def create_or_bump_for(activity) do + def create_or_bump_for(activity, opts \\ []) do with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), object <- Pleroma.Object.normalize(activity), "Create" <- activity.data["type"], @@ -58,7 +58,7 @@ def create_or_bump_for(activity) do participations = Enum.map(users, fn user -> {:ok, participation} = - Participation.create_for_user_and_conversation(user, conversation) + Participation.create_for_user_and_conversation(user, conversation, opts) participation end) @@ -84,7 +84,7 @@ def bump_for_all_activities do Repo.transaction( fn -> stream - |> Enum.each(&create_or_bump_for/1) + |> Enum.each(fn a -> create_or_bump_for(a, read: true) end) end, timeout: :infinity ) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 61021fb18..2a11f9069 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -22,15 +22,17 @@ defmodule Pleroma.Conversation.Participation do def creation_cng(struct, params) do struct - |> cast(params, [:user_id, :conversation_id]) + |> cast(params, [:user_id, :conversation_id, :read]) |> validate_required([:user_id, :conversation_id]) end - def create_for_user_and_conversation(user, conversation) do + def create_for_user_and_conversation(user, conversation, opts \\ []) do + read = !!opts[:read] + %__MODULE__{} - |> creation_cng(%{user_id: user.id, conversation_id: conversation.id}) + |> creation_cng(%{user_id: user.id, conversation_id: conversation.id, read: read}) |> Repo.insert( - on_conflict: [set: [read: false, updated_at: NaiveDateTime.utc_now()]], + on_conflict: [set: [read: read, updated_at: NaiveDateTime.utc_now()]], returning: true, conflict_target: [:user_id, :conversation_id] ) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8137ac83b..728761ebd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1056,5 +1056,6 @@ def fetch_direct_messages_query do Activity |> restrict_type(%{"type" => "Create"}) |> restrict_visibility(%{visibility: "direct"}) + |> order_by([activity], asc: activity.id) end end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 59368b0e7..cdec18f0f 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -24,7 +24,9 @@ test "it goes through old direct conversations" do Conversation.bump_for_all_activities() assert Repo.one(Conversation) - assert length(Repo.all(Conversation.Participation)) == 2 + [participation, _p2] = Repo.all(Conversation.Participation) + + assert participation.read end test "it creates a conversation for given ap_id" do From 786f2c7a849bc4fa2bd4aac18de59ef6b2ed18c5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 May 2019 11:16:54 -0500 Subject: [PATCH 7/9] Update shortdoc description --- lib/mix/tasks/pleroma/conversations.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex index 125d8851a..b52b9921a 100644 --- a/lib/mix/tasks/pleroma/conversations.ex +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -7,7 +7,7 @@ defmodule Mix.Tasks.Pleroma.Conversations do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation - @shortdoc "Manages Pleroma users" + @shortdoc "Manages Pleroma conversations." @moduledoc """ Manages Pleroma conversations. From efa61c161085fd24f7e85ccf7f32ef823e335d7b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2019 13:14:48 -0500 Subject: [PATCH 8/9] Move to pleroma.database mix task --- lib/mix/tasks/pleroma/conversations.ex | 23 ----------------------- lib/mix/tasks/pleroma/database.ex | 11 +++++++++++ 2 files changed, 11 insertions(+), 23 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/conversations.ex diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex deleted file mode 100644 index b52b9921a..000000000 --- a/lib/mix/tasks/pleroma/conversations.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Conversations do - use Mix.Task - alias Mix.Tasks.Pleroma.Common - alias Pleroma.Conversation - - @shortdoc "Manages Pleroma conversations." - @moduledoc """ - Manages Pleroma conversations. - - ## Create a conversation for all existing DMs. Can be safely re-run. - - mix pleroma.conversations bump_all - - """ - def run(["bump_all"]) do - Common.start_pleroma() - Conversation.bump_for_all_activities() - end -end diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ab9a3a7ff..42753a1a4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation require Logger use Mix.Task @@ -19,6 +20,11 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.database bump_all_conversations + """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -48,4 +54,9 @@ def run(["remove_embedded_objects" | args]) do ) end end + + def run(["bump_all_conversations"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end end From cd127d2fa43e6b1ecd7fadfc1d6293b2a432be18 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2019 13:17:09 -0500 Subject: [PATCH 9/9] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d8e1e0c..fe6ab002c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction +- Mix Tasks: `mix pleroma.database bump_all_conversations` - Mix Tasks: `mix pleroma.database remove_embedded_objects` - Mix Tasks: `mix pleroma.user toggle_confirmed` - Federation: Support for reports