From 663007b42c185efce41db73854ff1376a5dae0e5 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 18:44:04 +0000 Subject: [PATCH 01/62] activity: clean up direct use of object data --- lib/pleroma/activity.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index c065f3b6c..3c7e150ee 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Activity do use Ecto.Schema - alias Pleroma.{Repo, Activity, Notification} + alias Pleroma.{Repo, Activity, Notification, Object} import Ecto.Query schema "activities" do @@ -83,9 +83,13 @@ def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) def normalize(_), do: nil - def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do + defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do get_create_activity_by_object_ap_id(ap_id) end - def get_in_reply_to_activity(_), do: nil + defp get_in_reply_to_activity_from_object(_), do: nil + + def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do + get_in_reply_to_activity_from_object(Object.normalize(object)) + end end From 0522b26883534fb93210ece32aed60fcfa4b7a5b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 18:48:37 +0000 Subject: [PATCH 02/62] gopher: use Object.normalize() --- lib/pleroma/gopher/server.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 3b0569a99..1ab15611c 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -37,6 +37,7 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.HTML + alias Pleroma.Object def start_link(ref, socket, transport, opts) do pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts]) @@ -70,14 +71,14 @@ def render_activities(activities) do |> Enum.map(fn activity -> user = User.get_cached_by_ap_id(activity.data["actor"]) - object = activity.data["object"] + object = Object.normalize(activity.data["object"]) like_count = object["like_count"] || 0 announcement_count = object["announcement_count"] || 0 link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <> info("#{like_count} likes, #{announcement_count} repeats") <> "i\tfake\t(NULL)\t0\r\n" <> - info(HTML.strip_tags(String.replace(activity.data["object"]["content"], "
", "\r"))) + info(HTML.strip_tags(String.replace(object["content"], "
", "\r"))) end) |> Enum.join("i\tfake\t(NULL)\t0\r\n") end From d6b266163b9161bf28df7919ffd1391c81b142e3 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 18:57:38 +0000 Subject: [PATCH 03/62] common api: fetch visibility from normalized object --- lib/pleroma/web/common_api/common_api.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e3385310f..e0c9dedd5 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -64,7 +64,10 @@ def get_visibility(%{"in_reply_to_status_id" => status_id}) when not is_nil(stat "public" inReplyTo -> - Pleroma.Web.MastodonAPI.StatusView.get_visibility(inReplyTo.data["object"]) + # XXX: these heuristics should be moved out of MastodonAPI. + with %Object{} = object <- Object.normalize(inReplyTo.data["object"]) do + Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data) + end end end From 67038ae15e1f4a0b136388416af23547d82cbd0d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 19:00:05 +0000 Subject: [PATCH 04/62] common api: utils: access inReplyTo object ID correctly --- lib/pleroma/web/common_api/utils.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 728f24c7e..ec66452c2 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -177,8 +177,10 @@ def make_note_data( } if inReplyTo do + inReplyToObject = Object.normalize(inReplyTo.data["object"]) + object - |> Map.put("inReplyTo", inReplyTo.data["object"]["id"]) + |> Map.put("inReplyTo", inReplyToObject.data["id"]) |> Map.put("inReplyToStatusId", inReplyTo.id) else object From d3fde9b5f28b802b9150e1f8a763f4a67b42aadb Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 19:05:17 +0000 Subject: [PATCH 05/62] ostatus: note handler: appropriately use Object.normalize() --- lib/pleroma/web/ostatus/handlers/note_handler.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 0d4080291..39004367a 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -106,7 +106,8 @@ def handle_note(entry, doc \\ nil) do cw <- OStatus.get_cw(entry), inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo), - inReplyTo <- (inReplyToActivity && inReplyToActivity.data["object"]["id"]) || inReplyTo, + inReplyToObject <- (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil, + inReplyTo <- (inReplyToObject && inReplyToObject.data["id"]) || inReplyTo, attachments <- OStatus.get_attachments(entry), context <- get_context(entry, inReplyTo), tags <- OStatus.get_tags(entry), From 4482ce7e2dbd23e3b7948ee8d084c29d84ae5a1e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:09:28 +0000 Subject: [PATCH 06/62] activitypub: normalize objects when streaming them out --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7e207c620..b11d3221f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -81,6 +81,8 @@ def stream_out(activity) do public = "https://www.w3.org/ns/activitystreams#Public" if activity.data["type"] in ["Create", "Announce"] do + object = Object.normalize(activity.data["object"]) + Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) @@ -91,12 +93,12 @@ def stream_out(activity) do Pleroma.Web.Streamer.stream("public:local", activity) end - activity.data["object"] + object.data |> Map.get("tag", []) |> Enum.filter(fn tag -> is_bitstring(tag) end) |> Enum.map(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - if activity.data["object"]["attachment"] != [] do + if object.data["attachment"] != [] do Pleroma.Web.Streamer.stream("public:media", activity) if activity.local do From 95a458f3925fec4ae31a59face1bc130abdcde28 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:14:00 +0000 Subject: [PATCH 07/62] twitterapi: more object normalization work --- .../representers/activity_representer.ex | 34 ++++++++++--------- .../web/twitter_api/views/activity_view.ex | 6 ++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index fbd33f07e..4b4e202b3 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do use Pleroma.Web.TwitterAPI.Representers.BaseRepresenter alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter - alias Pleroma.{Activity, User} + alias Pleroma.{Activity, User, Object} alias Pleroma.Web.TwitterAPI.{TwitterAPI, UserView, ActivityView} alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Formatter @@ -144,11 +144,13 @@ def to_map( %Activity{data: %{"object" => %{"content" => content} = object}} = activity, %{user: user} = opts ) do - created_at = object["published"] |> Utils.date_to_asctime() - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) - repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) + object = Object.normalize(object) + + created_at = object.data["published"] |> Utils.date_to_asctime() + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || []) mentions = opts[:mentioned] || [] @@ -160,8 +162,8 @@ def to_map( conversation_id = conversation_id(activity) - tags = activity.data["object"]["tag"] || [] - possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags @@ -169,16 +171,16 @@ def to_map( html = HTML.filter_tags(content, User.html_filter_policy(opts[:for])) - |> Formatter.emojify(object["emoji"]) + |> Formatter.emojify(object.data["emoji"]) video = if object["type"] == "Video" do - vid = [object] + vid = [object.data] else [] end - attachments = (object["attachment"] || []) ++ video + attachments = (object.data["attachment"] || []) ++ video reply_parent = Activity.get_in_reply_to_activity(activity) @@ -186,14 +188,14 @@ def to_map( %{ "id" => activity.id, - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user, for: opts[:for]}), "statusnet_html" => html, "text" => HTML.strip_tags(content), "is_local" => activity.local, "is_post_verb" => true, "created_at" => created_at, - "in_reply_to_status_id" => object["inReplyToStatusId"], + "in_reply_to_status_id" => object.data["inReplyToStatusId"], "in_reply_to_screen_name" => reply_user && reply_user.nickname, "in_reply_to_profileurl" => User.profile_url(reply_user), "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id, @@ -205,12 +207,12 @@ def to_map( "repeat_num" => announcement_count, "favorited" => to_boolean(favorited), "repeated" => to_boolean(repeated), - "external_url" => object["external_url"] || object["id"], + "external_url" => object.data["external_url"] || object.data["id"], "tags" => tags, "activity_type" => "post", "possibly_sensitive" => possibly_sensitive, - "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object), - "summary" => object["summary"] + "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data), + "summary" => object.data["summary"] } end diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 83e8fb765..7839fe878 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -225,8 +225,8 @@ def render( conversation_id = get_context_id(activity, opts) - tags = activity.data["object"]["tag"] || [] - possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags @@ -242,7 +242,7 @@ def render( %{ "id" => activity.id, - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user, for: opts[:for]}), "statusnet_html" => html, "text" => HTML.strip_tags(content), From e9b718cea2ee21a38f6c2137f32e3da6f5893cc2 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:21:42 +0000 Subject: [PATCH 08/62] mastodon api: status view: use Object.normalize() --- .../web/mastodon_api/views/status_view.ex | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2d9a915f0..3b4911d53 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do use Pleroma.Web, :view alias Pleroma.Web.MastodonAPI.{AccountView, StatusView} - alias Pleroma.{User, Activity} + alias Pleroma.{User, Activity, Object} alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MediaProxy alias Pleroma.Repo @@ -11,8 +11,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_replied_to_activities(activities) do activities |> Enum.map(fn - %{data: %{"type" => "Create", "object" => %{"inReplyTo" => inReplyTo}}} -> - inReplyTo != "" && inReplyTo + %{data: %{"type" => "Create", "object" => object}} -> + object = Object.normalize(object) + object.data["inReplyTo"] != "" && object.data["inReplyTo"] _ -> nil @@ -21,7 +22,8 @@ defp get_replied_to_activities(activities) do |> Activity.create_activity_by_object_id_query() |> Repo.all() |> Enum.reduce(%{}, fn activity, acc -> - Map.put(acc, activity.data["object"]["id"], activity) + object = Object.normalize(activity.data["object"]) + Map.put(acc, object.data["id"], activity) end) end @@ -85,13 +87,15 @@ def render( end def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do + object = Object.normalize(object) + user = User.get_cached_by_ap_id(activity.data["actor"]) - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 - tags = object["tag"] || [] - sensitive = object["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") mentions = activity.recipients @@ -99,20 +103,20 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) - repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || []) + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) - attachment_data = object["attachment"] || [] - attachment_data = attachment_data ++ if object["type"] == "Video", do: [object], else: [] + attachment_data = object.data["attachment"] || [] + attachment_data = attachment_data ++ if object.data["type"] == "Video", do: [object], else: [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) - created_at = Utils.to_masto_date(object["published"]) + created_at = Utils.to_masto_date(object.data["published"]) reply_to = get_reply_to(activity, opts) reply_to_user = reply_to && User.get_cached_by_ap_id(reply_to.data["actor"]) emojis = - (activity.data["object"]["emoji"] || []) + (object.data["emoji"] || []) |> Enum.map(fn {name, url} -> name = HTML.strip_tags(name) @@ -124,13 +128,13 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} end) content = - render_content(object) + render_content(object.data) |> HTML.filter_tags(User.html_filter_policy(opts[:for])) %{ id: to_string(activity.id), - uri: object["id"], - url: object["external_url"] || object["id"], + uri: object.data["id"], + url: object.data["external_url"] || object["id"], account: AccountView.render("account.json", %{user: user}), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), @@ -144,7 +148,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} favourited: !!favorited, muted: false, sensitive: sensitive, - spoiler_text: object["summary"] || "", + spoiler_text: object.data["summary"] || "", visibility: get_visibility(object), media_attachments: attachments |> Enum.take(4), mentions: mentions, @@ -190,13 +194,15 @@ def render("attachment.json", %{attachment: attachment}) do end def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do - _id = activity.data["object"]["inReplyTo"] - replied_to_activities[activity.data["object"]["inReplyTo"]] + object = Object.normalize(activity.data["object"]) + replied_to_activities[object.data["inReplyTo"]] end def get_reply_to(%{data: %{"object" => object}}, _) do - if object["inReplyTo"] && object["inReplyTo"] != "" do - Activity.get_create_activity_by_object_ap_id(object["inReplyTo"]) + object = Object.normalize(object) + + if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do + Activity.get_create_activity_by_object_ap_id(object.data["inReplyTo"]) else nil end From 5d4a71906a15d4d6bff714ef0bb63fba1c5f4fb3 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:27:00 +0000 Subject: [PATCH 09/62] mastodon api: use Object.normalize() in a few missing spots --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 543fdf416..90225460b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -464,7 +464,8 @@ def upload(%{assigns: %{user: _}} = conn, %{"file" => file} = data) do end def favourited_by(conn, %{"id" => id}) do - with %Activity{data: %{"object" => %{"likes" => likes}}} <- Repo.get(Activity, id) do + with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), + %Object{data: %{"likes" => likes}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^likes) users = Repo.all(q) render(conn, AccountView, "accounts.json", %{users: users, as: :user}) @@ -474,7 +475,8 @@ def favourited_by(conn, %{"id" => id}) do end def reblogged_by(conn, %{"id" => id}) do - with %Activity{data: %{"object" => %{"announcements" => announces}}} <- Repo.get(Activity, id) do + with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), + %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^announces) users = Repo.all(q) render(conn, AccountView, "accounts.json", %{users: users, as: :user}) From dd66cc2ca6c99d5b1586133dd85d8d36d5cbaaa7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:40:16 +0000 Subject: [PATCH 10/62] ostatus: use Object.normalize() where appropriate when representing activities --- .../web/ostatus/activity_representer.ex | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 537bd9f77..ceb4d79db 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -84,11 +84,13 @@ def to_simple_form(activity, user, with_author \\ false) def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do h = fn str -> [to_charlist(str)] end - updated_at = activity.data["object"]["published"] - inserted_at = activity.data["object"]["published"] + object = Object.normalize(activity.data["object"]) + + updated_at = object.data["published"] + inserted_at = object.data["published"] attachments = - Enum.map(activity.data["object"]["attachment"] || [], fn attachment -> + Enum.map(object.data["attachment"] || [], fn attachment -> url = hd(attachment["url"]) {:link, @@ -101,7 +103,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, mentions = activity.recipients |> get_mentions categories = - (activity.data["object"]["tag"] || []) + (object.data["tag"] || []) |> Enum.map(fn tag -> if is_binary(tag) do {:category, [term: to_charlist(tag)], []} @@ -111,11 +113,11 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, end) |> Enum.filter(& &1) - emoji_links = get_emoji_links(activity.data["object"]["emoji"] || %{}) + emoji_links = get_emoji_links(object.data["emoji"] || %{}) summary = - if activity.data["object"]["summary"] do - [{:summary, [], h.(activity.data["object"]["summary"])}] + if object.data["summary"] do + [{:summary, [], h.(object.data["summary"])}] else [] end @@ -124,10 +126,9 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, # For notes, federate the object id. - {:id, h.(activity.data["object"]["id"])}, + {:id, h.(object.data["id"])}, {:title, ['New note by #{user.nickname}']}, - {:content, [type: 'html'], - h.(activity.data["object"]["content"] |> String.replace(~r/[\n\r]/, ""))}, + {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))}, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [ref: h.(activity.data["context"])], From 3b8e5bcbeb03f3cfa5de1c4e7a4f3a04871094d1 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:56:12 +0000 Subject: [PATCH 11/62] fix most tests --- lib/pleroma/web/mastodon_api/views/status_view.ex | 4 ++-- .../web/twitter_api/representers/activity_representer.ex | 4 ++-- lib/pleroma/web/twitter_api/views/activity_view.ex | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3b4911d53..31f4675c3 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -134,7 +134,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} %{ id: to_string(activity.id), uri: object.data["id"], - url: object.data["external_url"] || object["id"], + url: object.data["external_url"] || object.data["id"], account: AccountView.render("account.json", %{user: user}), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), @@ -149,7 +149,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} muted: false, sensitive: sensitive, spoiler_text: object.data["summary"] || "", - visibility: get_visibility(object), + visibility: get_visibility(object.data), media_attachments: attachments |> Enum.take(4), mentions: mentions, # fix, diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index 4b4e202b3..436f9bf92 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -167,14 +167,14 @@ def to_map( tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags - {summary, content} = ActivityView.render_content(object) + {summary, content} = ActivityView.render_content(object.data) html = HTML.filter_tags(content, User.html_filter_policy(opts[:for])) |> Formatter.emojify(object.data["emoji"]) video = - if object["type"] == "Video" do + if object.data["type"] == "Video" do vid = [object.data] else [] diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 7839fe878..f202b6e97 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -265,7 +265,7 @@ def render( "tags" => tags, "activity_type" => "post", "possibly_sensitive" => possibly_sensitive, - "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object), + "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data), "summary" => summary } end From 6f5f589f73da05ee09eef61b6a76a2ef611efa7c Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:08:55 +0000 Subject: [PATCH 12/62] test: fix mastodon api test failure --- test/web/mastodon_api/status_view_test.exs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 31554a07d..4f58ce8af 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -2,21 +2,21 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do use Pleroma.DataCase alias Pleroma.Web.MastodonAPI.{StatusView, AccountView} - alias Pleroma.User + alias Pleroma.{Repo, User, Object} alias Pleroma.Web.OStatus alias Pleroma.Web.CommonAPI import Pleroma.Factory test "a note with null content" do note = insert(:note_activity) + note_object = Object.normalize(note.data["object"]) data = - note.data - |> put_in(["object", "content"], nil) + note_object.data + |> Map.put("content", nil) - note = - note - |> Map.put(:data, data) + Object.change(note_object, %{data: data}) + |> Repo.update() user = User.get_cached_by_ap_id(note.data["actor"]) From e8570758f90f0ef040eab011d0584c59379ba743 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:16:44 +0000 Subject: [PATCH 13/62] twitterapi: fix remaining test failures --- .../representers/activity_representer.ex | 2 +- .../activity_representer_test.exs | 41 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index 436f9bf92..8f91aeaf0 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -141,7 +141,7 @@ def to_map( end def to_map( - %Activity{data: %{"object" => %{"content" => content} = object}} = activity, + %Activity{data: %{"object" => object}} = activity, %{user: user} = opts ) do object = Object.normalize(object) diff --git a/test/web/twitter_api/representers/activity_representer_test.exs b/test/web/twitter_api/representers/activity_representer_test.exs index 7cae4e4a1..314f2b51f 100644 --- a/test/web/twitter_api/representers/activity_representer_test.exs +++ b/test/web/twitter_api/representers/activity_representer_test.exs @@ -87,6 +87,26 @@ test "an activity" do {:ok, convo_object} = Object.context_mapping("2hu") |> Repo.insert() + note_object = %{ + "id" => "https://example.com/id/1", + "published" => date, + "type" => "Note", + "content" => content_html, + "summary" => "2hu", + "inReplyToStatusId" => 213_123, + "attachment" => [object.data], + "external_url" => "some url", + "like_count" => 5, + "announcement_count" => 3, + "context" => "2hu", + "tag" => ["content", "mentioning", "nsfw"], + "emoji" => %{ + "2hu" => "corndog.png" + } + } + + Object.create(note_object) + to = [ User.ap_followers(user), "https://www.w3.org/ns/activitystreams#Public", @@ -100,24 +120,7 @@ test "an activity" do "id" => "id", "to" => to, "actor" => User.ap_id(user), - "object" => %{ - "published" => date, - "type" => "Note", - "content" => content_html, - "summary" => "2hu", - "inReplyToStatusId" => 213_123, - "attachment" => [ - object - ], - "external_url" => "some url", - "like_count" => 5, - "announcement_count" => 3, - "context" => "2hu", - "tag" => ["content", "mentioning", "nsfw"], - "emoji" => %{ - "2hu" => "corndog.png" - } - }, + "object" => note_object["id"], "published" => date, "context" => "2hu" }, @@ -158,7 +161,7 @@ test "an activity" do "tags" => ["nsfw", "content", "mentioning"], "activity_type" => "post", "possibly_sensitive" => true, - "uri" => activity.data["object"]["id"], + "uri" => note_object["id"], "visibility" => "direct", "summary" => "2hu" } From 5ba5df1321b8c177419ae5342d37bc524f8f6656 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:20:29 +0000 Subject: [PATCH 14/62] object: normalize(): use object cache --- lib/pleroma/object.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 03a75dfbd..57a8b1d6b 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -27,8 +27,8 @@ def get_by_ap_id(ap_id) do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id) + def normalize(obj) when is_map(obj), do: normalize(obj["id"]) + def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) def normalize(_), do: nil if Mix.env() == :test do From 5ea64f4bf958d9801d89d1165fa20d0e17bf0570 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:34:36 +0000 Subject: [PATCH 15/62] activity: minor cleanups to normalization functions to align them with the object normalizers --- lib/pleroma/activity.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 3c7e150ee..34d0a34b8 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -79,8 +79,8 @@ def get_create_activity_by_object_ap_id(ap_id) when is_binary(ap_id) do def get_create_activity_by_object_ap_id(_), do: nil - def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) + def normalize(obj) when is_map(obj), do: normalize(obj["id"]) + def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id(ap_id) def normalize(_), do: nil defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do From 57d90e7afe46a54cc7e1ef14a9b76b0650ed3db6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:43:26 +0000 Subject: [PATCH 16/62] activitypub: relay: fix improper use of Object.normalize() --- lib/pleroma/web/activity_pub/relay.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index fcdc6b1c0..a48a91ef7 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -35,7 +35,7 @@ def unfollow(target_instance) do def publish(%Activity{data: %{"type" => "Create"}} = activity) do with %User{} = user <- get_actor(), - %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do + %Object{} = object <- Object.normalize(activity.data["object"]) do ActivityPub.announce(user, object) else e -> Logger.error("error: #{inspect(e)}") From d13d953385b7659d7a9eb381ac9303a41e6fc294 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:44:03 +0000 Subject: [PATCH 17/62] activitypub: implement activity flattening --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b11d3221f..dcf670afb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -57,7 +57,7 @@ def insert(map, local \\ true) when is_map(map) do map <- lazy_put_activity_defaults(map), :ok <- check_actor_is_active(map["actor"]), {:ok, map} <- MRF.filter(map), - :ok <- insert_full_object(map) do + {:ok, map} <- insert_full_object(map) do {recipients, _, _} = get_recipients(map) {:ok, activity} = diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 549148989..bc5b98f1a 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -180,14 +180,18 @@ def lazy_put_object_defaults(map, activity \\ %{}) do @doc """ Inserts a full object if it is contained in an activity. """ - def insert_full_object(%{"object" => %{"type" => type} = object_data}) + def insert_full_object(%{"object" => %{"type" => type} = object_data} = map) when is_map(object_data) and type in @supported_object_types do - with {:ok, _} <- Object.create(object_data) do - :ok + with {:ok, object} <- Object.create(object_data) do + map = + map + |> Map.put("object", object.data["id"]) + + {:ok, map} end end - def insert_full_object(_), do: :ok + def insert_full_object(map), do: {:ok, map} def update_object_in_activities(%{data: %{"id" => id}} = object) do # TODO From d6e65f9304c1598087a6dacc640bcd0bb4057009 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:50:46 +0000 Subject: [PATCH 18/62] common api: fix up improper Object.normalize() calls --- lib/pleroma/web/common_api/common_api.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e0c9dedd5..c83f8a6a9 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.CommonAPI do import Pleroma.Web.CommonAPI.Utils def delete(activity_id, user) do - with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id), + with %Activity{data: %{"object" => object_id}} <- Repo.get(Activity, activity_id), %Object{} = object <- Object.normalize(object_id), true <- user.info.is_moderator || user.ap_id == object.data["actor"], {:ok, delete} <- ActivityPub.delete(object) do @@ -16,7 +16,7 @@ def delete(activity_id, user) do def repeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.announce(user, object) else _ -> @@ -26,7 +26,7 @@ def repeat(id_or_ap_id, user) do def unrepeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.unannounce(user, object) else _ -> @@ -36,7 +36,7 @@ def unrepeat(id_or_ap_id, user) do def favorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.like(user, object) else _ -> @@ -46,7 +46,7 @@ def favorite(id_or_ap_id, user) do def unfavorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.unlike(user, object) else _ -> From cf139e3eec9bf978cfff24044dce5b5c388fa303 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 22:11:10 +0000 Subject: [PATCH 19/62] activitypub: transmogrifier: ensure we send nested object in Create --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 17b063609..5e3d40d9f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -616,9 +616,9 @@ def prepare_object(object) do # internal -> Mastodon # """ - def prepare_outgoing(%{"type" => "Create", "object" => %{"type" => "Note"} = object} = data) do + def prepare_outgoing(%{"type" => "Create", "object" => object_id} = data) do object = - object + Object.normalize(object_id).data |> prepare_object data = From 4ca4c83871a6026ba0e4eb148e99b352edbadb5b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 22:31:07 +0000 Subject: [PATCH 20/62] tests: fix most remaining failures --- test/web/activity_pub/activity_pub_test.exs | 6 +- test/web/activity_pub/transmogrifier_test.exs | 49 ++++++----- test/web/common_api/common_api_test.exs | 18 ++-- test/web/ostatus/ostatus_test.exs | 88 +++++++++++-------- 4 files changed, 95 insertions(+), 66 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 1d561d38d..afdf5c06a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -77,8 +77,10 @@ test "adds an id to a given object if it lacks one and is a note and inserts it } {:ok, %Activity{} = activity} = ActivityPub.insert(data) - assert is_binary(activity.data["object"]["id"]) - assert %Object{} = Object.get_by_ap_id(activity.data["object"]["id"]) + object = Object.normalize(activity.data["object"]) + + assert is_binary(object.data["id"]) + assert %Object{} = Object.get_by_ap_id(activity.data["object"]) end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index e74b8f9a1..ef89752f5 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.OStatus - alias Pleroma.Activity + alias Pleroma.{Activity, Object} alias Pleroma.User alias Pleroma.Repo alias Pleroma.Web.Websub.WebsubClientSubscription @@ -40,16 +40,16 @@ test "it fetches replied-to activities if we don't have them" do |> Map.put("object", object) {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + returned_object = Object.normalize(returned_activity.data["object"]) assert activity = Activity.get_create_activity_by_object_ap_id( "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" ) - assert returned_activity.data["object"]["inReplyToAtomUri"] == - "https://shitposter.club/notice/2827873" + assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" - assert returned_activity.data["object"]["inReplyToStatusId"] == activity.id + assert returned_object.data["inReplyToStatusId"] == activity.id end test "it works for incoming notices" do @@ -72,7 +72,7 @@ test "it works for incoming notices" do assert data["actor"] == "http://mastodon.example.org/users/admin" - object = data["object"] + object = Object.normalize(data["object"]).data assert object["id"] == "http://mastodon.example.org/users/admin/statuses/99512778738411822" assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"] @@ -99,7 +99,9 @@ test "it works for incoming notices with hashtags" do data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - assert Enum.at(data["object"]["tag"], 2) == "moo" + object = Object.normalize(data["object"]) + + assert Enum.at(object.data["tag"], 2) == "moo" end test "it works for incoming notices with contentMap" do @@ -107,8 +109,9 @@ test "it works for incoming notices with contentMap" do File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["content"] == + assert object.data["content"] == "

@lain

" end @@ -116,8 +119,9 @@ test "it works for incoming notices with to/cc not being an array (kroeg)" do data = File.read!("test/fixtures/kroeg-post-activity.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["content"] == + assert object.data["content"] == "

henlo from my Psion netBook

message sent from my Psion netBook

" end @@ -133,24 +137,27 @@ test "it works for incoming notices with tag not being an array (kroeg)" do data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["emoji"] == %{ + assert object.data["emoji"] == %{ "icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png" } data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert "test" in data["object"]["tag"] + assert "test" in object.data["tag"] end test "it works for incoming notices with url not being a string (prismo)" do data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["url"] == "https://prismo.news/posts/83" + assert object.data["url"] == "https://prismo.news/posts/83" end test "it works for incoming follow requests" do @@ -193,14 +200,14 @@ test "it works for incoming likes" do data = File.read!("test/fixtures/mastodon-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) assert data["actor"] == "http://mastodon.example.org/users/admin" assert data["type"] == "Like" assert data["id"] == "http://mastodon.example.org/users/admin#likes/2" - assert data["object"] == activity.data["object"]["id"] + assert data["object"] == activity.data["object"] end test "it returns an error for incoming unlikes wihout a like activity" do @@ -210,7 +217,7 @@ test "it returns an error for incoming unlikes wihout a like activity" do data = File.read!("test/fixtures/mastodon-undo-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) assert Transmogrifier.handle_incoming(data) == :error end @@ -222,7 +229,7 @@ test "it works for incoming unlikes with an existing like activity" do like_data = File.read!("test/fixtures/mastodon-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: like_data, local: false}} = Transmogrifier.handle_incoming(like_data) @@ -264,7 +271,7 @@ test "it works for incoming announces with an existing activity" do data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) @@ -274,7 +281,7 @@ test "it works for incoming announces with an existing activity" do assert data["id"] == "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity" - assert data["object"] == activity.data["object"]["id"] + assert data["object"] == activity.data["object"] assert Activity.get_create_activity_by_object_ap_id(data["object"]).id == activity.id end @@ -349,7 +356,7 @@ test "it works for incoming deletes" do object = data["object"] - |> Map.put("id", activity.data["object"]["id"]) + |> Map.put("id", activity.data["object"]) data = data @@ -370,7 +377,7 @@ test "it fails for incoming deletes with spoofed origin" do object = data["object"] - |> Map.put("id", activity.data["object"]["id"]) + |> Map.put("id", activity.data["object"]) data = data @@ -388,7 +395,7 @@ test "it works for incoming unannounces with an existing notice" do announce_data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: announce_data, local: false}} = Transmogrifier.handle_incoming(announce_data) @@ -403,7 +410,7 @@ test "it works for incoming unannounces with an existing notice" do assert data["type"] == "Undo" assert data["object"]["type"] == "Announce" - assert data["object"]["object"] == activity.data["object"]["id"] + assert data["object"]["object"] == activity.data["object"] assert data["object"]["id"] == "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity" diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 8fc65f4c0..3f81d952c 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -1,7 +1,7 @@ defmodule Pleroma.Web.CommonAPI.Test do use Pleroma.DataCase alias Pleroma.Web.CommonAPI - alias Pleroma.User + alias Pleroma.{User, Object} import Pleroma.Factory @@ -9,7 +9,9 @@ test "it de-duplicates tags" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "#2hu #2HU"}) - assert activity.data["object"]["tag"] == ["2hu"] + object = Object.normalize(activity.data["object"]) + + assert object.data["tag"] == ["2hu"] end test "it adds emoji when updating profiles" do @@ -34,8 +36,10 @@ test "it filters out obviously bad tags when accepting a post as HTML" do "content_type" => "text/html" }) - content = activity.data["object"]["content"] - assert content == "

2hu

alert('xss')" + object = + Object.normalize(activity.data["object"]) + + assert object.data["content"] == "

2hu

alert('xss')" end test "it filters out obviously bad tags when accepting a post as Markdown" do @@ -49,8 +53,10 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do "content_type" => "text/markdown" }) - content = activity.data["object"]["content"] - assert content == "

2hu

alert('xss')" + object = + Object.normalize(activity.data["object"]) + + assert object.data["content"] == "

2hu

alert('xss')" end end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 32bf6691b..73e2b5482 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -15,34 +15,36 @@ test "don't insert create notes twice" do test "handle incoming note - GS, Salmon" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) user = User.get_by_ap_id(activity.data["actor"]) assert user.info.note_count == 1 assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" + assert object.data["type"] == "Note" - assert activity.data["object"]["id"] == + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" - assert activity.data["object"]["published"] == "2017-04-23T14:51:03+00:00" + assert object.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] - assert activity.data["object"]["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"} + assert object.data["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"} assert activity.local == false end test "handle incoming notes - GS, subscription" do incoming = File.read!("test/fixtures/ostatus_incoming_post.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["content"] == "Will it blend?" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["content"] == "Will it blend?" user = User.get_cached_by_ap_id(activity.data["actor"]) assert User.ap_followers(user) in activity.data["to"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -51,20 +53,22 @@ test "handle incoming notes - GS, subscription" do test "handle incoming notes with attachments - GS, subscription" do incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["attachment"] |> length == 2 - assert activity.data["object"]["external_url"] == "https://social.heldscal.la/notice/2020923" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["attachment"] |> length == 2 + assert object.data["external_url"] == "https://social.heldscal.la/notice/2020923" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end test "handle incoming notes with tags" do incoming = File.read!("test/fixtures/ostatus_incoming_post_tag.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) - assert activity.data["object"]["tag"] == ["nsfw"] + assert object.data["tag"] == ["nsfw"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end @@ -79,10 +83,11 @@ test "handle incoming notes - Mastodon, salmon, reply" do incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" assert activity.data["context"] == "2hu" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end @@ -90,42 +95,47 @@ test "handle incoming notes - Mastodon, salmon, reply" do test "handle incoming notes - Mastodon, with CW" do incoming = File.read!("test/fixtures/mastodon-note-cw.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["object"]["summary"] == "technologic" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" + assert object.data["summary"] == "technologic" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end test "handle incoming unlisted messages, put public into cc" do incoming = File.read!("test/fixtures/mastodon-note-unlisted.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) + refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["cc"] - refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["to"] - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in object.data["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in object.data["cc"] end test "handle incoming retweets - Mastodon, with CW" do incoming = File.read!("test/fixtures/cw_retweet.xml") {:ok, [[_activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) - assert retweeted_activity.data["object"]["summary"] == "Hey." + assert retweeted_object.data["summary"] == "Hey." end test "handle incoming notes - GS, subscription, reply" do incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["content"] == + assert object.data["content"] == "@shpbot why not indeed." - assert activity.data["object"]["inReplyTo"] == + assert object.data["inReplyTo"] == "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -137,17 +147,19 @@ test "handle incoming retweets - GS, subscription" do assert activity.data["type"] == "Announce" assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == retweeted_activity.data["object"]["id"] + assert activity.data["object"] == retweeted_activity.data["object"] assert "https://pleroma.soykaf.com/users/lain" in activity.data["to"] refute activity.local retweeted_activity = Repo.get(Activity, retweeted_activity.id) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) + assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" refute retweeted_activity.local - assert retweeted_activity.data["object"]["announcement_count"] == 1 - assert String.contains?(retweeted_activity.data["object"]["content"], "mastodon") - refute String.contains?(retweeted_activity.data["object"]["content"], "Test account") + assert retweeted_object.data["announcement_count"] == 1 + assert String.contains?(retweeted_object.data["content"], "mastodon") + refute String.contains?(retweeted_object.data["content"], "Test account") end test "handle incoming retweets - GS, subscription - local message" do @@ -179,10 +191,11 @@ test "handle incoming retweets - GS, subscription - local message" do test "handle incoming retweets - Mastodon, salmon" do incoming = File.read!("test/fixtures/share.xml") {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) assert activity.data["type"] == "Announce" assert activity.data["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["object"] == retweeted_activity.data["object"]["id"] + assert activity.data["object"] == retweeted_activity.data["object"] assert activity.data["id"] == "tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status" @@ -191,7 +204,7 @@ test "handle incoming retweets - Mastodon, salmon" do assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" refute retweeted_activity.local - refute String.contains?(retweeted_activity.data["object"]["content"], "Test account") + refute String.contains?(retweeted_object.data["content"], "Test account") end test "handle incoming favorites - GS, websub" do @@ -201,7 +214,7 @@ test "handle incoming favorites - GS, websub" do assert activity.data["type"] == "Like" assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == favorited_activity.data["object"]["id"] + assert activity.data["object"] == favorited_activity.data["object"] assert activity.data["id"] == "tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00" @@ -210,7 +223,7 @@ test "handle incoming favorites - GS, websub" do assert favorited_activity.data["type"] == "Create" assert favorited_activity.data["actor"] == "https://shitposter.club/user/1" - assert favorited_activity.data["object"]["id"] == + assert favorited_activity.data["object"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" refute favorited_activity.local @@ -245,16 +258,17 @@ test "handle incoming favorites with locally available object - GS, websub" do test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" + assert object.data["type"] == "Note" - assert activity.data["object"]["inReplyTo"] == + assert object.data["inReplyTo"] == "http://pleroma.example.org:4000/objects/55bce8fc-b423-46b1-af71-3759ab4670bc" assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] - assert activity.data["object"]["id"] == + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -466,7 +480,7 @@ test "it builds a missing status from an html url" do assert activity.data["actor"] == "https://shitposter.club/user/1" - assert activity.data["object"]["id"] == + assert activity.data["object"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" end) end @@ -475,7 +489,7 @@ test "it works for atom notes, too" do url = "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056" {:ok, [activity]} = OStatus.fetch_activity_from_url(url) assert activity.data["actor"] == "https://social.sakamoto.gq/users/eal" - assert activity.data["object"]["id"] == url + assert activity.data["object"] == url end end From f7e15d3257c0428ba9d37bdbbd45aa5fe6829a1b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 22:49:39 +0000 Subject: [PATCH 21/62] tests: fix a lot of the remaining test failures --- test/web/common_api/common_api_test.exs | 6 ++--- .../mastodon_api_controller_test.exs | 5 ++-- test/web/ostatus/ostatus_test.exs | 6 ++--- test/web/twitter_api/twitter_api_test.exs | 26 ++++++++++--------- .../twitter_api/views/activity_view_test.exs | 18 +++++++------ 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 3f81d952c..3dc5f6f84 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -36,8 +36,7 @@ test "it filters out obviously bad tags when accepting a post as HTML" do "content_type" => "text/html" }) - object = - Object.normalize(activity.data["object"]) + object = Object.normalize(activity.data["object"]) assert object.data["content"] == "

2hu

alert('xss')" end @@ -53,8 +52,7 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do "content_type" => "text/markdown" }) - object = - Object.normalize(activity.data["object"]) + object = Object.normalize(activity.data["object"]) assert object.data["content"] == "

2hu

alert('xss')" end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 098acb59f..c30f253d9 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Web.TwitterAPI.TwitterAPI - alias Pleroma.{Repo, User, Activity, Notification} + alias Pleroma.{Repo, User, Activity, Notification, Object} alias Pleroma.Web.{OStatus, CommonAPI} alias Pleroma.Web.ActivityPub.ActivityPub @@ -219,9 +219,10 @@ test "replying to a status", %{conn: conn} do assert %{"content" => "xD", "id" => id} = json_response(conn, 200) activity = Repo.get(Activity, id) + object = Object.normalize(activity.data["object"]) assert activity.data["context"] == replied_to.data["context"] - assert activity.data["object"]["inReplyToStatusId"] == replied_to.id + assert object.data["inReplyToStatusId"] == replied_to.id end test "posting a status with an invalid in_reply_to_id", %{conn: conn} do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 73e2b5482..b5805c668 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -22,8 +22,7 @@ test "handle incoming note - GS, Salmon" do assert activity.data["type"] == "Create" assert object.data["type"] == "Note" - assert object.data["id"] == - "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert object.data["published"] == "2017-04-23T14:51:03+00:00" @@ -268,8 +267,7 @@ test "handle incoming replies" do assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] - assert object.data["id"] == - "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 28230699f..bc53fe68a 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -33,13 +33,14 @@ test "create a status" do } {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input) + object = Object.normalize(activity.data["object"]) expected_text = "Hello again, @shp.<script></script>
This is on another :moominmamma: line.
image.jpg" - assert get_in(activity.data, ["object", "content"]) == expected_text - assert get_in(activity.data, ["object", "type"]) == "Note" - assert get_in(activity.data, ["object", "actor"]) == user.ap_id + assert get_in(object.data, ["content"]) == expected_text + assert get_in(object.data, ["type"]) == "Note" + assert get_in(object.data, ["actor"]) == user.ap_id assert get_in(activity.data, ["actor"]) == user.ap_id assert Enum.member?(get_in(activity.data, ["cc"]), User.ap_followers(user)) @@ -52,18 +53,18 @@ test "create a status" do assert activity.local == true assert %{"moominmamma" => "http://localhost:4001/finmoji/128px/moominmamma-128.png"} = - activity.data["object"]["emoji"] + object.data["emoji"] # hashtags - assert activity.data["object"]["tag"] == ["2hu", "epic", "phantasmagoric"] + assert object.data["tag"] == ["2hu", "epic", "phantasmagoric"] # Add a context assert is_binary(get_in(activity.data, ["context"])) - assert is_binary(get_in(activity.data, ["object", "context"])) + assert is_binary(get_in(object.data, ["context"])) - assert is_list(activity.data["object"]["attachment"]) + assert is_list(object.data["attachment"]) - assert activity.data["object"] == Object.get_by_ap_id(activity.data["object"]["id"]).data + assert activity.data["object"] == object.data["id"] user = User.get_by_ap_id(user.ap_id) @@ -78,6 +79,7 @@ test "create a status that is a reply" do } {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input) + object = Object.normalize(activity.data["object"]) input = %{ "status" => "Here's your (you).", @@ -85,14 +87,14 @@ test "create a status that is a reply" do } {:ok, reply = %Activity{}} = TwitterAPI.create_status(user, input) + reply_object = Object.normalize(reply.data["object"]) assert get_in(reply.data, ["context"]) == get_in(activity.data, ["context"]) - assert get_in(reply.data, ["object", "context"]) == - get_in(activity.data, ["object", "context"]) + assert get_in(reply_object.data, ["context"]) == get_in(object.data, ["context"]) - assert get_in(reply.data, ["object", "inReplyTo"]) == get_in(activity.data, ["object", "id"]) - assert get_in(reply.data, ["object", "inReplyToStatusId"]) == activity.id + assert get_in(reply_object.data, ["inReplyTo"]) == get_in(activity.data, ["object"]) + assert get_in(reply_object.data, ["inReplyToStatusId"]) == activity.id end test "Follow another user using user_id" do diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index 5cef06f88..f4741cf24 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do alias Pleroma.Web.TwitterAPI.UserView alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Repo - alias Pleroma.Activity + alias Pleroma.{Activity, Object} alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -19,10 +19,11 @@ test "a create activity with a note" do other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + object = Object.normalize(activity.data["object"]) result = ActivityView.render("activity.json", activity: activity) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = TwitterAPI.context_to_conversation_id(object.data["context"]) expected = %{ "activity_type" => "post", @@ -30,8 +31,8 @@ test "a create activity with a note" do "attentions" => [ UserView.render("show.json", %{user: other_user}) ], - "created_at" => activity.data["object"]["published"] |> Utils.date_to_asctime(), - "external_url" => activity.data["object"]["id"], + "created_at" => object.data["published"] |> Utils.date_to_asctime(), + "external_url" => object.data["id"], "fave_num" => 0, "favorited" => false, "id" => activity.id, @@ -50,7 +51,7 @@ test "a create activity with a note" do "Hey @shp!", "tags" => [], "text" => "Hey @shp!", - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user}), "visibility" => "direct", "summary" => nil @@ -63,8 +64,9 @@ test "a list of activities" do user = insert(:user) other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) + object = Object.normalize(activity.data["object"]) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = TwitterAPI.context_to_conversation_id(object.data["context"]) mocks = [ { @@ -162,9 +164,9 @@ test "an announce activity" do other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) - {:ok, announce, _object} = CommonAPI.repeat(activity.id, other_user) + {:ok, announce, object} = CommonAPI.repeat(activity.id, other_user) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = TwitterAPI.context_to_conversation_id(object.data["context"]) activity = Repo.get(Activity, activity.id) From f168a2add66a312aa9911c880806534899f3fe08 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 03:06:20 +0000 Subject: [PATCH 22/62] ostatus: fix representing external objects --- .../web/ostatus/activity_representer.ex | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index ceb4d79db..fefd9459a 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -47,23 +47,16 @@ defp get_mentions(to) do end) end - defp get_links(%{local: true, data: data}) do + defp get_links(%{local: true}, %{"id" => object_id}) do h = fn str -> [to_charlist(str)] end [ - {:link, [type: ['application/atom+xml'], href: h.(data["object"]["id"]), rel: 'self'], []}, - {:link, [type: ['text/html'], href: h.(data["object"]["id"]), rel: 'alternate'], []} + {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []}, + {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []} ] end - defp get_links(%{ - local: false, - data: %{ - "object" => %{ - "external_url" => external_url - } - } - }) do + defp get_links(%{local: false}, %{"external_url" => external_url}) do h = fn str -> [to_charlist(str)] end [ @@ -71,7 +64,7 @@ defp get_links(%{ ] end - defp get_links(_activity), do: [] + defp get_links(_activity, _object_data), do: [] defp get_emoji_links(emojis) do Enum.map(emojis, fn {emoji, file} -> @@ -81,7 +74,7 @@ defp get_emoji_links(emojis) do def to_simple_form(activity, user, with_author \\ false) - def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do + def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do h = fn str -> [to_charlist(str)] end object = Object.normalize(activity.data["object"]) @@ -136,7 +129,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} ] ++ summary ++ - get_links(activity) ++ + get_links(activity, object.data) ++ categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links end From fe000f82f8e1a37a063c7260033f6c3fd275c8cc Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 03:27:59 +0000 Subject: [PATCH 23/62] tests: activitypub: fix broken test due to invalid child object --- test/web/activity_pub/activity_pub_test.exs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index afdf5c06a..231a334f9 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -93,7 +93,11 @@ test "removes doubled 'to' recipients" do to: ["user1", "user1", "user2"], actor: user, context: "", - object: %{} + object: %{ + "to" => ["user1", "user1", "user2"], + "type" => "Note", + "content" => "testing" + } }) assert activity.data["to"] == ["user1", "user2"] From 6f90f2c3ac70c74b9d06debb09530d5f479b5a8c Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 03:34:57 +0000 Subject: [PATCH 24/62] activitypub: rework thread filtering for split object view --- lib/pleroma/web/activity_pub/activity_pub.ex | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index dcf670afb..34a84b045 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -799,18 +799,21 @@ def visible_for_user?(activity, user) do # guard def entire_thread_visible_for_user?(nil, user), do: false - # child + # child / root def entire_thread_visible_for_user?( - %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, + %Activity{data: %{"object" => object_id}} = tail, user - ) - when is_binary(parent_id) do + ) do parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - end - # root - def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user) + cond do + !is_nil(parent) -> + visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) + + true -> + visible_for_user?(tail, user) + end + end # filter out broken threads def contain_broken_threads(%Activity{} = activity, %User{} = user) do From 5d753e1c7cb1b26ad224255c31b0b64ad917ebaa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 06:52:54 +0000 Subject: [PATCH 25/62] activity: add helpers for updating activities in the database --- lib/pleroma/activity.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 34d0a34b8..e3aa4eb97 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Activity do use Ecto.Schema alias Pleroma.{Repo, Activity, Notification, Object} - import Ecto.Query + import Ecto.{Query, Changeset} schema "activities" do field(:data, :map) @@ -22,6 +22,13 @@ def get_by_ap_id(ap_id) do ) end + def change(struct, params \\ %{}) do + struct + |> cast(params, [:data]) + |> validate_required([:data]) + |> unique_constraint(:ap_id, name: :activities_unique_apid_index) + end + # TODO: # Go through these and fix them everywhere. # Wrong name, only returns create activities From ef56488349a257def67d6c906a1f71e9bbed397e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 06:53:17 +0000 Subject: [PATCH 26/62] mix: add task to compact the database --- lib/mix/tasks/compact_database.ex | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/mix/tasks/compact_database.ex diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex new file mode 100644 index 000000000..b84b340ac --- /dev/null +++ b/lib/mix/tasks/compact_database.ex @@ -0,0 +1,57 @@ +defmodule Mix.Tasks.CompactDatabase do + @moduledoc """ + Compact the database by flattening the object graph. + """ + + require Logger + + use Mix.Task + import Mix.Ecto + import Ecto.Query + alias Pleroma.{Repo, Object, Activity} + + defp maybe_compact(%Activity{data: %{"object" => %{"id" => object_id}}} = activity) do + data = + activity.data + |> Map.put("object", object_id) + + {:ok, activity} = + Activity.change(activity, %{data: data}) + |> Repo.update() + + {:ok, activity} + end + + defp maybe_compact(%Activity{} = activity), do: {:ok, activity} + + defp activity_query(min_id, max_id) do + from( + a in Activity, + where: fragment("?->>'type' = 'Create'", a.data), + where: a.id >= ^min_id, + where: a.id < ^max_id + ) + end + + def run(args) do + Application.ensure_all_started(:pleroma) + + max = Repo.aggregate(Activity, :max, :id) + Logger.info("Considering #{max} activities") + + chunks = 0..(round(max / 100)) + + Enum.each(chunks, fn (i) -> + min = i * 100 + max = min + 100 + + activity_query(min, max) + |> Repo.all() + |> Enum.each(&maybe_compact/1) + + IO.write(".") + end) + + Logger.info("Finished.") + end +end From 1a360a4eaa57d57ccc4f03dfa25e82a240d0175a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 22:17:28 +0000 Subject: [PATCH 27/62] compact database task: fix formatting --- lib/mix/tasks/compact_database.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex index b84b340ac..7de50812a 100644 --- a/lib/mix/tasks/compact_database.ex +++ b/lib/mix/tasks/compact_database.ex @@ -39,9 +39,9 @@ def run(args) do max = Repo.aggregate(Activity, :max, :id) Logger.info("Considering #{max} activities") - chunks = 0..(round(max / 100)) + chunks = 0..round(max / 100) - Enum.each(chunks, fn (i) -> + Enum.each(chunks, fn i -> min = i * 100 max = min + 100 From e8caecb5c7f49a829b857131ff98a46c705e3a80 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 22:29:41 +0000 Subject: [PATCH 28/62] object: move object containment out of transmogrifier into it's own module --- lib/pleroma/object/containment.ex | 64 +++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 5 +- .../web/activity_pub/transmogrifier.ex | 71 +++---------------- test/object/containment_test.exs | 66 +++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 57 --------------- 5 files changed, 144 insertions(+), 119 deletions(-) create mode 100644 lib/pleroma/object/containment.ex create mode 100644 test/object/containment_test.exs diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex new file mode 100644 index 000000000..010b768bd --- /dev/null +++ b/lib/pleroma/object/containment.ex @@ -0,0 +1,64 @@ +defmodule Pleroma.Object.Containment do + @moduledoc """ + # Object Containment + + This module contains some useful functions for containing objects to specific + origins and determining those origins. They previously lived in the + ActivityPub `Transmogrifier` module. + + Object containment is an important step in validating remote objects to prevent + spoofing, therefore removal of object containment functions is NOT recommended. + """ + + require Logger + + def get_actor(%{"actor" => actor}) when is_binary(actor) do + actor + end + + def get_actor(%{"actor" => actor}) when is_list(actor) do + if is_binary(Enum.at(actor, 0)) do + Enum.at(actor, 0) + else + Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) + |> Map.get("id") + end + end + + def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do + id + end + + def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do + get_actor(%{"actor" => actor}) + end + + @doc """ + Checks that an imported AP object's actor matches the domain it came from. + """ + def contain_origin(id, %{"actor" => nil}), do: :error + + def contain_origin(id, %{"actor" => actor} = params) do + id_uri = URI.parse(id) + actor_uri = URI.parse(get_actor(params)) + + if id_uri.host == actor_uri.host do + :ok + else + :error + end + end + + def contain_origin_from_id(id, %{"id" => nil}), do: :error + + def contain_origin_from_id(id, %{"id" => other_id} = params) do + id_uri = URI.parse(id) + other_uri = URI.parse(other_id) + + if id_uri.host == other_uri.host do + :ok + else + :error + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 34a84b045..517cf4b46 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} + alias Pleroma.Object.Containment alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.WebFinger alias Pleroma.Web.Federator @@ -739,7 +740,7 @@ def fetch_object_from_id(id) do "actor" => data["actor"] || data["attributedTo"], "object" => data }, - :ok <- Transmogrifier.contain_origin(id, params), + :ok <- Containment.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do {:ok, Object.normalize(activity.data["object"])} else @@ -773,7 +774,7 @@ def fetch_and_contain_remote_object_from_id(id) do recv_timeout: 20000 ), {:ok, data} <- Jason.decode(body), - :ok <- Transmogrifier.contain_origin_from_id(id, data) do + :ok <- Containment.contain_origin_from_id(id, data) do {:ok, data} else e -> diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5e3d40d9f..1b5e57294 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.User alias Pleroma.Object + alias Pleroma.Object.Containment alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.Web.ActivityPub.ActivityPub @@ -13,56 +14,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Logger - def get_actor(%{"actor" => actor}) when is_binary(actor) do - actor - end - - def get_actor(%{"actor" => actor}) when is_list(actor) do - if is_binary(Enum.at(actor, 0)) do - Enum.at(actor, 0) - else - Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) - |> Map.get("id") - end - end - - def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do - id - end - - def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do - get_actor(%{"actor" => actor}) - end - - @doc """ - Checks that an imported AP object's actor matches the domain it came from. - """ - def contain_origin(id, %{"actor" => nil}), do: :error - - def contain_origin(id, %{"actor" => actor} = params) do - id_uri = URI.parse(id) - actor_uri = URI.parse(get_actor(params)) - - if id_uri.host == actor_uri.host do - :ok - else - :error - end - end - - def contain_origin_from_id(id, %{"id" => nil}), do: :error - - def contain_origin_from_id(id, %{"id" => other_id} = params) do - id_uri = URI.parse(id) - other_uri = URI.parse(other_id) - - if id_uri.host == other_uri.host do - :ok - else - :error - end - end - @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -99,7 +50,7 @@ def fix_addressing(map) do def fix_actor(%{"attributedTo" => actor} = object) do object - |> Map.put("actor", get_actor(%{"actor" => actor})) + |> Map.put("actor", Containment.get_actor(%{"actor" => actor})) end def fix_likes(%{"likes" => likes} = object) @@ -277,7 +228,7 @@ def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) when objtype in ["Article", "Note", "Video", "Page"] do - actor = get_actor(data) + actor = Containment.get_actor(data) data = Map.put(data, "actor", actor) @@ -360,7 +311,7 @@ defp get_follow_activity(follow_object, followed) do def handle_incoming( %{"type" => "Accept", "object" => follow_object, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = followed <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"), @@ -386,7 +337,7 @@ def handle_incoming( def handle_incoming( %{"type" => "Reject", "object" => follow_object, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = followed <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"), @@ -410,7 +361,7 @@ def handle_incoming( def handle_incoming( %{"type" => "Like", "object" => object_id, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do @@ -423,7 +374,7 @@ def handle_incoming( def handle_incoming( %{"type" => "Announce", "object" => object_id, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do @@ -477,10 +428,10 @@ def handle_incoming( ) do object_id = Utils.get_ap_id(object_id) - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), - :ok <- contain_origin(actor.ap_id, object.data), + :ok <- Containment.contain_origin(actor.ap_id, object.data), {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} else @@ -496,7 +447,7 @@ def handle_incoming( "id" => id } = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do @@ -566,7 +517,7 @@ def handle_incoming( "id" => id } = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs new file mode 100644 index 000000000..fcedb2283 --- /dev/null +++ b/test/object/containment_test.exs @@ -0,0 +1,66 @@ +defmodule Pleroma.Object.ContainmentTest do + use Pleroma.DataCase + + alias Pleroma.User + alias Pleroma.Object.Containment + alias Pleroma.Web.ActivityPub.ActivityPub + + import Pleroma.Factory + + describe "general origin containment" do + test "contain_origin_from_id() catches obvious spoofing attempts" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :error = + Containment.contain_origin_from_id( + "http://example.org/~alyssa/activities/1234.json", + data + ) + end + + test "contain_origin_from_id() allows alternate IDs within the same origin domain" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :ok = + Containment.contain_origin_from_id( + "http://example.com/~alyssa/activities/1234", + data + ) + end + + test "contain_origin_from_id() allows matching IDs" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :ok = + Containment.contain_origin_from_id( + "http://example.com/~alyssa/activities/1234.json", + data + ) + end + + test "users cannot be collided through fake direction spoofing attempts" do + user = + insert(:user, %{ + nickname: "rye@niu.moe", + local: false, + ap_id: "https://niu.moe/users/rye", + follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) + }) + + {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") + end + + test "all objects with fake directions are rejected by the object fetcher" do + {:error, _} = + ActivityPub.fetch_and_contain_remote_object_from_id( + "https://info.pleroma.site/activity4.json" + ) + end + end +end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index ef89752f5..a71a2c5b1 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -945,61 +945,4 @@ test "it rejects activities which reference objects that have an incorrect attri :error = Transmogrifier.handle_incoming(data) end end - - describe "general origin containment" do - test "contain_origin_from_id() catches obvious spoofing attempts" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :error = - Transmogrifier.contain_origin_from_id( - "http://example.org/~alyssa/activities/1234.json", - data - ) - end - - test "contain_origin_from_id() allows alternate IDs within the same origin domain" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234", - data - ) - end - - test "contain_origin_from_id() allows matching IDs" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234.json", - data - ) - end - - test "users cannot be collided through fake direction spoofing attempts" do - user = - insert(:user, %{ - nickname: "rye@niu.moe", - local: false, - ap_id: "https://niu.moe/users/rye", - follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) - }) - - {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") - end - - test "all objects with fake directions are rejected by the object fetcher" do - {:error, _} = - ActivityPub.fetch_and_contain_remote_object_from_id( - "https://info.pleroma.site/activity4.json" - ) - end - end end From f85949cc696685496a8e4e17aebeb81168ede41a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 22:53:10 +0000 Subject: [PATCH 29/62] object: factor out fetching functions into Pleroma.Object.Fetcher module --- lib/pleroma/object/fetcher.ex | 69 +++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 63 +---------------- .../activity_pub/activity_pub_controller.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 6 +- .../mastodon_api/mastodon_api_controller.ex | 7 +- test/web/activity_pub/activity_pub_test.exs | 55 --------------- test/web/activity_pub/transmogrifier_test.exs | 12 ---- 7 files changed, 81 insertions(+), 134 deletions(-) create mode 100644 lib/pleroma/object/fetcher.ex diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex new file mode 100644 index 000000000..c27cb1577 --- /dev/null +++ b/lib/pleroma/object/fetcher.ex @@ -0,0 +1,69 @@ +defmodule Pleroma.Object.Fetcher do + alias Pleroma.{Object, Repo} + alias Pleroma.Object.Containment + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.OStatus + + require Logger + + @httpoison Application.get_env(:pleroma, :httpoison) + + # TODO: + # This will create a Create activity, which we need internally at the moment. + def fetch_object_from_id(id) do + if object = Object.get_cached_by_ap_id(id) do + {:ok, object} + else + Logger.info("Fetching #{id} via AP") + + with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), + nil <- Object.normalize(data), + params <- %{ + "type" => "Create", + "to" => data["to"], + "cc" => data["cc"], + "actor" => data["actor"] || data["attributedTo"], + "object" => data + }, + :ok <- Containment.contain_origin(id, params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, Object.normalize(activity.data["object"])} + else + {:error, {:reject, nil}} -> + {:reject, nil} + + object = %Object{} -> + {:ok, object} + + _e -> + Logger.info("Couldn't get object via AP, trying out OStatus fetching...") + + case OStatus.fetch_activity_from_url(id) do + {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} + e -> e + end + end + end + end + + def fetch_and_contain_remote_object_from_id(id) do + Logger.info("Fetching #{id} via AP") + + with true <- String.starts_with?(id, "http"), + {:ok, %{body: body, status_code: code}} when code in 200..299 <- + @httpoison.get( + id, + [Accept: "application/activity+json"], + follow_redirect: true, + timeout: 10000, + recv_timeout: 20000 + ), + {:ok, data} <- Jason.decode(body), + :ok <- Containment.contain_origin_from_id(id, data) do + {:ok, data} + else + e -> + {:error, e} + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 517cf4b46..fefefc320 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} - alias Pleroma.Object.Containment + alias Pleroma.Object.Fetcher alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.WebFinger alias Pleroma.Web.Federator @@ -629,7 +629,7 @@ def user_data_from_user_object(data) do end def fetch_and_prepare_user_from_ap_id(ap_id) do - with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do user_data_from_user_object(data) else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") @@ -723,65 +723,6 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do ) end - # TODO: - # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do - if object = Object.get_cached_by_ap_id(id) do - {:ok, object} - else - Logger.info("Fetching #{id} via AP") - - with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - nil <- Object.normalize(data), - params <- %{ - "type" => "Create", - "to" => data["to"], - "cc" => data["cc"], - "actor" => data["actor"] || data["attributedTo"], - "object" => data - }, - :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"])} - else - {:error, {:reject, nil}} -> - {:reject, nil} - - object = %Object{} -> - {:ok, object} - - _e -> - Logger.info("Couldn't get object via AP, trying out OStatus fetching...") - - case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} - e -> e - end - end - end - end - - def fetch_and_contain_remote_object_from_id(id) do - Logger.info("Fetching #{id} via AP") - - with true <- String.starts_with?(id, "http"), - {:ok, %{body: body, status_code: code}} when code in 200..299 <- - @httpoison.get( - id, - [Accept: "application/activity+json"], - follow_redirect: true, - timeout: 10000, - recv_timeout: 20000 - ), - {:ok, data} <- Jason.decode(body), - :ok <- Containment.contain_origin_from_id(id, data) do - {:ok, data} - else - e -> - {:error, e} - end - end - def is_public?(activity) do "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++ (activity.data["cc"] || [])) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3570a75cb..7b7c0e090 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do use Pleroma.Web, :controller alias Pleroma.{User, Object} + alias Pleroma.Object.Fetcher alias Pleroma.Web.ActivityPub.{ObjectView, UserView} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -122,7 +123,7 @@ def inbox(conn, %{"type" => "Create"} = params) do "Signature missing or not from author, relayed Create message, fetching object from source" ) - ActivityPub.fetch_object_from_id(params["object"]["id"]) + Fetcher.fetch_object_from_id(params["object"]["id"]) json(conn, "ok") end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 1b5e57294..e76e29b95 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.User alias Pleroma.Object - alias Pleroma.Object.Containment + alias Pleroma.Object.{Containment, Fetcher} alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.Web.ActivityPub.ActivityPub @@ -529,8 +529,8 @@ def handle_incoming( def handle_incoming(_), do: :error - def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id) - def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"]) + def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id) + def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"]) def get_obj_helper(id) do if object = Object.normalize(id), do: {:ok, object}, else: nil diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 90225460b..71390be0d 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller alias Pleroma.{Repo, Object, Activity, User, Notification, Stats} + alias Pleroma.Object.Fetcher alias Pleroma.Web alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView} alias Pleroma.Web.ActivityPub.ActivityPub @@ -658,7 +659,7 @@ def unblock_domain(%{assigns: %{user: blocker}} = conn, %{"domain" => domain}) d def status_search(query) do fetched = if Regex.match?(~r/https?:/, query) do - with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do + with {:ok, object} <- Fetcher.fetch_object_from_id(query) do [Activity.get_create_activity_by_object_ap_id(object.data["id"])] else _e -> [] @@ -986,7 +987,9 @@ def login(conn, %{"code" => code}) do def login(conn, _) do with {:ok, app} <- get_or_make_app() do path = - o_auth_path(conn, :authorize, + o_auth_path( + conn, + :authorize, response_type: "code", client_id: app.client_id, redirect_uri: ".", diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 231a334f9..bc9fcc75d 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -372,43 +372,6 @@ test "fetches the latest Follow activity" do end end - describe "fetching an object" do - test "it fetches an object" do - {:ok, object} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert [attachment] = object.data["attachment"] - assert is_list(attachment["url"]) - - assert object == object_again - end - - test "it works with objects only available via Ostatus" do - {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - - assert object == object_again - end - - test "it correctly stitches up conversations between ostatus and ap" do - last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - {:ok, object} = ActivityPub.fetch_object_from_id(last) - - object = Object.get_by_ap_id(object.data["inReplyTo"]) - assert object - end - end - describe "following / unfollowing" do test "creates a follow activity" do follower = insert(:user) @@ -530,15 +493,6 @@ test "it filters broken threads" do end end - test "it can fetch plume articles" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" - ) - - assert object - end - describe "update" do test "it creates an update activity with the new user data" do user = insert(:user) @@ -560,15 +514,6 @@ test "it creates an update activity with the new user data" do end end - test "it can fetch peertube videos" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - ) - - assert object - end - def data_uri do File.read!("test/fixtures/avatar_data_uri") end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index a71a2c5b1..ea9d9fe58 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -894,10 +894,6 @@ test "it fixes the actor URL property to be a proper URI" do end describe "actor origin containment" do - test "it rejects objects with a bogus origin" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json") - end - test "it rejects activities which reference objects with bogus origins" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -911,10 +907,6 @@ test "it rejects activities which reference objects with bogus origins" do :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 1)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -928,10 +920,6 @@ test "it rejects activities which reference objects that have an incorrect attri :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 2)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", From 7a57db0d3a18dbeb9fb3682b98f741ef6ba5f518 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 23:02:32 +0000 Subject: [PATCH 30/62] federator: fix up contain_origin_from_id() call --- lib/pleroma/web/federator/federator.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index ac3d7c132..0644f8d0a 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -2,6 +2,7 @@ defmodule Pleroma.Web.Federator do use GenServer alias Pleroma.User alias Pleroma.Activity + alias Pleroma.Object.Containment alias Pleroma.Web.{WebFinger, Websub} alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.ActivityPub.ActivityPub @@ -106,7 +107,7 @@ def handle(:incoming_ap_doc, params) do # actor shouldn't be acting on objects outside their own AP server. with {:ok, _user} <- ap_enabled_actor(params["actor"]), nil <- Activity.normalize(params["id"]), - :ok <- Transmogrifier.contain_origin_from_id(params["actor"], params), + :ok <- Containment.contain_origin_from_id(params["actor"], params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do {:ok, activity} else From f0439617ef233c150bd3e9dff620b47cfb1f8895 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 23:03:03 +0000 Subject: [PATCH 31/62] tests: some minor cleanups --- test/object/containment_test.exs | 7 --- test/object/fetcher_test.exs | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 test/object/fetcher_test.exs diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index fcedb2283..268675c86 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -55,12 +55,5 @@ test "users cannot be collided through fake direction spoofing attempts" do {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") end - - test "all objects with fake directions are rejected by the object fetcher" do - {:error, _} = - ActivityPub.fetch_and_contain_remote_object_from_id( - "https://info.pleroma.site/activity4.json" - ) - end end end diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs new file mode 100644 index 000000000..3bbade9d1 --- /dev/null +++ b/test/object/fetcher_test.exs @@ -0,0 +1,85 @@ +defmodule Pleroma.Object.FetcherTest do + use Pleroma.DataCase + + alias Pleroma.{Activity, Object} + alias Pleroma.Object.Fetcher + + import Pleroma.Factory + + describe "actor origin containment" do + test "it rejects objects with a bogus origin" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json") + end + + test "it rejects objects when attributedTo is wrong (variant 1)" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity2.json") + end + + test "it rejects objects when attributedTo is wrong (variant 2)" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity3.json") + end + end + + describe "fetching an object" do + test "it fetches an object" do + {:ok, object} = + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity.data["id"] + + {:ok, object_again} = + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + assert [attachment] = object.data["attachment"] + assert is_list(attachment["url"]) + + assert object == object_again + end + + test "it works with objects only available via Ostatus" do + {:ok, object} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") + assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity.data["id"] + + {:ok, object_again} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") + + assert object == object_again + end + + test "it correctly stitches up conversations between ostatus and ap" do + last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" + {:ok, object} = Fetcher.fetch_object_from_id(last) + + object = Object.get_by_ap_id(object.data["inReplyTo"]) + assert object + end + end + + describe "implementation quirks" do + test "it can fetch plume articles" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" + ) + + assert object + end + + test "it can fetch peertube videos" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" + ) + + assert object + end + + test "all objects with fake directions are rejected by the object fetcher" do + {:error, _} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://info.pleroma.site/activity4.json" + ) + end + end +end From 02288b5f1cd428c12f92b9ea4fb0b0a935bbdaaf Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 23:08:23 +0000 Subject: [PATCH 32/62] twitterapi: fix bad rebase --- .../web/twitter_api/views/activity_view.ex | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index f202b6e97..18b2ebb0b 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -207,15 +207,17 @@ def render("activity.json", %{activity: %{data: %{"type" => "Like"}} = activity} def render( "activity.json", - %{activity: %{data: %{"type" => "Create", "object" => object}} = activity} = opts + %{activity: %{data: %{"type" => "Create", "object" => object_id}} = activity} = opts ) do user = get_user(activity.data["actor"], opts) - created_at = object["published"] |> Utils.date_to_asctime() - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) - repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) + object = Object.normalize(object_id) + + created_at = object.data["published"] |> Utils.date_to_asctime() + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || []) attentions = activity.recipients @@ -230,11 +232,11 @@ def render( tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags - {summary, content} = render_content(object) + {summary, content} = render_content(object.data) html = HTML.filter_tags(content, User.html_filter_policy(opts[:for])) - |> Formatter.emojify(object["emoji"]) + |> Formatter.emojify(object.data["emoji"]) reply_parent = Activity.get_in_reply_to_activity(activity) @@ -249,19 +251,19 @@ def render( "is_local" => activity.local, "is_post_verb" => true, "created_at" => created_at, - "in_reply_to_status_id" => object["inReplyToStatusId"], + "in_reply_to_status_id" => object.data["inReplyToStatusId"], "in_reply_to_screen_name" => reply_user && reply_user.nickname, "in_reply_to_profileurl" => User.profile_url(reply_user), "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id, "in_reply_to_user_id" => reply_user && reply_user.id, "statusnet_conversation_id" => conversation_id, - "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), + "attachments" => (object.data["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), "attentions" => attentions, "fave_num" => like_count, "repeat_num" => announcement_count, "favorited" => !!favorited, "repeated" => !!repeated, - "external_url" => object["external_url"] || object["id"], + "external_url" => object.data["external_url"] || object.data["id"], "tags" => tags, "activity_type" => "post", "possibly_sensitive" => possibly_sensitive, From fed9b5404c1f30a695b60b67348604fc262c14c5 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 03:17:25 +0000 Subject: [PATCH 33/62] object: rework Object.normalize() a bit to support transparent fetching --- lib/pleroma/object.ex | 10 +++++++--- lib/pleroma/object/fetcher.ex | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 57a8b1d6b..0e9aefb63 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Object do use Ecto.Schema alias Pleroma.{Repo, Object, Activity} + alias Pleroma.Object.Fetcher import Ecto.{Query, Changeset} schema "objects" do @@ -27,9 +28,12 @@ def get_by_ap_id(ap_id) do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(obj) when is_map(obj), do: normalize(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) - def normalize(_), do: nil + def normalize(_, fetch_remote \\ true) + + def normalize(obj, fetch_remote) when is_map(obj), do: normalize(obj["id"], fetch_remote) + def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id) + def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) + def normalize(obj, _), do: nil if Mix.env() == :test do def get_cached_by_ap_id(ap_id) do diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index c27cb1577..c98722f39 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -17,7 +17,7 @@ def fetch_object_from_id(id) do Logger.info("Fetching #{id} via AP") with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - nil <- Object.normalize(data), + nil <- Object.normalize(data, false), params <- %{ "type" => "Create", "to" => data["to"], @@ -27,7 +27,7 @@ def fetch_object_from_id(id) do }, :ok <- Containment.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"])} + {:ok, Object.normalize(activity.data["object"], false)} else {:error, {:reject, nil}} -> {:reject, nil} @@ -39,13 +39,22 @@ def fetch_object_from_id(id) do Logger.info("Couldn't get object via AP, trying out OStatus fetching...") case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} + {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"], false)} e -> e end end end end + def fetch_object_from_id!(id) do + with {:ok, object} <- fetch_object_from_id(id) do + object + else + _e -> + nil + end + end + def fetch_and_contain_remote_object_from_id(id) do Logger.info("Fetching #{id} via AP") From b3b52b58c3b31b22d3b3227d06d9a5336fa8edd0 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 03:18:10 +0000 Subject: [PATCH 34/62] activitypub: transmogrifier: remove obsolete fetch_obj_helper() --- lib/pleroma/web/activity_pub/transmogrifier.ex | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index e76e29b95..c4567193f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -85,7 +85,7 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object) "" end - case fetch_obj_helper(in_reply_to_id) do + case get_obj_helper(in_reply_to_id) do {:ok, replied_object} -> with %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(replied_object.data["id"]) do @@ -363,7 +363,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do {:ok, activity} else @@ -376,7 +376,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do {:ok, activity} else @@ -430,7 +430,7 @@ def handle_incoming( with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} @@ -449,7 +449,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do {:ok, activity} else @@ -519,7 +519,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do {:ok, activity} else @@ -529,9 +529,6 @@ def handle_incoming( def handle_incoming(_), do: :error - def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id) - def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"]) - def get_obj_helper(id) do if object = Object.normalize(id), do: {:ok, object}, else: nil end @@ -629,7 +626,7 @@ def prepare_outgoing(%{"type" => _type} = data) do def maybe_fix_object_url(data) do if is_binary(data["object"]) and not String.starts_with?(data["object"], "http") do - case fetch_obj_helper(data["object"]) do + case get_obj_helper(data["object"]) do {:ok, relative_object} -> if relative_object.data["external_url"] do _data = From 419d4bd5e43248957ea66039231e3389c608368d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 05:00:11 +0000 Subject: [PATCH 35/62] tests: add tests for Object.normalize() --- test/object_test.exs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/object_test.exs b/test/object_test.exs index 909605560..3907748f7 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -49,4 +49,22 @@ test "ensures cache is cleared for the object" do refute object == cached_object end end + + describe "normalizer" do + test "fetches unknown objects by default" do + %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367") + + assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" + end + + test "fetches unknown objects when fetch_remote is explicitly true" do + %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true) + + assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" + end + + test "does not fetch unknown objects when fetch_remote is false" do + assert is_nil(Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)) + end + end end From d6ab701a14f7c9fb4d59953648c425e04725fc62 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 05:01:21 +0000 Subject: [PATCH 36/62] formatting --- lib/pleroma/web/ostatus/handlers/note_handler.ex | 3 ++- test/object_test.exs | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 39004367a..ba232b0ec 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -106,7 +106,8 @@ def handle_note(entry, doc \\ nil) do cw <- OStatus.get_cw(entry), inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo), - inReplyToObject <- (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil, + inReplyToObject <- + (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil, inReplyTo <- (inReplyToObject && inReplyToObject.data["id"]) || inReplyTo, attachments <- OStatus.get_attachments(entry), context <- get_context(entry, inReplyTo), diff --git a/test/object_test.exs b/test/object_test.exs index 3907748f7..dac6c3be7 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -52,19 +52,23 @@ test "ensures cache is cleared for the object" do describe "normalizer" do test "fetches unknown objects by default" do - %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367") + %Object{} = + object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367") assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" end test "fetches unknown objects when fetch_remote is explicitly true" do - %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true) + %Object{} = + object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true) assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" end test "does not fetch unknown objects when fetch_remote is false" do - assert is_nil(Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)) + assert is_nil( + Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false) + ) end end end From dda4e0e2a8826e400cab0d9f34a699e4248e5e18 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 12:27:29 +0300 Subject: [PATCH 37/62] Fix warnings in object tests --- test/object/containment_test.exs | 3 +-- test/object/fetcher_test.exs | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index 268675c86..cb74c9f4f 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -3,7 +3,6 @@ defmodule Pleroma.Object.ContainmentTest do alias Pleroma.User alias Pleroma.Object.Containment - alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory @@ -45,7 +44,7 @@ test "contain_origin_from_id() allows matching IDs" do end test "users cannot be collided through fake direction spoofing attempts" do - user = + _user = insert(:user, %{ nickname: "rye@niu.moe", local: false, diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 3bbade9d1..568c5cc4c 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -1,11 +1,10 @@ defmodule Pleroma.Object.FetcherTest do use Pleroma.DataCase - alias Pleroma.{Activity, Object} + alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Object.Fetcher - import Pleroma.Factory - describe "actor origin containment" do test "it rejects objects with a bogus origin" do {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json") From 462028688b7050bb2335914b0987632082fdf3c8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 12:34:19 +0300 Subject: [PATCH 38/62] Fix pinned posts relying on embded objects --- lib/pleroma/web/common_api/common_api.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 9c3daac2c..d6eb843f7 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -217,8 +217,10 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do with %Activity{ actor: ^user_ap_id, data: %{ - "type" => "Create", - "object" => %{ + "type" => "Create" + }, + object: %Object{ + data: %{ "to" => object_to, "type" => "Note" } From b09ae02c04d66e58c2bcc6ce10277c88d5fed576 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 13:04:58 +0300 Subject: [PATCH 39/62] Added some more normalization calls all in mastodon api controller --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 +++-- lib/pleroma/web/mastodon_api/views/status_view.ex | 8 ++++---- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index d6eb843f7..6458a3449 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -127,7 +127,7 @@ def get_visibility(%{"in_reply_to_status_id" => status_id}) when not is_nil(stat in_reply_to -> # XXX: these heuristics should be moved out of MastodonAPI. with %Object{} = object <- Object.normalize(in_reply_to) do - Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data) + Pleroma.Web.MastodonAPI.StatusView.get_visibility(object) end end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 24a2d4cb9..d2e3da449 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -543,10 +543,11 @@ def unpin_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do end def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{} = activity <- Activity.get_by_id(id), + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{} = object <- Object.normalize(activity), %User{} = user <- User.get_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do + {:ok, user} <- User.bookmark(user, object.data["id"]) do conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e4de5ecfb..03dc587d9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -125,8 +125,8 @@ def render( } end - def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do - object = Object.normalize(object) + def render("status.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do + object = Object.normalize(activity) user = get_user(activity.data["actor"]) @@ -320,8 +320,8 @@ def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do end end - def get_reply_to(%{data: %{"object" => object}}, _) do - object = Object.normalize(object) + def get_reply_to(%{data: %{"object" => _object}} = activity, _) do + object = Object.normalize(activity) if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do Activity.get_create_by_object_ap_id(object.data["inReplyTo"]) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index f21cf677d..70ab92386 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1879,7 +1879,7 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do capture_log(fn -> conn = conn - |> get("/api/v1/search", %{"q" => activity.data["object"]["id"]}) + |> get("/api/v1/search", %{"q" => Object.normalize(activity).id}) assert results = json_response(conn, 200) From e7c3c367667e96ef7fe31ef9dd8337b563a3ccaa Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:21:39 +0300 Subject: [PATCH 40/62] Update functions in object fetcher for tesla and set up a proper mock for tests --- lib/pleroma/object/fetcher.ex | 11 ++++------- test/object/fetcher_test.exs | 10 ++++++++-- test/web/mastodon_api/status_view_test.exs | 3 +-- test/web/twitter_api/views/activity_view_test.exs | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 19d9c51af..138e7866f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -27,7 +27,7 @@ def fetch_object_from_id(id) do }, :ok <- Containment.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"], false)} + {:ok, Object.normalize(activity, false)} else {:error, {:reject, nil}} -> {:reject, nil} @@ -56,16 +56,13 @@ def fetch_object_from_id!(id) do end def fetch_and_contain_remote_object_from_id(id) do - Logger.info("Fetching #{id} via AP") + Logger.info("Fetching object #{id} via AP") with true <- String.starts_with?(id, "http"), - {:ok, %{body: body, status_code: code}} when code in 200..299 <- + {:ok, %{body: body, status: code}} when code in 200..299 <- @httpoison.get( id, - [Accept: "application/activity+json"], - follow_redirect: true, - timeout: 10000, - recv_timeout: 20000 + [{:Accept, "application/activity+json"}] ), {:ok, data} <- Jason.decode(body), :ok <- Containment.contain_origin_from_id(id, data) do diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 568c5cc4c..72f616782 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -4,6 +4,12 @@ defmodule Pleroma.Object.FetcherTest do alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Object.Fetcher + import Tesla.Mock + + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end describe "actor origin containment" do test "it rejects objects with a bogus origin" do @@ -24,7 +30,7 @@ test "it fetches an object" do {:ok, object} = Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) assert activity.data["id"] {:ok, object_again} = @@ -38,7 +44,7 @@ test "it fetches an object" do test "it works with objects only available via Ostatus" do {:ok, object} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) assert activity.data["id"] {:ok, object_again} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 4ea50c7c6..cc5a84b5d 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -9,7 +9,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.User alias Pleroma.Repo alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView @@ -232,7 +231,7 @@ test "a peertube video" do user = insert(:user) {:ok, object} = - ActivityPub.fetch_object_from_id( + Pleroma.Object.Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index 7ef0270cc..b5440c612 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -360,7 +360,7 @@ test "a delete activity" do test "a peertube video" do {:ok, object} = - ActivityPub.fetch_object_from_id( + Pleroma.Object.Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) From e641651e2b0ba9e6c4d0a99ebf4f7c3b25f7ce67 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:27:02 +0300 Subject: [PATCH 41/62] Fix unbookmarking --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index d2e3da449..1e82b2f68 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -555,10 +555,11 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do end def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{} = activity <- Activity.get_by_id(id), + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{} = object <- Object.normalize(activity), %User{} = user <- User.get_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do + {:ok, user} <- User.unbookmark(user, object.data["id"]) do conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) From ad681877df46c151ee20b58401bda9f84d884109 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:52:01 +0300 Subject: [PATCH 42/62] Make credo happy --- lib/pleroma/web/activity_pub/transmogrifier.ex | 5 ++--- lib/pleroma/web/common_api/utils.ex | 8 ++++---- lib/pleroma/web/federator/federator.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- test/object/containment_test.exs | 2 +- test/object_test.exs | 6 ++++++ test/web/common_api/common_api_test.exs | 2 +- test/web/mastodon_api/status_view_test.exs | 4 ++-- 9 files changed, 19 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0637b18dc..a80aa52c6 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -6,13 +6,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do @moduledoc """ A module to handle coding from internal to wire ActivityPub and back. """ - alias Pleroma.User - alias Pleroma.Object - alias Pleroma.Object.Containment alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Containment alias Pleroma.Repo alias Pleroma.User + alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 7781f1635..50a72aee5 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -208,7 +208,7 @@ def make_note_data( context, content_html, attachments, - inReplyTo, + in_reply_to, tags, cw \\ nil, cc \\ [] @@ -225,11 +225,11 @@ def make_note_data( "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if inReplyTo do - inReplyToObject = Object.normalize(inReplyTo.data["object"]) + if in_reply_to do + in_reply_to_object = Object.normalize(inReplyTo.data["object"]) object - |> Map.put("inReplyTo", inReplyToObject.data["id"]) + |> Map.put("inReplyTo", in_reply_to_object.data["id"]) else object end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index a1f6373a4..1b4deb6dc 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Activity + alias Pleroma.Object.Containment alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -12,7 +13,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.OStatus - alias Pleroma.Object.Containment alias Pleroma.Web.Salmon alias Pleroma.Web.WebFinger alias Pleroma.Web.Websub diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1e82b2f68..4cec26c9b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -4,13 +4,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller - alias Pleroma.Object.Fetcher alias Ecto.Changeset alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Filter alias Pleroma.Notification alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.Repo alias Pleroma.ScheduledActivity diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 03dc587d9..f8961eb6c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -7,8 +7,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.HTML - alias Pleroma.Repo alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index cb74c9f4f..452064093 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -1,8 +1,8 @@ defmodule Pleroma.Object.ContainmentTest do use Pleroma.DataCase - alias Pleroma.User alias Pleroma.Object.Containment + alias Pleroma.User import Pleroma.Factory diff --git a/test/object_test.exs b/test/object_test.exs index a30efd48c..d138ee091 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -5,9 +5,15 @@ defmodule Pleroma.ObjectTest do use Pleroma.DataCase import Pleroma.Factory + import Tesla.Mock alias Pleroma.Object alias Pleroma.Repo + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + test "returns an object by it's AP id" do object = insert(:note) found_object = Object.get_by_ap_id(object.data["id"]) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index b9ed088e4..e12cc04c8 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -5,8 +5,8 @@ defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase alias Pleroma.Activity - alias Pleroma.User alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index cc5a84b5d..ad6344006 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -6,9 +6,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do use Pleroma.DataCase alias Pleroma.Activity - alias Pleroma.User - alias Pleroma.Repo alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView From a53b917e7ff2df74f53f1f4578c212fe539acb00 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:55:26 +0300 Subject: [PATCH 43/62] oof --- lib/pleroma/web/common_api/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 50a72aee5..25f498fcb 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -226,7 +226,7 @@ def make_note_data( } if in_reply_to do - in_reply_to_object = Object.normalize(inReplyTo.data["object"]) + in_reply_to_object = Object.normalize(in_reply_to.data["object"]) object |> Map.put("inReplyTo", in_reply_to_object.data["id"]) From 54b82f236bac154f520442c4c5cd70323cb5cee6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:03:17 +0300 Subject: [PATCH 44/62] Fix note count test --- test/web/activity_pub/activity_pub_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 68bfb3858..0ab29742a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -775,10 +775,10 @@ test "decrements user note count only for public activities" do {:ok, a4} = CommonAPI.post(User.get_by_id(user.id), %{"status" => "yeah", "visibility" => "direct"}) - {:ok, _} = a1.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a2.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a3.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a4.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() + {:ok, _} = Object.normalize(a1) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a2) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a3) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a4) |> ActivityPub.delete() user = User.get_by_id(user.id) assert user.info.note_count == 10 From c3a20528060e4fb95292dd93768d9afc8926e66e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:11:22 +0300 Subject: [PATCH 45/62] If it's an object struct it is already normalized --- lib/pleroma/object.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 3f1d0fea1..740d687a3 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -44,6 +44,7 @@ def get_by_ap_id(ap_id) do def normalize(_, fetch_remote \\ true) # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. # Use this whenever possible, especially when walking graphs in an O(N) loop! + def normalize(%Object{} = object, _), do: object def normalize(%Activity{object: %Object{} = object}, _), do: object # A hack for fake activities From ff8d76c670eb25213948974d03bbc389421558a6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:46:59 +0300 Subject: [PATCH 46/62] Refactor all tests that acessed the embeded object --- test/scheduled_activity_worker_test.exs | 2 +- test/user_test.exs | 7 ++++--- test/web/activity_pub/activity_pub_test.exs | 8 -------- test/web/activity_pub/transmogrifier_test.exs | 10 +++++----- test/web/common_api/common_api_test.exs | 2 +- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 ++-- test/web/mastodon_api/status_view_test.exs | 2 +- 7 files changed, 14 insertions(+), 21 deletions(-) diff --git a/test/scheduled_activity_worker_test.exs b/test/scheduled_activity_worker_test.exs index b9c91dda6..e3ad1244e 100644 --- a/test/scheduled_activity_worker_test.exs +++ b/test/scheduled_activity_worker_test.exs @@ -14,6 +14,6 @@ test "creates a status from the scheduled activity" do refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert activity.data["object"]["content"] == "hi" + assert Pleroma.Object.normalize(activity).data["content"] == "hi" end end diff --git a/test/user_test.exs b/test/user_test.exs index d2167a970..6ce5b9cf5 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Builders.UserBuilder alias Pleroma.Repo alias Pleroma.User @@ -256,7 +257,7 @@ test "it sends a welcome message if it is set" do activity = Repo.one(Pleroma.Activity) assert registered_user.ap_id in activity.recipients - assert activity.data["object"]["content"] =~ "cool site" + assert Object.normalize(activity).data["content"] =~ "cool site" assert activity.actor == welcome_user.ap_id Pleroma.Config.put([:instance, :welcome_user_nickname], nil) @@ -1132,14 +1133,14 @@ test "bookmarks" do "status" => "heweoo!" }) - id1 = activity1.data["object"]["id"] + id1 = Object.normalize(activity1).data["id"] {:ok, activity2} = CommonAPI.post(user, %{ "status" => "heweoo!" }) - id2 = activity2.data["object"]["id"] + id2 = Object.normalize(activity2).data["id"] assert {:ok, user_state1} = User.bookmark(user, id1) assert user_state1.bookmarks == [id1] diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0ab29742a..122690184 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -250,25 +250,21 @@ test "increases replies count" do # public {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "public")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 1 assert object.data["repliesCount"] == 1 # unlisted {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "unlisted")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 # private {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "private")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 # direct {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "direct")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 end end @@ -820,22 +816,18 @@ test "decreases reply count" do _ = CommonAPI.delete(direct_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 _ = CommonAPI.delete(private_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 _ = CommonAPI.delete(public_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 1 assert object.data["repliesCount"] == 1 _ = CommonAPI.delete(unlisted_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 0 assert object.data["repliesCount"] == 0 end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 5559cdf87..b062b273f 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -187,15 +187,15 @@ test "it cleans up incoming notices which are not really DMs" do data = Map.put(data, "object", object) - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data) assert data["to"] == [] assert data["cc"] == to - object = data["object"] + object_data = Object.normalize(activity).data - assert object["to"] == [] - assert object["cc"] == to + assert object_data["to"] == [] + assert object_data["cc"] == to end test "it works for incoming follow requests" do @@ -331,7 +331,7 @@ test "it does not clobber the addressing on announce activities" do data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", Object.normalize(activity).data["id"]) |> Map.put("to", ["http://mastodon.example.org/users/admin/followers"]) |> Map.put("cc", []) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index e12cc04c8..3d2bb8929 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -42,7 +42,7 @@ test "it adds emoji in the object" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => ":moominmamma:"}) - assert activity.data["object"]["emoji"]["moominmamma"] + assert Object.normalize(activity).data["emoji"]["moominmamma"] end test "it adds emoji when updating profiles" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 70ab92386..245887ff8 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2791,9 +2791,9 @@ test "Repeated posts that are replies incorrectly have in_reply_to_id null", %{c assert %{"content" => "xD", "id" => id} = json_response(conn1, 200) - activity = Activity.get_by_id(id) + activity = Activity.get_by_id_with_object(id) - assert activity.data["object"]["inReplyTo"] == replied_to.data["object"]["id"] + assert Object.normalize(activity).data["inReplyTo"] == Object.normalize(replied_to).data["id"] assert Activity.get_in_reply_to_activity(activity).id == replied_to.id # Reblog from the third user diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index ad6344006..a02c7c210 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -61,7 +61,7 @@ test "a note with null content" do |> Map.put("content", nil) Object.change(note_object, %{data: data}) - |> Repo.update() + |> Object.update_and_set_cache() User.get_cached_by_ap_id(note.data["actor"]) From 8e4d950f31ec3ea956f6892f9f36b419344bf930 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:54:09 +0300 Subject: [PATCH 47/62] Remove updating reply count for embeded objects --- lib/pleroma/activity.ex | 50 -------------------- lib/pleroma/web/activity_pub/activity_pub.ex | 2 - test/activity_test.exs | 14 ------ 3 files changed, 66 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 99cc9c077..478d16356 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -257,54 +257,4 @@ def all_by_actor_and_id(actor, status_ids) do |> where([s], s.actor == ^actor) |> Repo.all() end - - def increase_replies_count(nil), do: nil - - def increase_replies_count(object_ap_id) do - from(a in create_by_object_ap_id(object_ap_id), - update: [ - set: [ - data: - fragment( - """ - jsonb_set(?, '{object, repliesCount}', - (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true) - """, - a.data, - a.data - ) - ] - ] - ) - |> Repo.update_all([]) - |> case do - {1, [activity]} -> activity - _ -> {:error, "Not found"} - end - end - - def decrease_replies_count(nil), do: nil - - def decrease_replies_count(object_ap_id) do - from(a in create_by_object_ap_id(object_ap_id), - update: [ - set: [ - data: - fragment( - """ - jsonb_set(?, '{object, repliesCount}', - (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true) - """, - a.data, - a.data - ) - ] - ] - ) - |> Repo.update_all([]) - |> case do - {1, [activity]} -> activity - _ -> {:error, "Not found"} - end - end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a3b47cb3..28fca6116 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -95,7 +95,6 @@ def increase_replies_count_if_reply(%{ "type" => "Create" }) do if is_public?(object) do - Activity.increase_replies_count(reply_ap_id) Object.increase_replies_count(reply_ap_id) end end @@ -106,7 +105,6 @@ def decrease_replies_count_if_reply(%Object{ data: %{"inReplyTo" => reply_ap_id} = object }) do if is_public?(object) do - Activity.decrease_replies_count(reply_ap_id) Object.decrease_replies_count(reply_ap_id) end end diff --git a/test/activity_test.exs b/test/activity_test.exs index dc9c56a21..ad889f544 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -28,18 +28,4 @@ test "returns the activity that created an object" do assert activity == found_activity end - - test "reply count" do - %{id: id, data: %{"object" => %{"id" => object_ap_id}}} = activity = insert(:note_activity) - - replies_count = activity.data["object"]["repliesCount"] || 0 - expected_increase = replies_count + 1 - Activity.increase_replies_count(object_ap_id) - %{data: %{"object" => %{"repliesCount" => actual_increase}}} = Activity.get_by_id(id) - assert expected_increase == actual_increase - expected_decrease = expected_increase - 1 - Activity.decrease_replies_count(object_ap_id) - %{data: %{"object" => %{"repliesCount" => actual_decrease}}} = Activity.get_by_id(id) - assert expected_decrease == actual_decrease - end end From 4c289e924e2ef7863a2c95b74f71fd83969b7827 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 16:35:01 +0300 Subject: [PATCH 48/62] Fix delete-by_ap_id to expect not only embeded objects --- lib/pleroma/activity.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 478d16356..9043530c9 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -230,6 +230,7 @@ def delete_by_ap_id(id) when is_binary(id) do |> Repo.delete_all() |> elem(1) |> Enum.find(fn + %{data: %{"type" => "Create", "object" => ap_id}} when is_binary(ap_id) -> ap_id == id %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id _ -> nil end) From d1eb578a5763638cb9959834814f8632a810f8b2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 17:03:35 +0300 Subject: [PATCH 49/62] Refactor tests that used ActivityPub.fetch_object_from_id --- test/web/activity_pub/activity_pub_test.exs | 46 ------------------- test/web/activity_pub/transmogrifier_test.exs | 5 +- 2 files changed, 3 insertions(+), 48 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 122690184..291f3df4b 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -647,43 +647,6 @@ test "fetches the latest Follow activity" do end end - describe "fetching an object" do - test "it fetches an object" do - {:ok, object} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert [attachment] = object.data["attachment"] - assert is_list(attachment["url"]) - - assert object == object_again - end - - test "it works with objects only available via Ostatus" do - {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - - assert object == object_again - end - - test "it correctly stitches up conversations between ostatus and ap" do - last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - {:ok, object} = ActivityPub.fetch_object_from_id(last) - - object = Object.get_by_ap_id(object.data["inReplyTo"]) - assert object - end - end - describe "following / unfollowing" do test "creates a follow activity" do follower = insert(:user) @@ -900,15 +863,6 @@ test "it creates an update activity with the new user data" do end end - test "it can fetch peertube videos" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - ) - - assert object - end - test "returned pinned statuses" do Pleroma.Config.put([:instance, :max_pinned_statuses], 3) user = insert(:user) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index b062b273f..34ae3a20e 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -790,7 +791,7 @@ test "it rejects activities without a valid ID" do test "it remaps video URLs as attachments if necessary" do {:ok, object} = - ActivityPub.fetch_object_from_id( + Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) @@ -1185,7 +1186,7 @@ test "users cannot be collided through fake direction spoofing attempts" do test "all objects with fake directions are rejected by the object fetcher" do {:error, _} = - ActivityPub.fetch_and_contain_remote_object_from_id( + Fetcher.fetch_and_contain_remote_object_from_id( "https://info.pleroma.site/activity4.json" ) end From 35ac672b8d4a6711754a5f88ad65e52d356c4c67 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 17:59:15 +0300 Subject: [PATCH 50/62] Remove containment tests from transmogrifier and fix thread visibility solver --- lib/pleroma/object/containment.ex | 3 - lib/pleroma/web/activity_pub/visibility.ex | 21 ++++--- test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/activity_pub/transmogrifier_test.exs | 56 ------------------- 4 files changed, 15 insertions(+), 69 deletions(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 27e89d87f..25bd911fb 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -9,9 +9,6 @@ defmodule Pleroma.Object.Containment do Object containment is an important step in validating remote objects to prevent spoofing, therefore removal of object containment functions is NOT recommended. """ - - require Logger - def get_actor(%{"actor" => actor}) when is_binary(actor) do actor end diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index db52fe933..3da709b3d 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -41,16 +41,19 @@ def visible_for_user?(activity, user) do # guard def entire_thread_visible_for_user?(nil, _user), do: false - # child + # XXX: Probably even more inefficient than the previous implementation, intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop def entire_thread_visible_for_user?( - %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, + %Activity{} = tail, + # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, user - ) - when is_binary(parent_id) do - parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - end + ) do + case Object.normalize(tail) do + %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) -> + parent = Activity.get_in_reply_to_activity(tail) + visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - # root - def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user) + _ -> + visible_for_user?(tail, user) + end + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 291f3df4b..4a9acae69 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -832,7 +832,9 @@ test "it filters broken threads" do activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) - assert [public_activity, private_activity_1, private_activity_3] == activities + assert [public_activity, private_activity_1, private_activity_3] == + activities + assert length(activities) == 3 activities = ActivityPub.contain_timeline(activities, user1) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 34ae3a20e..6bb81a054 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1136,62 +1136,6 @@ test "it rejects activities which reference objects that have an incorrect attri end end - describe "general origin containment" do - test "contain_origin_from_id() catches obvious spoofing attempts" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :error = - Transmogrifier.contain_origin_from_id( - "http://example.org/~alyssa/activities/1234.json", - data - ) - end - - test "contain_origin_from_id() allows alternate IDs within the same origin domain" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234", - data - ) - end - - test "contain_origin_from_id() allows matching IDs" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234.json", - data - ) - end - - test "users cannot be collided through fake direction spoofing attempts" do - insert(:user, %{ - nickname: "rye@niu.moe", - local: false, - ap_id: "https://niu.moe/users/rye", - follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) - }) - - {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") - end - - test "all objects with fake directions are rejected by the object fetcher" do - {:error, _} = - Fetcher.fetch_and_contain_remote_object_from_id( - "https://info.pleroma.site/activity4.json" - ) - end - end - describe "reserialization" do test "successfully reserializes a message with inReplyTo == nil" do user = insert(:user) From 2abc09570f40352e949f0142d11778f89a70c920 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 01:37:04 +0300 Subject: [PATCH 51/62] Use the preloaded object in tag queries --- lib/pleroma/web/activity_pub/activity_pub.ex | 16 ++++++++-------- test/web/activity_pub/activity_pub_test.exs | 9 +++++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 28fca6116..6b2fb17a4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -574,8 +574,8 @@ defp restrict_since(query, _), do: query defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) when is_list(tag_reject) and tag_reject != [] do from( - activity in query, - where: fragment(~s(\(not \(? #> '{"object","tag"}'\) \\?| ?\)), activity.data, ^tag_reject) + [_activity, object] in query, + where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) ) end @@ -584,8 +584,8 @@ defp restrict_tag_reject(query, _), do: query defp restrict_tag_all(query, %{"tag_all" => tag_all}) when is_list(tag_all) and tag_all != [] do from( - activity in query, - where: fragment(~s(\(? #> '{"object","tag"}'\) \\?& ?), activity.data, ^tag_all) + [_activity, object] in query, + where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) ) end @@ -593,15 +593,15 @@ defp restrict_tag_all(query, _), do: query defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( - activity in query, - where: fragment(~s(\(? #> '{"object","tag"}'\) \\?| ?), activity.data, ^tag) + [_activity, object] in query, + where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) ) end defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do from( - activity in query, - where: fragment(~s(? <@ (? #> '{"object","tag"}'\)), ^tag, activity.data) + [_activity, object] in query, + where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) ) end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 4a9acae69..aacafc60a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -84,17 +84,21 @@ test "it fetches the appropriate tag-restricted posts" do {:ok, status_two} = CommonAPI.post(user, %{"status" => ". #essais"}) {:ok, status_three} = CommonAPI.post(user, %{"status" => ". #test #reject"}) - fetch_one = ActivityPub.fetch_activities([], %{"tag" => "test"}) - fetch_two = ActivityPub.fetch_activities([], %{"tag" => ["test", "essais"]}) + fetch_one = ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => "test"}) + + fetch_two = + ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => ["test", "essais"]}) fetch_three = ActivityPub.fetch_activities([], %{ + "type" => "Create", "tag" => ["test", "essais"], "tag_reject" => ["reject"] }) fetch_four = ActivityPub.fetch_activities([], %{ + "type" => "Create", "tag" => ["test"], "tag_all" => ["test", "reject"] }) @@ -832,6 +836,7 @@ test "it filters broken threads" do activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) + assert [public_activity, private_activity_1, private_activity_3] == activities From e31a22043bcf74d1d85f6fe007bd4606291d41e9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 08:31:08 +0300 Subject: [PATCH 52/62] Fix media timeline depending on embeded object and add some guards --- lib/pleroma/web/activity_pub/activity_pub.ex | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6b2fb17a4..03be8b06f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -571,6 +571,10 @@ defp restrict_since(query, %{"since_id" => since_id}) do defp restrict_since(query, _), do: query + defp restrict_tag_reject(_query, %{"tag_reject" => _tag_reject, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) when is_list(tag_reject) and tag_reject != [] do from( @@ -581,6 +585,10 @@ defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) defp restrict_tag_reject(query, _), do: query + defp restrict_tag_all(_query, %{"tag_all" => _tag_all, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag_all(query, %{"tag_all" => tag_all}) when is_list(tag_all) and tag_all != [] do from( @@ -591,6 +599,10 @@ defp restrict_tag_all(query, %{"tag_all" => tag_all}) defp restrict_tag_all(query, _), do: query + defp restrict_tag(_query, %{"tag" => _tag, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( [_activity, object] in query, @@ -666,10 +678,14 @@ defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do defp restrict_favorited_by(query, _), do: query + defp restrict_media(_query, %{"only_media" => _val, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_media(query, %{"only_media" => val}) when val == "true" or val == "1" do from( - activity in query, - where: fragment(~s(not (? #> '{"object","attachment"}' = ?\)), activity.data, ^[]) + [_activity, object] in query, + where: fragment("not (?)->'attachment' = (?)", object.data, ^[]) ) end From 6069d0fd361a9971f2297d3babc596d60634eb43 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 09:28:20 +0300 Subject: [PATCH 53/62] Fix object search depending on embeded object --- lib/pleroma/web/activity_pub/visibility.ex | 4 +++- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 6 +++--- test/user_test.exs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 3da709b3d..6dee61dd6 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -41,7 +41,9 @@ def visible_for_user?(activity, user) do # guard def entire_thread_visible_for_user?(nil, _user), do: false - # XXX: Probably even more inefficient than the previous implementation, intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop + # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop + # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + def entire_thread_visible_for_user?( %Activity{} = tail, # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 4cec26c9b..3916d7c41 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1012,13 +1012,13 @@ def status_search(user, query) do q = from( - a in Activity, + [a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)", - a.data, + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, ^query ), limit: 20, diff --git a/test/user_test.exs b/test/user_test.exs index 6ce5b9cf5..eee6881eb 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -4,8 +4,8 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity - alias Pleroma.Object alias Pleroma.Builders.UserBuilder + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI From ac04311b3f0a611b5008747037d6cd5874fa3ae9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 10:24:06 +0300 Subject: [PATCH 54/62] Fix search in TwitterAPI --- lib/pleroma/web/twitter_api/twitter_api.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index d6ce0a7c6..8e44dbeb8 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -266,6 +266,7 @@ defp parse_int(string, default) when is_binary(string) do defp parse_int(_, default), do: default + # TODO: unify the search query with MastoAPI one and do only pagination here def search(_user, %{"q" => query} = params) do limit = parse_int(params["rpp"], 20) page = parse_int(params["page"], 1) @@ -273,13 +274,13 @@ def search(_user, %{"q" => query} = params) do q = from( - a in Activity, + [a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)", - a.data, + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, ^query ), limit: ^limit, From a11ca87f40fd85341afa4445d3b7303ae8e92b76 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 13:10:38 +0300 Subject: [PATCH 55/62] Add a migration to remove embeded objects --- lib/mix/tasks/compact_database.ex | 57 ------------------- .../20190418072951_remove_embeded_objects.exs | 10 ++++ 2 files changed, 10 insertions(+), 57 deletions(-) delete mode 100644 lib/mix/tasks/compact_database.ex create mode 100644 priv/repo/migrations/20190418072951_remove_embeded_objects.exs diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex deleted file mode 100644 index 17b9721f7..000000000 --- a/lib/mix/tasks/compact_database.ex +++ /dev/null @@ -1,57 +0,0 @@ -defmodule Mix.Tasks.CompactDatabase do - @moduledoc """ - Compact the database by flattening the object graph. - """ - - require Logger - - use Mix.Task - import Ecto.Query - alias Pleroma.Activity - alias Pleroma.Repo - - defp maybe_compact(%Activity{data: %{"object" => %{"id" => object_id}}} = activity) do - data = - activity.data - |> Map.put("object", object_id) - - {:ok, activity} = - Activity.change(activity, %{data: data}) - |> Repo.update() - - {:ok, activity} - end - - defp maybe_compact(%Activity{} = activity), do: {:ok, activity} - - defp activity_query(min_id, max_id) do - from( - a in Activity, - where: fragment("?->>'type' = 'Create'", a.data), - where: a.id >= ^min_id, - where: a.id < ^max_id - ) - end - - def run(_args) do - Application.ensure_all_started(:pleroma) - - max = Repo.aggregate(Activity, :max, :id) - Logger.info("Considering #{max} activities") - - chunks = 0..round(max / 100) - - Enum.each(chunks, fn i -> - min = i * 100 - max = min + 100 - - activity_query(min, max) - |> Repo.all() - |> Enum.each(&maybe_compact/1) - - IO.write(".") - end) - - Logger.info("Finished.") - end -end diff --git a/priv/repo/migrations/20190418072951_remove_embeded_objects.exs b/priv/repo/migrations/20190418072951_remove_embeded_objects.exs new file mode 100644 index 000000000..128094278 --- /dev/null +++ b/priv/repo/migrations/20190418072951_remove_embeded_objects.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.RemoveEmbededObjects do + use Ecto.Migration + + # TODO: bench on a real DB and add clippy if it takes too long + def change do + execute """ + update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->>'type' = 'Create' and data->'object'->>'id' is not null; + """ + end +end From 4ef237f26a41652014105b8a7ad3a9d5c7b41202 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 14:37:57 +0300 Subject: [PATCH 56/62] Fix my incorrect search test fix --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 245887ff8..786af2088 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1879,7 +1879,7 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do capture_log(fn -> conn = conn - |> get("/api/v1/search", %{"q" => Object.normalize(activity).id}) + |> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]}) assert results = json_response(conn, 200) From 9238dccac1310fc2e281b242768a2de79f405f35 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 21:40:40 +0300 Subject: [PATCH 57/62] Add a guard to fetching reply activity --- lib/pleroma/activity.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 9043530c9..4a2ded518 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -204,12 +204,14 @@ def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do def create_by_object_ap_id_with_object(_), do: nil - def get_create_by_object_ap_id_with_object(ap_id) do + def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do ap_id |> create_by_object_ap_id_with_object() |> Repo.one() end + def get_create_by_object_ap_id_with_object(_), do: nil + defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do get_create_by_object_ap_id_with_object(ap_id) end From 099f89367efaf4032b8e937258b2c1a90f16b047 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 23:34:01 +0300 Subject: [PATCH 58/62] Replace embedded object migration with a mix task --- lib/mix/tasks/pleroma/database.ex | 45 +++++++++++++++++++ .../20190418072951_remove_embeded_objects.exs | 10 ----- 2 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 lib/mix/tasks/pleroma/database.ex delete mode 100644 priv/repo/migrations/20190418072951_remove_embeded_objects.exs diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex new file mode 100644 index 000000000..ce3252af5 --- /dev/null +++ b/lib/mix/tasks/pleroma/database.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Database do + alias Mix.Tasks.Pleroma.Common + use Mix.Task + + @shortdoc "A collection of database related tasks" + @moduledoc """ + A collection of database related tasks + + ## Replace embedded objects with their references + + Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration. + + mix pleroma.database remove_embedded_objects + + Options: + - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + """ + def run(["remove_embedded_objects" | args]) do + {options, [], []} = + OptionParser.parse( + args, + strict: [ + vacuum: :boolean + ] + ) + + Common.start_pleroma() + + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;" + ) + + if Keyword.get(options, :vacuum) do + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "vacuum full;" + ) + end + end +end diff --git a/priv/repo/migrations/20190418072951_remove_embeded_objects.exs b/priv/repo/migrations/20190418072951_remove_embeded_objects.exs deleted file mode 100644 index 128094278..000000000 --- a/priv/repo/migrations/20190418072951_remove_embeded_objects.exs +++ /dev/null @@ -1,10 +0,0 @@ -defmodule Pleroma.Repo.Migrations.RemoveEmbededObjects do - use Ecto.Migration - - # TODO: bench on a real DB and add clippy if it takes too long - def change do - execute """ - update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->>'type' = 'Create' and data->'object'->>'id' is not null; - """ - end -end From 945325013af6dde3f1da2417753bb97f55911a84 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 23:58:59 +0300 Subject: [PATCH 59/62] remove query timeouts --- lib/mix/tasks/pleroma/database.ex | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ce3252af5..d657c1ef0 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -30,15 +30,17 @@ def run(["remove_embedded_objects" | args]) do Common.start_pleroma() - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;" + Pleroma.Repo.query!( + "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", + [], + timeout: :infinity ) if Keyword.get(options, :vacuum) do - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "vacuum full;" + Pleroma.Repo.query!( + "vacuum full;", + [], + timeout: :infinity ) end end From 73b8c5387b25caaf27734f7018dc4702d49af7de Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 19 Apr 2019 00:17:37 +0300 Subject: [PATCH 60/62] Add some logging --- lib/mix/tasks/pleroma/database.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index d657c1ef0..ab9a3a7ff 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common + require Logger use Mix.Task @shortdoc "A collection of database related tasks" @@ -29,6 +30,7 @@ def run(["remove_embedded_objects" | args]) do ) Common.start_pleroma() + Logger.info("Removing embedded objects") Pleroma.Repo.query!( "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", @@ -37,6 +39,8 @@ def run(["remove_embedded_objects" | args]) do ) if Keyword.get(options, :vacuum) do + Logger.info("Runnning VACUUM FULL") + Pleroma.Repo.query!( "vacuum full;", [], From 17d94ae2676efad3a78ddfcd9dbb86d52e26b929 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 19 Apr 2019 00:26:46 +0300 Subject: [PATCH 61/62] Add a changelog entry for removing embded objects mix task --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ad83a01..648dceabe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction +- Mix Tasks: `mix pleroma.database remove_embedded_objects` - Federation: Support for reports - Configuration: `safe_dm_mentions` option - Configuration: `link_name` option From f9865cf9439e2e9273f55d3b82c1e68166178b07 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 19 Apr 2019 00:47:02 +0300 Subject: [PATCH 62/62] Stream out deletes, mistakingly removed when resolving merge conflicts --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9e2574419..0b99a169c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -167,7 +167,7 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do def stream_out(activity) do public = "https://www.w3.org/ns/activitystreams#Public" - if activity.data["type"] in ["Create", "Announce"] do + if activity.data["type"] in ["Create", "Announce", "Delete"] do object = Object.normalize(activity.data["object"]) Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity)