From acb03d591bea1b20a715201f479f1ad7bf7bb67b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 31 Jul 2020 16:46:35 +0200 Subject: [PATCH 1/6] Insert text representation of hashtags into object["hashtags"] Includes a new mix task: pleroma.database fill_old_hashtags --- CHANGELOG.md | 2 +- docs/administration/CLI_tasks/database.md | 10 +++++ lib/mix/tasks/pleroma/database.ex | 43 +++++++++++++++++++ lib/pleroma/activity/ir/topics.ex | 4 ++ lib/pleroma/constants.ex | 3 +- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++-- .../web/activity_pub/mrf/simple_policy.ex | 8 ++-- .../web/activity_pub/transmogrifier.ex | 15 +++---- lib/pleroma/web/common_api/utils.ex | 11 ++++- .../web/mastodon_api/views/status_view.ex | 2 +- .../templates/feed/feed/_activity.atom.eex | 4 +- .../web/templates/feed/feed/_activity.rss.eex | 4 +- .../feed/feed/_tag_activity.atom.eex | 4 +- ...31165800_add_hashtags_index_to_objects.exs | 11 +++++ test/pleroma/activity/ir/topics_test.exs | 2 +- .../activity_pub/mrf/simple_policy_test.exs | 6 +-- .../transmogrifier/note_handling_test.exs | 4 +- .../web/activity_pub/transmogrifier_test.exs | 37 +++++++++------- test/pleroma/web/common_api/utils_test.exs | 3 +- test/pleroma/web/common_api_test.exs | 3 +- .../mastodon_api/views/status_view_test.exs | 4 +- test/support/factory.ex | 2 +- 22 files changed, 139 insertions(+), 51 deletions(-) create mode 100644 priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index c6bf38ee0..a5e5f5ecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed +- **Breaking:** Changed storage of hashtags in plain-text to `object->hashtags`, run [`pleroma.database fill_old_hashtags` mix task](docs/administration/CLI_tasks/database.md) for old objects (works while pleroma is running). - Polls now always return a `voters_count`, even if they are single-choice. - Admin Emails: The ap id is used as the user link in emails now. - Improved registration workflow for email confirmation and account approval modes. @@ -432,7 +433,6 @@ switched to a new configuration mechanism, however it was not officially removed - Static-FE: Fix remote posts not being sanitized ### Fixed -======= - Rate limiter crashes when there is no explicitly specified ip in the config - 500 errors when no `Accept` header is present if Static-FE is enabled - Instance panel not being updated immediately due to wrong `Cache-Control` headers diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md index 6dca83167..a2d2e8cdd 100644 --- a/docs/administration/CLI_tasks/database.md +++ b/docs/administration/CLI_tasks/database.md @@ -91,6 +91,16 @@ Can be safely re-run mix pleroma.database fix_likes_collections ``` +## Fill hashtags for old objects + +```sh tab="OTP" +./bin/pleroma_ctl database fill_old_hashtags +``` + +```sh tab="From Source" +mix pleroma.database fill_old_hashtags +``` + ## Vacuum the database ### Analyze diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 22151ce08..0c1343313 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -128,6 +128,49 @@ def run(["fix_likes_collections"]) do |> Stream.run() end + def run(["fill_old_hashtags"]) do + import Ecto.Query + + start_pleroma() + + from( + o in Object, + where: fragment("(?)->>'hashtags' is null", o.data), + where: fragment("(?)->>'tag' != '[]'", o.data), + select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)}, + order_by: [:desc, o.id] + ) + |> Pleroma.Repo.chunk_stream(200, :batches) + |> Stream.each(fn objects -> + Repo.transaction(fn -> + objects_first = objects |> List.first() + objects_last = objects |> List.last() + + Logger.info( + "fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{ + objects_last.id + } (#{objects_last.inserted_at})" + ) + + objects + |> Enum.map(fn object -> + tags = + object.tag + |> Jason.decode!() + |> Enum.filter(&is_bitstring(&1)) + + Object + |> where([o], o.id == ^object.id) + |> update([o], + set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)] + ) + |> Repo.update_all([], timeout: :infinity) + end) + end) + end) + |> Stream.run() + end + def run(["vacuum", args]) do start_pleroma() diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index fe2e8cb5c..2cdecf1e4 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -48,6 +48,10 @@ defp item_creation_tags(tags, _, _) do tags end + defp hashtags_to_topics(%{data: %{"hashtags" => tags}}) do + Enum.map(tags, fn tag -> "hashtag:" <> tag end) + end + defp hashtags_to_topics(%{data: %{"tag" => tags}}) do tags |> Enum.filter(&is_bitstring(&1)) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index cf8182d55..8f265715c 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -18,7 +18,8 @@ defmodule Pleroma.Constants do "emoji", "context_id", "deleted_activity_id", - "pleroma_internal" + "pleroma_internal", + "hashtags" ] ) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1c91bc074..61c1043ed 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -666,7 +666,7 @@ defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do from( [_activity, object] in query, - where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) + where: fragment("not (?)->'hashtags' \\?| (?)", object.data, ^tag_reject) ) end @@ -679,7 +679,7 @@ defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do from( [_activity, object] in query, - where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) + where: fragment("(?)->'hashtags' \\?& (?)", object.data, ^tag_all) ) end @@ -692,14 +692,14 @@ defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do defp restrict_tag(query, %{tag: tag}) when is_list(tag) do from( [_activity, object] in query, - where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) + where: fragment("(?)->'hashtags' \\?| (?)", object.data, ^tag) ) end defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do from( [_activity, object] in query, - where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) + where: fragment("(?)->'hashtags' \\? (?)", object.data, ^tag) ) end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 6cd91826d..2fa7b3194 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -74,9 +74,11 @@ defp check_media_nsfw( object = if MRF.subdomain_match?(media_nsfw, actor_host) do - tags = (child_object["tag"] || []) ++ ["nsfw"] - child_object = Map.put(child_object, "tag", tags) - child_object = Map.put(child_object, "sensitive", true) + child_object = + child_object + |> Map.put("hashtags", (child_object["hashtags"] || []) ++ ["nsfw"]) + |> Map.put("sensitive", true) + Map.put(object, "object", child_object) else object diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 565d32433..d3dc637da 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -312,16 +312,15 @@ def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do def fix_emoji(object), do: object def fix_tag(%{"tag" => tag} = object) when is_list(tag) do - tags = + hashtags = tag |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) - |> Enum.map(fn %{"name" => name} -> - name - |> String.slice(1..-1) - |> String.downcase() + |> Enum.map(fn + %{"name" => "#" <> hashtag} -> String.downcase(hashtag) + %{"name" => hashtag} -> String.downcase(hashtag) end) - Map.put(object, "tag", tag ++ tags) + Map.put(object, "hashtags", hashtags) end def fix_tag(%{"tag" => %{} = tag} = object) do @@ -865,7 +864,7 @@ def maybe_fix_object_url(data), do: data def add_hashtags(object) do tags = - (object["tag"] || []) + ((object["hashtags"] || []) ++ (object["tag"] || [])) |> Enum.map(fn # Expand internal representation tags into AS2 tags. tag when is_binary(tag) -> @@ -936,7 +935,7 @@ def set_sensitive(%{"sensitive" => _} = object) do end def set_sensitive(object) do - tags = object["tag"] || [] + tags = object["hashtags"] || object["tag"] || [] Map.put(object, "sensitive", "nsfw" in tags) end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1c74ea787..880b5d78f 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -310,7 +310,16 @@ def make_note_data(%ActivityDraft{} = draft) do "context" => draft.context, "attachment" => draft.attachments, "actor" => draft.user.ap_id, - "tag" => Keyword.values(draft.tags) |> Enum.uniq() + "tag" => Enum.filter(draft.tags, &is_map(&1)) |> Enum.uniq(), + "hashtags" => + draft.tags + |> Enum.reduce([], fn + # Why so many formats + {:name, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc + {"#" <> _, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc + x, acc -> if is_bitstring(x), do: [x | acc], else: acc + end) + |> Enum.uniq() } |> add_in_reply_to(draft.in_reply_to) |> Map.merge(draft.extra) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2301e21cf..6fc6272c2 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -347,7 +347,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} media_attachments: attachments, poll: render(PollView, "show.json", object: object, for: opts[:for]), mentions: mentions, - tags: build_tags(tags), + tags: build_tags(object.data["hashtags"] || tags), application: %{ name: "Web", website: nil diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 3fd150c4e..12a9545f3 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -22,8 +22,8 @@ <% end %> - <%= for tag <- @data["tag"] || [] do %> - + <%= for hashtag <- @data["hashtags"] || [] do %> + <% end %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 42960de7d..00872b4b7 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -21,8 +21,8 @@ <%= @data["external_url"] %> <% end %> - <%= for tag <- @data["tag"] || [] do %> - + <%= for hashtag <- @data["hashtags"] || [] do %> + <% end %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index cf5874a91..1377a6bbc 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -41,8 +41,8 @@ <% end %> <% end %> - <%= for tag <- @data["tag"] || [] do %> - + <%= for hashtag <- @data["hashtags"] || [] do %> + <% end %> <%= for {emoji, file} <- @data["emoji"] || %{} do %> diff --git a/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs b/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs new file mode 100644 index 000000000..b78682821 --- /dev/null +++ b/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddHashtagsIndexToObjects do + use Ecto.Migration + + def change do + drop_if_exists(index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags)) + + create_if_not_exists( + index(:objects, ["(data->'hashtags')"], using: :gin, name: :objects_hashtags) + ) + end +end diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs index 5e5c2f8da..eb098ee95 100644 --- a/test/pleroma/activity/ir/topics_test.exs +++ b/test/pleroma/activity/ir/topics_test.exs @@ -78,7 +78,7 @@ test "with no attachments doesn't produce public:media topics", %{activity: acti end test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do - tagged_data = Map.put(data, "tag", ["foo", "bar"]) + tagged_data = Map.put(data, "hashtags", ["foo", "bar"]) activity = %{activity | object: %{object | data: tagged_data}} topics = Topics.get_activity_topics(activity) diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index d7dde62c4..9777fcde1 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -78,7 +78,7 @@ test "has a matching host" do assert SimplePolicy.filter(media_message) == {:ok, media_message - |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "hashtags"], ["foo", "nsfw"]) |> put_in(["object", "sensitive"], true)} assert SimplePolicy.filter(local_message) == {:ok, local_message} @@ -92,7 +92,7 @@ test "match with wildcard domain" do assert SimplePolicy.filter(media_message) == {:ok, media_message - |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "hashtags"], ["foo", "nsfw"]) |> put_in(["object", "sensitive"], true)} assert SimplePolicy.filter(local_message) == {:ok, local_message} @@ -105,7 +105,7 @@ defp build_media_message do "type" => "Create", "object" => %{ "attachment" => [%{}], - "tag" => ["foo"], + "hashtags" => ["foo"], "sensitive" => false } } diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index b4a006aec..528636f04 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -39,7 +39,7 @@ test "it works for incoming notices with tag not being an array (kroeg)" do {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) object = Object.normalize(data["object"]) - assert "test" in object.data["tag"] + assert ["test"] == object.data["hashtags"] end test "it cleans up incoming notices which are not really DMs" do @@ -220,7 +220,7 @@ test "it works for incoming notices with hashtags" do {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) object = Object.normalize(data["object"]) - assert Enum.at(object.data["tag"], 2) == "moo" + assert object.data["hashtags"] == ["moo"] end test "it works for incoming notices with contentMap" do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 66ea7664a..d0bd00b58 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -204,30 +204,37 @@ test "it strips internal fields" do {:ok, activity} = CommonAPI.post(user, %{status: "#2hu :firefox:"}) - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) + {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data) - assert length(modified["object"]["tag"]) == 2 + assert [ + %{"name" => "#2hu", "type" => "Hashtag"}, + %{"name" => ":firefox:", "type" => "Emoji"} + ] = modified_object["tag"] - assert is_nil(modified["object"]["emoji"]) - assert is_nil(modified["object"]["like_count"]) - assert is_nil(modified["object"]["announcements"]) - assert is_nil(modified["object"]["announcement_count"]) - assert is_nil(modified["object"]["context_id"]) + refute Map.has_key?(modified_object, "hashtags") + refute Map.has_key?(modified_object, "emoji") + refute Map.has_key?(modified_object, "like_count") + refute Map.has_key?(modified_object, "announcements") + refute Map.has_key?(modified_object, "announcement_count") + refute Map.has_key?(modified_object, "context_id") end test "it strips internal fields of article" do activity = insert(:article_activity) - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) + {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data) - assert length(modified["object"]["tag"]) == 2 + assert [ + %{"name" => "#2hu", "type" => "Hashtag"}, + %{"name" => ":2hu:", "type" => "Emoji"} + ] = modified_object["tag"] - assert is_nil(modified["object"]["emoji"]) - assert is_nil(modified["object"]["like_count"]) - assert is_nil(modified["object"]["announcements"]) - assert is_nil(modified["object"]["announcement_count"]) - assert is_nil(modified["object"]["context_id"]) - assert is_nil(modified["object"]["likes"]) + refute Map.has_key?(modified_object, "hashtags") + refute Map.has_key?(modified_object, "emoji") + refute Map.has_key?(modified_object, "like_count") + refute Map.has_key?(modified_object, "announcements") + refute Map.has_key?(modified_object, "announcement_count") + refute Map.has_key?(modified_object, "context_id") end test "the directMessage flag is present" do diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 4d6c9ea26..211042192 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -591,7 +591,8 @@ test "returns note data" do "context" => "2hu", "sensitive" => false, "summary" => "test summary", - "tag" => ["jimm"], + "hashtags" => ["jimm"], + "tag" => [], "to" => [user2.ap_id], "type" => "Note", "custom_tag" => "test" diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 585b2c174..3b7ac2033 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -493,7 +493,8 @@ test "it de-duplicates tags" do object = Object.normalize(activity) - assert object.data["tag"] == ["2hu"] + assert object.data["tag"] == [] + assert object.data["hashtags"] == ["2hu"] end test "it adds emoji in the object" do diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index f2a7469ed..ecce26261 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -262,8 +262,8 @@ test "a note activity" do mentions: [], tags: [ %{ - name: "#{object_data["tag"]}", - url: "/tag/#{object_data["tag"]}" + name: "2hu", + url: "/tag/2hu" } ], application: %{ diff --git a/test/support/factory.ex b/test/support/factory.ex index 8eb07dc3c..a709d0dae 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -93,7 +93,7 @@ def note_factory(attrs \\ %{}) do "like_count" => 0, "context" => "2hu", "summary" => "2hu", - "tag" => ["2hu"], + "hashtags" => ["2hu"], "emoji" => %{ "2hu" => "corndog.png" } From 87b13c543039859007d9e2ba27c0236ab4092a9d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 22 Dec 2020 03:54:57 +0100 Subject: [PATCH 2/6] Create Object.hashtags/1 wrapper --- lib/pleroma/object.ex | 4 +++ .../web/activity_pub/mrf/simple_policy.ex | 3 ++- .../web/activity_pub/transmogrifier.ex | 25 ++++++++----------- .../web/mastodon_api/views/status_view.ex | 2 +- .../templates/feed/feed/_activity.atom.eex | 2 +- .../web/templates/feed/feed/_activity.rss.eex | 2 +- .../feed/feed/_tag_activity.atom.eex | 2 +- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 052ad413b..00d561d1b 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -344,4 +344,8 @@ def replies(object, opts \\ []) do def self_replies(object, opts \\ []), do: replies(object, Keyword.put(opts, :self_only, true)) + + def hashtags(%{"hashtags" => hashtags}), do: hashtags || [] + def hashtags(%{"tag" => tags}), do: Enum.filter(tags, &is_bitstring(&1)) + def hashtags(_), do: [] end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 2fa7b3194..8e0514dc8 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do alias Pleroma.Config alias Pleroma.FollowingRelationship + alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF @@ -76,7 +77,7 @@ defp check_media_nsfw( if MRF.subdomain_match?(media_nsfw, actor_host) do child_object = child_object - |> Map.put("hashtags", (child_object["hashtags"] || []) ++ ["nsfw"]) + |> Map.put("hashtags", Object.hashtags(child_object) ++ ["nsfw"]) |> Map.put("sensitive", true) Map.put(object, "object", child_object) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index d3dc637da..36ef6a454 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -863,23 +863,18 @@ def maybe_fix_object_url(%{"object" => object} = data) when is_binary(object) do def maybe_fix_object_url(data), do: data def add_hashtags(object) do - tags = - ((object["hashtags"] || []) ++ (object["tag"] || [])) - |> Enum.map(fn - # Expand internal representation tags into AS2 tags. - tag when is_binary(tag) -> - %{ - "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}", - "name" => "##{tag}", - "type" => "Hashtag" - } - - # Do not process tags which are already AS2 tag objects. - tag when is_map(tag) -> - tag + hashtags = + object + |> Object.hashtags() + |> Enum.map(fn tag -> + %{ + "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}", + "name" => "##{tag}", + "type" => "Hashtag" + } end) - Map.put(object, "tag", tags) + Map.put(object, "tag", hashtags ++ (object["tag"] || [])) end # TODO These should be added on our side on insertion, it doesn't make much diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6fc6272c2..b39f05d13 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -347,7 +347,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} media_attachments: attachments, poll: render(PollView, "show.json", object: object, for: opts[:for]), mentions: mentions, - tags: build_tags(object.data["hashtags"] || tags), + tags: build_tags(Object.hashtags(object.data)), application: %{ name: "Web", website: nil diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 12a9545f3..cb18abb5d 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -22,7 +22,7 @@ <% end %> - <%= for hashtag <- @data["hashtags"] || [] do %> + <%= for hashtag <- Object.hashtags(@data) do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 00872b4b7..6ef24d0ef 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -21,7 +21,7 @@ <%= @data["external_url"] %> <% end %> - <%= for hashtag <- @data["hashtags"] || [] do %> + <%= for hashtag <- Object.hashtags(@data) do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 1377a6bbc..098e28205 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -41,7 +41,7 @@ <% end %> <% end %> - <%= for hashtag <- @data["hashtags"] || [] do %> + <%= for hashtag <- Object.hashtags(@data) do %> <% end %> From 18b536c176d3b51f3a91f42ba5a001711ab85490 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 28 Dec 2020 10:33:28 +0100 Subject: [PATCH 3/6] Pleroma.Object/1: take %Object{} as argument instead --- lib/pleroma/activity/ir/topics.ex | 12 +++--------- lib/pleroma/object.ex | 4 ++-- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/feed/feed_view.ex | 1 + lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- .../web/templates/feed/feed/_activity.atom.eex | 2 +- .../web/templates/feed/feed/_activity.rss.eex | 2 +- .../web/templates/feed/feed/_tag_activity.atom.eex | 2 +- 9 files changed, 12 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index 2cdecf1e4..b7553c728 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -48,18 +48,12 @@ defp item_creation_tags(tags, _, _) do tags end - defp hashtags_to_topics(%{data: %{"hashtags" => tags}}) do - Enum.map(tags, fn tag -> "hashtag:" <> tag end) - end - - defp hashtags_to_topics(%{data: %{"tag" => tags}}) do - tags - |> Enum.filter(&is_bitstring(&1)) + defp hashtags_to_topics(object) do + object + |> Object.hashtags() |> Enum.map(fn tag -> "hashtag:" <> tag end) end - defp hashtags_to_topics(_), do: [] - defp remote_topics(%{local: true}), do: [] defp remote_topics(%{actor: actor}) when is_binary(actor), diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 00d561d1b..9cbc1c6a6 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -345,7 +345,7 @@ def replies(object, opts \\ []) do def self_replies(object, opts \\ []), do: replies(object, Keyword.put(opts, :self_only, true)) - def hashtags(%{"hashtags" => hashtags}), do: hashtags || [] - def hashtags(%{"tag" => tags}), do: Enum.filter(tags, &is_bitstring(&1)) + def hashtags(%Object{data: %{"hashtags" => hashtags}}), do: hashtags || [] + def hashtags(%Object{data: %{"tag" => tags}}), do: Enum.filter(tags, &is_bitstring(&1)) def hashtags(_), do: [] end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 8e0514dc8..94933ce99 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -77,7 +77,7 @@ defp check_media_nsfw( if MRF.subdomain_match?(media_nsfw, actor_host) do child_object = child_object - |> Map.put("hashtags", Object.hashtags(child_object) ++ ["nsfw"]) + |> Map.put("hashtags", Object.hashtags(%Object{data: child_object}) ++ ["nsfw"]) |> Map.put("sensitive", true) Map.put(object, "object", child_object) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 36ef6a454..109f03641 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -864,7 +864,7 @@ def maybe_fix_object_url(data), do: data def add_hashtags(object) do hashtags = - object + %Object{data: object} |> Object.hashtags() |> Enum.map(fn tag -> %{ diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 30e0a2a55..1155c6a39 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -32,6 +32,7 @@ def prepare_activity(activity, opts \\ []) do %{ activity: activity, + object: object, data: Map.get(object, :data), actor: actor } diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b39f05d13..3ba453d1f 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -347,7 +347,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} media_attachments: attachments, poll: render(PollView, "show.json", object: object, for: opts[:for]), mentions: mentions, - tags: build_tags(Object.hashtags(object.data)), + tags: build_tags(Object.hashtags(object)), application: %{ name: "Web", website: nil diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index cb18abb5d..4a1a3e993 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -22,7 +22,7 @@ <% end %> - <%= for hashtag <- Object.hashtags(@data) do %> + <%= for hashtag <- Object.hashtags(@object) do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 6ef24d0ef..9ebaa3300 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -21,7 +21,7 @@ <%= @data["external_url"] %> <% end %> - <%= for hashtag <- Object.hashtags(@data) do %> + <%= for hashtag <- Object.hashtags(@object) do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 098e28205..8d78520cf 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -41,7 +41,7 @@ <% end %> <% end %> - <%= for hashtag <- Object.hashtags(@data) do %> + <%= for hashtag <- Object.hashtags(@object) do %> <% end %> From d0c2479710b40a88b3a74c85c9ed9e72e9a2bfaf Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 22 Dec 2020 05:11:19 +0100 Subject: [PATCH 4/6] pleroma.database fill_old_hashtags: Add month_limit argument --- docs/administration/CLI_tasks/database.md | 6 +- lib/mix/tasks/pleroma/database.ex | 70 +++++++++++++---------- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md index a2d2e8cdd..eb85381ff 100644 --- a/docs/administration/CLI_tasks/database.md +++ b/docs/administration/CLI_tasks/database.md @@ -93,12 +93,14 @@ Can be safely re-run ## Fill hashtags for old objects +Migrate hashags fields for old objects, from now to `months_limit` months. + ```sh tab="OTP" -./bin/pleroma_ctl database fill_old_hashtags +./bin/pleroma_ctl database fill_old_hashtags ``` ```sh tab="From Source" -mix pleroma.database fill_old_hashtags +mix pleroma.database fill_old_hashtags ``` ## Vacuum the database diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 0c1343313..a09852503 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -128,47 +128,55 @@ def run(["fix_likes_collections"]) do |> Stream.run() end - def run(["fill_old_hashtags"]) do + def run(["fill_old_hashtags", month_limit]) do import Ecto.Query start_pleroma() - from( - o in Object, - where: fragment("(?)->>'hashtags' is null", o.data), - where: fragment("(?)->>'tag' != '[]'", o.data), - select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)}, - order_by: [:desc, o.id] - ) - |> Pleroma.Repo.chunk_stream(200, :batches) - |> Stream.each(fn objects -> - Repo.transaction(fn -> - objects_first = objects |> List.first() - objects_last = objects |> List.last() + month_limit = String.to_integer(month_limit) - Logger.info( - "fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{ - objects_last.id - } (#{objects_last.inserted_at})" - ) + if month_limit < 1 do + shell_error("Invalid `month_limit` argument, needs to be greater than 1") + else + time_limit = DateTime.utc_now() |> Timex.shift(months: -month_limit) - objects - |> Enum.map(fn object -> - tags = - object.tag - |> Jason.decode!() - |> Enum.filter(&is_bitstring(&1)) + from( + o in Object, + where: fragment("(?)->>'hashtags' is null", o.data), + where: fragment("(?)->>'tag' != '[]'", o.data), + where: o.inserted_at < ^time_limit, + select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)} + ) + |> Pleroma.Repo.chunk_stream(200, :batches) + |> Stream.each(fn objects -> + Repo.transaction(fn -> + objects_first = objects |> List.first() + objects_last = objects |> List.last() - Object - |> where([o], o.id == ^object.id) - |> update([o], - set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)] + Logger.info( + "fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{ + objects_last.id + } (#{objects_last.inserted_at})" ) - |> Repo.update_all([], timeout: :infinity) + + objects + |> Enum.map(fn object -> + tags = + object.tag + |> Jason.decode!() + |> Enum.filter(&is_bitstring(&1)) + + Object + |> where([o], o.id == ^object.id) + |> update([o], + set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)] + ) + |> Repo.update_all([], timeout: :infinity) + end) end) end) - end) - |> Stream.run() + |> Stream.run() + end end def run(["vacuum", args]) do From 3966add048fda791e6893540d8304b0e626ab9f4 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Mon, 28 Dec 2020 12:02:16 +0000 Subject: [PATCH 5/6] Revert "Merge branch 'features/hashtag-column' into 'develop'" This reverts merge request !2824 --- CHANGELOG.md | 2 +- docs/administration/CLI_tasks/database.md | 12 ----- lib/mix/tasks/pleroma/database.ex | 51 ------------------- lib/pleroma/activity/ir/topics.ex | 8 +-- lib/pleroma/constants.ex | 3 +- lib/pleroma/object.ex | 4 -- lib/pleroma/web/activity_pub/activity_pub.ex | 8 +-- .../web/activity_pub/mrf/simple_policy.ex | 9 ++-- .../web/activity_pub/transmogrifier.ex | 38 ++++++++------ lib/pleroma/web/common_api/utils.ex | 11 +--- lib/pleroma/web/feed/feed_view.ex | 1 - .../web/mastodon_api/views/status_view.ex | 2 +- .../templates/feed/feed/_activity.atom.eex | 4 +- .../web/templates/feed/feed/_activity.rss.eex | 4 +- .../feed/feed/_tag_activity.atom.eex | 4 +- ...31165800_add_hashtags_index_to_objects.exs | 11 ---- test/pleroma/activity/ir/topics_test.exs | 2 +- .../activity_pub/mrf/simple_policy_test.exs | 6 +-- .../transmogrifier/note_handling_test.exs | 4 +- .../web/activity_pub/transmogrifier_test.exs | 37 ++++++-------- test/pleroma/web/common_api/utils_test.exs | 3 +- test/pleroma/web/common_api_test.exs | 3 +- .../mastodon_api/views/status_view_test.exs | 4 +- test/support/factory.ex | 2 +- 24 files changed, 70 insertions(+), 163 deletions(-) delete mode 100644 priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 38972067c..e1604ab3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed -- **Breaking:** Changed storage of hashtags in plain-text to `object->hashtags`, run [`pleroma.database fill_old_hashtags` mix task](docs/administration/CLI_tasks/database.md) for old objects (works while pleroma is running). - Polls now always return a `voters_count`, even if they are single-choice. - Admin Emails: The ap id is used as the user link in emails now. - Improved registration workflow for email confirmation and account approval modes. @@ -444,6 +443,7 @@ switched to a new configuration mechanism, however it was not officially removed - Static-FE: Fix remote posts not being sanitized ### Fixed +======= - Rate limiter crashes when there is no explicitly specified ip in the config - 500 errors when no `Accept` header is present if Static-FE is enabled - Instance panel not being updated immediately due to wrong `Cache-Control` headers diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md index eb85381ff..6dca83167 100644 --- a/docs/administration/CLI_tasks/database.md +++ b/docs/administration/CLI_tasks/database.md @@ -91,18 +91,6 @@ Can be safely re-run mix pleroma.database fix_likes_collections ``` -## Fill hashtags for old objects - -Migrate hashags fields for old objects, from now to `months_limit` months. - -```sh tab="OTP" -./bin/pleroma_ctl database fill_old_hashtags -``` - -```sh tab="From Source" -mix pleroma.database fill_old_hashtags -``` - ## Vacuum the database ### Analyze diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index a09852503..22151ce08 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -128,57 +128,6 @@ def run(["fix_likes_collections"]) do |> Stream.run() end - def run(["fill_old_hashtags", month_limit]) do - import Ecto.Query - - start_pleroma() - - month_limit = String.to_integer(month_limit) - - if month_limit < 1 do - shell_error("Invalid `month_limit` argument, needs to be greater than 1") - else - time_limit = DateTime.utc_now() |> Timex.shift(months: -month_limit) - - from( - o in Object, - where: fragment("(?)->>'hashtags' is null", o.data), - where: fragment("(?)->>'tag' != '[]'", o.data), - where: o.inserted_at < ^time_limit, - select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)} - ) - |> Pleroma.Repo.chunk_stream(200, :batches) - |> Stream.each(fn objects -> - Repo.transaction(fn -> - objects_first = objects |> List.first() - objects_last = objects |> List.last() - - Logger.info( - "fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{ - objects_last.id - } (#{objects_last.inserted_at})" - ) - - objects - |> Enum.map(fn object -> - tags = - object.tag - |> Jason.decode!() - |> Enum.filter(&is_bitstring(&1)) - - Object - |> where([o], o.id == ^object.id) - |> update([o], - set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)] - ) - |> Repo.update_all([], timeout: :infinity) - end) - end) - end) - |> Stream.run() - end - end - def run(["vacuum", args]) do start_pleroma() diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index b7553c728..fe2e8cb5c 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -48,12 +48,14 @@ defp item_creation_tags(tags, _, _) do tags end - defp hashtags_to_topics(object) do - object - |> Object.hashtags() + defp hashtags_to_topics(%{data: %{"tag" => tags}}) do + tags + |> Enum.filter(&is_bitstring(&1)) |> Enum.map(fn tag -> "hashtag:" <> tag end) end + defp hashtags_to_topics(_), do: [] + defp remote_topics(%{local: true}), do: [] defp remote_topics(%{actor: actor}) when is_binary(actor), diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 8f265715c..cf8182d55 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -18,8 +18,7 @@ defmodule Pleroma.Constants do "emoji", "context_id", "deleted_activity_id", - "pleroma_internal", - "hashtags" + "pleroma_internal" ] ) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 8836beaac..b4a994da9 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -346,8 +346,4 @@ def replies(object, opts \\ []) do def self_replies(object, opts \\ []), do: replies(object, Keyword.put(opts, :self_only, true)) - - def hashtags(%Object{data: %{"hashtags" => hashtags}}), do: hashtags || [] - def hashtags(%Object{data: %{"tag" => tags}}), do: Enum.filter(tags, &is_bitstring(&1)) - def hashtags(_), do: [] end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 07a23b09b..5059bff03 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -669,7 +669,7 @@ defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do from( [_activity, object] in query, - where: fragment("not (?)->'hashtags' \\?| (?)", object.data, ^tag_reject) + where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) ) end @@ -682,7 +682,7 @@ defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do from( [_activity, object] in query, - where: fragment("(?)->'hashtags' \\?& (?)", object.data, ^tag_all) + where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) ) end @@ -695,14 +695,14 @@ defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do defp restrict_tag(query, %{tag: tag}) when is_list(tag) do from( [_activity, object] in query, - where: fragment("(?)->'hashtags' \\?| (?)", object.data, ^tag) + where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) ) end defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do from( [_activity, object] in query, - where: fragment("(?)->'hashtags' \\? (?)", object.data, ^tag) + where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) ) end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 94933ce99..6cd91826d 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do alias Pleroma.Config alias Pleroma.FollowingRelationship - alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF @@ -75,11 +74,9 @@ defp check_media_nsfw( object = if MRF.subdomain_match?(media_nsfw, actor_host) do - child_object = - child_object - |> Map.put("hashtags", Object.hashtags(%Object{data: child_object}) ++ ["nsfw"]) - |> Map.put("sensitive", true) - + tags = (child_object["tag"] || []) ++ ["nsfw"] + child_object = Map.put(child_object, "tag", tags) + child_object = Map.put(child_object, "sensitive", true) Map.put(object, "object", child_object) else object diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 109f03641..565d32433 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -312,15 +312,16 @@ def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do def fix_emoji(object), do: object def fix_tag(%{"tag" => tag} = object) when is_list(tag) do - hashtags = + tags = tag |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) - |> Enum.map(fn - %{"name" => "#" <> hashtag} -> String.downcase(hashtag) - %{"name" => hashtag} -> String.downcase(hashtag) + |> Enum.map(fn %{"name" => name} -> + name + |> String.slice(1..-1) + |> String.downcase() end) - Map.put(object, "hashtags", hashtags) + Map.put(object, "tag", tag ++ tags) end def fix_tag(%{"tag" => %{} = tag} = object) do @@ -863,18 +864,23 @@ def maybe_fix_object_url(%{"object" => object} = data) when is_binary(object) do def maybe_fix_object_url(data), do: data def add_hashtags(object) do - hashtags = - %Object{data: object} - |> Object.hashtags() - |> Enum.map(fn tag -> - %{ - "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}", - "name" => "##{tag}", - "type" => "Hashtag" - } + tags = + (object["tag"] || []) + |> Enum.map(fn + # Expand internal representation tags into AS2 tags. + tag when is_binary(tag) -> + %{ + "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}", + "name" => "##{tag}", + "type" => "Hashtag" + } + + # Do not process tags which are already AS2 tag objects. + tag when is_map(tag) -> + tag end) - Map.put(object, "tag", hashtags ++ (object["tag"] || [])) + Map.put(object, "tag", tags) end # TODO These should be added on our side on insertion, it doesn't make much @@ -930,7 +936,7 @@ def set_sensitive(%{"sensitive" => _} = object) do end def set_sensitive(object) do - tags = object["hashtags"] || object["tag"] || [] + tags = object["tag"] || [] Map.put(object, "sensitive", "nsfw" in tags) end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 880b5d78f..1c74ea787 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -310,16 +310,7 @@ def make_note_data(%ActivityDraft{} = draft) do "context" => draft.context, "attachment" => draft.attachments, "actor" => draft.user.ap_id, - "tag" => Enum.filter(draft.tags, &is_map(&1)) |> Enum.uniq(), - "hashtags" => - draft.tags - |> Enum.reduce([], fn - # Why so many formats - {:name, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc - {"#" <> _, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc - x, acc -> if is_bitstring(x), do: [x | acc], else: acc - end) - |> Enum.uniq() + "tag" => Keyword.values(draft.tags) |> Enum.uniq() } |> add_in_reply_to(draft.in_reply_to) |> Map.merge(draft.extra) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 1155c6a39..30e0a2a55 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -32,7 +32,6 @@ def prepare_activity(activity, opts \\ []) do %{ activity: activity, - object: object, data: Map.get(object, :data), actor: actor } diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3ba453d1f..2301e21cf 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -347,7 +347,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} media_attachments: attachments, poll: render(PollView, "show.json", object: object, for: opts[:for]), mentions: mentions, - tags: build_tags(Object.hashtags(object)), + tags: build_tags(tags), application: %{ name: "Web", website: nil diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 4a1a3e993..3fd150c4e 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -22,8 +22,8 @@ <% end %> - <%= for hashtag <- Object.hashtags(@object) do %> - + <%= for tag <- @data["tag"] || [] do %> + <% end %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 9ebaa3300..42960de7d 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -21,8 +21,8 @@ <%= @data["external_url"] %> <% end %> - <%= for hashtag <- Object.hashtags(@object) do %> - + <%= for tag <- @data["tag"] || [] do %> + <% end %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 8d78520cf..cf5874a91 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -41,8 +41,8 @@ <% end %> <% end %> - <%= for hashtag <- Object.hashtags(@object) do %> - + <%= for tag <- @data["tag"] || [] do %> + <% end %> <%= for {emoji, file} <- @data["emoji"] || %{} do %> diff --git a/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs b/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs deleted file mode 100644 index b78682821..000000000 --- a/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AddHashtagsIndexToObjects do - use Ecto.Migration - - def change do - drop_if_exists(index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags)) - - create_if_not_exists( - index(:objects, ["(data->'hashtags')"], using: :gin, name: :objects_hashtags) - ) - end -end diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs index dd1611bbe..b464822d9 100644 --- a/test/pleroma/activity/ir/topics_test.exs +++ b/test/pleroma/activity/ir/topics_test.exs @@ -78,7 +78,7 @@ test "with no attachments doesn't produce public:media topics", %{activity: acti end test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do - tagged_data = Map.put(data, "hashtags", ["foo", "bar"]) + tagged_data = Map.put(data, "tag", ["foo", "bar"]) activity = %{activity | object: %{object | data: tagged_data}} topics = Topics.get_activity_topics(activity) diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index 9777fcde1..d7dde62c4 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -78,7 +78,7 @@ test "has a matching host" do assert SimplePolicy.filter(media_message) == {:ok, media_message - |> put_in(["object", "hashtags"], ["foo", "nsfw"]) + |> put_in(["object", "tag"], ["foo", "nsfw"]) |> put_in(["object", "sensitive"], true)} assert SimplePolicy.filter(local_message) == {:ok, local_message} @@ -92,7 +92,7 @@ test "match with wildcard domain" do assert SimplePolicy.filter(media_message) == {:ok, media_message - |> put_in(["object", "hashtags"], ["foo", "nsfw"]) + |> put_in(["object", "tag"], ["foo", "nsfw"]) |> put_in(["object", "sensitive"], true)} assert SimplePolicy.filter(local_message) == {:ok, local_message} @@ -105,7 +105,7 @@ defp build_media_message do "type" => "Create", "object" => %{ "attachment" => [%{}], - "hashtags" => ["foo"], + "tag" => ["foo"], "sensitive" => false } } diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 528636f04..b4a006aec 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -39,7 +39,7 @@ test "it works for incoming notices with tag not being an array (kroeg)" do {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) object = Object.normalize(data["object"]) - assert ["test"] == object.data["hashtags"] + assert "test" in object.data["tag"] end test "it cleans up incoming notices which are not really DMs" do @@ -220,7 +220,7 @@ test "it works for incoming notices with hashtags" do {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) object = Object.normalize(data["object"]) - assert object.data["hashtags"] == ["moo"] + assert Enum.at(object.data["tag"], 2) == "moo" end test "it works for incoming notices with contentMap" do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index d0bd00b58..66ea7664a 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -204,37 +204,30 @@ test "it strips internal fields" do {:ok, activity} = CommonAPI.post(user, %{status: "#2hu :firefox:"}) - {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data) + {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - assert [ - %{"name" => "#2hu", "type" => "Hashtag"}, - %{"name" => ":firefox:", "type" => "Emoji"} - ] = modified_object["tag"] + assert length(modified["object"]["tag"]) == 2 - refute Map.has_key?(modified_object, "hashtags") - refute Map.has_key?(modified_object, "emoji") - refute Map.has_key?(modified_object, "like_count") - refute Map.has_key?(modified_object, "announcements") - refute Map.has_key?(modified_object, "announcement_count") - refute Map.has_key?(modified_object, "context_id") + assert is_nil(modified["object"]["emoji"]) + assert is_nil(modified["object"]["like_count"]) + assert is_nil(modified["object"]["announcements"]) + assert is_nil(modified["object"]["announcement_count"]) + assert is_nil(modified["object"]["context_id"]) end test "it strips internal fields of article" do activity = insert(:article_activity) - {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data) + {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - assert [ - %{"name" => "#2hu", "type" => "Hashtag"}, - %{"name" => ":2hu:", "type" => "Emoji"} - ] = modified_object["tag"] + assert length(modified["object"]["tag"]) == 2 - refute Map.has_key?(modified_object, "hashtags") - refute Map.has_key?(modified_object, "emoji") - refute Map.has_key?(modified_object, "like_count") - refute Map.has_key?(modified_object, "announcements") - refute Map.has_key?(modified_object, "announcement_count") - refute Map.has_key?(modified_object, "context_id") + assert is_nil(modified["object"]["emoji"]) + assert is_nil(modified["object"]["like_count"]) + assert is_nil(modified["object"]["announcements"]) + assert is_nil(modified["object"]["announcement_count"]) + assert is_nil(modified["object"]["context_id"]) + assert is_nil(modified["object"]["likes"]) end test "the directMessage flag is present" do diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 211042192..4d6c9ea26 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -591,8 +591,7 @@ test "returns note data" do "context" => "2hu", "sensitive" => false, "summary" => "test summary", - "hashtags" => ["jimm"], - "tag" => [], + "tag" => ["jimm"], "to" => [user2.ap_id], "type" => "Note", "custom_tag" => "test" diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 3b7ac2033..585b2c174 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -493,8 +493,7 @@ test "it de-duplicates tags" do object = Object.normalize(activity) - assert object.data["tag"] == [] - assert object.data["hashtags"] == ["2hu"] + assert object.data["tag"] == ["2hu"] end test "it adds emoji in the object" do diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index d4378ccbc..fa9066716 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -262,8 +262,8 @@ test "a note activity" do mentions: [], tags: [ %{ - name: "2hu", - url: "/tag/2hu" + name: "#{object_data["tag"]}", + url: "/tag/#{object_data["tag"]}" } ], application: %{ diff --git a/test/support/factory.ex b/test/support/factory.ex index a709d0dae..8eb07dc3c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -93,7 +93,7 @@ def note_factory(attrs \\ %{}) do "like_count" => 0, "context" => "2hu", "summary" => "2hu", - "hashtags" => ["2hu"], + "tag" => ["2hu"], "emoji" => %{ "2hu" => "corndog.png" } From a1a58f0a53a2508abe0b32faee11335f13d1cd65 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 28 Dec 2020 17:52:18 -0600 Subject: [PATCH 6/6] Switch to local fork of crypt until upstream fixes ability to build on aarch64 https://github.com/msantos/crypt/pull/8 --- mix.exs | 4 ++-- mix.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index a596e34ea..f26a5391a 100644 --- a/mix.exs +++ b/mix.exs @@ -147,8 +147,8 @@ defp deps do {:earmark, "1.4.3"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, - git: "https://github.com/msantos/crypt.git", - ref: "f63a705f92c26955977ee62a313012e309a4d77a"}, + git: "https://git.pleroma.social/pleroma/elixir-libraries/crypt.git", + ref: "cf2aa3f11632e8b0634810a15b3e612c7526f6a3"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3"}, {:swoosh, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 32b2e1391..01caf319e 100644 --- a/mix.lock +++ b/mix.lock @@ -22,7 +22,7 @@ "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"}, "credo": {:hex, :credo, "1.4.1", "16392f1edd2cdb1de9fe4004f5ab0ae612c92e230433968eab00aafd976282fc", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "155f8a2989ad77504de5d8291fa0d41320fdcaa6a1030472e9967f285f8c7692"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, - "crypt": {:git, "https://github.com/msantos/crypt.git", "f63a705f92c26955977ee62a313012e309a4d77a", [ref: "f63a705f92c26955977ee62a313012e309a4d77a"]}, + "crypt": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/crypt.git", "cf2aa3f11632e8b0634810a15b3e612c7526f6a3", [ref: "cf2aa3f11632e8b0634810a15b3e612c7526f6a3"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},