Merge branch 'revert-b122b6ff' into 'develop'

Revert "Merge branch 'features/hashtag-column' into 'develop'"

See merge request pleroma/pleroma!3222
This commit is contained in:
Haelwenn 2020-12-28 12:12:19 +00:00
commit 607cac1947
24 changed files with 70 additions and 163 deletions

View File

@ -8,7 +8,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed ### Changed
- **Breaking:** Changed storage of hashtags in plain-text to `object->hashtags`, run [`pleroma.database fill_old_hashtags` mix task](docs/administration/CLI_tasks/database.md) for old objects (works while pleroma is running).
- Polls now always return a `voters_count`, even if they are single-choice. - Polls now always return a `voters_count`, even if they are single-choice.
- Admin Emails: The ap id is used as the user link in emails now. - Admin Emails: The ap id is used as the user link in emails now.
- Improved registration workflow for email confirmation and account approval modes. - Improved registration workflow for email confirmation and account approval modes.
@ -444,6 +443,7 @@ switched to a new configuration mechanism, however it was not officially removed
- Static-FE: Fix remote posts not being sanitized - Static-FE: Fix remote posts not being sanitized
### Fixed ### Fixed
=======
- Rate limiter crashes when there is no explicitly specified ip in the config - Rate limiter crashes when there is no explicitly specified ip in the config
- 500 errors when no `Accept` header is present if Static-FE is enabled - 500 errors when no `Accept` header is present if Static-FE is enabled
- Instance panel not being updated immediately due to wrong `Cache-Control` headers - Instance panel not being updated immediately due to wrong `Cache-Control` headers

View File

@ -91,18 +91,6 @@ Can be safely re-run
mix pleroma.database fix_likes_collections mix pleroma.database fix_likes_collections
``` ```
## Fill hashtags for old objects
Migrate hashags fields for old objects, from now to `months_limit` months.
```sh tab="OTP"
./bin/pleroma_ctl database fill_old_hashtags <months_limit>
```
```sh tab="From Source"
mix pleroma.database fill_old_hashtags <months_limit>
```
## Vacuum the database ## Vacuum the database
### Analyze ### Analyze

View File

@ -128,57 +128,6 @@ def run(["fix_likes_collections"]) do
|> Stream.run() |> Stream.run()
end end
def run(["fill_old_hashtags", month_limit]) do
import Ecto.Query
start_pleroma()
month_limit = String.to_integer(month_limit)
if month_limit < 1 do
shell_error("Invalid `month_limit` argument, needs to be greater than 1")
else
time_limit = DateTime.utc_now() |> Timex.shift(months: -month_limit)
from(
o in Object,
where: fragment("(?)->>'hashtags' is null", o.data),
where: fragment("(?)->>'tag' != '[]'", o.data),
where: o.inserted_at < ^time_limit,
select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)}
)
|> Pleroma.Repo.chunk_stream(200, :batches)
|> Stream.each(fn objects ->
Repo.transaction(fn ->
objects_first = objects |> List.first()
objects_last = objects |> List.last()
Logger.info(
"fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{
objects_last.id
} (#{objects_last.inserted_at})"
)
objects
|> Enum.map(fn object ->
tags =
object.tag
|> Jason.decode!()
|> Enum.filter(&is_bitstring(&1))
Object
|> where([o], o.id == ^object.id)
|> update([o],
set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)]
)
|> Repo.update_all([], timeout: :infinity)
end)
end)
end)
|> Stream.run()
end
end
def run(["vacuum", args]) do def run(["vacuum", args]) do
start_pleroma() start_pleroma()

View File

@ -48,12 +48,14 @@ defp item_creation_tags(tags, _, _) do
tags tags
end end
defp hashtags_to_topics(object) do defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
object tags
|> Object.hashtags() |> Enum.filter(&is_bitstring(&1))
|> Enum.map(fn tag -> "hashtag:" <> tag end) |> Enum.map(fn tag -> "hashtag:" <> tag end)
end end
defp hashtags_to_topics(_), do: []
defp remote_topics(%{local: true}), do: [] defp remote_topics(%{local: true}), do: []
defp remote_topics(%{actor: actor}) when is_binary(actor), defp remote_topics(%{actor: actor}) when is_binary(actor),

View File

@ -18,8 +18,7 @@ defmodule Pleroma.Constants do
"emoji", "emoji",
"context_id", "context_id",
"deleted_activity_id", "deleted_activity_id",
"pleroma_internal", "pleroma_internal"
"hashtags"
] ]
) )

View File

@ -346,8 +346,4 @@ def replies(object, opts \\ []) do
def self_replies(object, opts \\ []), def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true)) do: replies(object, Keyword.put(opts, :self_only, true))
def hashtags(%Object{data: %{"hashtags" => hashtags}}), do: hashtags || []
def hashtags(%Object{data: %{"tag" => tags}}), do: Enum.filter(tags, &is_bitstring(&1))
def hashtags(_), do: []
end end

View File

@ -669,7 +669,7 @@ defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true})
defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
from( from(
[_activity, object] in query, [_activity, object] in query,
where: fragment("not (?)->'hashtags' \\?| (?)", object.data, ^tag_reject) where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
) )
end end
@ -682,7 +682,7 @@ defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
from( from(
[_activity, object] in query, [_activity, object] in query,
where: fragment("(?)->'hashtags' \\?& (?)", object.data, ^tag_all) where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
) )
end end
@ -695,14 +695,14 @@ defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
defp restrict_tag(query, %{tag: tag}) when is_list(tag) do defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
from( from(
[_activity, object] in query, [_activity, object] in query,
where: fragment("(?)->'hashtags' \\?| (?)", object.data, ^tag) where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
) )
end end
defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
from( from(
[_activity, object] in query, [_activity, object] in query,
where: fragment("(?)->'hashtags' \\? (?)", object.data, ^tag) where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
) )
end end

View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.FollowingRelationship alias Pleroma.FollowingRelationship
alias Pleroma.Object
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF
@ -75,11 +74,9 @@ defp check_media_nsfw(
object = object =
if MRF.subdomain_match?(media_nsfw, actor_host) do if MRF.subdomain_match?(media_nsfw, actor_host) do
child_object = tags = (child_object["tag"] || []) ++ ["nsfw"]
child_object child_object = Map.put(child_object, "tag", tags)
|> Map.put("hashtags", Object.hashtags(%Object{data: child_object}) ++ ["nsfw"]) child_object = Map.put(child_object, "sensitive", true)
|> Map.put("sensitive", true)
Map.put(object, "object", child_object) Map.put(object, "object", child_object)
else else
object object

View File

@ -312,15 +312,16 @@ def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do
def fix_emoji(object), do: object def fix_emoji(object), do: object
def fix_tag(%{"tag" => tag} = object) when is_list(tag) do def fix_tag(%{"tag" => tag} = object) when is_list(tag) do
hashtags = tags =
tag tag
|> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
|> Enum.map(fn |> Enum.map(fn %{"name" => name} ->
%{"name" => "#" <> hashtag} -> String.downcase(hashtag) name
%{"name" => hashtag} -> String.downcase(hashtag) |> String.slice(1..-1)
|> String.downcase()
end) end)
Map.put(object, "hashtags", hashtags) Map.put(object, "tag", tag ++ tags)
end end
def fix_tag(%{"tag" => %{} = tag} = object) do def fix_tag(%{"tag" => %{} = tag} = object) do
@ -863,18 +864,23 @@ def maybe_fix_object_url(%{"object" => object} = data) when is_binary(object) do
def maybe_fix_object_url(data), do: data def maybe_fix_object_url(data), do: data
def add_hashtags(object) do def add_hashtags(object) do
hashtags = tags =
%Object{data: object} (object["tag"] || [])
|> Object.hashtags() |> Enum.map(fn
|> Enum.map(fn tag -> # Expand internal representation tags into AS2 tags.
%{ tag when is_binary(tag) ->
"href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}", %{
"name" => "##{tag}", "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}",
"type" => "Hashtag" "name" => "##{tag}",
} "type" => "Hashtag"
}
# Do not process tags which are already AS2 tag objects.
tag when is_map(tag) ->
tag
end) end)
Map.put(object, "tag", hashtags ++ (object["tag"] || [])) Map.put(object, "tag", tags)
end end
# TODO These should be added on our side on insertion, it doesn't make much # TODO These should be added on our side on insertion, it doesn't make much
@ -930,7 +936,7 @@ def set_sensitive(%{"sensitive" => _} = object) do
end end
def set_sensitive(object) do def set_sensitive(object) do
tags = object["hashtags"] || object["tag"] || [] tags = object["tag"] || []
Map.put(object, "sensitive", "nsfw" in tags) Map.put(object, "sensitive", "nsfw" in tags)
end end

View File

@ -310,16 +310,7 @@ def make_note_data(%ActivityDraft{} = draft) do
"context" => draft.context, "context" => draft.context,
"attachment" => draft.attachments, "attachment" => draft.attachments,
"actor" => draft.user.ap_id, "actor" => draft.user.ap_id,
"tag" => Enum.filter(draft.tags, &is_map(&1)) |> Enum.uniq(), "tag" => Keyword.values(draft.tags) |> Enum.uniq()
"hashtags" =>
draft.tags
|> Enum.reduce([], fn
# Why so many formats
{:name, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
{"#" <> _, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
x, acc -> if is_bitstring(x), do: [x | acc], else: acc
end)
|> Enum.uniq()
} }
|> add_in_reply_to(draft.in_reply_to) |> add_in_reply_to(draft.in_reply_to)
|> Map.merge(draft.extra) |> Map.merge(draft.extra)

View File

@ -32,7 +32,6 @@ def prepare_activity(activity, opts \\ []) do
%{ %{
activity: activity, activity: activity,
object: object,
data: Map.get(object, :data), data: Map.get(object, :data),
actor: actor actor: actor
} }

View File

@ -347,7 +347,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
media_attachments: attachments, media_attachments: attachments,
poll: render(PollView, "show.json", object: object, for: opts[:for]), poll: render(PollView, "show.json", object: object, for: opts[:for]),
mentions: mentions, mentions: mentions,
tags: build_tags(Object.hashtags(object)), tags: build_tags(tags),
application: %{ application: %{
name: "Web", name: "Web",
website: nil website: nil

View File

@ -22,8 +22,8 @@
<link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/> <link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
<% end %> <% end %>
<%= for hashtag <- Object.hashtags(@object) do %> <%= for tag <- @data["tag"] || [] do %>
<category term="<%= hashtag %>"></category> <category term="<%= tag %>"></category>
<% end %> <% end %>
<%= for attachment <- @data["attachment"] || [] do %> <%= for attachment <- @data["attachment"] || [] do %>

View File

@ -21,8 +21,8 @@
<link><%= @data["external_url"] %></link> <link><%= @data["external_url"] %></link>
<% end %> <% end %>
<%= for hashtag <- Object.hashtags(@object) do %> <%= for tag <- @data["tag"] || [] do %>
<category term="<%= hashtag %>"></category> <category term="<%= tag %>"></category>
<% end %> <% end %>
<%= for attachment <- @data["attachment"] || [] do %> <%= for attachment <- @data["attachment"] || [] do %>

View File

@ -41,8 +41,8 @@
<% end %> <% end %>
<% end %> <% end %>
<%= for hashtag <- Object.hashtags(@object) do %> <%= for tag <- @data["tag"] || [] do %>
<category term="<%= hashtag %>"></category> <category term="<%= tag %>"></category>
<% end %> <% end %>
<%= for {emoji, file} <- @data["emoji"] || %{} do %> <%= for {emoji, file} <- @data["emoji"] || %{} do %>

View File

@ -1,11 +0,0 @@
defmodule Pleroma.Repo.Migrations.AddHashtagsIndexToObjects do
use Ecto.Migration
def change do
drop_if_exists(index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags))
create_if_not_exists(
index(:objects, ["(data->'hashtags')"], using: :gin, name: :objects_hashtags)
)
end
end

View File

@ -78,7 +78,7 @@ test "with no attachments doesn't produce public:media topics", %{activity: acti
end end
test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do
tagged_data = Map.put(data, "hashtags", ["foo", "bar"]) tagged_data = Map.put(data, "tag", ["foo", "bar"])
activity = %{activity | object: %{object | data: tagged_data}} activity = %{activity | object: %{object | data: tagged_data}}
topics = Topics.get_activity_topics(activity) topics = Topics.get_activity_topics(activity)

View File

@ -78,7 +78,7 @@ test "has a matching host" do
assert SimplePolicy.filter(media_message) == assert SimplePolicy.filter(media_message) ==
{:ok, {:ok,
media_message media_message
|> put_in(["object", "hashtags"], ["foo", "nsfw"]) |> put_in(["object", "tag"], ["foo", "nsfw"])
|> put_in(["object", "sensitive"], true)} |> put_in(["object", "sensitive"], true)}
assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(local_message) == {:ok, local_message}
@ -92,7 +92,7 @@ test "match with wildcard domain" do
assert SimplePolicy.filter(media_message) == assert SimplePolicy.filter(media_message) ==
{:ok, {:ok,
media_message media_message
|> put_in(["object", "hashtags"], ["foo", "nsfw"]) |> put_in(["object", "tag"], ["foo", "nsfw"])
|> put_in(["object", "sensitive"], true)} |> put_in(["object", "sensitive"], true)}
assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(local_message) == {:ok, local_message}
@ -105,7 +105,7 @@ defp build_media_message do
"type" => "Create", "type" => "Create",
"object" => %{ "object" => %{
"attachment" => [%{}], "attachment" => [%{}],
"hashtags" => ["foo"], "tag" => ["foo"],
"sensitive" => false "sensitive" => false
} }
} }

View File

@ -39,7 +39,7 @@ test "it works for incoming notices with tag not being an array (kroeg)" do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"]) object = Object.normalize(data["object"])
assert ["test"] == object.data["hashtags"] assert "test" in object.data["tag"]
end end
test "it cleans up incoming notices which are not really DMs" do test "it cleans up incoming notices which are not really DMs" do
@ -220,7 +220,7 @@ test "it works for incoming notices with hashtags" do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"]) object = Object.normalize(data["object"])
assert object.data["hashtags"] == ["moo"] assert Enum.at(object.data["tag"], 2) == "moo"
end end
test "it works for incoming notices with contentMap" do test "it works for incoming notices with contentMap" do

View File

@ -204,37 +204,30 @@ test "it strips internal fields" do
{:ok, activity} = CommonAPI.post(user, %{status: "#2hu :firefox:"}) {:ok, activity} = CommonAPI.post(user, %{status: "#2hu :firefox:"})
{:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data) {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
assert [ assert length(modified["object"]["tag"]) == 2
%{"name" => "#2hu", "type" => "Hashtag"},
%{"name" => ":firefox:", "type" => "Emoji"}
] = modified_object["tag"]
refute Map.has_key?(modified_object, "hashtags") assert is_nil(modified["object"]["emoji"])
refute Map.has_key?(modified_object, "emoji") assert is_nil(modified["object"]["like_count"])
refute Map.has_key?(modified_object, "like_count") assert is_nil(modified["object"]["announcements"])
refute Map.has_key?(modified_object, "announcements") assert is_nil(modified["object"]["announcement_count"])
refute Map.has_key?(modified_object, "announcement_count") assert is_nil(modified["object"]["context_id"])
refute Map.has_key?(modified_object, "context_id")
end end
test "it strips internal fields of article" do test "it strips internal fields of article" do
activity = insert(:article_activity) activity = insert(:article_activity)
{:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data) {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
assert [ assert length(modified["object"]["tag"]) == 2
%{"name" => "#2hu", "type" => "Hashtag"},
%{"name" => ":2hu:", "type" => "Emoji"}
] = modified_object["tag"]
refute Map.has_key?(modified_object, "hashtags") assert is_nil(modified["object"]["emoji"])
refute Map.has_key?(modified_object, "emoji") assert is_nil(modified["object"]["like_count"])
refute Map.has_key?(modified_object, "like_count") assert is_nil(modified["object"]["announcements"])
refute Map.has_key?(modified_object, "announcements") assert is_nil(modified["object"]["announcement_count"])
refute Map.has_key?(modified_object, "announcement_count") assert is_nil(modified["object"]["context_id"])
refute Map.has_key?(modified_object, "context_id") assert is_nil(modified["object"]["likes"])
end end
test "the directMessage flag is present" do test "the directMessage flag is present" do

View File

@ -591,8 +591,7 @@ test "returns note data" do
"context" => "2hu", "context" => "2hu",
"sensitive" => false, "sensitive" => false,
"summary" => "test summary", "summary" => "test summary",
"hashtags" => ["jimm"], "tag" => ["jimm"],
"tag" => [],
"to" => [user2.ap_id], "to" => [user2.ap_id],
"type" => "Note", "type" => "Note",
"custom_tag" => "test" "custom_tag" => "test"

View File

@ -493,8 +493,7 @@ test "it de-duplicates tags" do
object = Object.normalize(activity) object = Object.normalize(activity)
assert object.data["tag"] == [] assert object.data["tag"] == ["2hu"]
assert object.data["hashtags"] == ["2hu"]
end end
test "it adds emoji in the object" do test "it adds emoji in the object" do

View File

@ -262,8 +262,8 @@ test "a note activity" do
mentions: [], mentions: [],
tags: [ tags: [
%{ %{
name: "2hu", name: "#{object_data["tag"]}",
url: "/tag/2hu" url: "/tag/#{object_data["tag"]}"
} }
], ],
application: %{ application: %{

View File

@ -93,7 +93,7 @@ def note_factory(attrs \\ %{}) do
"like_count" => 0, "like_count" => 0,
"context" => "2hu", "context" => "2hu",
"summary" => "2hu", "summary" => "2hu",
"hashtags" => ["2hu"], "tag" => ["2hu"],
"emoji" => %{ "emoji" => %{
"2hu" => "corndog.png" "2hu" => "corndog.png"
} }