Merge branch 'features/emoji_stealer' into 'develop'

MRF.StealEmojiPolicy: New Policy

See merge request pleroma/pleroma!2385
This commit is contained in:
lain 2020-05-21 13:31:49 +00:00
commit a42a0716ec
5 changed files with 171 additions and 0 deletions

View File

@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mix task to create trusted OAuth App. - Mix task to create trusted OAuth App.
- Notifications: Added `follow_request` notification type. - Notifications: Added `follow_request` notification type.
- Added `:reject_deletes` group to SimplePolicy - Added `:reject_deletes` group to SimplePolicy
- MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>
- Mastodon API: Extended `/api/v1/instance`. - Mastodon API: Extended `/api/v1/instance`.

View File

@ -149,6 +149,11 @@ config :pleroma, :mrf_user_allowlist,
* `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines * `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines
* `:reject` rejects the message entirely * `:reject` rejects the message entirely
#### mrf_steal_emoji
* `hosts`: List of hosts to steal emojis from
* `rejected_shortcodes`: Regex-list of shortcodes to reject
* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk
### :activitypub ### :activitypub
* `unfollow_blocked`: Whether blocks result in people getting unfollowed * `unfollow_blocked`: Whether blocks result in people getting unfollowed
* `outgoing_blocks`: Whether to federate blocks to other instances * `outgoing_blocks`: Whether to federate blocks to other instances

View File

@ -0,0 +1,97 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
require Logger
alias Pleroma.Config
@moduledoc "Detect new emojis by their shortcode and steals them"
@behaviour Pleroma.Web.ActivityPub.MRF
defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host])
defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
defp steal_emoji({shortcode, url}) do
url = Pleroma.Web.MediaProxy.url(url)
{:ok, response} = Pleroma.HTTP.get(url)
size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
if byte_size(response.body) <= size_limit do
emoji_dir_path =
Config.get(
[:mrf_steal_emoji, :path],
Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
)
extension =
url
|> URI.parse()
|> Map.get(:path)
|> Path.basename()
|> Path.extname()
file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")])
try do
:ok = File.write(file_path, response.body)
shortcode
rescue
e ->
Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
nil
end
else
Logger.debug(
"MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
size_limit
} B)"
)
nil
end
rescue
e ->
Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
nil
end
@impl true
def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
host = URI.parse(actor).host
if remote_host?(host) and accept_host?(host) do
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
new_emojis =
foreign_emojis
|> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end)
|> Enum.filter(fn {shortcode, _url} ->
reject_emoji? =
Config.get([:mrf_steal_emoji, :rejected_shortcodes], [])
|> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
!reject_emoji?
end)
|> Enum.map(&steal_emoji(&1))
|> Enum.filter(& &1)
if !Enum.empty?(new_emojis) do
Logger.info("Stole new emojis: #{inspect(new_emojis)}")
Pleroma.Emoji.reload()
end
end
{:ok, message}
end
def filter(message), do: {:ok, message}
@impl true
def describe do
{:ok, %{}}
end
end

View File

@ -1291,6 +1291,10 @@ def get("https://skippers-bin.com/notes/7x9tmrp97i", _, _, _) do
}} }}
end end
def get("https://example.org/emoji/firedfox.png", _, _, _) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}}
end
def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}} {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}}
end end

View File

@ -0,0 +1,64 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
use Pleroma.DataCase
alias Pleroma.Config
alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy
setup_all do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
:ok
end
setup do
clear_config(:mrf_steal_emoji)
emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
File.rm_rf!(emoji_path)
File.mkdir!(emoji_path)
Pleroma.Emoji.reload()
end
test "does nothing by default" do
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
refute "firedfox" in installed_emoji
message = %{
"type" => "Create",
"object" => %{
"emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
"actor" => "https://example.org/users/admin"
}
}
assert {:ok, message} == StealEmojiPolicy.filter(message)
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
refute "firedfox" in installed_emoji
end
test "Steals emoji on unknown shortcode from allowed remote host" do
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
refute "firedfox" in installed_emoji
message = %{
"type" => "Create",
"object" => %{
"emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
"actor" => "https://example.org/users/admin"
}
}
Config.put([:mrf_steal_emoji, :hosts], ["example.org"])
Config.put([:mrf_steal_emoji, :size_limit], 284_468)
assert {:ok, message} == StealEmojiPolicy.filter(message)
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
assert "firedfox" in installed_emoji
end
end