Merge branch 'bugfix/mrf-ingestion' into 'develop'

Bugfix: MRF and Pipeline Ingestion

See merge request pleroma/secteam/pleroma!15
This commit is contained in:
rinpatch 2020-09-17 12:13:36 +00:00
parent eff7f9892d
commit 22d49993d9
12 changed files with 121 additions and 50 deletions

View File

@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## unreleased-patch - ???
### Security
- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event)
### Fixed
- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance
- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user
## [2.1.1] - 2020-09-08 ## [2.1.1] - 2020-09-08
### Security ### Security

View File

@ -5,16 +5,34 @@
defmodule Pleroma.Web.ActivityPub.MRF do defmodule Pleroma.Web.ActivityPub.MRF do
@callback filter(Map.t()) :: {:ok | :reject, Map.t()} @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
def filter(policies, %{} = object) do def filter(policies, %{} = message) do
policies policies
|> Enum.reduce({:ok, object}, fn |> Enum.reduce({:ok, message}, fn
policy, {:ok, object} -> policy.filter(object) policy, {:ok, message} -> policy.filter(message)
_, error -> error _, error -> error
end) end)
end end
def filter(%{} = object), do: get_policies() |> filter(object) def filter(%{} = object), do: get_policies() |> filter(object)
def pipeline_filter(%{} = message, meta) do
object = meta[:object_data]
ap_id = message["object"]
if object && ap_id do
with {:ok, message} <- filter(Map.put(message, "object", object)) do
meta = Keyword.put(meta, :object_data, message["object"])
{:ok, Map.put(message, "object", ap_id), meta}
else
{err, message} -> {err, message, meta}
end
else
{err, message} = filter(message)
{err, message, meta}
end
end
def get_policies do def get_policies do
Pleroma.Config.get([:mrf, :policies], []) |> get_policies() Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
end end

View File

@ -20,9 +20,17 @@ defp string_matches?(string, pattern) do
String.match?(string, pattern) String.match?(string, pattern)
end end
defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do defp object_payload(%{} = object) do
[object["content"], object["summary"], object["name"]]
|> Enum.filter(& &1)
|> Enum.join("\n")
end
defp check_reject(%{"object" => %{} = object} = message) do
payload = object_payload(object)
if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
string_matches?(content, pattern) or string_matches?(summary, pattern) string_matches?(payload, pattern)
end) do end) do
{:reject, "[KeywordPolicy] Matches with rejected keyword"} {:reject, "[KeywordPolicy] Matches with rejected keyword"}
else else
@ -30,12 +38,12 @@ defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} =
end end
end end
defp check_ftl_removal( defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
%{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message payload = object_payload(object)
) do
if Pleroma.Constants.as_public() in to and if Pleroma.Constants.as_public() in to and
Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
string_matches?(content, pattern) or string_matches?(summary, pattern) string_matches?(payload, pattern)
end) do end) do
to = List.delete(to, Pleroma.Constants.as_public()) to = List.delete(to, Pleroma.Constants.as_public())
cc = [Pleroma.Constants.as_public() | message["cc"] || []] cc = [Pleroma.Constants.as_public() | message["cc"] || []]
@ -51,35 +59,24 @@ defp check_ftl_removal(
end end
end end
defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do defp check_replace(%{"object" => %{} = object} = message) do
content = object =
if is_binary(content) do ["content", "name", "summary"]
content |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
else |> Enum.reduce(object, fn field, object ->
"" data =
end
summary =
if is_binary(summary) do
summary
else
""
end
{content, summary} =
Enum.reduce( Enum.reduce(
Pleroma.Config.get([:mrf_keyword, :replace]), Pleroma.Config.get([:mrf_keyword, :replace]),
{content, summary}, object[field],
fn {pattern, replacement}, {content_acc, summary_acc} -> fn {pat, repl}, acc -> String.replace(acc, pat, repl) end
{String.replace(content_acc, pattern, replacement),
String.replace(summary_acc, pattern, replacement)}
end
) )
{:ok, Map.put(object, field, data)
message end)
|> put_in(["object", "content"], content)
|> put_in(["object", "summary"], summary)} message = Map.put(message, "object", object)
{:ok, message}
end end
@impl true @impl true

View File

@ -28,8 +28,7 @@ def filter(%{"actor" => actor} = message) do
}" }"
) )
subchain MRF.filter(subchain, message)
|> MRF.filter(message)
else else
_e -> {:ok, message} _e -> {:ok, message}
end end

View File

@ -26,13 +26,17 @@ def common_pipeline(object, meta) do
{:error, e} -> {:error, e} ->
{:error, e} {:error, e}
{:reject, e} ->
{:reject, e}
end end
end end
def do_common_pipeline(object, meta) do def do_common_pipeline(object, meta) do
with {_, {:ok, validated_object, meta}} <- with {_, {:ok, validated_object, meta}} <-
{:validate_object, ObjectValidator.validate(object, meta)}, {:validate_object, ObjectValidator.validate(object, meta)},
{_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)}, {_, {:ok, mrfd_object, meta}} <-
{:mrf_object, MRF.pipeline_filter(validated_object, meta)},
{_, {:ok, activity, meta}} <- {_, {:ok, activity, meta}} <-
{:persist_object, ActivityPub.persist(mrfd_object, meta)}, {:persist_object, ActivityPub.persist(mrfd_object, meta)},
{_, {:ok, activity, meta}} <- {_, {:ok, activity, meta}} <-
@ -40,7 +44,7 @@ def do_common_pipeline(object, meta) do
{_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
{:ok, activity, meta} {:ok, activity, meta}
else else
{:mrf_object, {:reject, _}} -> {:ok, nil, meta} {:mrf_object, {:reject, message, _}} -> {:reject, message}
e -> {:error, e} e -> {:error, e}
end end
end end

View File

@ -184,7 +184,8 @@ def post_chat_message_operation do
"application/json", "application/json",
ChatMessage ChatMessage
), ),
400 => Operation.response("Bad Request", "application/json", ApiError) 400 => Operation.response("Bad Request", "application/json", ApiError),
422 => Operation.response("MRF Rejection", "application/json", ApiError)
}, },
security: [ security: [
%{ %{

View File

@ -55,7 +55,7 @@ def create_operation do
"application/json", "application/json",
%Schema{oneOf: [Status, ScheduledStatus]} %Schema{oneOf: [Status, ScheduledStatus]}
), ),
422 => Operation.response("Bad Request", "application/json", ApiError) 422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError)
} }
} }
end end

View File

@ -49,6 +49,9 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ [])
local: true local: true
)} do )} do
{:ok, activity} {:ok, activity}
else
{:common_pipeline, {:reject, _} = e} -> e
e -> e
end end
end end

View File

@ -90,6 +90,16 @@ def post_chat_message(
conn conn
|> put_view(MessageReferenceView) |> put_view(MessageReferenceView)
|> render("show.json", chat_message_reference: cm_ref) |> render("show.json", chat_message_reference: cm_ref)
else
{:reject, message} ->
conn
|> put_status(:unprocessable_entity)
|> json(%{error: message})
{:error, message} ->
conn
|> put_status(:bad_request)
|> json(%{error: message})
end end
end end

View File

@ -26,7 +26,7 @@ test "when given an `object_data` in meta, Federation will receive a the origina
{ {
Pleroma.Web.ActivityPub.MRF, Pleroma.Web.ActivityPub.MRF,
[], [],
[filter: fn o -> {:ok, o} end] [pipeline_filter: fn o, m -> {:ok, o, m} end]
}, },
{ {
Pleroma.Web.ActivityPub.ActivityPub, Pleroma.Web.ActivityPub.ActivityPub,
@ -51,7 +51,7 @@ test "when given an `object_data` in meta, Federation will receive a the origina
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta) Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity)) assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta)) assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
refute called(Pleroma.Web.Federator.publish(activity)) refute called(Pleroma.Web.Federator.publish(activity))
@ -68,7 +68,7 @@ test "it goes through validation, filtering, persisting, side effects and federa
{ {
Pleroma.Web.ActivityPub.MRF, Pleroma.Web.ActivityPub.MRF,
[], [],
[filter: fn o -> {:ok, o} end] [pipeline_filter: fn o, m -> {:ok, o, m} end]
}, },
{ {
Pleroma.Web.ActivityPub.ActivityPub, Pleroma.Web.ActivityPub.ActivityPub,
@ -93,7 +93,7 @@ test "it goes through validation, filtering, persisting, side effects and federa
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta) Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity)) assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta)) assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
assert_called(Pleroma.Web.Federator.publish(activity)) assert_called(Pleroma.Web.Federator.publish(activity))
@ -109,7 +109,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
{ {
Pleroma.Web.ActivityPub.MRF, Pleroma.Web.ActivityPub.MRF,
[], [],
[filter: fn o -> {:ok, o} end] [pipeline_filter: fn o, m -> {:ok, o, m} end]
}, },
{ {
Pleroma.Web.ActivityPub.ActivityPub, Pleroma.Web.ActivityPub.ActivityPub,
@ -131,7 +131,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta) Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity)) assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta)) assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
end end
@ -148,7 +148,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
{ {
Pleroma.Web.ActivityPub.MRF, Pleroma.Web.ActivityPub.MRF,
[], [],
[filter: fn o -> {:ok, o} end] [pipeline_filter: fn o, m -> {:ok, o, m} end]
}, },
{ {
Pleroma.Web.ActivityPub.ActivityPub, Pleroma.Web.ActivityPub.ActivityPub,
@ -170,7 +170,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta) Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity)) assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta)) assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta)) assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
end end

View File

@ -213,6 +213,17 @@ test "it reject messages over the local limit" do
assert message == :content_too_long assert message == :content_too_long
end end
test "it reject messages via MRF" do
clear_config([:mrf_keyword, :reject], ["GNO"])
clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
author = insert(:user)
recipient = insert(:user)
assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} ==
CommonAPI.post_chat_message(author, recipient, "GNO/Linux")
end
end end
describe "unblocking" do describe "unblocking" do

View File

@ -100,7 +100,7 @@ test "it fails if there is no content", %{conn: conn, user: user} do
|> post("/api/v1/pleroma/chats/#{chat.id}/messages") |> post("/api/v1/pleroma/chats/#{chat.id}/messages")
|> json_response_and_validate_schema(400) |> json_response_and_validate_schema(400)
assert result assert %{"error" => "no_content"} == result
end end
test "it works with an attachment", %{conn: conn, user: user} do test "it works with an attachment", %{conn: conn, user: user} do
@ -126,6 +126,23 @@ test "it works with an attachment", %{conn: conn, user: user} do
assert result["attachment"] assert result["attachment"]
end end
test "gets MRF reason when rejected", %{conn: conn, user: user} do
clear_config([:mrf_keyword, :reject], ["GNO"])
clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
other_user = insert(:user)
{:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
result =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "GNO/Linux"})
|> json_response_and_validate_schema(422)
assert %{"error" => "[KeywordPolicy] Matches with rejected keyword"} == result
end
end end
describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do