From b1fc4fe0ca6abab97be69e0b1bf138e8b5c1c303 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 18 Aug 2020 02:01:40 +0200 Subject: [PATCH 1/5] fetcher: fallback to [] when to/cc is nil Related: https://git.pleroma.social/pleroma/pleroma/-/issues/2063 --- lib/pleroma/object/fetcher.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 3ff25118d..6fdbc8efd 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -125,8 +125,8 @@ def fetch_object_from_id(id, options \\ []) do defp prepare_activity_params(data) do %{ "type" => "Create", - "to" => data["to"], - "cc" => data["cc"], + "to" => data["to"] || [], + "cc" => data["cc"] || [], # Should we seriously keep this attributedTo thing? "actor" => data["actor"] || data["attributedTo"], "object" => data From 2bc08d5573782ba491c36450b817aa352264fb27 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 18 Aug 2020 01:48:42 +0200 Subject: [PATCH 2/5] Pipeline Ingestion: Audio --- .../web/activity_pub/object_validator.ex | 17 ++- .../object_validators/audio_validator.ex | 109 ++++++++++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 4 +- 4 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/audio_validator.ex diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 3f1dffe2b..d770ce1be 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -16,6 +16,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator @@ -137,6 +138,16 @@ def validate(%{"type" => "Question"} = object, meta) do end end + def validate(%{"type" => "Audio"} = object, meta) do + with {:ok, object} <- + object + |> AudioValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def validate(%{"type" => "Answer"} = object, meta) do with {:ok, object} <- object @@ -176,7 +187,7 @@ def validate( %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ["Question", "Answer"] do + when objtype in ~w[Question Answer Audio] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -210,6 +221,10 @@ def cast_and_apply(%{"type" => "Answer"} = object) do AnswerValidator.cast_and_apply(object) end + def cast_and_apply(%{"type" => "Audio"} = object) do + AudioValidator.cast_and_apply(object) + end + def cast_and_apply(o), do: {:error, {:validator_not_set, o}} # is_struct/1 isn't present in Elixir 1.8.x diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex new file mode 100644 index 000000000..5ff9e3832 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Utils + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + # Extends from NoteValidator + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + field(:content, :string) + field(:context, :string) + + # TODO: Remove actor on objects + field(:actor, ObjectValidators.ObjectID) + + field(:attributedTo, ObjectValidators.ObjectID) + field(:summary, :string) + field(:published, ObjectValidators.DateTime) + # TODO: Write type + field(:emoji, :map, default: %{}) + field(:sensitive, :boolean, default: false) + embeds_many(:attachment, AttachmentValidator) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inReplyTo, :string) + field(:uri, ObjectValidators.Uri) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) + + field(:likes, {:array, :string}, default: []) + field(:announcements, {:array, :string}, default: []) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults + defp fix_defaults(data) do + %{data: %{"id" => context}, id: context_id} = + Utils.create_context(data["context"] || data["conversation"]) + + data + |> Map.put_new_lazy("published", &Utils.make_date/0) + |> Map.put_new("context", context) + |> Map.put_new("context_id", context_id) + end + + defp fix_attribution(data) do + data + |> Map.put_new("actor", data["attributedTo"]) + end + + defp fix(data) do + data + |> fix_defaults() + |> fix_attribution() + end + + def changeset(struct, data) do + data = fix(data) + + struct + |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast_embed(:attachment) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Audio"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_host_match() + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index bcd6fd2fb..3dc66c60b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -340,7 +340,8 @@ def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do end end - def handle_object_creation(%{"type" => "Question"} = object, meta) do + def handle_object_creation(%{"type" => objtype} = object, meta) + when objtype in ~w[Audio Question] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 544f3f3b6..6be17e0ed 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -461,7 +461,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Event", "Note", "Video", "Page", "Audio"] do + when objtype in ~w{Article Event Note Video Page} do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -555,7 +555,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype}} = data, _options ) - when objtype in ["Question", "Answer", "ChatMessage"] do + when objtype in ~w{Question Answer ChatMessage Audio} do with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} From c9d6638461e62a5b9e357f55a6d6d4e468b6bc92 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 18 Aug 2020 02:11:38 +0200 Subject: [PATCH 3/5] common_fixes: Get fixes common from Audio and Question --- .../object_validators/audio_validator.ex | 23 +++---------------- .../object_validators/common_fixes.ex | 23 +++++++++++++++++++ .../object_validators/question_validator.ex | 22 +++--------------- 3 files changed, 29 insertions(+), 39 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/common_fixes.ex diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex index 5ff9e3832..5d9bf345f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex @@ -7,15 +7,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations - alias Pleroma.Web.ActivityPub.Utils import Ecto.Changeset @primary_key false @derive Jason.Encoder - # Extends from NoteValidator embedded_schema do field(:id, ObjectValidators.ObjectID, primary_key: true) field(:to, ObjectValidators.Recipients, default: []) @@ -67,26 +66,10 @@ def cast_data(data) do |> changeset(data) end - # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults - defp fix_defaults(data) do - %{data: %{"id" => context}, id: context_id} = - Utils.create_context(data["context"] || data["conversation"]) - - data - |> Map.put_new_lazy("published", &Utils.make_date/0) - |> Map.put_new("context", context) - |> Map.put_new("context_id", context_id) - end - - defp fix_attribution(data) do - data - |> Map.put_new("actor", data["attributedTo"]) - end - defp fix(data) do data - |> fix_defaults() - |> fix_attribution() + |> CommonFixes.fix_defaults() + |> CommonFixes.fix_attribution() end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex new file mode 100644 index 000000000..f13c16eca --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do + alias Pleroma.Web.ActivityPub.Utils + + # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults + def fix_defaults(data) do + %{data: %{"id" => context}, id: context_id} = + Utils.create_context(data["context"] || data["conversation"]) + + data + |> Map.put_new_lazy("published", &Utils.make_date/0) + |> Map.put_new("context", context) + |> Map.put_new("context_id", context_id) + end + + def fix_attribution(data) do + data + |> Map.put_new("actor", data["attributedTo"]) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index f47acf606..0aa70ee30 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -7,9 +7,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator - alias Pleroma.Web.ActivityPub.Utils import Ecto.Changeset @@ -81,27 +81,11 @@ defp fix_closed(data) do end end - # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults - defp fix_defaults(data) do - %{data: %{"id" => context}, id: context_id} = - Utils.create_context(data["context"] || data["conversation"]) - - data - |> Map.put_new_lazy("published", &Utils.make_date/0) - |> Map.put_new("context", context) - |> Map.put_new("context_id", context_id) - end - - defp fix_attribution(data) do - data - |> Map.put_new("actor", data["attributedTo"]) - end - defp fix(data) do data - |> fix_attribution() + |> CommonFixes.fix_defaults() + |> CommonFixes.fix_attribution() |> fix_closed() - |> fix_defaults() end def changeset(struct, data) do From 2f8c3c842dd48c26009e1272a28220175d0b1f06 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 18 Aug 2020 02:12:13 +0200 Subject: [PATCH 4/5] common_fixes: Remove Utils.make_date call --- lib/pleroma/web/activity_pub/object_validators/common_fixes.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index f13c16eca..721749de0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -11,7 +11,6 @@ def fix_defaults(data) do Utils.create_context(data["context"] || data["conversation"]) data - |> Map.put_new_lazy("published", &Utils.make_date/0) |> Map.put_new("context", context) |> Map.put_new("context_id", context_id) end From f0a8d723bb4c4ec31dd2ab5ce7a1606aa280efbb Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 18 Aug 2020 13:37:28 +0200 Subject: [PATCH 5/5] Transmogrifier Test: Extract audio tests. --- .../transmogrifier/audio_handling_test.exs | 45 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 29 ------------ 2 files changed, 45 insertions(+), 29 deletions(-) create mode 100644 test/web/activity_pub/transmogrifier/audio_handling_test.exs diff --git a/test/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/web/activity_pub/transmogrifier/audio_handling_test.exs new file mode 100644 index 000000000..c74a9c45d --- /dev/null +++ b/test/web/activity_pub/transmogrifier/audio_handling_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Pleroma.Factory + + test "it works for incoming listens" do + _user = insert(:user, ap_id: "http://mastodon.example.org/users/admin") + + data = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], + "type" => "Listen", + "id" => "http://mastodon.example.org/users/admin/listens/1234/activity", + "actor" => "http://mastodon.example.org/users/admin", + "object" => %{ + "type" => "Audio", + "id" => "http://mastodon.example.org/users/admin/listens/1234", + "attributedTo" => "http://mastodon.example.org/users/admin", + "title" => "lain radio episode 1", + "artist" => "lain", + "album" => "lain radio", + "length" => 180_000 + } + } + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(activity) + + assert object.data["title"] == "lain radio episode 1" + assert object.data["artist"] == "lain" + assert object.data["album"] == "lain radio" + assert object.data["length"] == 180_000 + end +end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 0dd4e6e47..3fa41b0c7 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -225,35 +225,6 @@ test "it works for incoming notices with hashtags" do assert Enum.at(object.data["tag"], 2) == "moo" end - test "it works for incoming listens" do - data = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "cc" => [], - "type" => "Listen", - "id" => "http://mastodon.example.org/users/admin/listens/1234/activity", - "actor" => "http://mastodon.example.org/users/admin", - "object" => %{ - "type" => "Audio", - "id" => "http://mastodon.example.org/users/admin/listens/1234", - "attributedTo" => "http://mastodon.example.org/users/admin", - "title" => "lain radio episode 1", - "artist" => "lain", - "album" => "lain radio", - "length" => 180_000 - } - } - - {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) - - object = Object.normalize(activity) - - assert object.data["title"] == "lain radio episode 1" - assert object.data["artist"] == "lain" - assert object.data["album"] == "lain radio" - assert object.data["length"] == 180_000 - end - test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!()