From f9cae0d04fe4609f85d800351566080b5a4f34b7 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 19 Dec 2018 17:21:35 +0100 Subject: [PATCH 1/7] [WIP,MastoAPI] Multi-tag timelines --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++++++++- .../mastodon_api/mastodon_api_controller.ex | 18 +++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6b4682e35..e0d020fab 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -426,10 +426,18 @@ defp restrict_since(query, %{"since_id" => since_id}) do defp restrict_since(query, _), do: query + defp restrict_tag(query, %{"tag" => tag, "tag_reject" => tag_reject}) when tag_reject != [] do + from( + activity in query, + where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data), + where: fragment("? @> (? #> '{\"object\",\"tag\"}')", ^tag_reject, activity.data) + ) + end + defp restrict_tag(query, %{"tag" => tag}) do from( activity in query, - where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data) + where: fragment("? && jsonb_array_elements_text((? #> '{\"object\",\"tag\"}'))", ^tag, activity.data) ) end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index f4736fcb5..6811f827e 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -540,15 +540,27 @@ def reblogged_by(conn, %{"id" => id}) do def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do local_only = params["local"] in [true, "True", "true", "1"] - params = + tags = + ([params["tag"]] ++ (params["all"] || []) ++ (params["any"] || [])) + |> Enum.uniq() + |> Enum.filter(& &1) + |> Enum.map(&String.downcase(&1)) + + tag_reject = + params["none"] || + [] + |> Enum.map(&String.downcase(&1)) + + query_params = params |> Map.put("type", "Create") |> Map.put("local_only", local_only) |> Map.put("blocking_user", user) - |> Map.put("tag", String.downcase(params["tag"])) + |> Map.put("tag", tags) + |> Map.put("tag_reject", tag_reject) activities = - ActivityPub.fetch_public_activities(params) + ActivityPub.fetch_public_activities(query_params) |> Enum.reverse() conn From 1a9bb4daa0df06ac0f3d06ddacec71fa25f64db5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 21 Dec 2018 18:24:13 +0100 Subject: [PATCH 2/7] [Web.ActivityPub.ActivityPub]: Fix restrict_tag() Thanks to Senko-san for the help on array-matching --- lib/pleroma/web/activity_pub/activity_pub.ex | 33 +++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e0d020fab..d414ecc46 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -426,18 +426,41 @@ defp restrict_since(query, %{"since_id" => since_id}) do defp restrict_since(query, _), do: query - defp restrict_tag(query, %{"tag" => tag, "tag_reject" => tag_reject}) when tag_reject != [] do + defp restrict_tag(query, %{"tag" => tag, "tag_reject" => tag_reject}) + when is_list(tag) and tag_reject != [] do from( activity in query, - where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data), - where: fragment("? @> (? #> '{\"object\",\"tag\"}')", ^tag_reject, activity.data) + where: + fragment( + "? && ARRAY(SELECT jsonb_array_elements_text((? #> '{\"object\",\"tag\"}')))", + ^tag, + activity.data + ), + where: + fragment( + "(not ? && ARRAY(SELECT jsonb_array_elements_text((? #> '{\"object\",\"tag\"}'))))", + ^tag_reject, + activity.data + ) ) end - defp restrict_tag(query, %{"tag" => tag}) do + defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( activity in query, - where: fragment("? && jsonb_array_elements_text((? #> '{\"object\",\"tag\"}'))", ^tag, activity.data) + where: + fragment( + "? && ARRAY(SELECT jsonb_array_elements_text((? #> '{\"object\",\"tag\"}')))", + ^tag, + activity.data + ) + ) + end + + defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do + from( + activity in query, + where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data) ) end From d8f446f438716cc1a474c9352e8cca8778a48d85 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 21 Dec 2018 19:34:08 +0100 Subject: [PATCH 3/7] Web.MastodonAPI.MastodonAPIControllerTest: Add test against multi-hashtag timeline --- .../mastodon_api_controller_test.exs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 6004285d6..be868c081 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1044,6 +1044,30 @@ test "hashtag timeline", %{conn: conn} do end) end + test "multi-hashtag timeline", %{conn: conn} do + user = insert(:user) + + {:ok, activity_test} = CommonAPI.post(user, %{"status" => "#test"}) + {:ok, activity_test1} = CommonAPI.post(user, %{"status" => "#test1"}) + {:ok, activity_none} = CommonAPI.post(user, %{"status" => "#test #none"}) + + all_test = + conn + |> get("/api/v1/timelines/tag/test", %{"all" => ["test1"]}) + + assert [status_none, status_test1, status_test] = json_response(all_test, 200) + + assert to_string(activity_test.id) == status_test["id"] + assert to_string(activity_test1.id) == status_test1["id"] + assert to_string(activity_none.id) == status_none["id"] + + restricted_test = + conn + |> get("/api/v1/timelines/tag/test", %{"all" => ["test1"], "none" => ["none"]}) + + assert [status_test1, status_test] == json_response(restricted_test, 200) + end + test "getting followers", %{conn: conn} do user = insert(:user) other_user = insert(:user) From 4ad0ad14ed2d8a10bbf642fd989b3f7f55f9017d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Jan 2019 16:07:32 +0100 Subject: [PATCH 4/7] Web.ActivityPub.ActivityPub: Simplify multi-hashtag, add tests --- lib/pleroma/web/activity_pub/activity_pub.ex | 21 +++----------------- test/web/activity_pub/activity_pub_test.exs | 21 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d414ecc46..62f4a33c8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -430,30 +430,15 @@ defp restrict_tag(query, %{"tag" => tag, "tag_reject" => tag_reject}) when is_list(tag) and tag_reject != [] do from( activity in query, - where: - fragment( - "? && ARRAY(SELECT jsonb_array_elements_text((? #> '{\"object\",\"tag\"}')))", - ^tag, - activity.data - ), - where: - fragment( - "(not ? && ARRAY(SELECT jsonb_array_elements_text((? #> '{\"object\",\"tag\"}'))))", - ^tag_reject, - activity.data - ) + where: fragment("(? #> '{\"object\",\"tag\"}') \\?| ?", activity.data, ^tag), + where: fragment("(not (? #> '{\"object\",\"tag\"}') \\?| ?)", activity.data, ^tag_reject) ) end defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( activity in query, - where: - fragment( - "? && ARRAY(SELECT jsonb_array_elements_text((? #> '{\"object\",\"tag\"}')))", - ^tag, - activity.data - ) + where: fragment("(? #> '{\"object\",\"tag\"}') \\?| ?", activity.data, ^tag) ) end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d2e54d804..acece36f0 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -64,6 +64,27 @@ test "it returns a user" do assert user.info.ap_enabled assert user.follower_address == "http://mastodon.example.org/users/admin/followers" end + + test "it fetches the appropriate tag-restricted posts" do + user = insert(:user) + + {:ok, status_one} = CommonAPI.post(user, %{"status" => ". #test"}) + {:ok, status_two} = CommonAPI.post(user, %{"status" => ". #essais"}) + {:ok, status_three} = CommonAPI.post(user, %{"status" => ". #test #reject"}) + + fetch_one = ActivityPub.fetch_activities([], %{"tag" => "test"}) + fetch_two = ActivityPub.fetch_activities([], %{"tag" => ["test", "essais"]}) + + fetch_three = + ActivityPub.fetch_activities([], %{ + "tag" => ["test", "essais"], + "tag_reject" => ["reject"] + }) + + assert fetch_one == [status_one, status_three] + assert fetch_two == [status_one, status_two, status_three] + assert fetch_three == [status_one, status_two] + end end describe "insertion" do From 5a84def6a6cd6ac782e16b2aace220a99c31ace7 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Jan 2019 16:44:28 +0100 Subject: [PATCH 5/7] Fix the logic in multi-hashtag TLs --- lib/pleroma/web/activity_pub/activity_pub.ex | 19 ++++++++++++++++--- .../mastodon_api/mastodon_api_controller.ex | 8 +++++++- test/web/activity_pub/activity_pub_test.exs | 7 +++++++ .../mastodon_api_controller_test.exs | 10 +++++++--- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 62f4a33c8..d94ad9748 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -426,15 +426,26 @@ defp restrict_since(query, %{"since_id" => since_id}) do defp restrict_since(query, _), do: query - defp restrict_tag(query, %{"tag" => tag, "tag_reject" => tag_reject}) - when is_list(tag) and tag_reject != [] do + defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) + when is_list(tag_reject) and tag_reject != [] do from( activity in query, - where: fragment("(? #> '{\"object\",\"tag\"}') \\?| ?", activity.data, ^tag), where: fragment("(not (? #> '{\"object\",\"tag\"}') \\?| ?)", activity.data, ^tag_reject) ) end + defp restrict_tag_reject(query, _), do: query + + defp restrict_tag_all(query, %{"tag_all" => tag_all}) + when is_list(tag_all) and tag_all != [] do + from( + activity in query, + where: fragment("(? #> '{\"object\",\"tag\"}') \\?& ?", activity.data, ^tag_all) + ) + end + + defp restrict_tag_all(query, _), do: query + defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( activity in query, @@ -591,6 +602,8 @@ def fetch_activities_query(recipients, opts \\ %{}) do base_query |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) + |> restrict_tag_reject(opts) + |> restrict_tag_all(opts) |> restrict_since(opts) |> restrict_local(opts) |> restrict_limit(opts) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 6811f827e..4c5f1e7a9 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -541,11 +541,16 @@ def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do local_only = params["local"] in [true, "True", "true", "1"] tags = - ([params["tag"]] ++ (params["all"] || []) ++ (params["any"] || [])) + ([params["tag"]] ++ (params["any"] || [])) |> Enum.uniq() |> Enum.filter(& &1) |> Enum.map(&String.downcase(&1)) + tag_all = + params["all"] || + [] + |> Enum.map(&String.downcase(&1)) + tag_reject = params["none"] || [] @@ -557,6 +562,7 @@ def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do |> Map.put("local_only", local_only) |> Map.put("blocking_user", user) |> Map.put("tag", tags) + |> Map.put("tag_all", tag_all) |> Map.put("tag_reject", tag_reject) activities = diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index acece36f0..48eed3f13 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -81,9 +81,16 @@ test "it fetches the appropriate tag-restricted posts" do "tag_reject" => ["reject"] }) + fetch_four = + ActivityPub.fetch_activities([], %{ + "tag" => ["test"], + "tag_all" => ["test", "reject"] + }) + assert fetch_one == [status_one, status_three] assert fetch_two == [status_one, status_two, status_three] assert fetch_three == [status_one, status_two] + assert fetch_four == [status_three] end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index be868c081..b4870e0b2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1051,11 +1051,11 @@ test "multi-hashtag timeline", %{conn: conn} do {:ok, activity_test1} = CommonAPI.post(user, %{"status" => "#test1"}) {:ok, activity_none} = CommonAPI.post(user, %{"status" => "#test #none"}) - all_test = + any_test = conn - |> get("/api/v1/timelines/tag/test", %{"all" => ["test1"]}) + |> get("/api/v1/timelines/tag/test", %{"any" => ["none"]}) - assert [status_none, status_test1, status_test] = json_response(all_test, 200) + [status_none, status_test1, status_test] = json_response(any_test, 200) assert to_string(activity_test.id) == status_test["id"] assert to_string(activity_test1.id) == status_test1["id"] @@ -1066,6 +1066,10 @@ test "multi-hashtag timeline", %{conn: conn} do |> get("/api/v1/timelines/tag/test", %{"all" => ["test1"], "none" => ["none"]}) assert [status_test1, status_test] == json_response(restricted_test, 200) + + all_test = conn |> get("/api/v1/timelines/tag/test", %{"all" => ["none"]}) + + assert [status_none] == json_response(all_test, 200) end test "getting followers", %{conn: conn} do From 22f2687f176a90d0d7d54d002e972d5f036c4579 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 22 Jan 2019 15:42:46 +0100 Subject: [PATCH 6/7] Web.MastodonAPI.MastodonAPIControllerTest: Update for difference between all and any parameters --- test/web/mastodon_api/mastodon_api_controller_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index b4870e0b2..3ca8d4651 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1048,12 +1048,12 @@ test "multi-hashtag timeline", %{conn: conn} do user = insert(:user) {:ok, activity_test} = CommonAPI.post(user, %{"status" => "#test"}) - {:ok, activity_test1} = CommonAPI.post(user, %{"status" => "#test1"}) + {:ok, activity_test1} = CommonAPI.post(user, %{"status" => "#test #test1"}) {:ok, activity_none} = CommonAPI.post(user, %{"status" => "#test #none"}) any_test = conn - |> get("/api/v1/timelines/tag/test", %{"any" => ["none"]}) + |> get("/api/v1/timelines/tag/test", %{"any" => ["test1"]}) [status_none, status_test1, status_test] = json_response(any_test, 200) @@ -1065,7 +1065,7 @@ test "multi-hashtag timeline", %{conn: conn} do conn |> get("/api/v1/timelines/tag/test", %{"all" => ["test1"], "none" => ["none"]}) - assert [status_test1, status_test] == json_response(restricted_test, 200) + assert [status_test1] == json_response(restricted_test, 200) all_test = conn |> get("/api/v1/timelines/tag/test", %{"all" => ["none"]}) From de956b9e04f9c3706d50522bf19a0fb6477feecc Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 26 Jan 2019 16:46:20 +0100 Subject: [PATCH 7/7] Web.MastodonAPI.MastodonAPIController: tag+any bookmark params in a array and flatten it --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 4c5f1e7a9..39e0f8492 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -541,7 +541,8 @@ def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do local_only = params["local"] in [true, "True", "true", "1"] tags = - ([params["tag"]] ++ (params["any"] || [])) + [params["tag"], params["any"]] + |> List.flatten() |> Enum.uniq() |> Enum.filter(& &1) |> Enum.map(&String.downcase(&1))