Merge branch 'fetch-posts-on-first-federation' into 'develop'

Fetch user's outbox posts on first federation with that user

Closes #131

See merge request pleroma/pleroma!907
This commit is contained in:
kaniini 2019-03-08 22:33:31 +00:00
commit 130fc9eae6
5 changed files with 149 additions and 10 deletions

View File

@ -348,6 +348,10 @@
federator_outgoing: [max_jobs: 50], federator_outgoing: [max_jobs: 50],
mailer: [max_jobs: 10] mailer: [max_jobs: 10]
config :pleroma, :fetch_initial_posts,
enabled: false,
pages: 5
config :auto_linker, config :auto_linker,
opts: [ opts: [
scheme: true, scheme: true,

View File

@ -285,6 +285,10 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
## :rich_media ## :rich_media
* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews * `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
## :fetch_initial_posts
* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts
* `pages`: the amount of pages to fetch
## :hackney_pools ## :hackney_pools
Advanced. Tweaks Hackney (http client) connections pools. Advanced. Tweaks Hackney (http client) connections pools.

View File

@ -532,6 +532,10 @@ def get_or_fetch_by_nickname(nickname) do
_e -> _e ->
with [_nick, _domain] <- String.split(nickname, "@"), with [_nick, _domain] <- String.split(nickname, "@"),
{:ok, user} <- fetch_by_nickname(nickname) do {:ok, user} <- fetch_by_nickname(nickname) do
if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
{:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
end
user user
else else
_e -> nil _e -> nil
@ -539,6 +543,17 @@ def get_or_fetch_by_nickname(nickname) do
end end
end end
@doc "Fetch some posts when the user has just been federated with"
def fetch_initial_posts(user) do
pages = Pleroma.Config.get!([:fetch_initial_posts, :pages])
Enum.each(
# Insert all the posts in reverse order, so they're in the right order on the timeline
Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)),
&Pleroma.Web.Federator.incoming_ap_doc/1
)
end
def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
from( from(
u in User, u in User,
@ -1121,24 +1136,36 @@ def html_filter_policy(%User{info: %{no_rich_text: true}}) do
def html_filter_policy(_), do: @default_scrubbers def html_filter_policy(_), do: @default_scrubbers
def fetch_by_ap_id(ap_id) do
ap_try = ActivityPub.make_user_from_ap_id(ap_id)
case ap_try do
{:ok, user} ->
user
_ ->
case OStatus.make_user(ap_id) do
{:ok, user} -> user
_ -> {:error, "Could not fetch by AP id"}
end
end
end
def get_or_fetch_by_ap_id(ap_id) do def get_or_fetch_by_ap_id(ap_id) do
user = get_by_ap_id(ap_id) user = get_by_ap_id(ap_id)
if !is_nil(user) and !User.needs_update?(user) do if !is_nil(user) and !User.needs_update?(user) do
user user
else else
ap_try = ActivityPub.make_user_from_ap_id(ap_id) user = fetch_by_ap_id(ap_id)
case ap_try do if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
{:ok, user} -> with %User{} = user do
user {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
end
_ ->
case OStatus.make_user(ap_id) do
{:ok, user} -> user
_ -> {:error, "Could not fetch by AP id"}
end
end end
user
end end
end end

View File

@ -633,4 +633,43 @@ def make_flag_data(params, additional) do
} }
|> Map.merge(additional) |> Map.merge(additional)
end end
@doc """
Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after
the first one to `pages_left` pages.
If the amount of pages is higher than the collection has, it returns whatever was there.
"""
def fetch_ordered_collection(from, pages_left, acc \\ []) do
with {:ok, response} <- Tesla.get(from),
{:ok, collection} <- Poison.decode(response.body) do
case collection["type"] do
"OrderedCollection" ->
# If we've encountered the OrderedCollection and not the page,
# just call the same function on the page address
fetch_ordered_collection(collection["first"], pages_left)
"OrderedCollectionPage" ->
if pages_left > 0 do
# There are still more pages
if Map.has_key?(collection, "next") do
# There are still more pages, go deeper saving what we have into the accumulator
fetch_ordered_collection(
collection["next"],
pages_left - 1,
acc ++ collection["orderedItems"]
)
else
# No more pages left, just return whatever we already have
acc ++ collection["orderedItems"]
end
else
# Got the amount of pages needed, add them all to the accumulator
acc ++ collection["orderedItems"]
end
_ ->
{:error, "Not an OrderedCollection or OrderedCollectionPage"}
end
end
end
end end

View File

@ -104,4 +104,69 @@ test "does not adress actor's follower address if the activity is not public", %
assert Enum.sort(cc) == expected_cc assert Enum.sort(cc) == expected_cc
end end
end end
describe "fetch_ordered_collection" do
import Tesla.Mock
test "fetches the first OrderedCollectionPage when an OrderedCollection is encountered" do
mock(fn
%{method: :get, url: "http://mastodon.com/outbox"} ->
json(%{"type" => "OrderedCollection", "first" => "http://mastodon.com/outbox?page=true"})
%{method: :get, url: "http://mastodon.com/outbox?page=true"} ->
json(%{"type" => "OrderedCollectionPage", "orderedItems" => ["ok"]})
end)
assert Utils.fetch_ordered_collection("http://mastodon.com/outbox", 1) == ["ok"]
end
test "fetches several pages in the right order one after another, but only the specified amount" do
mock(fn
%{method: :get, url: "http://example.com/outbox"} ->
json(%{
"type" => "OrderedCollectionPage",
"orderedItems" => [0],
"next" => "http://example.com/outbox?page=1"
})
%{method: :get, url: "http://example.com/outbox?page=1"} ->
json(%{
"type" => "OrderedCollectionPage",
"orderedItems" => [1],
"next" => "http://example.com/outbox?page=2"
})
%{method: :get, url: "http://example.com/outbox?page=2"} ->
json(%{"type" => "OrderedCollectionPage", "orderedItems" => [2]})
end)
assert Utils.fetch_ordered_collection("http://example.com/outbox", 0) == [0]
assert Utils.fetch_ordered_collection("http://example.com/outbox", 1) == [0, 1]
end
test "returns an error if the url doesn't have an OrderedCollection/Page" do
mock(fn
%{method: :get, url: "http://example.com/not-an-outbox"} ->
json(%{"type" => "NotAnOutbox"})
end)
assert {:error, _} = Utils.fetch_ordered_collection("http://example.com/not-an-outbox", 1)
end
test "returns the what was collected if there are less pages than specified" do
mock(fn
%{method: :get, url: "http://example.com/outbox"} ->
json(%{
"type" => "OrderedCollectionPage",
"orderedItems" => [0],
"next" => "http://example.com/outbox?page=1"
})
%{method: :get, url: "http://example.com/outbox?page=1"} ->
json(%{"type" => "OrderedCollectionPage", "orderedItems" => [1]})
end)
assert Utils.fetch_ordered_collection("http://example.com/outbox", 5) == [0, 1]
end
end
end end