Consolidate construction of Rich Media Parser HTTP requests

This commit is contained in:
Mark Felder 2020-08-03 12:37:31 -05:00
parent 56b5b9aa27
commit 016d8d6c56
3 changed files with 23 additions and 20 deletions

View File

@ -9,6 +9,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser alias Pleroma.Web.RichMedia.Parser
@rich_media_options [
pool: :media,
max_body: 2_000_000
]
@spec validate_page_url(URI.t() | binary()) :: :ok | :error @spec validate_page_url(URI.t() | binary()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
@ -77,4 +82,20 @@ def perform(:fetch, %Activity{} = activity) do
fetch_data_for_activity(activity) fetch_data_for_activity(activity)
:ok :ok
end end
def rich_media_get(url) do
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
options =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@rich_media_options,
recv_timeout: 2_000,
with_body: true
)
else
@rich_media_options
end
Pleroma.HTTP.get(url, headers, options)
end
end end

View File

@ -3,11 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do defmodule Pleroma.Web.RichMedia.Parser do
@options [
pool: :media,
max_body: 2_000_000
]
defp parsers do defp parsers do
Pleroma.Config.get([:rich_media, :parsers]) Pleroma.Config.get([:rich_media, :parsers])
end end
@ -75,21 +70,8 @@ defp get_ttl_from_image(data, url) do
end end
defp parse_url(url) do defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
try do try do
rich_media_agent = Pleroma.Application.user_agent() <> "; Bot" {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
{:ok, %Tesla.Env{body: html}} =
Pleroma.HTTP.get(url, [{"user-agent", rich_media_agent}], adapter: opts)
html html
|> parse_html() |> parse_html()

View File

@ -22,7 +22,7 @@ defp get_oembed_url([{"link", attributes, _children} | _]) do
end end
defp get_oembed_data(url) do defp get_oembed_data(url) do
with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do
Jason.decode(json) Jason.decode(json)
end end
end end