Merge branch 'media-proxy' into 'develop'

Media proxy

See merge request pleroma/pleroma!34
This commit is contained in:
lambda 2017-12-20 14:22:24 +00:00
commit 35ac549a99
10 changed files with 152 additions and 10 deletions

View File

@ -47,6 +47,11 @@
limit: 5000, limit: 5000,
registrations_open: true registrations_open: true
config :pleroma, :media_proxy,
enabled: false,
redirect_on_failure: true
#base_url: "https://cache.pleroma.social"
# Import environment specific config. This must remain at the bottom # Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above. # of this file so it overrides the configuration defined above.
import_config "#{Mix.env}.exs" import_config "#{Mix.env}.exs"

View File

@ -1,3 +1,6 @@
proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m max_size=10g
inactive=720m use_temp_path=off;
server { server {
listen 80; listen 80;
server_name example.tld; server_name example.tld;
@ -24,6 +27,13 @@ server {
proxy_set_header Connection "upgrade"; proxy_set_header Connection "upgrade";
proxy_pass http://localhost:4000; proxy_pass http://localhost:4000;
} }
location /proxy {
proxy_cache pleroma_media_cache;
proxy_cache_lock on;
proxy_pass http://localhost:4000;
}
include snippets/well-known.conf; include snippets/well-known.conf;
} }

View File

@ -1,5 +1,6 @@
defmodule Pleroma.Formatter do defmodule Pleroma.Formatter do
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.MediaProxy
@link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u
def linkify(text) do def linkify(text) do
@ -125,7 +126,7 @@ def emojify(text, additional \\ nil) do
end end
Enum.reduce(all_emoji, text, fn ({emoji, file}, text) -> Enum.reduce(all_emoji, text, fn ({emoji, file}, text) ->
String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{file}' />") String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />")
end) end)
end end

View File

@ -3,6 +3,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MediaProxy
defp image_url(%{"url" => [ %{ "href" => href } | _ ]}), do: href defp image_url(%{"url" => [ %{ "href" => href } | _ ]}), do: href
defp image_url(_), do: nil defp image_url(_), do: nil
@ -12,10 +13,10 @@ def render("accounts.json", %{users: users} = opts) do
end end
def render("account.json", %{user: user}) do def render("account.json", %{user: user}) do
image = User.avatar_url(user) image = User.avatar_url(user) |> MediaProxy.url()
user_info = User.user_info(user) user_info = User.user_info(user)
header = image_url(user.info["banner"]) || "https://placehold.it/700x335" header = (image_url(user.info["banner"]) || "https://placehold.it/700x335") |> MediaProxy.url()
%{ %{
id: to_string(user.id), id: to_string(user.id),

View File

@ -3,6 +3,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
alias Pleroma.Web.MastodonAPI.{AccountView, StatusView} alias Pleroma.Web.MastodonAPI.{AccountView, StatusView}
alias Pleroma.{User, Activity} alias Pleroma.{User, Activity}
alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MediaProxy
def render("index.json", opts) do def render("index.json", opts) do
render_many(opts.activities, StatusView, "status.json", opts) render_many(opts.activities, StatusView, "status.json", opts)
@ -121,9 +122,9 @@ def render("attachment.json", %{attachment: attachment}) do
%{ %{
id: to_string(attachment["id"] || hash_id), id: to_string(attachment["id"] || hash_id),
url: href, url: MediaProxy.url(href),
remote_url: href, remote_url: href,
preview_url: href, preview_url: MediaProxy.url(href),
text_url: href, text_url: href,
type: type type: type
} }

View File

@ -0,0 +1,83 @@
defmodule Pleroma.Web.MediaProxy.MediaProxyController do
use Pleroma.Web, :controller
require Logger
@max_body_length 25 * 1048576
@cache_control %{
default: "public, max-age=1209600",
error: "public, must-revalidate, max-age=160",
}
def remote(conn, %{"sig" => sig, "url" => url}) do
config = Application.get_env(:pleroma, :media_proxy, [])
with \
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- Pleroma.Web.MediaProxy.decode_url(sig, url),
url = URI.encode(url),
{:ok, content_type, body} <- proxy_request(url)
do
conn
|> put_resp_content_type(content_type)
|> set_cache_header(:default)
|> send_resp(200, body)
else
false -> send_error(conn, 404)
{:error, :invalid_signature} -> send_error(conn, 403)
{:error, {:http, _, url}} -> redirect_or_error(conn, url, Keyword.get(config, :redirect_on_failure, true))
end
end
defp proxy_request(link) do
headers = [{"user-agent", "Pleroma/MediaProxy; #{Pleroma.Web.base_url()} <#{Application.get_env(:pleroma, :instance)[:email]}>"}]
options = [:insecure, {:follow_redirect, true}]
with \
{:ok, 200, headers, client} <- :hackney.request(:get, link, headers, "", options),
headers = Enum.into(headers, Map.new),
{:ok, body} <- proxy_request_body(client),
content_type <- proxy_request_content_type(headers, body)
do
{:ok, content_type, body}
else
{:ok, status, _, _} ->
Logger.warn "MediaProxy: request failed, status #{status}, link: #{link}"
{:error, {:http, :bad_status, link}}
{:error, error} ->
Logger.warn "MediaProxy: request failed, error #{inspect error}, link: #{link}"
{:error, {:http, error, link}}
end
end
defp set_cache_header(conn, key) do
Plug.Conn.put_resp_header(conn, "cache-control", @cache_control[key])
end
defp redirect_or_error(conn, url, true), do: redirect(conn, external: url)
defp redirect_or_error(conn, url, _), do: send_error(conn, 502, "Media proxy error: " <> url)
defp send_error(conn, code, body \\ "") do
conn
|> set_cache_header(:error)
|> send_resp(code, body)
end
defp proxy_request_body(client), do: proxy_request_body(client, <<>>)
defp proxy_request_body(client, body) when byte_size(body) < @max_body_length do
case :hackney.stream_body(client) do
{:ok, data} -> proxy_request_body(client, <<body :: binary, data :: binary>>)
:done -> {:ok, body}
{:error, reason} -> {:error, reason}
end
end
defp proxy_request_body(client, _) do
:hackney.close(client)
{:error, :body_too_large}
end
# TODO: the body is passed here as well because some hosts do not provide a content-type.
# At some point we may want to use magic numbers to discover the content-type and reply a proper one.
defp proxy_request_content_type(headers, _body) do
headers["Content-Type"] || headers["content-type"] || "image/jpeg"
end
end

View File

@ -0,0 +1,32 @@
defmodule Pleroma.Web.MediaProxy do
@base64_opts [padding: false]
def url(nil), do: nil
def url(url = "/" <> _), do: url
def url(url) do
config = Application.get_env(:pleroma, :media_proxy, [])
if !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url) do
url
else
secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
base64 = Base.url_encode64(url, @base64_opts)
sig = :crypto.hmac(:sha, secret, base64)
sig64 = sig |> Base.url_encode64(@base64_opts)
Keyword.get(config, :base_url, Pleroma.Web.base_url) <> "/proxy/#{sig64}/#{base64}"
end
end
def decode_url(sig, url) do
secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
sig = Base.url_decode64!(sig, @base64_opts)
local_sig = :crypto.hmac(:sha, secret, url)
if local_sig == sig do
{:ok, Base.url_decode64!(url, @base64_opts)}
else
{:error, :invalid_signature}
end
end
end

View File

@ -238,6 +238,14 @@ def user_fetcher(username) do
delete "/auth/sign_out", MastodonAPIController, :logout delete "/auth/sign_out", MastodonAPIController, :logout
end end
pipeline :remote_media do
plug :accepts, ["html"]
end
scope "/proxy/", Pleroma.Web.MediaProxy do
pipe_through :remote_media
get "/:sig/:url", MediaProxyController, :remote
end
scope "/", Fallback do scope "/", Fallback do
get "/*path", RedirectController, :redirector get "/*path", RedirectController, :redirector
end end

View File

@ -6,7 +6,7 @@ def to_map(%Object{} = object, _opts) do
data = object.data data = object.data
url = List.first(data["url"]) url = List.first(data["url"])
%{ %{
url: url["href"], url: url["href"] |> Pleroma.Web.MediaProxy.url(),
mimetype: url["mediaType"], mimetype: url["mediaType"],
id: data["uuid"], id: data["uuid"],
oembed: false oembed: false

View File

@ -2,6 +2,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
use Pleroma.Web, :view use Pleroma.Web, :view
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MediaProxy
def render("show.json", %{user: user = %User{}} = assigns) do def render("show.json", %{user: user = %User{}} = assigns) do
render_one(user, Pleroma.Web.TwitterAPI.UserView, "user.json", assigns) render_one(user, Pleroma.Web.TwitterAPI.UserView, "user.json", assigns)
@ -12,7 +13,7 @@ def render("index.json", %{users: users, for: user}) do
end end
def render("user.json", %{user: user = %User{}} = assigns) do def render("user.json", %{user: user = %User{}} = assigns) do
image = User.avatar_url(user) image = User.avatar_url(user) |> MediaProxy.url()
{following, follows_you, statusnet_blocking} = if assigns[:for] do {following, follows_you, statusnet_blocking} = if assigns[:for] do
{ {
User.following?(assigns[:for], user), User.following?(assigns[:for], user),
@ -44,8 +45,8 @@ def render("user.json", %{user: user = %User{}} = assigns) do
"screen_name" => user.nickname, "screen_name" => user.nickname,
"statuses_count" => user_info[:note_count], "statuses_count" => user_info[:note_count],
"statusnet_profile_url" => user.ap_id, "statusnet_profile_url" => user.ap_id,
"cover_photo" => image_url(user.info["banner"]), "cover_photo" => image_url(user.info["banner"]) |> MediaProxy.url(),
"background_image" => image_url(user.info["background"]) "background_image" => image_url(user.info["background"]) |> MediaProxy.url(),
} }
if assigns[:token] do if assigns[:token] do