From 02d3dc6869f388388ea744ea4ee3b54279d55e86 Mon Sep 17 00:00:00 2001 From: href Date: Thu, 29 Nov 2018 21:11:45 +0100 Subject: [PATCH] Uploads fun, part. 2 --- config/config.exs | 12 +- lib/mix/tasks/migrate_local_uploads.ex | 61 ++-- lib/pleroma/mime.ex | 100 +++++++ lib/pleroma/reverse_proxy.ex | 6 +- lib/pleroma/upload.ex | 271 +++++++++--------- lib/pleroma/upload/filter.ex | 35 +++ lib/pleroma/upload/filter/dedupe.ex | 10 + lib/pleroma/upload/filter/mogrifun.ex | 60 ++++ lib/pleroma/upload/filter/mogrify.ex | 37 +++ lib/pleroma/uploaders/local.ex | 39 +-- lib/pleroma/uploaders/mdii.ex | 9 +- lib/pleroma/uploaders/s3.ex | 10 +- lib/pleroma/uploaders/swift/uploader.ex | 11 +- lib/pleroma/uploaders/uploader.ex | 25 +- .../mastodon_api/mastodon_api_controller.ex | 25 +- .../web/twitter_api/twitter_api_controller.ex | 19 +- test/upload_test.exs | 26 +- 17 files changed, 491 insertions(+), 265 deletions(-) create mode 100644 lib/pleroma/mime.ex create mode 100644 lib/pleroma/upload/filter.ex create mode 100644 lib/pleroma/upload/filter/dedupe.ex create mode 100644 lib/pleroma/upload/filter/mogrifun.ex create mode 100644 lib/pleroma/upload/filter/mogrify.ex diff --git a/config/config.exs b/config/config.exs index ee43071ea..d7869464e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,11 +10,19 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes +# Upload configuration config :pleroma, Pleroma.Upload, uploader: Pleroma.Uploaders.Local, - strip_exif: false, + # filters: [Pleroma.Upload.DedupeFilter, Pleroma.Upload.MogrifyFilter], + filters: [], proxy_remote: false, - proxy_opts: [inline_content_types: true, keep_user_agent: true] + proxy_opts: [] + +# Strip Exif +# Also put Pleroma.Upload.MogrifyFilter in the `filters` list of Pleroma.Upload configuration. +# config :pleroma, Pleroma.Upload.MogrifyFilter, +# args: "strip" +# Pleroma.Upload.MogrifyFilter: [args: "strip"] config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads" diff --git a/lib/mix/tasks/migrate_local_uploads.ex b/lib/mix/tasks/migrate_local_uploads.ex index 40117350c..8f9e210c0 100644 --- a/lib/mix/tasks/migrate_local_uploads.ex +++ b/lib/mix/tasks/migrate_local_uploads.ex @@ -34,33 +34,50 @@ def run([target_uploader | args]) do :timer.sleep(:timer.seconds(5)) end - uploads = File.ls!(local_path) + uploads = + File.ls!(local_path) + |> Enum.map(fn id -> + root_path = Path.join(local_path, id) + + cond do + File.dir?(root_path) -> + files = for file <- File.ls!(root_path), do: {id, file, Path.join([root_path, file])} + + case List.first(files) do + {id, file, path} -> + {%Pleroma.Upload{id: id, name: file, path: id <> "/" <> file, tempfile: path}, + root_path} + + _ -> + nil + end + + File.exists?(root_path) -> + file = Path.basename(id) + [hash, ext] = String.split(id, ".") + {%Pleroma.Upload{id: hash, name: file, path: file, tempfile: root_path}, root_path} + + true -> + nil + end + end) + |> Enum.filter(& &1) + total_count = length(uploads) + Logger.info("Found #{total_count} uploads") uploads |> Task.async_stream( - fn uuid -> - u_path = Path.join(local_path, uuid) + fn {upload, root_path} -> + case Upload.store(upload, uploader: uploader, filters: [], size_limit: nil) do + {:ok, _} -> + if delete?, do: File.rm_rf!(root_path) + Logger.debug("uploaded: #{inspect(upload.path)} #{inspect(upload)}") + :ok - {name, path} = - cond do - File.dir?(u_path) -> - files = for file <- File.ls!(u_path), do: {{file, uuid}, Path.join([u_path, file])} - List.first(files) - - File.exists?(u_path) -> - # {uuid, u_path} - raise "should_dedupe local storage not supported yet sorry" - end - - {:ok, _} = - Upload.store({:from_local, name, path}, should_dedupe: false, uploader: uploader) - - if delete? do - File.rm_rf!(u_path) + error -> + Logger.error("failed to upload #{inspect(upload.path)}: #{inspect(error)}") end - - Logger.debug("uploaded: #{inspect(name)}") end, timeout: 150_000 ) @@ -75,6 +92,6 @@ def run([target_uploader | args]) do end def run(_) do - Logger.error("Usage: migrate_local_uploads UploaderName [--delete]") + Logger.error("Usage: migrate_local_uploads S3|Swift [--delete]") end end diff --git a/lib/pleroma/mime.ex b/lib/pleroma/mime.ex new file mode 100644 index 000000000..377e6d11a --- /dev/null +++ b/lib/pleroma/mime.ex @@ -0,0 +1,100 @@ +defmodule Pleroma.MIME do + @moduledoc """ + Returns the mime-type of a binary and optionally a normalized file-name. Requires at least (the first) 8 bytes. + """ + @default "application/octet-stream" + + @spec file_mime_type(String.t()) :: + {:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error + def file_mime_type(path, filename) do + with {:ok, content_type} <- file_mime_type(path), + filename <- fix_extension(filename, content_type) do + {:ok, content_type, filename} + end + end + + @spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error + def file_mime_type(filename) do + File.open(filename, [:read], fn f -> + check_mime_type(IO.binread(f, 8)) + end) + end + + def bin_mime_type(binary, filename) do + with {:ok, content_type} <- bin_mime_type(binary), + filename <- fix_extension(filename, content_type) do + {:ok, content_type, filename} + end + end + + @spec bin_mime_type(binary()) :: {:ok, String.t()} | :error + def bin_mime_type(<>) do + {:ok, check_mime_type(head)} + end + + def mime_type(<<_::binary>>), do: {:ok, @default} + + def bin_mime_type(_), do: :error + + defp fix_extension(filename, content_type) do + parts = String.split(filename, ".") + + new_filename = + if length(parts) > 1 do + Enum.drop(parts, -1) |> Enum.join(".") + else + Enum.join(parts) + end + + cond do + content_type == "application/octet-stream" -> + filename + + ext = List.first(MIME.extensions(content_type)) -> + new_filename <> "." <> ext + + true -> + Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".") + end + end + + defp check_mime_type(<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A>>) do + "image/png" + end + + defp check_mime_type(<<0x47, 0x49, 0x46, 0x38, _, 0x61, _, _>>) do + "image/gif" + end + + defp check_mime_type(<<0xFF, 0xD8, 0xFF, _, _, _, _, _>>) do + "image/jpeg" + end + + defp check_mime_type(<<0x1A, 0x45, 0xDF, 0xA3, _, _, _, _>>) do + "video/webm" + end + + defp check_mime_type(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70>>) do + "video/mp4" + end + + defp check_mime_type(<<0x49, 0x44, 0x33, _, _, _, _, _>>) do + "audio/mpeg" + end + + defp check_mime_type(<<255, 251, _, 68, 0, 0, 0, 0>>) do + "audio/mpeg" + end + + defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00>>) do + "audio/ogg" + end + + defp check_mime_type(<<0x52, 0x49, 0x46, 0x46, _, _, _, _>>) do + "audio/wav" + end + + defp check_mime_type(_) do + @default + end +end diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 4f9f0b169..dc1c50d07 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -101,7 +101,7 @@ def call(conn = %{method: method}, url, opts \\ []) when method in @methods do end with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), - :ok <- header_lenght_constraint(headers, Keyword.get(opts, :max_body_length)) do + :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length)) do response(conn, client, url, code, headers, opts) else {:ok, code, headers} -> @@ -298,7 +298,7 @@ defp build_resp_content_disposition_header(headers, opts) do end end - defp header_lenght_constraint(headers, limit) when is_integer(limit) and limit > 0 do + defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do with {_, size} <- List.keyfind(headers, "content-length", 0), {size, _} <- Integer.parse(size), true <- size <= limit do @@ -312,7 +312,7 @@ defp header_lenght_constraint(headers, limit) when is_integer(limit) and limit > end end - defp header_lenght_constraint(_, _), do: :ok + defp header_length_constraint(_, _), do: :ok defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do {:error, :body_too_large} diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 16043a264..f2607b603 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -1,31 +1,73 @@ defmodule Pleroma.Upload do + @moduledoc """ + # Upload + + Options: + * `:type`: presets for activity type (defaults to Document) and size limits from app configuration + * `:description`: upload alternative text + * `:uploader`: override uploader + * `:filters`: override filters + * `:size_limit`: override size limit + * `:activity_type`: override activity type + + The `%Pleroma.Upload{}` struct: all documented fields are meant to be overwritten in filters: + + * `:id` - the upload id. + * `:name` - the upload file name. + * `:path` - the upload path: set at first to `id/name` but can be changed. Keep in mind that the path + is once created permanent and changing it (especially in uploaders) is probably a bad idea! + * `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the + path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. + + Related behaviors: + + * `Pleroma.Uploaders.Uploader` + * `Pleroma.Upload.Filter` + + """ alias Ecto.UUID require Logger - @type upload_option :: - {:dedupe, boolean()} | {:size_limit, non_neg_integer()} | {:uploader, module()} - @type upload_source :: - Plug.Upload.t() | data_uri_string() :: - String.t() | {:from_local, name :: String.t(), uuid :: String.t(), path :: String.t()} + @type source :: + Plug.Upload.t() | data_uri_string :: + String.t() | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()} - @spec store(upload_source, options :: [upload_option()]) :: {:ok, Map.t()} | {:error, any()} + @type option :: + {:type, :avatar | :banner | :background} + | {:description, String.t()} + | {:activity_type, String.t()} + | {:size_limit, nil | non_neg_integer()} + | {:uploader, module()} + | {:filters, [module()]} + + @type t :: %__MODULE__{ + id: String.t(), + name: String.t(), + tempfile: String.t(), + content_type: String.t(), + path: String.t() + } + defstruct [:id, :name, :tempfile, :content_type, :path] + + @spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()} def store(upload, opts \\ []) do opts = get_opts(opts) - with {:ok, name, uuid, path, content_type} <- process_upload(upload, opts), - _ <- strip_exif_data(content_type, path), - {:ok, url_spec} <- opts.uploader.put_file(name, uuid, path, content_type, opts) do + with {:ok, upload} <- prepare_upload(upload, opts), + upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"}, + {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload), + {:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do {:ok, %{ - "type" => "Image", + "type" => opts.activity_type, "url" => [ %{ "type" => "Link", - "mediaType" => content_type, + "mediaType" => upload.content_type, "href" => url_from_spec(url_spec) } ], - "name" => name + "name" => Map.get(opts, :description) || upload.name }} else {:error, error} -> @@ -38,40 +80,98 @@ def store(upload, opts \\ []) do end defp get_opts(opts) do - %{ - dedupe: Keyword.get(opts, :dedupe, Pleroma.Config.get([:instance, :dedupe_media])), - size_limit: Keyword.get(opts, :size_limit, Pleroma.Config.get([:instance, :upload_limit])), - uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])) + {size_limit, activity_type} = + case Keyword.get(opts, :type) do + :banner -> + {Pleroma.Config.get!([:instance, :banner_upload_limit]), "Image"} + + :avatar -> + {Pleroma.Config.get!([:instance, :avatar_upload_limit]), "Image"} + + :background -> + {Pleroma.Config.get!([:instance, :background_upload_limit]), "Image"} + + _ -> + {Pleroma.Config.get!([:instance, :upload_limit]), "Document"} + end + + opts = %{ + activity_type: Keyword.get(opts, :activity_type, activity_type), + size_limit: Keyword.get(opts, :size_limit, size_limit), + uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])), + filters: Keyword.get(opts, :filters, Pleroma.Config.get([__MODULE__, :filters])), + description: Keyword.get(opts, :description) } + + # TODO: 1.0+ : remove old config compatibility + opts = + if Pleroma.Config.get([__MODULE__, :strip_exif]) == true && + !Enum.member?(opts.filters, Pleroma.Upload.Filter.Mogrify) do + Logger.warn(""" + Pleroma: configuration `:instance, :strip_exif` is deprecated, please instead set: + + :instance, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]] + + :pleroma, Pleroma.Upload.Mogrify, args: "strip" + """) + + Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: "strip") + Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify]) + else + opts + end + + opts = + if Pleroma.Config.get([:instance, :dedupe_media]) == true && + !Enum.member?(opts.filters, Pleroma.Upload.Filter.Dedupe) do + Logger.warn(""" + Pleroma: configuration `:instance, :dedupe_media` is deprecated, please instead set: + + :instance, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Dedupe]] + """) + + Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Dedupe]) + else + opts + end end - defp process_upload(%Plug.Upload{} = file, opts) do + defp prepare_upload(%Plug.Upload{} = file, opts) do with :ok <- check_file_size(file.path, opts.size_limit), - uuid <- get_uuid(file, opts.dedupe), - content_type <- get_content_type(file.path), - name <- get_name(file, uuid, content_type, opts.dedupe) do - {:ok, name, uuid, file.path, content_type} + {:ok, content_type, name} <- Pleroma.MIME.file_mime_type(file.path, file.filename) do + {:ok, + %__MODULE__{ + id: UUID.generate(), + name: name, + tempfile: file.path, + content_type: content_type + }} end end - defp process_upload(%{"img" => "data:image/" <> image_data}, opts) do + defp prepare_upload(%{"img" => "data:image/" <> image_data}, opts) do parsed = Regex.named_captures(~r/(?jpeg|png|gif);base64,(?.*)/, image_data) data = Base.decode64!(parsed["data"], ignore: :whitespace) hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data))) with :ok <- check_binary_size(data, opts.size_limit), tmp_path <- tempfile_for_image(data), - content_type <- get_content_type(tmp_path), - uuid <- UUID.generate(), - name <- create_name(hash, parsed["filetype"], content_type) do - {:ok, name, uuid, tmp_path, content_type} + {:ok, content_type, name} <- + Pleroma.MIME.bin_mime_type(data, hash <> "." <> parsed["filetype"]) do + {:ok, + %__MODULE__{ + id: UUID.generate(), + name: name, + tempfile: tmp_path, + content_type: content_type + }} end end # For Mix.Tasks.MigrateLocalUploads - defp process_upload({:from_local, name, uuid, path}, _opts) do - with content_type <- get_content_type(path) do - {:ok, name, uuid, path, content_type} + defp prepare_upload(upload = %__MODULE__{tempfile: path}, _opts) do + with {:ok, content_type} <- Pleroma.MIME.file_mime_type(path) do + {:ok, %__MODULE__{upload | content_type: content_type}} end end @@ -104,119 +204,6 @@ defp tempfile_for_image(data) do tmp_path end - defp strip_exif_data(content_type, file) do - settings = Application.get_env(:pleroma, Pleroma.Upload) - do_strip = Keyword.fetch!(settings, :strip_exif) - [filetype, _ext] = String.split(content_type, "/") - - if filetype == "image" and do_strip == true do - Mogrify.open(file) |> Mogrify.custom("strip") |> Mogrify.save(in_place: true) - end - end - - defp create_name(uuid, ext, type) do - extension = - cond do - type == "application/octect-stream" -> ext - ext = mime_extension(ext) -> ext - true -> String.split(type, "/") |> List.last() - end - - [uuid, extension] - |> Enum.join(".") - |> String.downcase() - end - - defp mime_extension(type) do - List.first(MIME.extensions(type)) - end - - defp get_uuid(file, should_dedupe) do - if should_dedupe do - Base.encode16(:crypto.hash(:sha256, File.read!(file.path))) - else - UUID.generate() - end - end - - defp get_name(file, uuid, type, should_dedupe) do - if should_dedupe do - create_name(uuid, List.last(String.split(file.filename, ".")), type) - else - parts = String.split(file.filename, ".") - - new_filename = - if length(parts) > 1 do - Enum.drop(parts, -1) |> Enum.join(".") - else - Enum.join(parts) - end - - cond do - type == "application/octet-stream" -> - file.filename - - ext = mime_extension(type) -> - new_filename <> "." <> ext - - true -> - Enum.join([new_filename, String.split(type, "/") |> List.last()], ".") - end - end - end - - def get_content_type(file) do - match = - File.open(file, [:read], fn f -> - case IO.binread(f, 8) do - <<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A>> -> - "image/png" - - <<0x47, 0x49, 0x46, 0x38, _, 0x61, _, _>> -> - "image/gif" - - <<0xFF, 0xD8, 0xFF, _, _, _, _, _>> -> - "image/jpeg" - - <<0x1A, 0x45, 0xDF, 0xA3, _, _, _, _>> -> - "video/webm" - - <<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70>> -> - "video/mp4" - - <<0x49, 0x44, 0x33, _, _, _, _, _>> -> - "audio/mpeg" - - <<255, 251, _, 68, 0, 0, 0, 0>> -> - "audio/mpeg" - - <<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00>> -> - case IO.binread(f, 27) do - <<_::size(160), 0x80, 0x74, 0x68, 0x65, 0x6F, 0x72, 0x61>> -> - "video/ogg" - - _ -> - "audio/ogg" - end - - <<0x52, 0x49, 0x46, 0x46, _, _, _, _>> -> - "audio/wav" - - _ -> - "application/octet-stream" - end - end) - - case match do - {:ok, type} -> type - _e -> "application/octet-stream" - end - end - - defp uploader() do - Pleroma.Config.get!([Pleroma.Upload, :uploader]) - end - defp url_from_spec({:file, path}) do [Pleroma.Web.base_url(), "media", path] |> Path.join() diff --git a/lib/pleroma/upload/filter.ex b/lib/pleroma/upload/filter.ex new file mode 100644 index 000000000..d1384ddad --- /dev/null +++ b/lib/pleroma/upload/filter.ex @@ -0,0 +1,35 @@ +defmodule Pleroma.Upload.Filter do + @moduledoc """ + Upload Filter behaviour + + This behaviour allows to run filtering actions just before a file is uploaded. This allows to: + + * morph in place the temporary file + * change any field of a `Pleroma.Upload` struct + * cancel/stop the upload + """ + + require Logger + + @callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()} + + @spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()} + + def filter([], upload) do + {:ok, upload} + end + + def filter([filter | rest], upload) do + case filter.filter(upload) do + :ok -> + filter(rest, upload) + + {:ok, upload} -> + filter(rest, upload) + + error -> + Logger.error("#{__MODULE__}: Filter #{filter} failed: #{inspect(error)}") + error + end + end +end diff --git a/lib/pleroma/upload/filter/dedupe.ex b/lib/pleroma/upload/filter/dedupe.ex new file mode 100644 index 000000000..28091a627 --- /dev/null +++ b/lib/pleroma/upload/filter/dedupe.ex @@ -0,0 +1,10 @@ +defmodule Pleroma.Upload.Filter.Dedupe do + @behaviour Pleroma.Upload.Filter + + def filter(upload = %Pleroma.Upload{name: name, tempfile: path}) do + extension = String.split(name, ".") |> List.last() + shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower) + filename = shasum <> "." <> extension + {:ok, %Pleroma.Upload{upload | id: shasum, path: filename}} + end +end diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex new file mode 100644 index 000000000..4d4f0b401 --- /dev/null +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -0,0 +1,60 @@ +defmodule Pleroma.Upload.Filter.Mogrifun do + @behaviour Pleroma.Upload.Filter + + @filters [ + {"implode", "1"}, + {"-raise", "20"}, + {"+raise", "20"}, + [{"-interpolate", "nearest"}, {"-virtual-pixel", "mirror"}, {"-spread", "5"}], + "+polaroid", + {"-statistic", "Mode 10"}, + {"-emboss", "0x1.1"}, + {"-emboss", "0x2"}, + {"-colorspace", "Gray"}, + "-negate", + [{"-channel", "green"}, "-negate"], + [{"-channel", "red"}, "-negate"], + [{"-channel", "blue"}, "-negate"], + {"+level-colors", "green,gold"}, + {"+level-colors", ",DodgerBlue"}, + {"+level-colors", ",Gold"}, + {"+level-colors", ",Lime"}, + {"+level-colors", ",Red"}, + {"+level-colors", ",DarkGreen"}, + {"+level-colors", "firebrick,yellow"}, + {"+level-colors", "'rgb(102,75,25)',lemonchiffon"}, + [{"fill", "red"}, {"tint", "40"}], + [{"fill", "green"}, {"tint", "40"}], + [{"fill", "blue"}, {"tint", "40"}], + [{"fill", "yellow"}, {"tint", "40"}] + ] + + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do + filter = Enum.random(@filters) + + file + |> Mogrify.open() + |> mogrify_filter(filter) + |> Mogrify.save(in_place: true) + + :ok + end + + def filter(_), do: :ok + + defp mogrify_filter(mogrify, [filter | rest]) do + mogrify + |> mogrify_filter(filter) + |> mogrify_filter(rest) + end + + defp mogrify_filter(mogrify, []), do: mogrify + + defp mogrify_filter(mogrify, {action, options}) do + Mogrify.custom(mogrify, action, options) + end + + defp mogrify_filter(mogrify, string) when is_binary(string) do + Mogrify.custom(mogrify, string) + end +end diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex new file mode 100644 index 000000000..d6ed471ed --- /dev/null +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -0,0 +1,37 @@ +defmodule Pleroma.Upload.Filter.Mogrify do + @behaviour Pleroma.Uploader.Filter + + @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} + @type conversions :: conversion() | [conversion()] + + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do + filters = Pleroma.Config.get!([__MODULE__, :args]) + + file + |> Mogrify.open() + |> mogrify_filter(filters) + |> Mogrify.save(in_place: true) + + :ok + end + + def filter(_), do: :ok + + defp mogrify_filter(mogrify, nil), do: mogrify + + defp mogrify_filter(mogrify, [filter | rest]) do + mogrify + |> mogrify_filter(filter) + |> mogrify_filter(rest) + end + + defp mogrify_filter(mogrify, []), do: mogrify + + defp mogrify_filter(mogrify, {action, options}) do + Mogrify.custom(mogrify, action, options) + end + + defp mogrify_filter(mogrify, action) when is_binary(action) do + Mogrify.custom(mogrify, action) + end +end diff --git a/lib/pleroma/uploaders/local.ex b/lib/pleroma/uploaders/local.ex index 7ca1ba07d..434a6b515 100644 --- a/lib/pleroma/uploaders/local.ex +++ b/lib/pleroma/uploaders/local.ex @@ -7,39 +7,28 @@ def get_file(_) do {:ok, {:static_dir, upload_path()}} end - def put_file(name, uuid, tmpfile, _content_type, opts) do - upload_folder = get_upload_path(uuid, opts.dedupe) + def put_file(upload) do + {local_path, file} = + case Enum.reverse(String.split(upload.path, "/", trim: true)) do + [file] -> + {upload_path(), file} - File.mkdir_p!(upload_folder) + [file | folders] -> + path = Path.join([upload_path()] ++ Enum.reverse(folders)) + File.mkdir_p!(path) + {path, file} + end - result_file = Path.join(upload_folder, name) + result_file = Path.join(local_path, file) - if File.exists?(result_file) do - File.rm!(tmpfile) - else - File.cp!(tmpfile, result_file) + unless File.exists?(result_file) do + File.cp!(upload.tempfile, result_file) end - {:ok, {:file, get_url(name, uuid, opts.dedupe)}} + :ok end def upload_path do Pleroma.Config.get!([__MODULE__, :uploads]) end - - defp get_upload_path(uuid, should_dedupe) do - if should_dedupe do - upload_path() - else - Path.join(upload_path(), uuid) - end - end - - defp get_url(name, uuid, should_dedupe) do - if should_dedupe do - :cow_uri.urlencode(name) - else - Path.join(uuid, :cow_uri.urlencode(name)) - end - end end diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex index 1d93c8154..35d36d3e4 100644 --- a/lib/pleroma/uploaders/mdii.ex +++ b/lib/pleroma/uploaders/mdii.ex @@ -11,22 +11,21 @@ def get_file(file) do Pleroma.Uploaders.Local.get_file(file) end - def put_file(name, uuid, path, content_type, opts) do + def put_file(upload) do cgi = Pleroma.Config.get([Pleroma.Uploaders.MDII, :cgi]) files = Pleroma.Config.get([Pleroma.Uploaders.MDII, :files]) - {:ok, file_data} = File.read(path) + {:ok, file_data} = File.read(upload.tempfile) - extension = String.split(name, ".") |> List.last() + extension = String.split(upload.name, ".") |> List.last() query = "#{cgi}?#{extension}" with {:ok, %{status_code: 200, body: body}} <- @httpoison.post(query, file_data) do - File.rm!(path) remote_file_name = String.split(body) |> List.first() public_url = "#{files}/#{remote_file_name}.#{extension}" {:ok, {:url, public_url}} else - _ -> Pleroma.Uploaders.Local.put_file(name, uuid, path, content_type, opts) + _ -> Pleroma.Uploaders.Local.put_file(upload) end end end diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 2d1ddef75..19832a7ec 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -15,20 +15,18 @@ def get_file(file) do ])}} end - def put_file(name, uuid, path, content_type, _opts) do + def put_file(upload = %Pleroma.Upload{}) do config = Pleroma.Config.get([__MODULE__]) bucket = Keyword.get(config, :bucket) - {:ok, file_data} = File.read(path) + {:ok, file_data} = File.read(upload.tempfile) - File.rm!(path) - - s3_name = "#{uuid}/#{strict_encode(name)}" + s3_name = strict_encode(upload.path) op = ExAws.S3.put_object(bucket, s3_name, file_data, [ {:acl, :public_read}, - {:content_type, content_type} + {:content_type, upload.content_type} ]) case ExAws.request(op) do diff --git a/lib/pleroma/uploaders/swift/uploader.ex b/lib/pleroma/uploaders/swift/uploader.ex index 5db35fe50..b35b9807b 100644 --- a/lib/pleroma/uploaders/swift/uploader.ex +++ b/lib/pleroma/uploaders/swift/uploader.ex @@ -5,10 +5,11 @@ def get_file(name) do {:ok, {:url, Path.join([Pleroma.Config.get!([__MODULE__, :object_url]), name])}} end - def put_file(name, uuid, tmp_path, content_type, _opts) do - {:ok, file_data} = File.read(tmp_path) - remote_name = "#{uuid}/#{name}" - - Pleroma.Uploaders.Swift.Client.upload_file(remote_name, file_data, content_type) + def put_file(upload) do + Pleroma.Uploaders.Swift.Client.upload_file( + upload.path, + File.read!(upload.tmpfile), + upload.content_type + ) end end diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex index 8ef82b4ef..afda5609e 100644 --- a/lib/pleroma/uploaders/uploader.ex +++ b/lib/pleroma/uploaders/uploader.ex @@ -16,20 +16,25 @@ defmodule Pleroma.Uploaders.Uploader do Returns: + * `:ok` which assumes `{:ok, upload.path}` * `{:ok, spec}` where spec is: * `{:file, filename :: String.t}` to handle reads with `get_file/1` (recommended) - This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL. - - * `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity. + This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL. + * `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity. * `{:error, String.t}` error information if the file failed to be saved to the backend. + """ - @callback put_file( - name :: String.t(), - uuid :: String.t(), - file :: File.t(), - content_type :: String.t(), - options :: Map.t() - ) :: {:ok, {:file, String.t()} | {:url, String.t()}} | {:error, String.t()} + @callback put_file(Pleroma.Upload.t()) :: + :ok | {:ok, {:file | :url, String.t()}} | {:error, String.t()} + + @spec put_file(module(), Pleroma.Upload.t()) :: + {:ok, {:file | :url, String.t()}} | {:error, String.t()} + def put_file(uploader, upload) do + case uploader.put_file(upload) do + :ok -> {:ok, {:file, upload.path}} + other -> other + end + end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 9d50d906d..009be50e7 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -35,14 +35,6 @@ def create_app(conn, params) do def update_credentials(%{assigns: %{user: user}} = conn, params) do original_user = user - avatar_upload_limit = - Application.get_env(:pleroma, :instance) - |> Keyword.fetch(:avatar_upload_limit) - - banner_upload_limit = - Application.get_env(:pleroma, :instance) - |> Keyword.fetch(:banner_upload_limit) - params = if bio = params["note"] do Map.put(params, "bio", bio) @@ -60,7 +52,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do user = if avatar = params["avatar"] do with %Plug.Upload{} <- avatar, - {:ok, object} <- ActivityPub.upload(avatar, size_limit: avatar_upload_limit), + {:ok, object} <- ActivityPub.upload(avatar, type: :avatar), change = Ecto.Changeset.change(user, %{avatar: object.data}), {:ok, user} = User.update_and_set_cache(change) do user @@ -74,7 +66,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do user = if banner = params["header"] do with %Plug.Upload{} <- banner, - {:ok, object} <- ActivityPub.upload(banner, size_limit: banner_upload_limit), + {:ok, object} <- ActivityPub.upload(banner, type: :banner), new_info <- Map.put(user.info, "banner", object.data), change <- User.info_changeset(user, %{info: new_info}), {:ok, user} <- User.update_and_set_cache(change) do @@ -471,19 +463,12 @@ def update_media(%{assigns: %{user: _}} = conn, data) do end def upload(%{assigns: %{user: _}} = conn, %{"file" => file} = data) do - with {:ok, object} <- ActivityPub.upload(file) do - objdata = - if Map.has_key?(data, "description") do - Map.put(object.data, "name", data["description"]) - else - object.data - end - - change = Object.change(object, %{data: objdata}) + with {:ok, object} <- ActivityPub.upload(file, description: Map.get(data, "description")) do + change = Object.change(object, %{data: object.data}) {:ok, object} = Repo.update(change) objdata = - objdata + object.data |> Map.put("id", object.id) render(conn, StatusView, "attachment.json", %{attachment: objdata}) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index fa9ee9f99..064730867 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -290,11 +290,7 @@ def register(conn, params) do end def update_avatar(%{assigns: %{user: user}} = conn, params) do - upload_limit = - Application.get_env(:pleroma, :instance) - |> Keyword.fetch(:avatar_upload_limit) - - {:ok, object} = ActivityPub.upload(params, size_limit: upload_limit) + {:ok, object} = ActivityPub.upload(params, type: :avatar) change = Changeset.change(user, %{avatar: object.data}) {:ok, user} = User.update_and_set_cache(change) CommonAPI.update(user) @@ -303,12 +299,7 @@ def update_avatar(%{assigns: %{user: user}} = conn, params) do end def update_banner(%{assigns: %{user: user}} = conn, params) do - upload_limit = - Application.get_env(:pleroma, :instance) - |> Keyword.fetch(:banner_upload_limit) - - with {:ok, object} <- - ActivityPub.upload(%{"img" => params["banner"]}, size_limit: upload_limit), + with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner), new_info <- Map.put(user.info, "banner", object.data), change <- User.info_changeset(user, %{info: new_info}), {:ok, user} <- User.update_and_set_cache(change) do @@ -322,11 +313,7 @@ def update_banner(%{assigns: %{user: user}} = conn, params) do end def update_background(%{assigns: %{user: user}} = conn, params) do - upload_limit = - Application.get_env(:pleroma, :instance) - |> Keyword.fetch(:background_upload_limit) - - with {:ok, object} <- ActivityPub.upload(params, size_limit: upload_limit), + with {:ok, object} <- ActivityPub.upload(params, type: :background), new_info <- Map.put(user.info, "background", object.data), change <- User.info_changeset(user, %{info: new_info}), {:ok, _user} <- User.update_and_set_cache(change) do diff --git a/test/upload_test.exs b/test/upload_test.exs index 998245b29..65562cb2a 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -5,16 +5,23 @@ defmodule Pleroma.UploadTest do describe "Storing a file with the Local uploader" do setup do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + filters = Pleroma.Config.get([Pleroma.Upload, :filters]) + + unless uploader == Pleroma.Uploaders.Local || filters != [] do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + Pleroma.Config.put([Pleroma.Upload, :filters], []) - unless uploader == Pleroma.Uploaders.Local do on_exit(fn -> Pleroma.Config.put([Pleroma.Upload, :uploader], uploader) + Pleroma.Config.put([Pleroma.Upload, :filters], filters) end) end :ok end + OH - HELLO - EAL + test "returns a media url" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") @@ -40,10 +47,11 @@ test "copies the file to the configured folder with deduping" do filename: "an [image.jpg" } - {:ok, data} = Upload.store(file, dedupe: true) + {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe]) - assert data["name"] == - "e7a6d0cf595bff76f14c9a98b6c199539559e8b844e02e51e5efcfd1f614a2df.jpeg" + assert List.first(data["url"])["href"] == + Pleroma.Web.base_url() <> + "/media/e7a6d0cf595bff76f14c9a98b6c199539559e8b844e02e51e5efcfd1f614a2df.jpg" end test "copies the file to the configured folder without deduping" do @@ -55,7 +63,7 @@ test "copies the file to the configured folder without deduping" do filename: "an [image.jpg" } - {:ok, data} = Upload.store(file, dedupe: false) + {:ok, data} = Upload.store(file) assert data["name"] == "an [image.jpg" end @@ -68,7 +76,7 @@ test "fixes incorrect content type" do filename: "an [image.jpg" } - {:ok, data} = Upload.store(file, dedupe: true) + {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe]) assert hd(data["url"])["mediaType"] == "image/jpeg" end @@ -81,7 +89,7 @@ test "adds missing extension" do filename: "an [image" } - {:ok, data} = Upload.store(file, dedupe: false) + {:ok, data} = Upload.store(file) assert data["name"] == "an [image.jpg" end @@ -94,7 +102,7 @@ test "fixes incorrect file extension" do filename: "an [image.blah" } - {:ok, data} = Upload.store(file, dedupe: false) + {:ok, data} = Upload.store(file) assert data["name"] == "an [image.jpg" end @@ -107,7 +115,7 @@ test "don't modify filename of an unknown type" do filename: "test.txt" } - {:ok, data} = Upload.store(file, dedupe: false) + {:ok, data} = Upload.store(file) assert data["name"] == "test.txt" end end