Merge branch 'blurhash' into 'develop'

Upload filter for media metadata (Support blurhash, get WxH)

See merge request pleroma/pleroma!3404
This commit is contained in:
feld 2021-05-18 21:41:34 +00:00
commit 0db436789d
14 changed files with 164 additions and 12 deletions

View File

@ -15,9 +15,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded.
- Return OAuth token `id` (primary key) in POST `/oauth/token`. - Return OAuth token `id` (primary key) in POST `/oauth/token`.
- `AnalyzeMetadata` upload filter for extracting attachment dimensions.
- Attachment dimensions are federated when available.
### Fixed ### Fixed
- Don't crash so hard when email settings are invalid. - Don't crash so hard when email settings are invalid.
- Checking activated Upload Filters for required commands.
## Unreleased (Patch) ## Unreleased (Patch)

View File

@ -164,9 +164,11 @@ defp do_check_rum!(setting, migrate) do
defp check_system_commands!(:ok) do defp check_system_commands!(:ok) do
filter_commands_statuses = [ filter_commands_statuses = [
check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"),
check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify") check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),
check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert")
] ]
preview_proxy_commands_status = preview_proxy_commands_status =

View File

@ -23,6 +23,9 @@ defmodule Pleroma.Upload do
is once created permanent and changing it (especially in uploaders) is probably a bad idea! is once created permanent and changing it (especially in uploaders) is probably a bad idea!
* `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the * `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the
path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over.
* `:width` - width of the media in pixels
* `:height` - height of the media in pixels
* `:blurhash` - string hash of the image encoded with the blurhash algorithm (https://blurha.sh/)
Related behaviors: Related behaviors:
@ -32,6 +35,7 @@ defmodule Pleroma.Upload do
""" """
alias Ecto.UUID alias Ecto.UUID
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Maps
require Logger require Logger
@type source :: @type source ::
@ -53,9 +57,12 @@ defmodule Pleroma.Upload do
name: String.t(), name: String.t(),
tempfile: String.t(), tempfile: String.t(),
content_type: String.t(), content_type: String.t(),
width: integer(),
height: integer(),
blurhash: String.t(),
path: String.t() path: String.t()
} }
defstruct [:id, :name, :tempfile, :content_type, :path] defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]
defp get_description(opts, upload) do defp get_description(opts, upload) do
case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do
@ -89,9 +96,12 @@ def store(upload, opts \\ []) do
"mediaType" => upload.content_type, "mediaType" => upload.content_type,
"href" => url_from_spec(upload, opts.base_url, url_spec) "href" => url_from_spec(upload, opts.base_url, url_spec)
} }
|> Maps.put_if_present("width", upload.width)
|> Maps.put_if_present("height", upload.height)
], ],
"name" => description "name" => description
}} }
|> Maps.put_if_present("blurhash", upload.blurhash)}
else else
{:description_limit, _} -> {:description_limit, _} ->
{:error, :description_too_long} {:error, :description_too_long}

View File

@ -0,0 +1,45 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
@moduledoc """
Extracts metadata about the upload, such as width/height
"""
require Logger
@behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) ::
{:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do
try do
image =
file
|> Mogrify.open()
|> Mogrify.verbose()
upload =
upload
|> Map.put(:width, image.width)
|> Map.put(:height, image.height)
|> Map.put(:blurhash, get_blurhash(file))
{:ok, :filtered, upload}
rescue
e in ErlangError ->
Logger.warn("#{__MODULE__}: #{inspect(e)}")
{:ok, :noop}
end
end
def filter(_), do: {:ok, :noop}
defp get_blurhash(file) do
with {:ok, blurhash} <- :eblurhash.magick(file) do
blurhash
else
_ -> nil
end
end
end

View File

@ -20,6 +20,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
field(:type, :string) field(:type, :string)
field(:href, ObjectValidators.Uri) field(:href, ObjectValidators.Uri)
field(:mediaType, :string, default: "application/octet-stream") field(:mediaType, :string, default: "application/octet-stream")
field(:width, :integer)
field(:height, :integer)
end end
end end
@ -51,7 +53,7 @@ def url_changeset(struct, data) do
data = fix_media_type(data) data = fix_media_type(data)
struct struct
|> cast(data, [:type, :href, :mediaType]) |> cast(data, [:type, :href, :mediaType, :width, :height])
|> validate_inclusion(:type, ["Link"]) |> validate_inclusion(:type, ["Link"])
|> validate_required([:type, :href, :mediaType]) |> validate_required([:type, :href, :mediaType])
end end

View File

@ -244,6 +244,8 @@ def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachm
"type" => Map.get(url || %{}, "type", "Link") "type" => Map.get(url || %{}, "type", "Link")
} }
|> Maps.put_if_present("mediaType", media_type) |> Maps.put_if_present("mediaType", media_type)
|> Maps.put_if_present("width", (url || %{})["width"] || data["width"])
|> Maps.put_if_present("height", (url || %{})["height"] || data["height"])
%{ %{
"url" => [attachment_url], "url" => [attachment_url],
@ -949,7 +951,7 @@ def prepare_attachments(object) do
object object
|> Map.get("attachment", []) |> Map.get("attachment", [])
|> Enum.map(fn data -> |> Enum.map(fn data ->
[%{"mediaType" => media_type, "href" => href} | _] = data["url"] [%{"mediaType" => media_type, "href" => href} = url | _] = data["url"]
%{ %{
"url" => href, "url" => href,
@ -957,6 +959,9 @@ def prepare_attachments(object) do
"name" => data["name"], "name" => data["name"],
"type" => "Document" "type" => "Document"
} }
|> Maps.put_if_present("width", url["width"])
|> Maps.put_if_present("height", url["height"])
|> Maps.put_if_present("blurhash", data["blurhash"])
end) end)
Map.put(object, "attachment", attachments) Map.put(object, "attachment", attachments)

View File

@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.Maps
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
@ -417,6 +418,7 @@ def render("attachment.json", %{attachment: attachment}) do
media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image" media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image"
href = attachment_url["href"] |> MediaProxy.url() href = attachment_url["href"] |> MediaProxy.url()
href_preview = attachment_url["href"] |> MediaProxy.preview_url() href_preview = attachment_url["href"] |> MediaProxy.preview_url()
meta = render("attachment_meta.json", %{attachment: attachment})
type = type =
cond do cond do
@ -439,8 +441,24 @@ def render("attachment.json", %{attachment: attachment}) do
pleroma: %{mime_type: media_type}, pleroma: %{mime_type: media_type},
blurhash: attachment["blurhash"] blurhash: attachment["blurhash"]
} }
|> Maps.put_if_present(:meta, meta)
end end
def render("attachment_meta.json", %{
attachment: %{"url" => [%{"width" => width, "height" => height} | _]}
})
when is_integer(width) and is_integer(height) do
%{
original: %{
width: width,
height: height,
aspect: width / height
}
}
end
def render("attachment_meta.json", _), do: nil
def render("context.json", %{activity: activity, activities: activities, user: user}) do def render("context.json", %{activity: activity, activities: activities, user: user}) do
%{ancestors: ancestors, descendants: descendants} = %{ancestors: ancestors, descendants: descendants} =
activities activities

View File

@ -196,6 +196,9 @@ defp deps do
{:majic, {:majic,
git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git",
ref: "289cda1b6d0d70ccb2ba508a2b0bd24638db2880"}, ref: "289cda1b6d0d70ccb2ba508a2b0bd24638db2880"},
{:eblurhash,
git: "https://github.com/zotonic/eblurhash.git",
ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"},
{:open_api_spex, "~> 3.10"}, {:open_api_spex, "~> 3.10"},
## dev & test ## dev & test

View File

@ -29,6 +29,7 @@
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
"earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"}, "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"},
"earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"},
"eblurhash": {:git, "https://github.com/zotonic/eblurhash.git", "04a0b76eadf4de1be17726f39b6313b88708fd12", [ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"]},
"ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"},
"ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
"ecto_explain": {:hex, :ecto_explain, "0.1.2", "a9d504cbd4adc809911f796d5ef7ebb17a576a6d32286c3d464c015bd39d5541", [:mix], [], "hexpm", "1d0e7798ae30ecf4ce34e912e5354a0c1c832b7ebceba39298270b9a9f316330"}, "ecto_explain": {:hex, :ecto_explain, "0.1.2", "a9d504cbd4adc809911f796d5ef7ebb17a576a6d32286c3d464c015bd39d5541", [:mix], [], "hexpm", "1d0e7798ae30ecf4ce34e912e5354a0c1c832b7ebceba39298270b9a9f316330"},

View File

@ -0,0 +1,19 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Upload.Filter.AnalyzeMetadataTest do
use Pleroma.DataCase, async: true
alias Pleroma.Upload.Filter.AnalyzeMetadata
test "adds the image dimensions" do
upload = %Pleroma.Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image.jpg"),
tempfile: Path.absname("test/fixtures/image.jpg")
}
assert {:ok, :filtered, %{width: 1024, height: 768}} = AnalyzeMetadata.filter(upload)
end
end

View File

@ -72,5 +72,38 @@ test "it handles our own uploads" do
assert attachment.mediaType == "image/jpeg" assert attachment.mediaType == "image/jpeg"
end end
test "it handles image dimensions" do
attachment = %{
"url" => [
%{
"type" => "Link",
"mediaType" => "image/jpeg",
"href" => "https://example.com/images/1.jpg",
"width" => 200,
"height" => 100
}
],
"type" => "Document",
"name" => nil,
"mediaType" => "image/jpeg"
}
{:ok, attachment} =
AttachmentValidator.cast_and_validate(attachment)
|> Ecto.Changeset.apply_action(:insert)
assert [
%{
href: "https://example.com/images/1.jpg",
type: "Link",
mediaType: "image/jpeg",
width: 200,
height: 100
}
] = attachment.url
assert attachment.mediaType == "image/jpeg"
end
end end
end end

View File

@ -76,7 +76,9 @@ test "Funkwhale Audio object" do
"href" => "href" =>
"https://channels.tests.funkwhale.audio/api/v1/listen/3901e5d8-0445-49d5-9711-e096cf32e515/?upload=42342395-0208-4fee-a38d-259a6dae0871&download=false", "https://channels.tests.funkwhale.audio/api/v1/listen/3901e5d8-0445-49d5-9711-e096cf32e515/?upload=42342395-0208-4fee-a38d-259a6dae0871&download=false",
"mediaType" => "audio/ogg", "mediaType" => "audio/ogg",
"type" => "Link" "type" => "Link",
"width" => nil,
"height" => nil
} }
] ]
} }

View File

@ -60,7 +60,9 @@ test "it remaps video URLs as attachments if necessary" do
"href" => "href" =>
"https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4", "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4",
"mediaType" => "video/mp4", "mediaType" => "video/mp4",
"type" => "Link" "type" => "Link",
"width" => nil,
"height" => nil
} }
] ]
} }
@ -83,7 +85,9 @@ test "it remaps video URLs as attachments if necessary" do
"href" => "href" =>
"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4", "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4",
"mediaType" => "video/mp4", "mediaType" => "video/mp4",
"type" => "Link" "type" => "Link",
"width" => nil,
"height" => nil
} }
] ]
} }
@ -113,7 +117,9 @@ test "it works for peertube videos with only their mpegURL map" do
"href" => "href" =>
"https://peertube.stream/static/streaming-playlists/hls/abece3c3-b9c6-47f4-8040-f3eed8c602e6/abece3c3-b9c6-47f4-8040-f3eed8c602e6-1080-fragmented.mp4", "https://peertube.stream/static/streaming-playlists/hls/abece3c3-b9c6-47f4-8040-f3eed8c602e6/abece3c3-b9c6-47f4-8040-f3eed8c602e6-1080-fragmented.mp4",
"mediaType" => "video/mp4", "mediaType" => "video/mp4",
"type" => "Link" "type" => "Link",
"width" => nil,
"height" => nil
} }
] ]
} }

View File

@ -459,7 +459,9 @@ test "attachments" do
"url" => [ "url" => [
%{ %{
"mediaType" => "image/png", "mediaType" => "image/png",
"href" => "someurl" "href" => "someurl",
"width" => 200,
"height" => 100
} }
], ],
"blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn",
@ -475,6 +477,7 @@ test "attachments" do
text_url: "someurl", text_url: "someurl",
description: nil, description: nil,
pleroma: %{mime_type: "image/png"}, pleroma: %{mime_type: "image/png"},
meta: %{original: %{width: 200, height: 100, aspect: 2}},
blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn"
} }