Merge branch 'cache-plug' into 'develop'

Add Pleroma.Plugs.Cache

Closes #1174

See merge request pleroma/pleroma!1612
This commit is contained in:
feld 2019-09-09 18:53:09 +00:00
commit ceb2e09126
11 changed files with 457 additions and 7 deletions

View File

@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mix Tasks: `mix pleroma.database fix_likes_collections` - Mix Tasks: `mix pleroma.database fix_likes_collections`
- Federation: Remove `likes` from objects. - Federation: Remove `likes` from objects.
- Admin API: Added moderation log - Admin API: Added moderation log
- Web response cache (currently, enabled for ActivityPub)
### Changed ### Changed
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

View File

@ -560,6 +560,10 @@
config :pleroma, Pleroma.ActivityExpiration, enabled: true config :pleroma, Pleroma.ActivityExpiration, enabled: true
config :pleroma, :web_cache_ttl,
activity_pub: nil,
activity_pub_question: 30_000
# Import environment specific config. This must remain at the bottom # Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above. # of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs" import_config "#{Mix.env()}.exs"

View File

@ -690,3 +690,12 @@ Supported rate limiters:
* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) * `:relation_id_action` for actions on relation with a specific user (follow, unfollow)
* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses
* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user
## :web_cache_ttl
The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration.
Available caches:
* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration).
* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds).

View File

@ -308,10 +308,19 @@ def delete_by_ap_id(id) when is_binary(id) do
%{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
_ -> nil _ -> nil
end) end)
|> purge_web_resp_cache()
end end
def delete_by_ap_id(_), do: nil def delete_by_ap_id(_), do: nil
defp purge_web_resp_cache(%Activity{} = activity) do
%{path: path} = URI.parse(activity.data["id"])
Cachex.del(:web_resp_cache, path)
activity
end
defp purge_web_resp_cache(nil), do: nil
for {ap_type, type} <- @mastodon_notification_types do for {ap_type, type} <- @mastodon_notification_types do
def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}), def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
do: unquote(type) do: unquote(type)

View File

@ -116,7 +116,8 @@ defp cachex_children do
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500), build_cachex("scrubber", limit: 2500),
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500) build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500)
] ]
end end

View File

@ -130,14 +130,16 @@ def swap_object_with_tombstone(object) do
def delete(%Object{data: %{"id" => id}} = object) do def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object), with {:ok, _obj} = swap_object_with_tombstone(object),
deleted_activity = Activity.delete_by_ap_id(id), deleted_activity = Activity.delete_by_ap_id(id),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, object, deleted_activity} {:ok, object, deleted_activity}
end end
end end
def prune(%Object{data: %{"id" => id}} = object) do def prune(%Object{data: %{"id" => id}} = object) do
with {:ok, object} <- Repo.delete(object), with {:ok, object} <- Repo.delete(object),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, object} {:ok, object}
end end
end end

122
lib/pleroma/plugs/cache.ex Normal file
View File

@ -0,0 +1,122 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Plugs.Cache do
@moduledoc """
Caches successful GET responses.
To enable the cache add the plug to a router pipeline or controller:
plug(Pleroma.Plugs.Cache)
## Configuration
To configure the plug you need to pass settings as the second argument to the `plug/2` macro:
plug(Pleroma.Plugs.Cache, [ttl: nil, query_params: true])
Available options:
- `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`.
- `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`.
Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct:
def index(conn, _params) do
ttl = 60_000 # one minute
conn
|> assign(:cache_ttl, ttl)
|> render("index.html")
end
"""
import Phoenix.Controller, only: [current_path: 1, json: 2]
import Plug.Conn
@behaviour Plug
@defaults %{ttl: nil, query_params: true}
@impl true
def init([]), do: @defaults
def init(opts) do
opts = Map.new(opts)
Map.merge(@defaults, opts)
end
@impl true
def call(%{method: "GET"} = conn, opts) do
key = cache_key(conn, opts)
case Cachex.get(:web_resp_cache, key) do
{:ok, nil} ->
cache_resp(conn, opts)
{:ok, record} ->
send_cached(conn, record)
{atom, message} when atom in [:ignore, :error] ->
render_error(conn, message)
end
end
def call(conn, _), do: conn
# full path including query params
defp cache_key(conn, %{query_params: true}), do: current_path(conn)
# request path without query params
defp cache_key(conn, %{query_params: false}), do: conn.request_path
# request path with specific query params
defp cache_key(conn, %{query_params: query_params}) when is_list(query_params) do
query_string =
conn.params
|> Map.take(query_params)
|> URI.encode_query()
conn.request_path <> "?" <> query_string
end
defp cache_resp(conn, opts) do
register_before_send(conn, fn
%{status: 200, resp_body: body} = conn ->
ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl)
key = cache_key(conn, opts)
content_type = content_type(conn)
record = {content_type, body}
Cachex.put(:web_resp_cache, key, record, ttl: ttl)
put_resp_header(conn, "x-cache", "MISS from Pleroma")
conn ->
conn
end)
end
defp content_type(conn) do
conn
|> Plug.Conn.get_resp_header("content-type")
|> hd()
end
defp send_cached(conn, {content_type, body}) do
conn
|> put_resp_content_type(content_type, nil)
|> put_resp_header("x-cache", "HIT from Pleroma")
|> send_resp(:ok, body)
|> halt()
end
defp render_error(conn, message) do
conn
|> put_status(:internal_server_error)
|> json(%{error: message})
|> halt()
end
end

View File

@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
action_fallback(:errors) action_fallback(:errors)
plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object])
plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
plug(:set_requester_reachable when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay]) plug(:relay_active? when action in [:relay])
@ -53,8 +54,10 @@ def object(conn, %{"uuid" => uuid}) do
%Object{} = object <- Object.get_cached_by_ap_id(ap_id), %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
{_, true} <- {:public?, Visibility.is_public?(object)} do {_, true} <- {:public?, Visibility.is_public?(object)} do
conn conn
|> set_cache_ttl_for(object)
|> put_resp_content_type("application/activity+json") |> put_resp_content_type("application/activity+json")
|> json(ObjectView.render("object.json", %{object: object})) |> put_view(ObjectView)
|> render("object.json", object: object)
else else
{:public?, false} -> {:public?, false} ->
{:error, :not_found} {:error, :not_found}
@ -96,14 +99,36 @@ def activity(conn, %{"uuid" => uuid}) do
%Activity{} = activity <- Activity.normalize(ap_id), %Activity{} = activity <- Activity.normalize(ap_id),
{_, true} <- {:public?, Visibility.is_public?(activity)} do {_, true} <- {:public?, Visibility.is_public?(activity)} do
conn conn
|> set_cache_ttl_for(activity)
|> put_resp_content_type("application/activity+json") |> put_resp_content_type("application/activity+json")
|> json(ObjectView.render("object.json", %{object: activity})) |> put_view(ObjectView)
|> render("object.json", object: activity)
else else
{:public?, false} -> {:public?, false} -> {:error, :not_found}
{:error, :not_found} nil -> {:error, :not_found}
end end
end end
defp set_cache_ttl_for(conn, %Activity{object: object}) do
set_cache_ttl_for(conn, object)
end
defp set_cache_ttl_for(conn, entity) do
ttl =
case entity do
%Object{data: %{"type" => "Question"}} ->
Pleroma.Config.get([:web_cache_ttl, :activity_pub_question])
%Object{} ->
Pleroma.Config.get([:web_cache_ttl, :activity_pub])
_ ->
nil
end
assign(conn, :cache_ttl, ttl)
end
# GET /relay/following # GET /relay/following
def following(%{assigns: %{relay: true}} = conn, _params) do def following(%{assigns: %{relay: true}} = conn, _params) do
conn conn

View File

@ -53,9 +53,12 @@ test "ensures cache is cleared for the object" do
assert object == cached_object assert object == cached_object
Cachex.put(:web_resp_cache, URI.parse(object.data["id"]).path, "cofe")
Object.delete(cached_object) Object.delete(cached_object)
{:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}") {:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}")
{:ok, nil} = Cachex.get(:web_resp_cache, URI.parse(object.data["id"]).path)
cached_object = Object.get_cached_by_ap_id(object.data["id"]) cached_object = Object.get_cached_by_ap_id(object.data["id"])

186
test/plugs/cache_test.exs Normal file
View File

@ -0,0 +1,186 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Plugs.CacheTest do
use ExUnit.Case, async: true
use Plug.Test
alias Pleroma.Plugs.Cache
@miss_resp {200,
[
{"cache-control", "max-age=0, private, must-revalidate"},
{"content-type", "cofe/hot; charset=utf-8"},
{"x-cache", "MISS from Pleroma"}
], "cofe"}
@hit_resp {200,
[
{"cache-control", "max-age=0, private, must-revalidate"},
{"content-type", "cofe/hot; charset=utf-8"},
{"x-cache", "HIT from Pleroma"}
], "cofe"}
@ttl 5
setup do
Cachex.clear(:web_resp_cache)
:ok
end
test "caches a response" do
assert @miss_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
assert_raise(Plug.Conn.AlreadySentError, fn ->
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
end)
assert @hit_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: nil})
|> sent_resp()
end
test "ttl is set" do
assert @miss_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: @ttl})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
assert @hit_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: @ttl})
|> sent_resp()
:timer.sleep(@ttl + 1)
assert @miss_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: @ttl})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
end
test "set ttl via conn.assigns" do
assert @miss_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> assign(:cache_ttl, @ttl)
|> send_resp(:ok, "cofe")
|> sent_resp()
assert @hit_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: nil})
|> sent_resp()
:timer.sleep(@ttl + 1)
assert @miss_resp ==
conn(:get, "/")
|> Cache.call(%{query_params: false, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
end
test "ignore query string when `query_params` is false" do
assert @miss_resp ==
conn(:get, "/?cofe")
|> Cache.call(%{query_params: false, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
assert @hit_resp ==
conn(:get, "/?cofefe")
|> Cache.call(%{query_params: false, ttl: nil})
|> sent_resp()
end
test "take query string into account when `query_params` is true" do
assert @miss_resp ==
conn(:get, "/?cofe")
|> Cache.call(%{query_params: true, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
assert @miss_resp ==
conn(:get, "/?cofefe")
|> Cache.call(%{query_params: true, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
end
test "take specific query params into account when `query_params` is list" do
assert @miss_resp ==
conn(:get, "/?a=1&b=2&c=3&foo=bar")
|> fetch_query_params()
|> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
assert @hit_resp ==
conn(:get, "/?bar=foo&c=3&b=2&a=1")
|> fetch_query_params()
|> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
|> sent_resp()
assert @miss_resp ==
conn(:get, "/?bar=foo&c=3&b=2&a=2")
|> fetch_query_params()
|> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
end
test "ignore not GET requests" do
expected =
{200,
[
{"cache-control", "max-age=0, private, must-revalidate"},
{"content-type", "cofe/hot; charset=utf-8"}
], "cofe"}
assert expected ==
conn(:post, "/")
|> Cache.call(%{query_params: true, ttl: nil})
|> put_resp_content_type("cofe/hot")
|> send_resp(:ok, "cofe")
|> sent_resp()
end
test "ignore non-successful responses" do
expected =
{418,
[
{"cache-control", "max-age=0, private, must-revalidate"},
{"content-type", "tea/iced; charset=utf-8"}
], "🥤"}
assert expected ==
conn(:get, "/cofe")
|> Cache.call(%{query_params: true, ttl: nil})
|> put_resp_content_type("tea/iced")
|> send_resp(:im_a_teapot, "🥤")
|> sent_resp()
end
end

View File

@ -175,6 +175,49 @@ test "it returns 404 for tombstone objects", %{conn: conn} do
assert json_response(conn, 404) assert json_response(conn, 404)
end end
test "it caches a response", %{conn: conn} do
note = insert(:note)
uuid = String.split(note.data["id"], "/") |> List.last()
conn1 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/objects/#{uuid}")
assert json_response(conn1, :ok)
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
conn2 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/objects/#{uuid}")
assert json_response(conn1, :ok) == json_response(conn2, :ok)
assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"}))
end
test "cached purged after object deletion", %{conn: conn} do
note = insert(:note)
uuid = String.split(note.data["id"], "/") |> List.last()
conn1 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/objects/#{uuid}")
assert json_response(conn1, :ok)
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
Object.delete(note)
conn2 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/objects/#{uuid}")
assert "Not found" == json_response(conn2, :not_found)
end
end end
describe "/object/:uuid/likes" do describe "/object/:uuid/likes" do
@ -264,6 +307,51 @@ test "it returns 404 for non-public activities", %{conn: conn} do
assert json_response(conn, 404) assert json_response(conn, 404)
end end
test "it caches a response", %{conn: conn} do
activity = insert(:note_activity)
uuid = String.split(activity.data["id"], "/") |> List.last()
conn1 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/activities/#{uuid}")
assert json_response(conn1, :ok)
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
conn2 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/activities/#{uuid}")
assert json_response(conn1, :ok) == json_response(conn2, :ok)
assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"}))
end
test "cached purged after activity deletion", %{conn: conn} do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"status" => "cofe"})
uuid = String.split(activity.data["id"], "/") |> List.last()
conn1 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/activities/#{uuid}")
assert json_response(conn1, :ok)
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
Activity.delete_by_ap_id(activity.object.data["id"])
conn2 =
conn
|> put_req_header("accept", "application/activity+json")
|> get("/activities/#{uuid}")
assert "Not found" == json_response(conn2, :not_found)
end
end end
describe "/inbox" do describe "/inbox" do