Merge branch 'issue/1936' into 'develop'

[#1936] Ability to search for banned MediaProxy URLs 

See merge request pleroma/pleroma!2873
This commit is contained in:
lain 2020-08-17 13:05:20 +00:00
commit 34b099fffa
6 changed files with 121 additions and 79 deletions

View File

@ -1266,11 +1266,14 @@ Loads json generated from `config/descriptions.exs`.
- Params: - Params:
- *optional* `page`: **integer** page number - *optional* `page`: **integer** page number
- *optional* `page_size`: **integer** number of log entries per page (default is `50`) - *optional* `page_size`: **integer** number of log entries per page (default is `50`)
- *optional* `query`: **string** search term
- Response: - Response:
``` json ``` json
{ {
"page_size": integer,
"count": integer,
"urls": [ "urls": [
"http://example.com/media/a688346.jpg", "http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg" "http://example.com/media/fb1f4d.jpg"
@ -1290,12 +1293,7 @@ Loads json generated from `config/descriptions.exs`.
- Response: - Response:
``` json ``` json
{ { }
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
}
``` ```
@ -1311,11 +1309,6 @@ Loads json generated from `config/descriptions.exs`.
- Response: - Response:
``` json ``` json
{ { }
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
}
``` ```

View File

@ -26,29 +26,40 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
def index(%{assigns: %{user: _}} = conn, params) do def index(%{assigns: %{user: _}} = conn, params) do
cursor = entries = fetch_entries(params)
:banned_urls_cache urls = paginate_entries(entries, params.page, params.page_size)
|> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
|> :qlc.cursor()
urls = render(conn, "index.json",
case params.page do urls: urls,
1 -> page_size: params.page_size,
:qlc.next_answers(cursor, params.page_size) count: length(entries)
)
end
_ -> defp fetch_entries(params) do
:qlc.next_answers(cursor, (params.page - 1) * params.page_size) MediaProxy.cache_table()
:qlc.next_answers(cursor, params.page_size) |> Cachex.stream!(Cachex.Query.create(true, :key))
end |> filter_entries(params[:query])
end
:qlc.delete_cursor(cursor) defp filter_entries(stream, query) when is_binary(query) do
regex = ~r/#{query}/i
render(conn, "index.json", urls: urls) stream
|> Enum.filter(fn url -> String.match?(url, regex) end)
|> Enum.to_list()
end
defp filter_entries(stream, _), do: Enum.to_list(stream)
defp paginate_entries(entries, page, page_size) do
offset = page_size * (page - 1)
Enum.slice(entries, offset, page_size)
end end
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
MediaProxy.remove_from_banned_urls(urls) MediaProxy.remove_from_banned_urls(urls)
render(conn, "index.json", urls: urls) json(conn, %{})
end end
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
@ -58,6 +69,6 @@ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _
MediaProxy.put_in_banned_urls(urls) MediaProxy.put_in_banned_urls(urls)
end end
render(conn, "index.json", urls: urls) json(conn, %{})
end end
end end

View File

@ -5,7 +5,11 @@
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
use Pleroma.Web, :view use Pleroma.Web, :view
def render("index.json", %{urls: urls}) do def render("index.json", %{urls: urls, page_size: page_size, count: count}) do
%{urls: urls} %{
urls: urls,
count: count,
page_size: page_size
}
end end
end end

View File

@ -21,6 +21,12 @@ def index_operation do
operationId: "AdminAPI.MediaProxyCacheController.index", operationId: "AdminAPI.MediaProxyCacheController.index",
security: [%{"oAuth" => ["read:media_proxy_caches"]}], security: [%{"oAuth" => ["read:media_proxy_caches"]}],
parameters: [ parameters: [
Operation.parameter(
:query,
:query,
%Schema{type: :string, default: nil},
"Page"
),
Operation.parameter( Operation.parameter(
:page, :page,
:query, :query,
@ -36,7 +42,26 @@ def index_operation do
| admin_api_params() | admin_api_params()
], ],
responses: %{ responses: %{
200 => success_response() 200 =>
Operation.response(
"Array of banned MediaProxy URLs in Cachex",
"application/json",
%Schema{
type: :object,
properties: %{
count: %Schema{type: :integer},
page_size: %Schema{type: :integer},
urls: %Schema{
type: :array,
items: %Schema{
type: :string,
format: :uri,
description: "MediaProxy URLs"
}
}
}
}
)
} }
} }
end end
@ -61,7 +86,7 @@ def delete_operation do
required: true required: true
), ),
responses: %{ responses: %{
200 => success_response(), 200 => empty_object_response(),
400 => Operation.response("Error", "application/json", ApiError) 400 => Operation.response("Error", "application/json", ApiError)
} }
} }
@ -88,25 +113,9 @@ def purge_operation do
required: true required: true
), ),
responses: %{ responses: %{
200 => success_response(), 200 => empty_object_response(),
400 => Operation.response("Error", "application/json", ApiError) 400 => Operation.response("Error", "application/json", ApiError)
} }
} }
end end
defp success_response do
Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
type: :object,
properties: %{
urls: %Schema{
type: :array,
items: %Schema{
type: :string,
format: :uri,
description: "MediaProxy URLs"
}
}
}
})
end
end end

View File

@ -9,28 +9,31 @@ defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Web.MediaProxy.Invalidation alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false] @base64_opts [padding: false]
@cache_table :banned_urls_cache
def cache_table, do: @cache_table
@spec in_banned_urls(String.t()) :: boolean() @spec in_banned_urls(String.t()) :: boolean()
def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1) def in_banned_urls(url), do: elem(Cachex.exists?(@cache_table, url(url)), 1)
def remove_from_banned_urls(urls) when is_list(urls) do def remove_from_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache -> Cachex.execute!(@cache_table, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
end) end)
end end
def remove_from_banned_urls(url) when is_binary(url) do def remove_from_banned_urls(url) when is_binary(url) do
Cachex.del(:banned_urls_cache, url(url)) Cachex.del(@cache_table, url(url))
end end
def put_in_banned_urls(urls) when is_list(urls) do def put_in_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache -> Cachex.execute!(@cache_table, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
end) end)
end end
def put_in_banned_urls(url) when is_binary(url) do def put_in_banned_urls(url) when is_binary(url) do
Cachex.put(:banned_urls_cache, url(url), true) Cachex.put(@cache_table, url(url), true)
end end
def url(url) when is_nil(url) or url == "", do: nil def url(url) when is_nil(url) or url == "", do: nil

View File

@ -48,6 +48,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2") |> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|> json_response_and_validate_schema(200) |> json_response_and_validate_schema(200)
assert response["page_size"] == 2
assert response["count"] == 5
assert response["urls"] == [ assert response["urls"] == [
"http://localhost:4001/media/fb1f4d.jpg", "http://localhost:4001/media/fb1f4d.jpg",
"http://localhost:4001/media/a688346.jpg" "http://localhost:4001/media/a688346.jpg"
@ -63,6 +66,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
"http://localhost:4001/media/tb13f47.jpg" "http://localhost:4001/media/tb13f47.jpg"
] ]
assert response["page_size"] == 2
assert response["count"] == 5
response = response =
conn conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3") |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
@ -70,6 +76,30 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"] assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
end end
test "search banned MediaProxy URLs", %{conn: conn} do
MediaProxy.put_in_banned_urls([
"http://localhost:4001/media/a688346.jpg",
"http://localhost:4001/media/ff44b1f4d.jpg"
])
MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
response =
conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&query=F44")
|> json_response_and_validate_schema(200)
assert response["urls"] == [
"http://localhost:4001/media/gb1f44.jpg",
"http://localhost:4001/media/ff44b1f4d.jpg"
]
assert response["page_size"] == 2
assert response["count"] == 2
end
end end
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
@ -79,15 +109,13 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do
"http://localhost:4001/media/fb1f4d.jpg" "http://localhost:4001/media/fb1f4d.jpg"
]) ])
response = conn
conn |> put_req_header("content-type", "application/json")
|> put_req_header("content-type", "application/json") |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{ urls: ["http://localhost:4001/media/a688346.jpg"]
urls: ["http://localhost:4001/media/a688346.jpg"] })
}) |> json_response_and_validate_schema(200)
|> json_response_and_validate_schema(200)
assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg") refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg") assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
end end
@ -106,13 +134,10 @@ test "perform invalidates cache of MediaProxy", %{conn: conn} do
purge: fn _, _ -> {"ok", 0} end purge: fn _, _ -> {"ok", 0} end
]} ]}
] do ] do
response = conn
conn |> put_req_header("content-type", "application/json")
|> put_req_header("content-type", "application/json") |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false}) |> json_response_and_validate_schema(200)
|> json_response_and_validate_schema(200)
assert response["urls"] == urls
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg") refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
@ -126,16 +151,13 @@ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: c
] ]
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
response = conn
conn |> put_req_header("content-type", "application/json")
|> put_req_header("content-type", "application/json") |> post(
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{ "/api/pleroma/admin/media_proxy_caches/purge",
urls: urls, %{urls: urls, ban: true}
ban: true )
}) |> json_response_and_validate_schema(200)
|> json_response_and_validate_schema(200)
assert response["urls"] == urls
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg") assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")