Merge branch 'develop' into hellthread-filter-fix

This commit is contained in:
Karen Konou 2019-02-13 00:04:26 +01:00
commit adcdd21f16
6 changed files with 178 additions and 28 deletions

View File

@ -16,7 +16,8 @@
debug_errors: true, debug_errors: true,
code_reloader: true, code_reloader: true,
check_origin: false, check_origin: false,
watchers: [] watchers: [],
secure_cookie_flag: false
config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Local config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Local

View File

@ -26,60 +26,71 @@ Feel free to contact us to be added to this list!
- Source Code: <https://github.com/ReticentJohn/Amaroq> - Source Code: <https://github.com/ReticentJohn/Amaroq>
- Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy) - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy)
- Platforms: iOS - Platforms: iOS
- Features: No Streaming
### Nekonium ### Nekonium
- Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/) - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)
- Source: <https://git.gdgd.jp.net/lin/nekonium/> - Source: <https://git.gdgd.jp.net/lin/nekonium/>
- Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin) - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin)
- Platforms: Android - Platforms: Android
- Features: Streaming Ready
### Mastalab ### Mastalab
- Source Code: <https://gitlab.com/tom79/mastalab/> - Source Code: <https://gitlab.com/tom79/mastalab/>
- Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79) - Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79)
- Platforms: Android - Platforms: Android
- Features: Streaming Ready
### Roma ### Roma
- Homepage: <http://www.pleroma.com/> - Homepage: <http://www.pleroma.com/>
- Source Code: ??? - Source Code: ???
- Platforms: iOS, Android - Platforms: iOS, Android
- Features: No Streaming
### Tootdon ### Tootdon
- Homepage: <http://tootdon.club/>, <http://blog.mastodon-tootdon.com/> - Homepage: <http://tootdon.club/>, <http://blog.mastodon-tootdon.com/>
- Source Code: ??? - Source Code: ???
- Contact: [@tootdon@mstdn.jp](https://mstdn.jp/users/tootdon) - Contact: [@tootdon@mstdn.jp](https://mstdn.jp/users/tootdon)
- Platforms: Android, iOS - Platforms: Android, iOS
- Features: No Streaming
### Tusky ### Tusky
- Homepage: <https://tuskyapp.github.io/> - Homepage: <https://tuskyapp.github.io/>
- Source Code: <https://github.com/tuskyapp/Tusky> - Source Code: <https://github.com/tuskyapp/Tusky>
- Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck) - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck)
- Platforms: Android - Platforms: Android
- Features: No Streaming
### Twidere ### Twidere
- Homepage: <https://twidere.mariotaku.org/> - Homepage: <https://twidere.mariotaku.org/>
- Source Code: <https://github.com/TwidereProject/Twidere-Android/>, <https://github.com/TwidereProject/Twidere-iOS/> - Source Code: <https://github.com/TwidereProject/Twidere-Android/>, <https://github.com/TwidereProject/Twidere-iOS/>
- Contact: <me@mariotaku.org> - Contact: <me@mariotaku.org>
- Platform: Android, iOS - Platform: Android, iOS
- Features: No Streaming
## Alternative Web Interfaces ## Alternative Web Interfaces
### Brutaldon ### Brutaldon
- Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/> - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
- Source Code: <https://github.com/jfmcbrayer/brutaldon> - Source Code: <https://github.com/jfmcbrayer/brutaldon>
- Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc) - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
- Features: No Streaming
### Feather ### Feather
- Source Code: <https://github.com/kaniini/feather> - Source Code: <https://github.com/kaniini/feather>
- Contact: [@kaniini@pleroma.site](https://pleroma.site/kaniini) - Contact: [@kaniini@pleroma.site](https://pleroma.site/kaniini)
- Features: No Streaming
### Halcyon ### Halcyon
- Source Code: <https://notabug.org/halcyon-suite/halcyon> - Source Code: <https://notabug.org/halcyon-suite/halcyon>
- Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon) - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
- Features: Streaming Ready
### Pinafore ### Pinafore
- Homepage: <https://pinafore.social/> - Homepage: <https://pinafore.social/>
- Source Code: <https://github.com/nolanlawson/pinafore> - Source Code: <https://github.com/nolanlawson/pinafore>
- Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore) - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore)
- Note: Pleroma support is a secondary goal - Note: Pleroma support is a secondary goal
- Features: No Streaming
### Sengi ### Sengi
- Source Code: <https://github.com/NicolasConstant/sengi> - Source Code: <https://github.com/NicolasConstant/sengi>

View File

@ -33,7 +33,22 @@ defp headers do
end end
defp csp_string do defp csp_string do
protocol = Config.get([Pleroma.Web.Endpoint, :protocol]) scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
websocket_url = String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
connect_src =
if Mix.env() == :dev do
"connect-src 'self' http://localhost:3035/ " <> websocket_url
else
"connect-src 'self' " <> websocket_url
end
script_src =
if Mix.env() == :dev do
"script-src 'self' 'unsafe-eval'"
else
"script-src 'self'"
end
[ [
"default-src 'none'", "default-src 'none'",
@ -43,10 +58,10 @@ defp csp_string do
"media-src 'self' https:", "media-src 'self' https:",
"style-src 'self' 'unsafe-inline'", "style-src 'self' 'unsafe-inline'",
"font-src 'self'", "font-src 'self'",
"script-src 'self'",
"connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
"manifest-src 'self'", "manifest-src 'self'",
if protocol == "https" do connect_src,
script_src,
if scheme == "https" do
"upgrade-insecure-requests" "upgrade-insecure-requests"
end end
] ]

View File

@ -12,9 +12,9 @@ defp string_matches?(string, pattern) do
String.match?(string, pattern) String.match?(string, pattern)
end end
defp check_reject(%{"object" => %{"content" => content}} = message) do defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
string_matches?(content, pattern) string_matches?(content, pattern) or string_matches?(summary, pattern)
end) do end) do
{:reject, nil} {:reject, nil}
else else
@ -22,10 +22,12 @@ defp check_reject(%{"object" => %{"content" => content}} = message) do
end end
end end
defp check_ftl_removal(%{"to" => to, "object" => %{"content" => content}} = message) do defp check_ftl_removal(
%{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
) do
if "https://www.w3.org/ns/activitystreams#Public" in to and if "https://www.w3.org/ns/activitystreams#Public" in to and
Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
string_matches?(content, pattern) string_matches?(content, pattern) or string_matches?(summary, pattern)
end) do end) do
to = List.delete(to, "https://www.w3.org/ns/activitystreams#Public") to = List.delete(to, "https://www.w3.org/ns/activitystreams#Public")
cc = ["https://www.w3.org/ns/activitystreams#Public" | message["cc"] || []] cc = ["https://www.w3.org/ns/activitystreams#Public" | message["cc"] || []]
@ -41,14 +43,20 @@ defp check_ftl_removal(%{"to" => to, "object" => %{"content" => content}} = mess
end end
end end
defp check_replace(%{"object" => %{"content" => content}} = message) do defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
content = {content, summary} =
Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), content, fn {pattern, replacement}, Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), {content, summary}, fn {pattern,
acc -> replacement},
String.replace(acc, pattern, replacement) {content_acc,
summary_acc} ->
{String.replace(content_acc, pattern, replacement),
String.replace(summary_acc, pattern, replacement)}
end) end)
{:ok, put_in(message["object"]["content"], content)} {:ok,
message
|> put_in(["object", "content"], content)
|> put_in(["object", "summary"], summary)}
end end
@impl true @impl true

View File

@ -9,6 +9,13 @@ defmodule Pleroma.Web.RichMedia.Parser do
Pleroma.Web.RichMedia.Parsers.OEmbed Pleroma.Web.RichMedia.Parsers.OEmbed
] ]
@hackney_options [
pool: :media,
timeout: 2_000,
recv_timeout: 2_000,
max_body: 2_000_000
]
def parse(nil), do: {:error, "No URL provided"} def parse(nil), do: {:error, "No URL provided"}
if Mix.env() == :test do if Mix.env() == :test do
@ -28,7 +35,7 @@ def parse(url) do
defp parse_url(url) do defp parse_url(url) do
try do try do
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media]) {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data() html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data()
rescue rescue

View File

@ -12,18 +12,35 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do
end end
describe "rejecting based on keywords" do describe "rejecting based on keywords" do
test "rejects if string matches" do test "rejects if string matches in content" do
Pleroma.Config.put([:mrf_keyword, :reject], ["pun"]) Pleroma.Config.put([:mrf_keyword, :reject], ["pun"])
message = %{ message = %{
"type" => "Create", "type" => "Create",
"object" => %{"content" => "just a daily reminder that compLAINer is a good pun"} "object" => %{
"content" => "just a daily reminder that compLAINer is a good pun",
"summary" => ""
}
} }
assert {:reject, nil} == KeywordPolicy.filter(message) assert {:reject, nil} == KeywordPolicy.filter(message)
end end
test "rejects if regex matches" do test "rejects if string matches in summary" do
Pleroma.Config.put([:mrf_keyword, :reject], ["pun"])
message = %{
"type" => "Create",
"object" => %{
"summary" => "just a daily reminder that compLAINer is a good pun",
"content" => ""
}
}
assert {:reject, nil} == KeywordPolicy.filter(message)
end
test "rejects if regex matches in content" do
Pleroma.Config.put([:mrf_keyword, :reject], [~r/comp[lL][aA][iI][nN]er/]) Pleroma.Config.put([:mrf_keyword, :reject], [~r/comp[lL][aA][iI][nN]er/])
assert true == assert true ==
@ -31,7 +48,25 @@ test "rejects if regex matches" do
message = %{ message = %{
"type" => "Create", "type" => "Create",
"object" => %{ "object" => %{
"content" => "just a daily reminder that #{content} is a good pun" "content" => "just a daily reminder that #{content} is a good pun",
"summary" => ""
}
}
{:reject, nil} == KeywordPolicy.filter(message)
end)
end
test "rejects if regex matches in summary" do
Pleroma.Config.put([:mrf_keyword, :reject], [~r/comp[lL][aA][iI][nN]er/])
assert true ==
Enum.all?(["complainer", "compLainer", "compLAiNer", "compLAINer"], fn content ->
message = %{
"type" => "Create",
"object" => %{
"summary" => "just a daily reminder that #{content} is a good pun",
"content" => ""
} }
} }
@ -41,13 +76,16 @@ test "rejects if regex matches" do
end end
describe "delisting from ftl based on keywords" do describe "delisting from ftl based on keywords" do
test "delists if string matches" do test "delists if string matches in content" do
Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], ["pun"]) Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], ["pun"])
message = %{ message = %{
"to" => ["https://www.w3.org/ns/activitystreams#Public"], "to" => ["https://www.w3.org/ns/activitystreams#Public"],
"type" => "Create", "type" => "Create",
"object" => %{"content" => "just a daily reminder that compLAINer is a good pun"} "object" => %{
"content" => "just a daily reminder that compLAINer is a good pun",
"summary" => ""
}
} }
{:ok, result} = KeywordPolicy.filter(message) {:ok, result} = KeywordPolicy.filter(message)
@ -55,7 +93,24 @@ test "delists if string matches" do
refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"] refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"]
end end
test "delists if regex matches" do test "delists if string matches in summary" do
Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], ["pun"])
message = %{
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"type" => "Create",
"object" => %{
"summary" => "just a daily reminder that compLAINer is a good pun",
"content" => ""
}
}
{:ok, result} = KeywordPolicy.filter(message)
assert ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"]
refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"]
end
test "delists if regex matches in content" do
Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], [~r/comp[lL][aA][iI][nN]er/]) Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], [~r/comp[lL][aA][iI][nN]er/])
assert true == assert true ==
@ -64,7 +119,29 @@ test "delists if regex matches" do
"type" => "Create", "type" => "Create",
"to" => ["https://www.w3.org/ns/activitystreams#Public"], "to" => ["https://www.w3.org/ns/activitystreams#Public"],
"object" => %{ "object" => %{
"content" => "just a daily reminder that #{content} is a good pun" "content" => "just a daily reminder that #{content} is a good pun",
"summary" => ""
}
}
{:ok, result} = KeywordPolicy.filter(message)
["https://www.w3.org/ns/activitystreams#Public"] == result["cc"] and
not (["https://www.w3.org/ns/activitystreams#Public"] == result["to"])
end)
end
test "delists if regex matches in summary" do
Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], [~r/comp[lL][aA][iI][nN]er/])
assert true ==
Enum.all?(["complainer", "compLainer", "compLAiNer", "compLAINer"], fn content ->
message = %{
"type" => "Create",
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"object" => %{
"summary" => "just a daily reminder that #{content} is a good pun",
"content" => ""
} }
} }
@ -77,20 +154,33 @@ test "delists if regex matches" do
end end
describe "replacing keywords" do describe "replacing keywords" do
test "replaces keyword if string matches" do test "replaces keyword if string matches in content" do
Pleroma.Config.put([:mrf_keyword, :replace], [{"opensource", "free software"}]) Pleroma.Config.put([:mrf_keyword, :replace], [{"opensource", "free software"}])
message = %{ message = %{
"type" => "Create", "type" => "Create",
"to" => ["https://www.w3.org/ns/activitystreams#Public"], "to" => ["https://www.w3.org/ns/activitystreams#Public"],
"object" => %{"content" => "ZFS is opensource"} "object" => %{"content" => "ZFS is opensource", "summary" => ""}
} }
{:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message) {:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message)
assert result == "ZFS is free software" assert result == "ZFS is free software"
end end
test "replaces keyword if regex matches" do test "replaces keyword if string matches in summary" do
Pleroma.Config.put([:mrf_keyword, :replace], [{"opensource", "free software"}])
message = %{
"type" => "Create",
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"object" => %{"summary" => "ZFS is opensource", "content" => ""}
}
{:ok, %{"object" => %{"summary" => result}}} = KeywordPolicy.filter(message)
assert result == "ZFS is free software"
end
test "replaces keyword if regex matches in content" do
Pleroma.Config.put([:mrf_keyword, :replace], [ Pleroma.Config.put([:mrf_keyword, :replace], [
{~r/open(-|\s)?source\s?(software)?/, "free software"} {~r/open(-|\s)?source\s?(software)?/, "free software"}
]) ])
@ -100,12 +190,30 @@ test "replaces keyword if regex matches" do
message = %{ message = %{
"type" => "Create", "type" => "Create",
"to" => ["https://www.w3.org/ns/activitystreams#Public"], "to" => ["https://www.w3.org/ns/activitystreams#Public"],
"object" => %{"content" => "ZFS is #{content}"} "object" => %{"content" => "ZFS is #{content}", "summary" => ""}
} }
{:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message) {:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message)
result == "ZFS is free software" result == "ZFS is free software"
end) end)
end end
test "replaces keyword if regex matches in summary" do
Pleroma.Config.put([:mrf_keyword, :replace], [
{~r/open(-|\s)?source\s?(software)?/, "free software"}
])
assert true ==
Enum.all?(["opensource", "open-source", "open source"], fn content ->
message = %{
"type" => "Create",
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"object" => %{"summary" => "ZFS is #{content}", "content" => ""}
}
{:ok, %{"object" => %{"summary" => result}}} = KeywordPolicy.filter(message)
result == "ZFS is free software"
end)
end
end end
end end