Merge branch 'develop' into refactor/config-get

This commit is contained in:
Roman Chvanikov 2020-08-05 17:46:24 +03:00
commit 2299bfe4c1
24 changed files with 188 additions and 42 deletions

View File

@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Added ### Added
- Configuration: Added a blacklist for email servers.
- Chats: Added `accepts_chat_messages` field to user, exposed in APIs and federation. - Chats: Added `accepts_chat_messages` field to user, exposed in APIs and federation.
- Chats: Added support for federated chats. For details, see the docs. - Chats: Added support for federated chats. For details, see the docs.
- ActivityPub: Added support for existing AP ids for instances migrated from Mastodon. - ActivityPub: Added support for existing AP ids for instances migrated from Mastodon.

View File

@ -516,7 +516,8 @@
"user_exists", "user_exists",
"users", "users",
"web" "web"
] ],
email_blacklist: []
config :pleroma, Oban, config :pleroma, Oban,
repo: Pleroma.Repo, repo: Pleroma.Repo,

View File

@ -3056,6 +3056,7 @@
%{ %{
key: :restricted_nicknames, key: :restricted_nicknames,
type: {:list, :string}, type: {:list, :string},
description: "List of nicknames users may not register with.",
suggestions: [ suggestions: [
".well-known", ".well-known",
"~", "~",
@ -3088,6 +3089,12 @@
"users", "users",
"web" "web"
] ]
},
%{
key: :email_blacklist,
type: {:list, :string},
description: "List of email domains users may not register with.",
suggestions: ["mailinator.com", "maildrop.cc"]
} }
] ]
}, },

View File

@ -120,6 +120,8 @@
config :tzdata, :autoupdate, :disabled config :tzdata, :autoupdate, :disabled
config :pleroma, :mrf, policies: []
if File.exists?("./config/test.secret.exs") do if File.exists?("./config/test.secret.exs") do
import_config "test.secret.exs" import_config "test.secret.exs"
else else

View File

@ -207,6 +207,11 @@ config :pleroma, :mrf_user_allowlist, %{
* `sign_object_fetches`: Sign object fetches with HTTP signatures * `sign_object_fetches`: Sign object fetches with HTTP signatures
* `authorized_fetch_mode`: Require HTTP signatures for AP fetches * `authorized_fetch_mode`: Require HTTP signatures for AP fetches
## Pleroma.User
* `restricted_nicknames`: List of nicknames users may not register with.
* `email_blacklist`: List of email domains users may not register with.
## Pleroma.ScheduledActivity ## Pleroma.ScheduledActivity
* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) * `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`)

View File

@ -676,10 +676,19 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
|> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_required([:name, :nickname, :password, :password_confirmation])
|> validate_confirmation(:password) |> validate_confirmation(:password)
|> unique_constraint(:email) |> unique_constraint(:email)
|> validate_format(:email, @email_regex)
|> validate_change(:email, fn :email, email ->
valid? =
Config.get([User, :email_blacklist])
|> Enum.all?(fn blacklisted_domain ->
!String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain])
end)
if valid?, do: [], else: [email: "Invalid email"]
end)
|> unique_constraint(:nickname) |> unique_constraint(:nickname)
|> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
|> validate_format(:nickname, local_nickname_regex()) |> validate_format(:nickname, local_nickname_regex())
|> validate_format(:email, @email_regex)
|> validate_length(:bio, max: bio_limit) |> validate_length(:bio, max: bio_limit)
|> validate_length(:name, min: 1, max: name_limit) |> validate_length(:name, min: 1, max: name_limit)
|> validate_length(:registration_reason, max: reason_limit) |> validate_length(:registration_reason, max: reason_limit)

View File

@ -21,8 +21,8 @@ def filter(activity) do
@impl true @impl true
def describe, do: {:ok, %{}} def describe, do: {:ok, %{}}
defp local?(%{"id" => id}) do defp local?(%{"actor" => actor}) do
String.starts_with?(id, Pleroma.Web.Endpoint.url()) String.starts_with?(actor, Pleroma.Web.Endpoint.url())
end end
defp note?(activity) do defp note?(activity) do

View File

@ -34,9 +34,14 @@ def validate_actor_presence(cng, options \\ []) do
cng cng
|> validate_change(field_name, fn field_name, actor -> |> validate_change(field_name, fn field_name, actor ->
if User.get_cached_by_ap_id(actor) do case User.get_cached_by_ap_id(actor) do
%User{deactivated: true} ->
[{field_name, "user is deactivated"}]
%User{} ->
[] []
else
_ ->
[{field_name, "can't find user"}] [{field_name, "can't find user"}]
end end
end) end)

View File

@ -9,6 +9,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser alias Pleroma.Web.RichMedia.Parser
@rich_media_options [
pool: :media,
max_body: 2_000_000
]
@spec validate_page_url(URI.t() | binary()) :: :ok | :error @spec validate_page_url(URI.t() | binary()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
@ -77,4 +82,20 @@ def perform(:fetch, %Activity{} = activity) do
fetch_data_for_activity(activity) fetch_data_for_activity(activity)
:ok :ok
end end
def rich_media_get(url) do
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
options =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@rich_media_options,
recv_timeout: 2_000,
with_body: true
)
else
@rich_media_options
end
Pleroma.HTTP.get(url, headers, options)
end
end end

View File

@ -3,11 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do defmodule Pleroma.Web.RichMedia.Parser do
@options [
pool: :media,
max_body: 2_000_000
]
defp parsers do defp parsers do
Pleroma.Config.get([:rich_media, :parsers]) Pleroma.Config.get([:rich_media, :parsers])
end end
@ -75,21 +70,8 @@ defp get_ttl_from_image(data, url) do
end end
defp parse_url(url) do defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
try do try do
rich_media_agent = Pleroma.Application.user_agent() <> "; Bot" {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
{:ok, %Tesla.Env{body: html}} =
Pleroma.HTTP.get(url, [{"user-agent", rich_media_agent}], adapter: opts)
html html
|> parse_html() |> parse_html()

View File

@ -22,7 +22,7 @@ defp get_oembed_url([{"link", attributes, _children} | _]) do
end end
defp get_oembed_data(url) do defp get_oembed_data(url) do
with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do
Jason.decode(json) Jason.decode(json)
end end
end end

View File

@ -37,7 +37,7 @@
} }
a { a {
color: color: #d8a070; color: #d8a070;
text-decoration: none; text-decoration: none;
} }

View File

@ -214,7 +214,8 @@ defp aliases do
"ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
"ecto.reset": ["ecto.drop", "ecto.setup"], "ecto.reset": ["ecto.drop", "ecto.setup"],
test: ["ecto.create --quiet", "ecto.migrate", "test"], test: ["ecto.create --quiet", "ecto.migrate", "test"],
docs: ["pleroma.docs", "docs"] docs: ["pleroma.docs", "docs"],
analyze: ["credo --strict --only=warnings,todo,fixme,consistency,readability"]
] ]
end end

View File

@ -0,0 +1,19 @@
defmodule Pleroma.Repo.Migrations.RemoveNonlocalExpirations do
use Ecto.Migration
def up do
statement = """
DELETE FROM
activity_expirations A USING activities B
WHERE
A.activity_id = B.id
AND B.local = false;
"""
execute(statement)
end
def down do
:ok
end
end

View File

@ -0,0 +1,7 @@
defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAppClientId do
use Ecto.Migration
def change do
create(unique_index(:apps, [:client_id]))
end
end

View File

@ -50,13 +50,13 @@ test "with errors" do
defp assert_app(name, redirect, scopes) do defp assert_app(name, redirect, scopes) do
app = Repo.get_by(Pleroma.Web.OAuth.App, client_name: name) app = Repo.get_by(Pleroma.Web.OAuth.App, client_name: name)
assert_received {:mix_shell, :info, [message]} assert_receive {:mix_shell, :info, [message]}
assert message == "#{name} successfully created:" assert message == "#{name} successfully created:"
assert_received {:mix_shell, :info, [message]} assert_receive {:mix_shell, :info, [message]}
assert message == "App client_id: #{app.client_id}" assert message == "App client_id: #{app.client_id}"
assert_received {:mix_shell, :info, [message]} assert_receive {:mix_shell, :info, [message]}
assert message == "App client_secret: #{app.client_secret}" assert message == "App client_secret: #{app.client_secret}"
assert app.scopes == scopes assert app.scopes == scopes

View File

@ -513,6 +513,29 @@ test "it restricts certain nicknames" do
refute changeset.valid? refute changeset.valid?
end end
test "it blocks blacklisted email domains" do
clear_config([User, :email_blacklist], ["trolling.world"])
# Block with match
params = Map.put(@full_user_data, :email, "troll@trolling.world")
changeset = User.register_changeset(%User{}, params)
refute changeset.valid?
# Block with subdomain match
params = Map.put(@full_user_data, :email, "troll@gnomes.trolling.world")
changeset = User.register_changeset(%User{}, params)
refute changeset.valid?
# Pass with different domains that are similar
params = Map.put(@full_user_data, :email, "troll@gnomestrolling.world")
changeset = User.register_changeset(%User{}, params)
assert changeset.valid?
params = Map.put(@full_user_data, :email, "troll@trolling.world.us")
changeset = User.register_changeset(%User{}, params)
assert changeset.valid?
end
test "it sets the password_hash and ap_id" do test "it sets the password_hash and ap_id" do
changeset = User.register_changeset(%User{}, @full_user_data) changeset = User.register_changeset(%User{}, @full_user_data)

View File

@ -7,11 +7,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
alias Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy alias Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy
@id Pleroma.Web.Endpoint.url() <> "/activities/cofe" @id Pleroma.Web.Endpoint.url() <> "/activities/cofe"
@local_actor Pleroma.Web.Endpoint.url() <> "/users/cofe"
test "adds `expires_at` property" do test "adds `expires_at` property" do
assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} = assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} =
ActivityExpirationPolicy.filter(%{ ActivityExpirationPolicy.filter(%{
"id" => @id, "id" => @id,
"actor" => @local_actor,
"type" => "Create", "type" => "Create",
"object" => %{"type" => "Note"} "object" => %{"type" => "Note"}
}) })
@ -25,6 +27,7 @@ test "keeps existing `expires_at` if it less than the config setting" do
assert {:ok, %{"type" => "Create", "expires_at" => ^expires_at}} = assert {:ok, %{"type" => "Create", "expires_at" => ^expires_at}} =
ActivityExpirationPolicy.filter(%{ ActivityExpirationPolicy.filter(%{
"id" => @id, "id" => @id,
"actor" => @local_actor,
"type" => "Create", "type" => "Create",
"expires_at" => expires_at, "expires_at" => expires_at,
"object" => %{"type" => "Note"} "object" => %{"type" => "Note"}
@ -37,6 +40,7 @@ test "overwrites existing `expires_at` if it greater than the config setting" do
assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} = assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} =
ActivityExpirationPolicy.filter(%{ ActivityExpirationPolicy.filter(%{
"id" => @id, "id" => @id,
"actor" => @local_actor,
"type" => "Create", "type" => "Create",
"expires_at" => too_distant_future, "expires_at" => too_distant_future,
"object" => %{"type" => "Note"} "object" => %{"type" => "Note"}
@ -49,6 +53,7 @@ test "ignores remote activities" do
assert {:ok, activity} = assert {:ok, activity} =
ActivityExpirationPolicy.filter(%{ ActivityExpirationPolicy.filter(%{
"id" => "https://example.com/123", "id" => "https://example.com/123",
"actor" => "https://example.com/users/cofe",
"type" => "Create", "type" => "Create",
"object" => %{"type" => "Note"} "object" => %{"type" => "Note"}
}) })
@ -60,6 +65,7 @@ test "ignores non-Create/Note activities" do
assert {:ok, activity} = assert {:ok, activity} =
ActivityExpirationPolicy.filter(%{ ActivityExpirationPolicy.filter(%{
"id" => "https://example.com/123", "id" => "https://example.com/123",
"actor" => "https://example.com/users/cofe",
"type" => "Follow" "type" => "Follow"
}) })
@ -68,6 +74,7 @@ test "ignores non-Create/Note activities" do
assert {:ok, activity} = assert {:ok, activity} =
ActivityExpirationPolicy.filter(%{ ActivityExpirationPolicy.filter(%{
"id" => "https://example.com/123", "id" => "https://example.com/123",
"actor" => "https://example.com/users/cofe",
"type" => "Create", "type" => "Create",
"object" => %{"type" => "Cofe"} "object" => %{"type" => "Cofe"}
}) })

View File

@ -124,6 +124,24 @@ test "it fetches the actor if they aren't in our system" do
{:ok, %Activity{} = _activity} = Transmogrifier.handle_incoming(data) {:ok, %Activity{} = _activity} = Transmogrifier.handle_incoming(data)
end end
test "it doesn't work for deactivated users" do
data =
File.read!("test/fixtures/create-chat-message.json")
|> Poison.decode!()
_author =
insert(:user,
ap_id: data["actor"],
local: false,
last_refreshed_at: DateTime.utc_now(),
deactivated: true
)
_recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
assert {:error, _} = Transmogrifier.handle_incoming(data)
end
test "it inserts it and creates a chat" do test "it inserts it and creates a chat" do
data = data =
File.read!("test/fixtures/create-chat-message.json") File.read!("test/fixtures/create-chat-message.json")

View File

@ -163,6 +163,14 @@ test "it does not crash if the object in inReplyTo can't be fetched" do
end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil" end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil"
end end
test "it does not work for deactivated users" do
data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
insert(:user, ap_id: data["actor"], deactivated: true)
assert {:error, _} = Transmogrifier.handle_incoming(data)
end
test "it works for incoming notices" do test "it works for incoming notices" do
data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()

View File

@ -458,6 +458,11 @@ test "it adds emoji in the object" do
end end
describe "posting" do describe "posting" do
test "deactivated users can't post" do
user = insert(:user, deactivated: true)
assert {:error, _} = CommonAPI.post(user, %{status: "ye"})
end
test "it supports explicit addressing" do test "it supports explicit addressing" do
user = insert(:user) user = insert(:user)
user_two = insert(:user) user_two = insert(:user)

View File

@ -940,17 +940,32 @@ test "registers and logs in without :account_activation_required / :account_appr
assert refresh assert refresh
assert scope == "read write follow" assert scope == "read write follow"
conn = clear_config([User, :email_blacklist], ["example.org"])
build_conn()
|> put_req_header("content-type", "multipart/form-data") params = %{
|> put_req_header("authorization", "Bearer " <> token)
|> post("/api/v1/accounts", %{
username: "lain", username: "lain",
email: "lain@example.org", email: "lain@example.org",
password: "PlzDontHackLain", password: "PlzDontHackLain",
bio: "Test Bio", bio: "Test Bio",
agreement: true agreement: true
}) }
conn =
build_conn()
|> put_req_header("content-type", "multipart/form-data")
|> put_req_header("authorization", "Bearer " <> token)
|> post("/api/v1/accounts", params)
assert %{"error" => "{\"email\":[\"Invalid email\"]}"} =
json_response_and_validate_schema(conn, 400)
Pleroma.Config.put([User, :email_blacklist], [])
conn =
build_conn()
|> put_req_header("content-type", "multipart/form-data")
|> put_req_header("authorization", "Bearer " <> token)
|> post("/api/v1/accounts", params)
%{ %{
"access_token" => token, "access_token" => token,

View File

@ -17,8 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPITest do
test "returns error when followed user is deactivated" do test "returns error when followed user is deactivated" do
follower = insert(:user) follower = insert(:user)
user = insert(:user, local: true, deactivated: true) user = insert(:user, local: true, deactivated: true)
{:error, error} = MastodonAPI.follow(follower, user) assert {:error, _error} = MastodonAPI.follow(follower, user)
assert error == :rejected
end end
test "following for user" do test "following for user" do

View File

@ -29,5 +29,16 @@ test "gets exist app and updates scopes" do
assert exist_app.id == app.id assert exist_app.id == app.id
assert exist_app.scopes == ["read", "write", "follow", "push"] assert exist_app.scopes == ["read", "write", "follow", "push"]
end end
test "has unique client_id" do
insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop")
error =
catch_error(insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop"))
assert %Ecto.ConstraintError{} = error
assert error.constraint == "apps_client_id_index"
assert error.type == :unique
end
end end
end end