From 0673511d3939c04efd0e3fd350c47b4bfb3e322e Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 26 Nov 2017 20:57:49 +0300 Subject: [PATCH 01/18] first version of safe DB setup --- README.md | 6 ++---- lib/mix/tasks/generate_config.ex | 13 +++++++++---- lib/mix/tasks/sample_config.eex | 4 ++-- lib/mix/tasks/sample_psql.eex | 6 ++++++ 4 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 lib/mix/tasks/sample_psql.eex diff --git a/README.md b/README.md index b07593f67..01a56d6bf 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,14 @@ No release has been made yet, but several servers have been online for months al ### Dependencies * Postgresql version 9.6 or newer -* Elixir version 1.4 or newer +* Elixir version 1.4 or newer (you will also need erlang-dev, erlang-parsetools, erlang-xmerl packages) * Build-essential tools ### Configuration * Run `mix deps.get` to install elixir dependencies. - * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. - - * Configure your database settings in `{dev,prod}.secret.exs` and either create the database with psql or run `mix ecto.create`. + * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/dev.secret.exs` and `config/prod.secret.exs`. It will also create a `config/db_setup.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su - postgre -c "psql -f config/db_setup.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser. * Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates. diff --git a/lib/mix/tasks/generate_config.ex b/lib/mix/tasks/generate_config.ex index b3bc14d5b..f9399b114 100644 --- a/lib/mix/tasks/generate_config.ex +++ b/lib/mix/tasks/generate_config.ex @@ -4,14 +4,19 @@ defmodule Mix.Tasks.GenerateConfig do @shortdoc "Generates a new config" def run(_) do IO.puts("Answer a few questions to generate a new config\n") - IO.puts("--- THIS WILL OVERWRITE YOUR config/generated_config.exs! ---\n") + IO.puts("--- THIS WILL OVERWRITE YOUR config/dev.secret.exs AND config/prod.secret.exs! ---\n") domain = IO.gets("What is your domain name? (e.g. pleroma.soykaf.com): ") |> String.trim name = IO.gets("What is the name of your instance? (e.g. Pleroma/Soykaf): ") |> String.trim email = IO.gets("What's your admin email address: ") |> String.trim secret = :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64) + dbpass = :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64) - result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret]) - IO.puts("\nWriting config to config/generated_config.exs.\n\nCheck it and configure your database, then copy it to either config/dev.secret.exs or config/prod.secret.exs") - File.write("config/generated_config.exs", result) + resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass]) + result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass]) + IO.puts("\nWriting config to config/dev.secret.exs and config/prod.secret.exs") + IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e. su - postgre -c 'psql -f setup_db.psql'") + File.write("config/dev.secret.exs", result) + File.write("config/prod.secret.exs", result) + File.write("config/setup_db.psql", resultSql) end end diff --git a/lib/mix/tasks/sample_config.eex b/lib/mix/tasks/sample_config.eex index 62a9804fe..85a7c554e 100644 --- a/lib/mix/tasks/sample_config.eex +++ b/lib/mix/tasks/sample_config.eex @@ -13,8 +13,8 @@ config :pleroma, :instance, # Configure your database config :pleroma, Pleroma.Repo, adapter: Ecto.Adapters.Postgres, - username: "postgres", - password: "postgres", + username: "pleroma", + password: "<%= dbpass %>", database: "pleroma_dev", hostname: "localhost", pool_size: 10 diff --git a/lib/mix/tasks/sample_psql.eex b/lib/mix/tasks/sample_psql.eex new file mode 100644 index 000000000..5353fa216 --- /dev/null +++ b/lib/mix/tasks/sample_psql.eex @@ -0,0 +1,6 @@ +CREATE USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; +CREATE DATABASE pleroma_dev; +ALTER DATABASE pleroma_dev OWNER TO pleroma; +\c pleroma_dev; +--Extensions made by ecto.migrate that need superuser access +CREATE EXTENSION citext; From 40714031176f28ed1af2ed83f01b1d066badb871 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Tue, 28 Nov 2017 00:42:03 +0300 Subject: [PATCH 02/18] fixes --- README.md | 2 +- lib/mix/tasks/generate_config.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 01a56d6bf..175b90f88 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ No release has been made yet, but several servers have been online for months al * Run `mix deps.get` to install elixir dependencies. - * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/dev.secret.exs` and `config/prod.secret.exs`. It will also create a `config/db_setup.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su - postgre -c "psql -f config/db_setup.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser. + * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/dev.secret.exs` and `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su postgres -c "psql -f config/setup_db.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser. * Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates. diff --git a/lib/mix/tasks/generate_config.ex b/lib/mix/tasks/generate_config.ex index f9399b114..d06cc1a84 100644 --- a/lib/mix/tasks/generate_config.ex +++ b/lib/mix/tasks/generate_config.ex @@ -14,7 +14,7 @@ def run(_) do resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass]) result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass]) IO.puts("\nWriting config to config/dev.secret.exs and config/prod.secret.exs") - IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e. su - postgre -c 'psql -f setup_db.psql'") + IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e. su - postgres -c 'psql -f config/setup_db.psql'") File.write("config/dev.secret.exs", result) File.write("config/prod.secret.exs", result) File.write("config/setup_db.psql", resultSql) From a884e08271ec29ed42011c450655861b870f4745 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Tue, 28 Nov 2017 00:44:14 +0300 Subject: [PATCH 03/18] fix --- lib/mix/tasks/generate_config.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/generate_config.ex b/lib/mix/tasks/generate_config.ex index d06cc1a84..95cf394be 100644 --- a/lib/mix/tasks/generate_config.ex +++ b/lib/mix/tasks/generate_config.ex @@ -14,7 +14,7 @@ def run(_) do resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass]) result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass]) IO.puts("\nWriting config to config/dev.secret.exs and config/prod.secret.exs") - IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e. su - postgres -c 'psql -f config/setup_db.psql'") + IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e.: sudo su postgres -c 'psql -f config/setup_db.psql'") File.write("config/dev.secret.exs", result) File.write("config/prod.secret.exs", result) File.write("config/setup_db.psql", resultSql) From 071f518028e20d1f22811d6a10f120bebe26c2ce Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Mon, 4 Dec 2017 02:01:45 +0300 Subject: [PATCH 04/18] reverts config gen --- README.md | 2 +- lib/mix/tasks/generate_config.ex | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 175b90f88..664b8b475 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ No release has been made yet, but several servers have been online for months al * Run `mix deps.get` to install elixir dependencies. - * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/dev.secret.exs` and `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su postgres -c "psql -f config/setup_db.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser. + * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su postgres -c "psql -f config/setup_db.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser. * Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates. diff --git a/lib/mix/tasks/generate_config.ex b/lib/mix/tasks/generate_config.ex index 95cf394be..f20f93e4d 100644 --- a/lib/mix/tasks/generate_config.ex +++ b/lib/mix/tasks/generate_config.ex @@ -4,7 +4,7 @@ defmodule Mix.Tasks.GenerateConfig do @shortdoc "Generates a new config" def run(_) do IO.puts("Answer a few questions to generate a new config\n") - IO.puts("--- THIS WILL OVERWRITE YOUR config/dev.secret.exs AND config/prod.secret.exs! ---\n") + IO.puts("--- THIS WILL OVERWRITE YOUR config/generated_config.exs! ---\n") domain = IO.gets("What is your domain name? (e.g. pleroma.soykaf.com): ") |> String.trim name = IO.gets("What is the name of your instance? (e.g. Pleroma/Soykaf): ") |> String.trim email = IO.gets("What's your admin email address: ") |> String.trim @@ -13,10 +13,10 @@ def run(_) do resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass]) result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass]) - IO.puts("\nWriting config to config/dev.secret.exs and config/prod.secret.exs") + + IO.puts("\nWriting config to config/generated_config.exs.\n\nCheck it and configure your database, then copy it to either config/dev.secret.exs or config/prod.secret.exs") + File.write("config/generated_config.exs", result) IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e.: sudo su postgres -c 'psql -f config/setup_db.psql'") - File.write("config/dev.secret.exs", result) - File.write("config/prod.secret.exs", result) File.write("config/setup_db.psql", resultSql) end end From 6b60f5f64ab9d7598ccdb2545b0e5560991ff5f3 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Mon, 4 Dec 2017 02:06:05 +0300 Subject: [PATCH 05/18] made psql script compatible with migrations or generate_config reruns --- lib/mix/tasks/sample_psql.eex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/sample_psql.eex b/lib/mix/tasks/sample_psql.eex index 5353fa216..9eb7f65b0 100644 --- a/lib/mix/tasks/sample_psql.eex +++ b/lib/mix/tasks/sample_psql.eex @@ -1,6 +1,8 @@ -CREATE USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; -CREATE DATABASE pleroma_dev; +CREATE USER IF NOT EXISTS pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; +-- in case someone runs this second time accidentally +ALTER USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; +CREATE DATABASE IF NOT EXISTS pleroma_dev; ALTER DATABASE pleroma_dev OWNER TO pleroma; \c pleroma_dev; --Extensions made by ecto.migrate that need superuser access -CREATE EXTENSION citext; +CREATE EXTENSION IF NOT EXISTS citext; From 964e2dd2f055207ac8fa7f8b3612e50860a0472a Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Mon, 4 Dec 2017 04:06:52 +0300 Subject: [PATCH 06/18] unfuck --- lib/mix/tasks/sample_psql.eex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/sample_psql.eex b/lib/mix/tasks/sample_psql.eex index 9eb7f65b0..18e322efc 100644 --- a/lib/mix/tasks/sample_psql.eex +++ b/lib/mix/tasks/sample_psql.eex @@ -1,7 +1,7 @@ -CREATE USER IF NOT EXISTS pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; +CREATE USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; -- in case someone runs this second time accidentally ALTER USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; -CREATE DATABASE IF NOT EXISTS pleroma_dev; +CREATE DATABASE pleroma_dev; ALTER DATABASE pleroma_dev OWNER TO pleroma; \c pleroma_dev; --Extensions made by ecto.migrate that need superuser access From 66c3813ea6388e9933af2b15e903f1cf6254cd3a Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Tue, 5 Dec 2017 18:21:30 +0100 Subject: [PATCH 07/18] Add basic queue prioritization. --- lib/pleroma/web/federator/federator.ex | 21 +++++++++++++++------ test/web/federator_test.exs | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 test/web/federator_test.exs diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 9f6f983aa..f384b313c 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -15,8 +15,8 @@ def start_link do enqueue(:refresh_subscriptions, nil) end) GenServer.start_link(__MODULE__, %{ - in: {:sets.new(), :queue.new()}, - out: {:sets.new(), :queue.new()} + in: {:sets.new(), [], + out: {:sets.new(), []} }, name: __MODULE__) end @@ -88,8 +88,8 @@ def enqueue(type, payload) do end def maybe_start_job(running_jobs, queue) do - if (:sets.size(running_jobs) < @max_jobs) && !:queue.is_empty(queue) do - {{:value, {type, payload}}, queue} = :queue.out(queue) + if (:sets.size(running_jobs) < @max_jobs) && queue != [] do + {{:value, {type, payload}}, queue} = queue_pop(queue) {:ok, pid} = Task.start(fn -> handle(type, payload) end) mref = Process.monitor(pid) {:sets.add_element(mref, running_jobs), queue} @@ -100,14 +100,14 @@ def maybe_start_job(running_jobs, queue) do def handle_cast({:enqueue, type, payload}, state) when type in [:incoming_doc] do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - i_queue = :queue.in({type, payload}, i_queue) + i_queue = enqueue_sorted(i_queue, {type, payload}, 1) {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end def handle_cast({:enqueue, type, payload}, state) do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - o_queue = :queue.in({type, payload}, o_queue) + o_queue = enqueue_sorted(o_queue, {type, payload}, 1) {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end @@ -126,4 +126,13 @@ def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end + + def enqueue_sorted(queue, element, priority) do + [%{item: element, priority: priority} | queue] + |> Enum.sort_by(fn (%{priority: priority}) -> priority end) + end + + def queue_pop([%{item: element} | queue]) do + {element, queue} + end end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs new file mode 100644 index 000000000..09533362a --- /dev/null +++ b/test/web/federator_test.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Web.FederatorTest do + alias Pleroma.Web.Federator + use Pleroma.DataCase + + test "enqueues an element according to priority" do + queue = [%{item: 1, priority: 2}] + + new_queue = Federator.enqueue_sorted(queue, 2, 1) + assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + new_queue = Federator.enqueue_sorted(queue, 2, 3) + assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}] + end + + test "pop first item" do + queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue) + end +end From e31a5ff4af04156b9db8f032cf184e1da540f025 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Wed, 6 Dec 2017 16:51:11 +0100 Subject: [PATCH 08/18] Priority queue fixes. --- lib/pleroma/web/activity_pub/utils.ex | 7 ++++++- lib/pleroma/web/federator/federator.ex | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 51fac6fe2..ac20a2822 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -29,7 +29,12 @@ def generate_id(type) do Enqueues an activity for federation if it's local """ def maybe_federate(%Activity{local: true} = activity) do - Pleroma.Web.Federator.enqueue(:publish, activity) + priority = case activity.data["type"] do + "Delete" -> 10 + "Create" -> 1 + _ -> 5 + end + Pleroma.Web.Federator.enqueue(:publish, activity, priority) :ok end def maybe_federate(_), do: :ok diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f384b313c..b23ed5fcc 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -15,7 +15,7 @@ def start_link do enqueue(:refresh_subscriptions, nil) end) GenServer.start_link(__MODULE__, %{ - in: {:sets.new(), [], + in: {:sets.new(), []}, out: {:sets.new(), []} }, name: __MODULE__) end @@ -79,17 +79,17 @@ def handle(type, _) do {:error, "Don't know what do do with this"} end - def enqueue(type, payload) do + def enqueue(type, payload, priority \\ 1) do if Mix.env == :test do handle(type, payload) else - GenServer.cast(__MODULE__, {:enqueue, type, payload}) + GenServer.cast(__MODULE__, {:enqueue, type, payload, priority}) end end def maybe_start_job(running_jobs, queue) do if (:sets.size(running_jobs) < @max_jobs) && queue != [] do - {{:value, {type, payload}}, queue} = queue_pop(queue) + {{type, payload}, queue} = queue_pop(queue) {:ok, pid} = Task.start(fn -> handle(type, payload) end) mref = Process.monitor(pid) {:sets.add_element(mref, running_jobs), queue} @@ -98,14 +98,14 @@ def maybe_start_job(running_jobs, queue) do end end - def handle_cast({:enqueue, type, payload}, state) when type in [:incoming_doc] do + def handle_cast({:enqueue, type, payload, priority}, state) when type in [:incoming_doc] do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state i_queue = enqueue_sorted(i_queue, {type, payload}, 1) {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end - def handle_cast({:enqueue, type, payload}, state) do + def handle_cast({:enqueue, type, payload, priority}, state) do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state o_queue = enqueue_sorted(o_queue, {type, payload}, 1) {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) From 0ec5aeb8a76653935caefa0de92861269f98f343 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 17:41:34 +0100 Subject: [PATCH 09/18] Don't log in deactivated users. --- lib/pleroma/plugs/authentication_plug.ex | 1 + lib/pleroma/plugs/oauth_plug.ex | 3 ++- test/plugs/authentication_plug_test.exs | 27 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/authentication_plug.ex b/lib/pleroma/plugs/authentication_plug.ex index beb02eb88..60f6faf49 100644 --- a/lib/pleroma/plugs/authentication_plug.ex +++ b/lib/pleroma/plugs/authentication_plug.ex @@ -12,6 +12,7 @@ def call(%{assigns: %{user: %User{}}} = conn, _), do: conn def call(conn, opts) do with {:ok, username, password} <- decode_header(conn), {:ok, user} <- opts[:fetcher].(username), + false <- !!user.info["deactivated"], saved_user_id <- get_session(conn, :user_id), {:ok, verified_user} <- verify(user, password, saved_user_id) do diff --git a/lib/pleroma/plugs/oauth_plug.ex b/lib/pleroma/plugs/oauth_plug.ex index 775423bb1..be737dc9a 100644 --- a/lib/pleroma/plugs/oauth_plug.ex +++ b/lib/pleroma/plugs/oauth_plug.ex @@ -16,7 +16,8 @@ def call(conn, _) do end with token when not is_nil(token) <- token, %Token{user_id: user_id} <- Repo.get_by(Token, token: token), - %User{} = user <- Repo.get(User, user_id) do + %User{} = user <- Repo.get(User, user_id), + false <- !!user.info["deactivated"] do conn |> assign(:user, user) else diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index 9d6c2cd70..5480dab43 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -14,6 +14,13 @@ defp fetch_nil(_name) do password_hash: Comeonin.Pbkdf2.hashpwsalt("guy") } + @deactivated %User{ + id: 1, + name: "dude", + password_hash: Comeonin.Pbkdf2.hashpwsalt("guy"), + info: %{"deactivated" => true} + } + @session_opts [ store: :cookie, key: "_test", @@ -131,6 +138,26 @@ test "it assigns the user", %{conn: conn} do end end + describe "with a correct authorization header for an deactiviated user" do + test "it halts the appication", %{conn: conn} do + opts = %{ + optional: false, + fetcher: fn _ -> @deactivated end + } + + header = basic_auth_enc("dude", "guy") + + conn = conn + |> Plug.Session.call(Plug.Session.init(@session_opts)) + |> fetch_session + |> put_req_header("authorization", header) + |> AuthenticationPlug.call(opts) + + assert conn.status == 403 + assert conn.halted == true + end + end + describe "with a user_id in the session for an existing user" do test "it assigns the user", %{conn: conn} do opts = %{ From b727ecc5e75388e2b2d67c76cab6df1a7e6719b3 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 17:47:23 +0100 Subject: [PATCH 10/18] Add function to deactivate users. --- lib/pleroma/user.ex | 5 +++++ test/user_test.exs | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index afc62f265..779a89a12 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -334,4 +334,9 @@ def local_user_query() do where: u.local == true end + def deactivate (%User{} = user) do + new_info = Map.put(user.info, "deactivated", true) + cs = User.info_changeset(user, %{info: new_info}) + Repo.update(cs) + end end diff --git a/test/user_test.exs b/test/user_test.exs index 31c5962e2..6daf1ecb7 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -325,5 +325,11 @@ test "get recipients from activity" do assert user in recipients assert addressed in recipients end -end + test ".deactivate deactivates a user" do + user = insert(:user) + assert false == !!user.info["deactivated"] + {:ok, user} = User.deactivate(user) + assert true == user.info["deactivated"] + end +end From a78ae2a685769cbf7459eb347aca581d389018ad Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 17:51:55 +0100 Subject: [PATCH 11/18] Don't follow deactivated users. --- lib/pleroma/user.ex | 4 ++-- test/user_test.exs | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 779a89a12..b21caba9d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -142,9 +142,9 @@ def register_changeset(struct, params \\ %{}) do end end - def follow(%User{} = follower, %User{} = followed) do + def follow(%User{} = follower, %User{info: info} = followed) do ap_followers = followed.follower_address - if following?(follower, followed) do + if following?(follower, followed) or info["deactivated"] do {:error, "Could not follow user: #{followed.nickname} is already on your list."} else diff --git a/test/user_test.exs b/test/user_test.exs index 6daf1ecb7..486998907 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -39,6 +39,13 @@ test "follow takes a user and another user" do assert User.ap_followers(followed) in user.following end + test "can't follow a deactivated users" do + user = insert(:user) + followed = insert(:user, info: %{"deactivated" => true}) + + {:error, _} = User.follow(user, followed) + end + test "following a remote user will ensure a websub subscription is present" do user = insert(:user) {:ok, followed} = OStatus.make_user("shp@social.heldscal.la") From 6df6ad0b429150b00b065d95890bd62cd2778fad Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 18:13:05 +0100 Subject: [PATCH 12/18] User deletion: Remove relationships. --- lib/pleroma/user.ex | 15 +++++++++++++++ test/user_test.exs | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index b21caba9d..021207de9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -339,4 +339,19 @@ def deactivate (%User{} = user) do cs = User.info_changeset(user, %{info: new_info}) Repo.update(cs) end + + def delete (%User{} = user) do + {:ok, user} = User.deactivate(user) + + # Remove all relationships + {:ok, followers } = User.get_followers(user) + followers + |> Enum.each(fn (follower) -> User.unfollow(follower, user) end) + + {:ok, friends} = User.get_friends(user) + friends + |> Enum.each(fn (followed) -> User.unfollow(user, followed) end) + + :ok + end end diff --git a/test/user_test.exs b/test/user_test.exs index 486998907..0dd5afb18 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -339,4 +339,33 @@ test ".deactivate deactivates a user" do {:ok, user} = User.deactivate(user) assert true == user.info["deactivated"] end + + test ".delete deactivates a user, all follow relationships and all create activities" do + user = insert(:user) + followed = insert(:user) + follower = insert(:user) + + {:ok, user} = User.follow(user, followed) + {:ok, follower} = User.follow(follower, user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"}) + {:ok, activity_two} = CommonAPI.post(follower, %{"status" => "3hu"}) + + {:ok, _, _} = CommonAPI.favorite(activity_two.id, user) + {:ok, _, _} = CommonAPI.favorite(activity.id, follower) + {:ok, _, _} = CommonAPI.repeat(activity.id, follower) + + :ok = User.delete(user) + + followed = Repo.get(User, followed.id) + follower = Repo.get(User, follower.id) + user = Repo.get(User, user.id) + + assert user.info["deactivated"] + + refute User.following?(user, followed) + refute User.following?(followed, follower) + + # TODO: check for activities. + end end From afd0ea37f3284bdfa6ddce99162601615b7ad845 Mon Sep 17 00:00:00 2001 From: eal Date: Thu, 7 Dec 2017 20:07:51 +0200 Subject: [PATCH 13/18] Add websocket upgrade to example nginx config. --- installation/pleroma.nginx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx index 1a6e1d56f..6cf9f3fa0 100644 --- a/installation/pleroma.nginx +++ b/installation/pleroma.nginx @@ -19,6 +19,9 @@ server { server_name example.tld; location / { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; proxy_pass http://localhost:4000; } include snippets/well-known.conf; From 5436dbaeaf0765c5d651cd16b02018edbc6793d4 Mon Sep 17 00:00:00 2001 From: eal Date: Thu, 7 Dec 2017 20:44:09 +0200 Subject: [PATCH 14/18] MastoAPI: don't add attachment links. --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 8 ++++++-- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index dc94e5377..d3a9f7b85 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -56,7 +56,7 @@ def post(user, %{"status" => status} = data) do inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]), to <- to_for_user_and_mentions(user, mentions, inReplyTo), tags <- Formatter.parse_tags(status, data), - content_html <- make_content_html(status, mentions, attachments, tags), + content_html <- make_content_html(status, mentions, attachments, tags, data["no_attachment_links"]), context <- make_context(inReplyTo), cw <- data["spoiler_text"], object <- make_note_data(user.ap_id, to, context, content_html, attachments, inReplyTo, tags, cw), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1a23b1ad2..2d9fdaf6c 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -38,15 +38,19 @@ def to_for_user_and_mentions(user, mentions, inReplyTo) do end end - def make_content_html(status, mentions, attachments, tags) do + def make_content_html(status, mentions, attachments, tags, no_attachment_links \\ false) do status |> format_input(mentions, tags) - |> add_attachments(attachments) + |> maybe_add_attachments(attachments, no_attachment_links) end def make_context(%Activity{data: %{"context" => context}}), do: context def make_context(_), do: Utils.generate_context_id + def maybe_add_attachments(text, attachments, _no_links = true), do: text + def maybe_add_attachments(text, attachments, _no_links) do + add_attachments(text, attachments) + end def add_attachments(text, attachments) do attachment_text = Enum.map(attachments, fn (%{"url" => [%{"href" => href} | _]}) -> diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 61bf8b4b8..e50f53ba4 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -212,6 +212,7 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do params = params |> Map.put("in_reply_to_status_id", params["in_reply_to_id"]) + |> Map.put("no_attachment_links", true) {:ok, activity} = CommonAPI.post(user, params) render conn, StatusView, "status.json", %{activity: activity, for: user, as: :activity} From bf91e5659ffd03b15110b6f1094b30aed940e45b Mon Sep 17 00:00:00 2001 From: eal Date: Thu, 7 Dec 2017 21:34:25 +0200 Subject: [PATCH 15/18] Fix HTML escape breaking some links. --- lib/pleroma/formatter.ex | 9 +++++++++ lib/pleroma/web/common_api/utils.ex | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index fbcbca979..275c60f32 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -24,6 +24,15 @@ def parse_mentions(text) do |> Enum.filter(fn ({_match, user}) -> user end) end + def html_escape(text) do + Regex.split(@link_regex, text, include_captures: true) + |> Enum.map_every(2, fn chunk -> + {:safe, part} = Phoenix.HTML.html_escape(chunk) + part + end) + |> Enum.join("") + end + @finmoji [ "a_trusted_friend", "alandislands", diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1a23b1ad2..f46db4cf0 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -58,8 +58,8 @@ def add_attachments(text, attachments) do end def format_input(text, mentions, _tags) do - Phoenix.HTML.html_escape(text) - |> elem(1) + text + |> Formatter.html_escape |> Formatter.linkify |> String.replace("\n", "
") |> add_user_links(mentions) From a3e68f02330a3c38f1d3f568ad507e5111b82600 Mon Sep 17 00:00:00 2001 From: eal Date: Thu, 7 Dec 2017 21:38:31 +0200 Subject: [PATCH 16/18] Allow parentheses in links. --- lib/pleroma/formatter.ex | 2 +- test/formatter_test.exs | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 275c60f32..c98db2d94 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User - @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~]+[\w\/]/u + @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u def linkify(text) do Regex.replace(@link_regex, text, "\\0") end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index f91973881..cb7695e8e 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -25,6 +25,16 @@ test "turning urls into links" do expected = "http://www.cs.vu.nl/~ast/intel/" assert Formatter.linkify(text) == expected + + text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087" + expected = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087" + + assert Formatter.linkify(text) == expected + + text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul" + expected = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul" + + assert Formatter.linkify(text) == expected end end From bad499b3fd95d1643890feaedea07123c004be57 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Fri, 8 Dec 2017 17:50:11 +0100 Subject: [PATCH 17/18] Basic user deletion. --- lib/pleroma/user.ex | 42 ++++++++++++++++++++++++++++++++---------- test/user_test.exs | 6 ++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 021207de9..09bcf0cb4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -5,7 +5,7 @@ defmodule Pleroma.User do alias Pleroma.{Repo, User, Object, Web, Activity, Notification} alias Comeonin.Pbkdf2 alias Pleroma.Web.{OStatus, Websub} - alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.{Utils, ActivityPub} schema "users" do field :bio, :string @@ -113,7 +113,7 @@ def password_update_changeset(struct, params) do end def reset_password(user, data) do - Repo.update(password_update_changeset(user, data)) + update_and_set_cache(password_update_changeset(user, data)) end def register_changeset(struct, params \\ %{}) do @@ -157,7 +157,7 @@ def follow(%User{} = follower, %User{info: info} = followed) do follower = follower |> follow_changeset(%{following: following}) - |> Repo.update + |> update_and_set_cache {:ok, _} = update_follower_count(followed) @@ -173,7 +173,7 @@ def unfollow(%User{} = follower, %User{} = followed) do { :ok, follower } = follower |> follow_changeset(%{following: following}) - |> Repo.update + |> update_and_set_cache {:ok, followed} = update_follower_count(followed) @@ -191,6 +191,17 @@ def get_by_ap_id(ap_id) do Repo.get_by(User, ap_id: ap_id) end + def update_and_set_cache(changeset) do + with {:ok, user} <- Repo.update(changeset) do + Cachex.set(:user_cache, "ap_id:#{user.ap_id}", user) + Cachex.set(:user_cache, "nickname:#{user.nickname}", user) + Cachex.set(:user_cache, "user_info:#{user.id}", user_info(user)) + {:ok, user} + else + e -> e + end + end + def get_cached_by_ap_id(ap_id) do key = "ap_id:#{ap_id}" Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end) @@ -245,7 +256,7 @@ def increase_note_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def update_note_count(%User{} = user) do @@ -259,7 +270,7 @@ def update_note_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def update_follower_count(%User{} = user) do @@ -274,7 +285,7 @@ def update_follower_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def get_notified_from_activity(%Activity{data: %{"to" => to}}) do @@ -312,7 +323,7 @@ def block(user, %{ap_id: ap_id}) do new_info = Map.put(user.info, "blocks", new_blocks) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def unblock(user, %{ap_id: ap_id}) do @@ -321,7 +332,7 @@ def unblock(user, %{ap_id: ap_id}) do new_info = Map.put(user.info, "blocks", new_blocks) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def blocks?(user, %{ap_id: ap_id}) do @@ -337,7 +348,7 @@ def local_user_query() do def deactivate (%User{} = user) do new_info = Map.put(user.info, "deactivated", true) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def delete (%User{} = user) do @@ -352,6 +363,17 @@ def delete (%User{} = user) do friends |> Enum.each(fn (followed) -> User.unfollow(user, followed) end) + query = from a in Activity, + where: a.actor == ^user.ap_id + + Repo.all(query) + |> Enum.each(fn (activity) -> + case activity.data["type"] do + "Create" -> ActivityPub.delete(Object.get_by_ap_id(activity.data["object"]["id"])) + _ -> "Doing nothing" # TODO: Do something with likes, follows, repeats. + end + end) + :ok end end diff --git a/test/user_test.exs b/test/user_test.exs index 0dd5afb18..16d43e619 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1,6 +1,6 @@ defmodule Pleroma.UserTest do alias Pleroma.Builders.UserBuilder - alias Pleroma.{User, Repo} + alias Pleroma.{User, Repo, Activity} alias Pleroma.Web.OStatus alias Pleroma.Web.Websub.WebsubClientSubscription alias Pleroma.Web.CommonAPI @@ -366,6 +366,8 @@ test ".delete deactivates a user, all follow relationships and all create activi refute User.following?(user, followed) refute User.following?(followed, follower) - # TODO: check for activities. + # TODO: Remove favorites, repeats, delete activities. + + refute Repo.get(Activity, activity.id) end end From ef2322bdb7d46882eeae0271c4381efda6835c3c Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Fri, 8 Dec 2017 18:17:30 +0100 Subject: [PATCH 18/18] Delete notifications on activity deletion. --- lib/pleroma/activity.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a35ccc9b4..afd09982f 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Activity do field :data, :map field :local, :boolean, default: true field :actor, :string - has_many :notifications, Notification + has_many :notifications, Notification, on_delete: :delete_all timestamps() end