diff --git a/README.md b/README.md index b07593f67..664b8b475 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,14 @@ No release has been made yet, but several servers have been online for months al ### Dependencies * Postgresql version 9.6 or newer -* Elixir version 1.4 or newer +* Elixir version 1.4 or newer (you will also need erlang-dev, erlang-parsetools, erlang-xmerl packages) * Build-essential tools ### Configuration * Run `mix deps.get` to install elixir dependencies. - * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. - - * Configure your database settings in `{dev,prod}.secret.exs` and either create the database with psql or run `mix ecto.create`. + * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su postgres -c "psql -f config/setup_db.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser. * Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates. diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx index d882908e4..4e88a426d 100644 --- a/installation/pleroma.nginx +++ b/installation/pleroma.nginx @@ -22,6 +22,9 @@ server { server_name example.tld; location / { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; proxy_pass http://localhost:4000; } diff --git a/lib/mix/tasks/generate_config.ex b/lib/mix/tasks/generate_config.ex index b3bc14d5b..f20f93e4d 100644 --- a/lib/mix/tasks/generate_config.ex +++ b/lib/mix/tasks/generate_config.ex @@ -9,9 +9,14 @@ def run(_) do name = IO.gets("What is the name of your instance? (e.g. Pleroma/Soykaf): ") |> String.trim email = IO.gets("What's your admin email address: ") |> String.trim secret = :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64) + dbpass = :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64) + + resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass]) + result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass]) - result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret]) IO.puts("\nWriting config to config/generated_config.exs.\n\nCheck it and configure your database, then copy it to either config/dev.secret.exs or config/prod.secret.exs") File.write("config/generated_config.exs", result) + IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e.: sudo su postgres -c 'psql -f config/setup_db.psql'") + File.write("config/setup_db.psql", resultSql) end end diff --git a/lib/mix/tasks/sample_config.eex b/lib/mix/tasks/sample_config.eex index 62a9804fe..85a7c554e 100644 --- a/lib/mix/tasks/sample_config.eex +++ b/lib/mix/tasks/sample_config.eex @@ -13,8 +13,8 @@ config :pleroma, :instance, # Configure your database config :pleroma, Pleroma.Repo, adapter: Ecto.Adapters.Postgres, - username: "postgres", - password: "postgres", + username: "pleroma", + password: "<%= dbpass %>", database: "pleroma_dev", hostname: "localhost", pool_size: 10 diff --git a/lib/mix/tasks/sample_psql.eex b/lib/mix/tasks/sample_psql.eex new file mode 100644 index 000000000..18e322efc --- /dev/null +++ b/lib/mix/tasks/sample_psql.eex @@ -0,0 +1,8 @@ +CREATE USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; +-- in case someone runs this second time accidentally +ALTER USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB; +CREATE DATABASE pleroma_dev; +ALTER DATABASE pleroma_dev OWNER TO pleroma; +\c pleroma_dev; +--Extensions made by ecto.migrate that need superuser access +CREATE EXTENSION IF NOT EXISTS citext; diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a35ccc9b4..afd09982f 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Activity do field :data, :map field :local, :boolean, default: true field :actor, :string - has_many :notifications, Notification + has_many :notifications, Notification, on_delete: :delete_all timestamps() end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index fbcbca979..c98db2d94 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User - @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~]+[\w\/]/u + @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u def linkify(text) do Regex.replace(@link_regex, text, "\\0") end @@ -24,6 +24,15 @@ def parse_mentions(text) do |> Enum.filter(fn ({_match, user}) -> user end) end + def html_escape(text) do + Regex.split(@link_regex, text, include_captures: true) + |> Enum.map_every(2, fn chunk -> + {:safe, part} = Phoenix.HTML.html_escape(chunk) + part + end) + |> Enum.join("") + end + @finmoji [ "a_trusted_friend", "alandislands", diff --git a/lib/pleroma/plugs/authentication_plug.ex b/lib/pleroma/plugs/authentication_plug.ex index beb02eb88..60f6faf49 100644 --- a/lib/pleroma/plugs/authentication_plug.ex +++ b/lib/pleroma/plugs/authentication_plug.ex @@ -12,6 +12,7 @@ def call(%{assigns: %{user: %User{}}} = conn, _), do: conn def call(conn, opts) do with {:ok, username, password} <- decode_header(conn), {:ok, user} <- opts[:fetcher].(username), + false <- !!user.info["deactivated"], saved_user_id <- get_session(conn, :user_id), {:ok, verified_user} <- verify(user, password, saved_user_id) do diff --git a/lib/pleroma/plugs/oauth_plug.ex b/lib/pleroma/plugs/oauth_plug.ex index 775423bb1..be737dc9a 100644 --- a/lib/pleroma/plugs/oauth_plug.ex +++ b/lib/pleroma/plugs/oauth_plug.ex @@ -16,7 +16,8 @@ def call(conn, _) do end with token when not is_nil(token) <- token, %Token{user_id: user_id} <- Repo.get_by(Token, token: token), - %User{} = user <- Repo.get(User, user_id) do + %User{} = user <- Repo.get(User, user_id), + false <- !!user.info["deactivated"] do conn |> assign(:user, user) else diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index afc62f265..09bcf0cb4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -5,7 +5,7 @@ defmodule Pleroma.User do alias Pleroma.{Repo, User, Object, Web, Activity, Notification} alias Comeonin.Pbkdf2 alias Pleroma.Web.{OStatus, Websub} - alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.{Utils, ActivityPub} schema "users" do field :bio, :string @@ -113,7 +113,7 @@ def password_update_changeset(struct, params) do end def reset_password(user, data) do - Repo.update(password_update_changeset(user, data)) + update_and_set_cache(password_update_changeset(user, data)) end def register_changeset(struct, params \\ %{}) do @@ -142,9 +142,9 @@ def register_changeset(struct, params \\ %{}) do end end - def follow(%User{} = follower, %User{} = followed) do + def follow(%User{} = follower, %User{info: info} = followed) do ap_followers = followed.follower_address - if following?(follower, followed) do + if following?(follower, followed) or info["deactivated"] do {:error, "Could not follow user: #{followed.nickname} is already on your list."} else @@ -157,7 +157,7 @@ def follow(%User{} = follower, %User{} = followed) do follower = follower |> follow_changeset(%{following: following}) - |> Repo.update + |> update_and_set_cache {:ok, _} = update_follower_count(followed) @@ -173,7 +173,7 @@ def unfollow(%User{} = follower, %User{} = followed) do { :ok, follower } = follower |> follow_changeset(%{following: following}) - |> Repo.update + |> update_and_set_cache {:ok, followed} = update_follower_count(followed) @@ -191,6 +191,17 @@ def get_by_ap_id(ap_id) do Repo.get_by(User, ap_id: ap_id) end + def update_and_set_cache(changeset) do + with {:ok, user} <- Repo.update(changeset) do + Cachex.set(:user_cache, "ap_id:#{user.ap_id}", user) + Cachex.set(:user_cache, "nickname:#{user.nickname}", user) + Cachex.set(:user_cache, "user_info:#{user.id}", user_info(user)) + {:ok, user} + else + e -> e + end + end + def get_cached_by_ap_id(ap_id) do key = "ap_id:#{ap_id}" Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end) @@ -245,7 +256,7 @@ def increase_note_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def update_note_count(%User{} = user) do @@ -259,7 +270,7 @@ def update_note_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def update_follower_count(%User{} = user) do @@ -274,7 +285,7 @@ def update_follower_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def get_notified_from_activity(%Activity{data: %{"to" => to}}) do @@ -312,7 +323,7 @@ def block(user, %{ap_id: ap_id}) do new_info = Map.put(user.info, "blocks", new_blocks) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def unblock(user, %{ap_id: ap_id}) do @@ -321,7 +332,7 @@ def unblock(user, %{ap_id: ap_id}) do new_info = Map.put(user.info, "blocks", new_blocks) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def blocks?(user, %{ap_id: ap_id}) do @@ -334,4 +345,35 @@ def local_user_query() do where: u.local == true end + def deactivate (%User{} = user) do + new_info = Map.put(user.info, "deactivated", true) + cs = User.info_changeset(user, %{info: new_info}) + update_and_set_cache(cs) + end + + def delete (%User{} = user) do + {:ok, user} = User.deactivate(user) + + # Remove all relationships + {:ok, followers } = User.get_followers(user) + followers + |> Enum.each(fn (follower) -> User.unfollow(follower, user) end) + + {:ok, friends} = User.get_friends(user) + friends + |> Enum.each(fn (followed) -> User.unfollow(user, followed) end) + + query = from a in Activity, + where: a.actor == ^user.ap_id + + Repo.all(query) + |> Enum.each(fn (activity) -> + case activity.data["type"] do + "Create" -> ActivityPub.delete(Object.get_by_ap_id(activity.data["object"]["id"])) + _ -> "Doing nothing" # TODO: Do something with likes, follows, repeats. + end + end) + + :ok + end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 51fac6fe2..ac20a2822 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -29,7 +29,12 @@ def generate_id(type) do Enqueues an activity for federation if it's local """ def maybe_federate(%Activity{local: true} = activity) do - Pleroma.Web.Federator.enqueue(:publish, activity) + priority = case activity.data["type"] do + "Delete" -> 10 + "Create" -> 1 + _ -> 5 + end + Pleroma.Web.Federator.enqueue(:publish, activity, priority) :ok end def maybe_federate(_), do: :ok diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index dc94e5377..d3a9f7b85 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -56,7 +56,7 @@ def post(user, %{"status" => status} = data) do inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]), to <- to_for_user_and_mentions(user, mentions, inReplyTo), tags <- Formatter.parse_tags(status, data), - content_html <- make_content_html(status, mentions, attachments, tags), + content_html <- make_content_html(status, mentions, attachments, tags, data["no_attachment_links"]), context <- make_context(inReplyTo), cw <- data["spoiler_text"], object <- make_note_data(user.ap_id, to, context, content_html, attachments, inReplyTo, tags, cw), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1a23b1ad2..e60dff7dc 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -38,15 +38,19 @@ def to_for_user_and_mentions(user, mentions, inReplyTo) do end end - def make_content_html(status, mentions, attachments, tags) do + def make_content_html(status, mentions, attachments, tags, no_attachment_links \\ false) do status |> format_input(mentions, tags) - |> add_attachments(attachments) + |> maybe_add_attachments(attachments, no_attachment_links) end def make_context(%Activity{data: %{"context" => context}}), do: context def make_context(_), do: Utils.generate_context_id + def maybe_add_attachments(text, attachments, _no_links = true), do: text + def maybe_add_attachments(text, attachments, _no_links) do + add_attachments(text, attachments) + end def add_attachments(text, attachments) do attachment_text = Enum.map(attachments, fn (%{"url" => [%{"href" => href} | _]}) -> @@ -58,8 +62,8 @@ def add_attachments(text, attachments) do end def format_input(text, mentions, _tags) do - Phoenix.HTML.html_escape(text) - |> elem(1) + text + |> Formatter.html_escape |> Formatter.linkify |> String.replace("\n", "
") |> add_user_links(mentions) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 9f6f983aa..b23ed5fcc 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -15,8 +15,8 @@ def start_link do enqueue(:refresh_subscriptions, nil) end) GenServer.start_link(__MODULE__, %{ - in: {:sets.new(), :queue.new()}, - out: {:sets.new(), :queue.new()} + in: {:sets.new(), []}, + out: {:sets.new(), []} }, name: __MODULE__) end @@ -79,17 +79,17 @@ def handle(type, _) do {:error, "Don't know what do do with this"} end - def enqueue(type, payload) do + def enqueue(type, payload, priority \\ 1) do if Mix.env == :test do handle(type, payload) else - GenServer.cast(__MODULE__, {:enqueue, type, payload}) + GenServer.cast(__MODULE__, {:enqueue, type, payload, priority}) end end def maybe_start_job(running_jobs, queue) do - if (:sets.size(running_jobs) < @max_jobs) && !:queue.is_empty(queue) do - {{:value, {type, payload}}, queue} = :queue.out(queue) + if (:sets.size(running_jobs) < @max_jobs) && queue != [] do + {{type, payload}, queue} = queue_pop(queue) {:ok, pid} = Task.start(fn -> handle(type, payload) end) mref = Process.monitor(pid) {:sets.add_element(mref, running_jobs), queue} @@ -98,16 +98,16 @@ def maybe_start_job(running_jobs, queue) do end end - def handle_cast({:enqueue, type, payload}, state) when type in [:incoming_doc] do + def handle_cast({:enqueue, type, payload, priority}, state) when type in [:incoming_doc] do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - i_queue = :queue.in({type, payload}, i_queue) + i_queue = enqueue_sorted(i_queue, {type, payload}, 1) {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end - def handle_cast({:enqueue, type, payload}, state) do + def handle_cast({:enqueue, type, payload, priority}, state) do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - o_queue = :queue.in({type, payload}, o_queue) + o_queue = enqueue_sorted(o_queue, {type, payload}, 1) {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end @@ -126,4 +126,13 @@ def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end + + def enqueue_sorted(queue, element, priority) do + [%{item: element, priority: priority} | queue] + |> Enum.sort_by(fn (%{priority: priority}) -> priority end) + end + + def queue_pop([%{item: element} | queue]) do + {element, queue} + end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 61bf8b4b8..e50f53ba4 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -212,6 +212,7 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do params = params |> Map.put("in_reply_to_status_id", params["in_reply_to_id"]) + |> Map.put("no_attachment_links", true) {:ok, activity} = CommonAPI.post(user, params) render conn, StatusView, "status.json", %{activity: activity, for: user, as: :activity} diff --git a/test/formatter_test.exs b/test/formatter_test.exs index f91973881..cb7695e8e 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -25,6 +25,16 @@ test "turning urls into links" do expected = "http://www.cs.vu.nl/~ast/intel/" assert Formatter.linkify(text) == expected + + text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087" + expected = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087" + + assert Formatter.linkify(text) == expected + + text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul" + expected = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul" + + assert Formatter.linkify(text) == expected end end diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index 9d6c2cd70..5480dab43 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -14,6 +14,13 @@ defp fetch_nil(_name) do password_hash: Comeonin.Pbkdf2.hashpwsalt("guy") } + @deactivated %User{ + id: 1, + name: "dude", + password_hash: Comeonin.Pbkdf2.hashpwsalt("guy"), + info: %{"deactivated" => true} + } + @session_opts [ store: :cookie, key: "_test", @@ -131,6 +138,26 @@ test "it assigns the user", %{conn: conn} do end end + describe "with a correct authorization header for an deactiviated user" do + test "it halts the appication", %{conn: conn} do + opts = %{ + optional: false, + fetcher: fn _ -> @deactivated end + } + + header = basic_auth_enc("dude", "guy") + + conn = conn + |> Plug.Session.call(Plug.Session.init(@session_opts)) + |> fetch_session + |> put_req_header("authorization", header) + |> AuthenticationPlug.call(opts) + + assert conn.status == 403 + assert conn.halted == true + end + end + describe "with a user_id in the session for an existing user" do test "it assigns the user", %{conn: conn} do opts = %{ diff --git a/test/user_test.exs b/test/user_test.exs index 31c5962e2..16d43e619 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1,6 +1,6 @@ defmodule Pleroma.UserTest do alias Pleroma.Builders.UserBuilder - alias Pleroma.{User, Repo} + alias Pleroma.{User, Repo, Activity} alias Pleroma.Web.OStatus alias Pleroma.Web.Websub.WebsubClientSubscription alias Pleroma.Web.CommonAPI @@ -39,6 +39,13 @@ test "follow takes a user and another user" do assert User.ap_followers(followed) in user.following end + test "can't follow a deactivated users" do + user = insert(:user) + followed = insert(:user, info: %{"deactivated" => true}) + + {:error, _} = User.follow(user, followed) + end + test "following a remote user will ensure a websub subscription is present" do user = insert(:user) {:ok, followed} = OStatus.make_user("shp@social.heldscal.la") @@ -325,5 +332,42 @@ test "get recipients from activity" do assert user in recipients assert addressed in recipients end -end + test ".deactivate deactivates a user" do + user = insert(:user) + assert false == !!user.info["deactivated"] + {:ok, user} = User.deactivate(user) + assert true == user.info["deactivated"] + end + + test ".delete deactivates a user, all follow relationships and all create activities" do + user = insert(:user) + followed = insert(:user) + follower = insert(:user) + + {:ok, user} = User.follow(user, followed) + {:ok, follower} = User.follow(follower, user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"}) + {:ok, activity_two} = CommonAPI.post(follower, %{"status" => "3hu"}) + + {:ok, _, _} = CommonAPI.favorite(activity_two.id, user) + {:ok, _, _} = CommonAPI.favorite(activity.id, follower) + {:ok, _, _} = CommonAPI.repeat(activity.id, follower) + + :ok = User.delete(user) + + followed = Repo.get(User, followed.id) + follower = Repo.get(User, follower.id) + user = Repo.get(User, user.id) + + assert user.info["deactivated"] + + refute User.following?(user, followed) + refute User.following?(followed, follower) + + # TODO: Remove favorites, repeats, delete activities. + + refute Repo.get(Activity, activity.id) + end +end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs new file mode 100644 index 000000000..09533362a --- /dev/null +++ b/test/web/federator_test.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Web.FederatorTest do + alias Pleroma.Web.Federator + use Pleroma.DataCase + + test "enqueues an element according to priority" do + queue = [%{item: 1, priority: 2}] + + new_queue = Federator.enqueue_sorted(queue, 2, 1) + assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + new_queue = Federator.enqueue_sorted(queue, 2, 3) + assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}] + end + + test "pop first item" do + queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue) + end +end