diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 16ae65867..4e2056e20 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -81,6 +81,14 @@ defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(
 
   defp check_remote_limit(_), do: true
 
+  def increase_note_count_if_public(actor, object) do
+    if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor}
+  end
+
+  def decrease_note_count_if_public(actor, object) do
+    if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor}
+  end
+
   def insert(map, local \\ true) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map),
@@ -163,7 +171,7 @@ def create(%{to: to, actor: actor, context: context, object: object} = params) d
            ),
          {:ok, activity} <- insert(create_data, local),
          # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.increase_note_count(actor),
+         {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
@@ -316,7 +324,7 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru
     with {:ok, _} <- Object.delete(object),
          {:ok, activity} <- insert(data, local),
          # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.decrease_note_count(user),
+         {:ok, _actor} <- decrease_note_count_if_public(user, object),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index ac3a565de..70a98824d 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -205,6 +205,25 @@ test "removes doubled 'to' recipients" do
       assert activity.actor == user.ap_id
       assert activity.recipients == ["user1", "user2", user.ap_id]
     end
+
+    test "increases user note count only for public activities" do
+      user = insert(:user)
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "1", "visibility" => "public"})
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "2", "visibility" => "unlisted"})
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "2", "visibility" => "private"})
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "3", "visibility" => "direct"})
+
+      user = Repo.get(User, user.id)
+      assert user.info.note_count == 2
+    end
   end
 
   describe "fetch activities for recipients" do
@@ -640,6 +659,30 @@ test "it creates a delete activity and deletes the original object" do
 
       assert Repo.get(Object, object.id).data["type"] == "Tombstone"
     end
+
+    test "decrements user note count only for public activities" do
+      user = insert(:user, info: %{note_count: 10})
+
+      {:ok, a1} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "public"})
+
+      {:ok, a2} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "unlisted"})
+
+      {:ok, a3} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "private"})
+
+      {:ok, a4} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "direct"})
+
+      {:ok, _} = a1.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+      {:ok, _} = a2.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+      {:ok, _} = a3.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+      {:ok, _} = a4.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+
+      user = Repo.get(User, user.id)
+      assert user.info.note_count == 10
+    end
   end
 
   describe "timeline post-processing" do