diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index b33ebb565..5603d1f5f 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -454,11 +454,13 @@ def update_follower_count(%User{} = user) do
 
     follower_count = Repo.one(follower_count_query)
 
-    new_info = Map.put(user.info, "follower_count", follower_count)
+    info_cng = user.info
+    |> User.Info.set_follower_count(follower_count)
 
-    cs = info_changeset(user, %{info: new_info})
+    cng = change(user)
+    |> put_embed(:info, info_cng)
 
-    update_and_set_cache(cs)
+    update_and_set_cache(cng)
   end
 
   def get_users_from_set_query(ap_ids, false) do
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index 465a14d91..49b52d5da 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -32,4 +32,12 @@ def add_to_note_count(info, number) do
     |> cast(params, [:note_count])
     |> validate_required([:note_count])
   end
+
+  def set_follower_count(info, number) do
+    params = %{follower_count: Enum.max([0, number])}
+
+    info
+    |> cast(params, [:follower_count])
+    |> validate_required([:follower_count])
+  end
 end