diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index e8deee223..b9fa21648 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -102,7 +102,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   end
 
   def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct", _) do
-    if inReplyTo do
+    # If the OP is a DM already, add the implicit actor.
+    if inReplyTo && Visibility.is_direct?(inReplyTo) do
       {Enum.uniq([inReplyTo.data["actor"] | mentioned_users]), []}
     else
       {mentioned_users, []}
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 52e95397c..6014ffdac 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -335,6 +335,32 @@ defmodule Pleroma.Web.CommonAPITest do
       end)
     end
 
+    test "replying with a direct message will NOT auto-add the author of the reply to the recipient list" do
+      user = insert(:user)
+      other_user = insert(:user)
+      third_user = insert(:user)
+
+      {:ok, post} = CommonAPI.post(user, %{status: "I'm stupid"})
+
+      {:ok, open_answer} =
+        CommonAPI.post(other_user, %{status: "No ur smart", in_reply_to_status_id: post.id})
+
+      # The OP is implicitly added
+      assert user.ap_id in open_answer.recipients
+
+      {:ok, secret_answer} =
+        CommonAPI.post(other_user, %{
+          status: "lol, that guy really is stupid, right, @#{third_user.nickname}?",
+          in_reply_to_status_id: post.id,
+          visibility: "direct"
+        })
+
+      assert third_user.ap_id in secret_answer.recipients
+
+      # The OP is not added
+      refute user.ap_id in secret_answer.recipients
+    end
+
     test "it allows to address a list" do
       user = insert(:user)
       {:ok, list} = Pleroma.List.create("foo", user)
diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs
index 5708db6a4..d7d2d10d5 100644
--- a/test/web/common_api/common_api_utils_test.exs
+++ b/test/web/common_api/common_api_utils_test.exs
@@ -297,11 +297,10 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
 
       {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "private", nil)
 
-      assert length(to) == 3
+      assert length(to) == 2
       assert Enum.empty?(cc)
 
       assert mentioned_user.ap_id in to
-      assert third_user.ap_id in to
       assert user.follower_address in to
     end
 
@@ -327,6 +326,15 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
 
       {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "direct", nil)
 
+      assert length(to) == 1
+      assert Enum.empty?(cc)
+
+      assert mentioned_user.ap_id in to
+
+      {:ok, direct_activity} = CommonAPI.post(third_user, %{status: "uguu", visibility: "direct"})
+
+      {to, cc} = Utils.get_to_and_cc(user, mentions, direct_activity, "direct", nil)
+
       assert length(to) == 2
       assert Enum.empty?(cc)