Merge branch 'develop' into feature/local-only-scope

.gitignore

@@ -27,6 +27,8 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
+/config/*.env
+
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql

CHANGELOG.md

@@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
 - Support for local-only statuses
 
 ### Changed
@@ -24,11 +27,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
 
-### Added
-- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
-- Pleroma API: Importing the mutes users from CSV files.
-- Experimental websocket-based federation between Pleroma instances.
-
 <details>
   <summary>API Changes</summary>
 

docs/administration/CLI_tasks/instance.md

@@ -40,3 +40,5 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
 - `--anonymize-uploads <Y|N>` - randomize uploaded filenames
 - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
+- `--skip-release-env` - skip generation the release environment file
+- `--release-env-file` - release environment file path

docs/administration/CLI_tasks/release_environments.md

@@ -0,0 +1,9 @@
+# Generate release environment file
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl release_env gen
+```
+
+```sh tab="From Source"
+mix pleroma.release_env gen
+```

docs/installation/debian_based_en.md

@@ -101,6 +101,7 @@ sudo -Hu pleroma mix deps.get
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
+
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
 
 ```shell
@@ -181,6 +182,7 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se
 ```
 
 * Edit the service file and make sure that all paths fit your installation
+* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen`
 * Enable and start `pleroma.service`:
 
 ```shell

docs/installation/otp_en.md

@@ -149,6 +149,9 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
+# Run the environment file generator.
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
+
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
@@ -159,7 +162,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -311,4 +314,3 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
-

installation/init.d/pleroma

@@ -8,6 +8,7 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
+export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 

installation/pleroma.service

@@ -17,6 +17,8 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
+; Path to the environment file. the file contains RELEASE_COOKIE and etc 
+EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 

lib/mix/tasks/pleroma/instance.ex

@@ -36,7 +36,9 @@ defmodule Mix.Tasks.Pleroma.Instance do
           listen_port: :string,
           strip_uploads: :string,
           anonymize_uploads: :string,
-          dedupe_uploads: :string
+          dedupe_uploads: :string,
+          skip_release_env: :boolean,
+          release_env_file: :string
         ],
         aliases: [
           o: :output,
@@ -241,6 +243,24 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
       write_robots_txt(static_dir, indexable, template_dir)
 
+      if Keyword.get(options, :skip_release_env, false) do
+        shell_info("""
+        Release environment file is skip. Please generate the release env file before start.
+        `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen`
+        """)
+      else
+        shell_info("Generation the environment file:")
+
+        release_env_args =
+          with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do
+            ["gen", "--path", path]
+          else
+            _ -> ["gen"]
+          end
+
+        Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args)
+      end
+
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
       )

lib/mix/tasks/pleroma/release_env.ex

@@ -0,0 +1,76 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.ReleaseEnv do
+  use Mix.Task
+  import Mix.Pleroma
+
+  @shortdoc "Generate Pleroma environment file."
+  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
+
+  def run(["gen" | rest]) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [
+          force: :boolean,
+          path: :string
+        ],
+        aliases: [
+          p: :path,
+          f: :force
+        ]
+      )
+
+    file_path =
+      get_option(
+        options,
+        :path,
+        "Environment file path",
+        "./config/pleroma.env"
+      )
+
+    env_path = Path.expand(file_path)
+
+    proceed? =
+      if File.exists?(env_path) do
+        get_option(
+          options,
+          :force,
+          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
+          "n"
+        ) === "y"
+      else
+        true
+      end
+
+    if proceed? do
+      case do_generate(env_path) do
+        {:error, reason} ->
+          shell_error(
+            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
+          )
+
+        _ ->
+          shell_info("\nThe file generated: #{env_path}.\n")
+
+          shell_info("""
+          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
+            Example:
+              chmod 0444 #{file_path}
+              chattr +i #{file_path}
+          """)
+      end
+    else
+      shell_info("\nThe file is exist. #{env_path}.\n")
+    end
+  end
+
+  def do_generate(path) do
+    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
+
+    File.mkdir_p!(Path.dirname(path))
+    File.write(path, content)
+  end
+end

lib/pleroma/user.ex

@@ -136,7 +136,7 @@ defmodule Pleroma.User do
     field(:pleroma_settings_store, :map, default: %{})
     field(:fields, {:array, :map}, default: [])
     field(:raw_fields, {:array, :map}, default: [])
-    field(:discoverable, :boolean, default: false)
+    field(:is_discoverable, :boolean, default: false)
     field(:invisible, :boolean, default: false)
     field(:allow_following_move, :boolean, default: true)
     field(:skip_thread_containment, :boolean, default: false)
@@ -448,7 +448,7 @@ defmodule Pleroma.User do
         :follower_count,
         :fields,
         :following_count,
-        :discoverable,
+        :is_discoverable,
         :invisible,
         :actor_type,
         :also_known_as,
@@ -495,7 +495,7 @@ defmodule Pleroma.User do
         :fields,
         :raw_fields,
         :pleroma_settings_store,
-        :discoverable,
+        :is_discoverable,
         :actor_type,
         :also_known_as,
         :accepts_chat_messages
@@ -1618,7 +1618,7 @@ defmodule Pleroma.User do
       pleroma_settings_store: %{},
       fields: [],
       raw_fields: [],
-      discoverable: false,
+      is_discoverable: false,
       also_known_as: []
     })
   end

lib/pleroma/user/search.ex

@@ -164,7 +164,7 @@ defmodule Pleroma.User.Search do
   end
 
   defp filter_discoverable_users(query) do
-    from(q in query, where: q.discoverable == true)
+    from(q in query, where: q.is_discoverable == true)
   end
 
   defp filter_internal_users(query) do

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -827,7 +827,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     query =
       from([activity] in query,
         where: fragment("not (? = ANY(?))", activity.actor, ^mutes),
-        where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes)
+        where:
+          fragment(
+            "not (?->'to' \\?| ?) or ? = ?",
+            activity.data,
+            ^mutes,
+            activity.actor,
+            ^user.ap_id
+          )
       )
 
     unless opts[:skip_preload] do
@@ -1232,7 +1239,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     capabilities = data["capabilities"] || %{}
     accepts_chat_messages = capabilities["acceptsChatMessages"]
     data = Transmogrifier.maybe_fix_user_object(data)
-    discoverable = data["discoverable"] || false
+    is_discoverable = data["discoverable"] || false
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
@@ -1258,7 +1265,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       fields: fields,
       emoji: emojis,
       is_locked: is_locked,
-      discoverable: discoverable,
+      is_discoverable: is_discoverable,
       invisible: invisible,
       avatar: avatar,
       name: data["name"],

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -110,7 +110,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
-      "discoverable" => user.discoverable,
+      "discoverable" => user.is_discoverable,
       "capabilities" => capabilities
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -185,7 +185,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
         :show_role,
         :skip_thread_containment,
         :allow_following_move,
-        :discoverable,
         :accepts_chat_messages
       ]
       |> Enum.reduce(%{}, fn key, acc ->
@@ -210,6 +209,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
       end)
       |> Maps.put_if_present(:actor_type, params[:actor_type])
       |> Maps.put_if_present(:is_locked, params[:locked])
+      |> Maps.put_if_present(:is_discoverable, params[:discoverable])
 
     # What happens here:
     #

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -261,7 +261,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
-          discoverable: user.discoverable,
+          discoverable: user.is_discoverable,
           actor_type: user.actor_type
         }
       },

lib/pleroma/web/metadata/providers/restrict_indexing.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
   """
 
   @impl true
-  def build_tags(%{user: %{local: true, discoverable: true}}), do: []
+  def build_tags(%{user: %{local: true, is_discoverable: true}}), do: []
 
   def build_tags(_) do
     [

priv/repo/migrations/20201013144052_refactor_discoverable_user_field.exs

@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.RefactorDiscoverableUserField do
+  use Ecto.Migration
+
+  def up do
+    execute("ALTER TABLE users RENAME COLUMN discoverable TO is_discoverable;")
+  end
+
+  def down do
+    execute("ALTER TABLE users RENAME COLUMN is_discoverable TO discoverable;")
+  end
+end

test/mix/tasks/pleroma/instance_test.exs

@@ -5,6 +5,8 @@
 defmodule Mix.Tasks.Pleroma.InstanceTest do
   use ExUnit.Case
 
+  @release_env_file "./test/pleroma.test.env"
+
   setup do
     File.mkdir_p!(tmp_path())
 
@@ -16,6 +18,8 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do
         File.rm_rf(Path.join(static_dir, "robots.txt"))
       end
 
+      if File.exists?(@release_env_file), do: File.rm_rf(@release_env_file)
+
       Pleroma.Config.put([:instance, :static_dir], static_dir)
     end)
 
@@ -69,7 +73,9 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do
         "--dedupe-uploads",
         "n",
         "--anonymize-uploads",
-        "n"
+        "n",
+        "--release-env-file",
+        @release_env_file
       ])
     end
 
@@ -91,6 +97,9 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do
     assert generated_config =~ "filters: [Pleroma.Upload.Filter.ExifTool]"
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
     assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
+    assert File.exists?(@release_env_file)
+
+    assert File.read!(@release_env_file) =~ ~r/^RELEASE_COOKIE=.*/
   end
 
   defp generated_setup_psql do

test/mix/tasks/pleroma/release_env_test.exs

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.ReleaseEnvTest do
+  use ExUnit.Case
+  import ExUnit.CaptureIO, only: [capture_io: 1]
+
+  @path "config/pleroma.test.env"
+
+  def do_clean do
+    if File.exists?(@path) do
+      File.rm_rf(@path)
+    end
+  end
+
+  setup do
+    do_clean()
+    on_exit(fn -> do_clean() end)
+    :ok
+  end
+
+  test "generate pleroma.env" do
+    assert capture_io(fn ->
+             Mix.Tasks.Pleroma.ReleaseEnv.run(["gen", "--path", @path, "--force"])
+           end) =~ "The file generated"
+
+    assert File.read!(@path) =~ "RELEASE_COOKIE="
+  end
+end

test/pleroma/user_search_test.exs

@@ -66,7 +66,7 @@ defmodule Pleroma.UserSearchTest do
     end
 
     test "excludes users when discoverable is false" do
-      insert(:user, %{nickname: "john 3000", discoverable: false})
+      insert(:user, %{nickname: "john 3000", is_discoverable: false})
       insert(:user, %{nickname: "john 3001"})
 
       users = User.search("john")

test/pleroma/user_test.exs

@@ -1467,7 +1467,7 @@ defmodule Pleroma.UserTest do
         pleroma_settings_store: %{"q" => "x"},
         fields: [%{"gg" => "qq"}],
         raw_fields: [%{"gg" => "qq"}],
-        discoverable: true,
+        is_discoverable: true,
         also_known_as: ["https://lol.olo/users/loll"]
       })
 
@@ -1509,7 +1509,7 @@ defmodule Pleroma.UserTest do
              pleroma_settings_store: %{},
              fields: [],
              raw_fields: [],
-             discoverable: false,
+             is_discoverable: false,
              also_known_as: []
            } = user
   end

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -752,6 +752,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     refute repeat_activity in activities
   end
 
+  test "returns your own posts regardless of mute" do
+    user = insert(:user)
+    muted = insert(:user)
+
+    {:ok, muted_post} = CommonAPI.post(muted, %{status: "Im stupid"})
+
+    {:ok, reply} =
+      CommonAPI.post(user, %{status: "I'm muting you", in_reply_to_status_id: muted_post.id})
+
+    {:ok, _} = User.mute(user, muted)
+
+    [activity] = ActivityPub.fetch_activities([], %{muting_user: user, skip_preload: true})
+
+    assert activity.id == reply.id
+  end
+
   test "doesn't return muted activities" do
     activity_one = insert(:note_activity)
     activity_two = insert(:note_activity)

test/pleroma/web/admin_api/search_test.exs

@@ -180,7 +180,7 @@ defmodule Pleroma.Web.AdminAPI.SearchTest do
 
     test "it returns non-discoverable users" do
       insert(:user)
-      insert(:user, discoverable: false)
+      insert(:user, is_discoverable: false)
 
       {:ok, _results, total} = Search.user()
 

test/pleroma/web/metadata/providers/restrict_indexing_test.exs

@@ -14,13 +14,13 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexingTest do
 
     test "for local user" do
       assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
-               user: %Pleroma.User{local: true, discoverable: true}
+               user: %Pleroma.User{local: true, is_discoverable: true}
              }) == []
     end
 
     test "for local user when discoverable is false" do
       assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
-               user: %Pleroma.User{local: true, discoverable: false}
+               user: %Pleroma.User{local: true, is_discoverable: false}
              }) == [{:meta, [name: "robots", content: "noindex, noarchive"], []}]
     end
   end

test/pleroma/web/metadata_test.exs

@@ -16,14 +16,14 @@ defmodule Pleroma.Web.MetadataTest do
     end
 
     test "for local user" do
-      user = insert(:user, discoverable: false)
+      user = insert(:user, is_discoverable: false)
 
       assert Pleroma.Web.Metadata.build_tags(%{user: user}) =~
                "<meta content=\"noindex, noarchive\" name=\"robots\">"
     end
 
     test "for local user set to discoverable" do
-      user = insert(:user, discoverable: true)
+      user = insert(:user, is_discoverable: true)
 
       refute Pleroma.Web.Metadata.build_tags(%{user: user}) =~
                "<meta content=\"noindex, noarchive\" name=\"robots\">"
@@ -33,14 +33,14 @@ defmodule Pleroma.Web.MetadataTest do
   describe "no metadata for private instances" do
     test "for local user set to discoverable" do
       clear_config([:instance, :public], false)
-      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: true)
+      user = insert(:user, bio: "This is my secret fedi account bio", is_discoverable: true)
 
       assert "" = Pleroma.Web.Metadata.build_tags(%{user: user})
     end
 
     test "search exclusion metadata is included" do
       clear_config([:instance, :public], false)
-      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: false)
+      user = insert(:user, bio: "This is my secret fedi account bio", is_discoverable: false)
 
       assert ~s(<meta content="noindex, noarchive" name="robots">) ==
                Pleroma.Web.Metadata.build_tags(%{user: user})

test/support/factory.ex

@@ -31,7 +31,7 @@ defmodule Pleroma.Factory do
       nickname: sequence(:nickname, &"nick#{&1}"),
       password_hash: Pbkdf2.hash_pwd_salt("test"),
       bio: sequence(:bio, &"Tester Number #{&1}"),
-      discoverable: true,
+      is_discoverable: true,
       last_digest_emailed_at: NaiveDateTime.utc_now(),
       last_refreshed_at: NaiveDateTime.utc_now(),
       notification_settings: %Pleroma.User.NotificationSetting{},