Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

.gitlab-ci.yml

@@ -59,7 +59,7 @@ unit-testing:
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
-    - apt-get update && apt-get install -y libimage-exiftool-perl
+    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
@@ -93,7 +93,7 @@ unit-testing-rum:
     <<: *global_variables
     RUM_ENABLED: "true"
   script:
-    - apt-get update && apt-get install -y libimage-exiftool-perl
+    - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate

CHANGELOG.md

@@ -5,11 +5,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Added
+- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
+- Mix task option for force-unfollowing relays
+
 ### Changed
 
+- Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
+- The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
+- Users with the `discoverable` field set to false will not show up in searches.
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
+- Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
+
+### Added
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
+- Admin API: Importing emoji from a zip file
 
 ### Removed
 
@@ -19,12 +34,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
 switched to a new configuration mechanism, however it was not officially removed until now.
 
-## unreleased-patch - ???
+### Fixed
+
+- Add documented-but-missing chat pagination.
+- Allow sending out emails again.
+
+## [2.1.2] - 2020-09-17
+
+### Security
+
+- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event).
 
 ### Fixed
 
-- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance
+- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance.
-- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user
+- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user.
+- Mastodon Streaming API: Handler crashes on authentication failures, resulting in error logs.
+- Mastodon Streaming API: Error logs on client pings.
+- Rich media: Log spam on failures. Now the error is only logged once per attempt.
+
+### Changed
+
+- Rich Media: A HEAD request is now done to the url, to ensure it has the appropriate content type and size before proceeding with a GET.
+
+### Upgrade notes
+
+1. Restart Pleroma
 
 ## [2.1.1] - 2020-09-08
 
@@ -41,6 +76,12 @@ switched to a new configuration mechanism, however it was not officially removed
 ### Added
 - Rich media failure tracking (along with `:failure_backoff` option).
 
+<details>
+  <summary>Admin API Changes</summary>
+
+- Add `PATCH /api/pleroma/admin/instance_document/:document_name` to modify the Terms of Service and Instance Panel HTML pages via Admin API
+</details>
+
 ### Fixed
 - Default HTTP adapter not respecting pool setting, leading to possible OOM.
 - Fixed uploading webp images when the Exiftool Upload Filter is enabled by skipping them

README.md

@@ -18,15 +18,16 @@ If you are running Linux (glibc or musl) on x86/arm, the recommended way to inst
 ### From Source
 If your platform is not supported, or you just want to be able to edit the source code easily, you may install Pleroma from source.
 
-- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
-- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
 - [Alpine Linux](https://docs-develop.pleroma.social/backend/installation/alpine_linux_en/)
 - [Arch Linux](https://docs-develop.pleroma.social/backend/installation/arch_linux_en/)
+- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
+- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
+- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
+- [FreeBSD](https://docs-develop.pleroma.social/backend/installation/freebsd_en/)
 - [Gentoo Linux](https://docs-develop.pleroma.social/backend/installation/gentoo_en/)
 - [NetBSD](https://docs-develop.pleroma.social/backend/installation/netbsd_en/)
 - [OpenBSD](https://docs-develop.pleroma.social/backend/installation/openbsd_en/)
 - [OpenBSD (fi)](https://docs-develop.pleroma.social/backend/installation/openbsd_fi/)
-- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
 
 ### OS/Distro packages
 Currently Pleroma is not packaged by any OS/Distros, but if you want to package it for one, we can guide you through the process on our [community channels](#community-channels). If you want to change default options in your Pleroma package, please **discuss it with us first**.

config/benchmark.exs

@@ -59,8 +59,6 @@ config :web_push_encryption, :vapid_details,
     "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
   private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
 
-config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
-
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 2,
   total_user_limit: 3,

config/config.exs

@@ -130,6 +130,7 @@ config :pleroma, Pleroma.Web.Endpoint,
     dispatch: [
       {:_,
        [
+         {"/api/fedsocket/v1", Pleroma.Web.FedSockets.IncomingHandler, []},
          {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
          {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
           {Phoenix.Transports.WebSocket,
@@ -148,6 +149,16 @@ config :pleroma, Pleroma.Web.Endpoint,
     "SameSite=Lax"
   ]
 
+config :pleroma, :fed_sockets,
+  enabled: false,
+  connection_duration: :timer.hours(8),
+  rejection_duration: :timer.minutes(15),
+  fed_socket_fetches: [
+    default: 12_000,
+    interval: 3_000,
+    lazy: false
+  ]
+
 # Configures Elixir's Logger
 config :logger, :console,
   level: :debug,
@@ -426,6 +437,8 @@ config :pleroma, :media_proxy,
   proxy_opts: [
     redirect_on_failure: false,
     max_body_length: 25 * 1_048_576,
+    # Note: max_read_duration defaults to Pleroma.ReverseProxy.max_read_duration_default/1
+    max_read_duration: 30_000,
     http: [
       follow_redirect: true,
       pool: :media
@@ -440,6 +453,14 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
 
 config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
 
+# Note: media preview proxy depends on media proxy to be enabled
+config :pleroma, :media_preview_proxy,
+  enabled: false,
+  thumbnail_max_width: 600,
+  thumbnail_max_height: 600,
+  image_quality: 85,
+  min_content_length: 100 * 1024
+
 config :pleroma, :chat, enabled: true
 
 config :phoenix, :format_encoders, json: Jason
@@ -535,6 +556,7 @@ config :pleroma, Oban,
     token_expiration: 5,
     federator_incoming: 50,
     federator_outgoing: 50,
+    ingestion_queue: 50,
     web_push: 50,
     mailer: 10,
     transmogrifier: 20,
@@ -744,8 +766,8 @@ config :pleroma, :pools,
   ],
   media: [
     size: 50,
-    max_waiting: 10,
+    max_waiting: 20,
-    recv_timeout: 10_000
+    recv_timeout: 15_000
   ],
   upload: [
     size: 25,
@@ -790,6 +812,8 @@ config :tzdata, :http_client, Pleroma.HTTP.Tzdata
 
 config :ex_aws, http_client: Pleroma.HTTP.ExAws
 
+config :web_push_encryption, http_client: Pleroma.HTTP.WebPush
+
 config :pleroma, :instances_favicons, enabled: false
 
 config :floki, :html_parser, Floki.HTMLParser.FastHtml

config/description.exs

@@ -270,6 +270,19 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :fed_sockets,
+    type: :group,
+    description: "Websocket based federation",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "Enable FedSockets"
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Emails.Mailer,
@@ -1874,6 +1887,7 @@ config :pleroma, :config_description, [
         suggestions: [
           redirect_on_failure: false,
           max_body_length: 25 * 1_048_576,
+          max_read_duration: 30_000,
           http: [
             follow_redirect: true,
             pool: :media
@@ -1894,6 +1908,11 @@ config :pleroma, :config_description, [
               "Limits the content length to be approximately the " <>
                 "specified length. It is validated with the `content-length` header and also verified when proxying."
           },
+          %{
+            key: :max_read_duration,
+            type: :integer,
+            description: "Timeout (in milliseconds) of GET request to remote URI."
+          },
           %{
             key: :http,
             label: "HTTP",
@@ -1940,6 +1959,43 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :media_preview_proxy,
+    type: :group,
+    description: "Media preview proxy",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description:
+          "Enables proxying of remote media preview to the instance's proxy. Requires enabled media proxy."
+      },
+      %{
+        key: :thumbnail_max_width,
+        type: :integer,
+        description:
+          "Max width of preview thumbnail for images (video preview always has original dimensions)."
+      },
+      %{
+        key: :thumbnail_max_height,
+        type: :integer,
+        description:
+          "Max height of preview thumbnail for images (video preview always has original dimensions)."
+      },
+      %{
+        key: :image_quality,
+        type: :integer,
+        description: "Quality of the output. Ranges from 0 (min quality) to 100 (max quality)."
+      },
+      %{
+        key: :min_content_length,
+        type: :integer,
+        description:
+          "Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing."
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Web.MediaProxy.Invalidation.Http,
@@ -2389,7 +2445,7 @@ config :pleroma, :config_description, [
   %{
     group: :pleroma,
     key: Pleroma.Formatter,
-    label: "Auto Linker",
+    label: "Linkify",
     type: :group,
     description:
       "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.",

config/test.exs

@@ -19,6 +19,11 @@ config :logger, :console,
   level: :warn,
   format: "\n[$level] $message\n"
 
+config :pleroma, :fed_sockets,
+  enabled: false,
+  connection_duration: 5,
+  rejection_duration: 5
+
 config :pleroma, :auth, oauth_consumer_strategies: []
 
 config :pleroma, Pleroma.Upload,
@@ -78,8 +83,6 @@ config :web_push_encryption, :vapid_details,
     "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
   private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
 
-config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
-
 config :pleroma, Oban,
   queues: false,
   crontab: false,

docs/API/admin_api.md

@@ -349,9 +349,9 @@ Response:
 
 ### Unfollow a Relay
 
-Params:
+- Params:
-
+  - `relay_url`
-* `relay_url`
+  - *optional* `force`: forcefully unfollow a relay even when the relay is not available. (default is `false`)
 
 Response:
 
@@ -1334,3 +1334,166 @@ Loads json generated from `config/descriptions.exs`.
 { }
 
 ```
+
+## GET /api/pleroma/admin/users/:nickname/chats
+
+### List a user's chats
+
+- Params: None
+
+- Response:
+
+```json
+[
+   {
+      "sender": {
+        "id": "someflakeid",
+        "username": "somenick",
+        ...
+      },
+      "receiver": {
+        "id": "someflakeid",
+        "username": "somenick",
+        ...
+      },
+      "id" : "1",
+      "unread" : 2,
+      "last_message" : {...}, // The last message in that chat
+      "updated_at": "2020-04-21T15:11:46.000Z"
+   }
+]
+```
+
+## GET /api/pleroma/admin/chats/:chat_id
+
+### View a single chat
+
+- Params: None
+
+- Response:
+
+```json
+{
+  "sender": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "receiver": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "id" : "1",
+  "unread" : 2,
+  "last_message" : {...}, // The last message in that chat
+  "updated_at": "2020-04-21T15:11:46.000Z"
+}
+```
+
+## GET /api/pleroma/admin/chats/:chat_id/messages
+
+### List the messages in a chat
+
+- Params: `max_id`, `min_id`
+
+- Response:
+
+```json
+[
+  {
+    "account_id": "someflakeid",
+    "chat_id": "1",
+    "content": "Check this out :firefox:",
+    "created_at": "2020-04-21T15:11:46.000Z",
+    "emojis": [
+      {
+        "shortcode": "firefox",
+        "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+        "url": "https://dontbulling.me/emoji/Firefox.gif",
+        "visible_in_picker": false
+      }
+    ],
+    "id": "13",
+    "unread": true
+  },
+  {
+    "account_id": "someflakeid",
+    "chat_id": "1",
+    "content": "Whats' up?",
+    "created_at": "2020-04-21T15:06:45.000Z",
+    "emojis": [],
+    "id": "12",
+    "unread": false
+  }
+]
+```
+
+## DELETE /api/pleroma/admin/chats/:chat_id/messages/:message_id
+
+### Delete a single message
+
+- Params: None
+
+- Response:
+
+```json
+{
+  "account_id": "someflakeid",
+  "chat_id": "1",
+  "content": "Check this out :firefox:",
+  "created_at": "2020-04-21T15:11:46.000Z",
+  "emojis": [
+    {
+      "shortcode": "firefox",
+      "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+      "url": "https://dontbulling.me/emoji/Firefox.gif",
+      "visible_in_picker": false
+    }
+  ],
+  "id": "13",
+  "unread": false
+}
+```
+
+## `GET /api/pleroma/admin/instance_document/:document_name`
+
+### Get an instance document
+
+- Authentication: required
+
+- Response:
+
+Returns the content of the document
+
+```html
+<h1>Instance panel</h1>
+```
+
+## `PATCH /api/pleroma/admin/instance_document/:document_name`
+- Params:
+  - `file` (the file to be uploaded, using multipart form data.)
+
+### Update an instance document
+
+- Authentication: required
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```
+
+## `DELETE /api/pleroma/admin/instance_document/:document_name`
+
+### Delete an instance document
+
+- Response:
+
+``` json
+{
+  "url": "https://example.com/instance/panel.html"
+}
+```

docs/API/pleroma_api.md

@@ -44,6 +44,22 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: HTTP 200 on success, 500 on error
 * Note: Users that can't be followed are silently skipped.
 
+## `/api/pleroma/blocks_import`
+### Imports your blocks.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to block
+* Response: HTTP 200 on success, 500 on error
+
+## `/api/pleroma/mutes_import`
+### Imports your mutes.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to mute
+* Response: HTTP 200 on success, 500 on error
+
 ## `/api/pleroma/captcha`
 ### Get a new captcha
 * Method: `GET`

docs/administration/CLI_tasks/email.md

@@ -1,4 +1,4 @@
-# Managing emails
+# EMail administration tasks
 
 {! backend/administration/CLI_tasks/general_cli_task_info.include !}
 
@@ -30,3 +30,17 @@ Example:
     ```sh
     mix pleroma.email test --to root@example.org
     ```
+
+## Send confirmation emails to all unconfirmed user accounts
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl email send_confirmation_mails
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.email send_confirmation_mails
+    ```

docs/administration/CLI_tasks/user.md

@@ -224,9 +224,10 @@
     ```
 
 ### Options
+- `--admin`/`--no-admin` - whether the user should be an admin
+- `--confirmed`/`--no-confirmed` - whether the user account is confirmed
 - `--locked`/`--no-locked` - whether the user should be locked
 - `--moderator`/`--no-moderator` - whether the user should be a moderator
-- `--admin`/`--no-admin` - whether the user should be an admin
 
 ## Add tags to a user
 
@@ -271,3 +272,33 @@
     ```sh
     mix pleroma.user toggle_confirmed <nickname>
     ```
+
+## Set confirmation status for all regular active users
+*Admins and moderators are excluded*
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl user confirm_all
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user confirm_all
+    ```
+
+## Revoke confirmation status for all regular active users
+*Admins and moderators are excluded*
+
+=== "OTP"
+
+    ```sh
+     ./bin/pleroma_ctl user unconfirm_all
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.user unconfirm_all
+    ```

docs/configuration/cheatsheet.md

@@ -225,6 +225,16 @@ Enables the worker which processes posts scheduled for deletion. Pinned posts ar
 
 * `enabled`: whether expired activities will be sent to the job queue to be deleted
 
+## FedSockets
+FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options.
+
+### :fedsockets
+* `enabled`: Enables FedSockets for this instance. `false` by default.
+* `connection_duration`: Time an idle websocket is kept open.
+* `rejection_duration`: Failures to connect via FedSockets will not be retried for this period of time.
+* `fed_socket_fetches` and `fed_socket_rejections`: Settings passed to `cachex` for the fetch registry, and rejection stacks. See `Pleroma.Web.FedSockets` for more details.
+
+
 ## Frontends
 
 ### :frontend_configurations
@@ -314,6 +324,14 @@ This section describe PWA manifest instance-specific values. Currently this opti
   * `enabled`: Enables purge cache
   * `provider`: Which one of  the [purge cache strategy](#purge-cache-strategy) to use.
 
+## :media_preview_proxy
+
+* `enabled`: Enables proxying of remote media preview to the instance’s proxy. Requires enabled media proxy (`media_proxy/enabled`).
+* `thumbnail_max_width`: Max width of preview thumbnail for images (video preview always has original dimensions).
+* `thumbnail_max_height`: Max height of preview thumbnail for images (video preview always has original dimensions).
+* `image_quality`: Quality of the output. Ranges from 0 (min quality) to 100 (max quality).
+* `min_content_length`: Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing.
+
 ### Purge cache strategy
 
 #### Pleroma.Web.MediaProxy.Invalidation.Script

docs/installation/alpine_linux_en.md

@@ -20,6 +20,9 @@ It assumes that you have administrative rights, either as root or a user with [s
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -29,7 +32,6 @@ It assumes that you have administrative rights, either as root or a user with [s
 awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories
 ```
 
-
 * Then update the system, if not already done:
 
 ```shell
@@ -56,6 +58,7 @@ sudo apk add erlang erlang-runtime-tools erlang-xmerl elixir
 ```shell
 sudo apk add erlang-eldap
 ```
+
 ### Install PostgreSQL
 
 * Install Postgresql server:
@@ -76,6 +79,12 @@ sudo /etc/init.d/postgresql start
 sudo rc-update add postgresql
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+sudo apk add ffmpeg imagemagick exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/arch_linux_en.md

@@ -15,6 +15,9 @@ This guide will assume that you have administrative rights, either as root or a
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -52,6 +55,12 @@ sudo -iu postgres initdb -D /var/lib/postgres/data
 sudo systemctl enable --now postgresql.service
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+sudo pacman -S ffmpeg imagemagick perl-image-exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/debian_based_en.md

@@ -18,6 +18,9 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 
 * `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `certbot` (or any other ACME client for Let’s Encrypt certificates)
+* `ImageMagick`
+* `ffmpeg`
+* `exiftool`
 
 ### Prepare the system
 
@@ -50,6 +53,12 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
+### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+
+```shell
+sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service:

docs/installation/debian_based_jp.md

@@ -22,6 +22,9 @@
 
 - `nginx` (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください)
 - `certbot` (または何らかのLet's Encrypt向けACMEクライアント)
+- `ImageMagick`
+- `ffmpeg`
+- `exiftool`
 
 ### システムを準備する
 
@@ -33,10 +36,9 @@ sudo apt full-upgrade
 
 * 上記に挙げたパッケージをインストールしておきます。
 ```
-sudo apt install git build-essential postgresql postgresql-contrib cmake
+sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick
 ```
 
-
 ### ElixirとErlangをインストールします
 
 * Erlangのリポジトリをダウンロードおよびインストールします。
@@ -51,6 +53,12 @@ sudo apt update
 sudo apt install elixir erlang-dev erlang-nox
 ```
 
+### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)
+
+```shell
+sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+```
+
 ### Pleroma BE (バックエンド) をインストールします
 
 *  Pleroma用に新しいユーザーを作ります。

docs/installation/freebsd_en.md

@@ -26,6 +26,12 @@ Setup the required services to automatically start at boot, using `sysrc(8)`.
 # service postgresql start
 ```
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+# pkg install imagemagick ffmpeg p5-Image-ExifTool
+```
+
 ## Configuring Pleroma
 
 Create a user for Pleroma:

docs/installation/gentoo_en.md

@@ -35,6 +35,9 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `www-servers/nginx` (preferred, example configs for other reverse proxies can be found in the repo)
 * `app-crypt/certbot` (or any other ACME client for Let’s Encrypt certificates)
 * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot)
+* `media-gfx/imagemagick`
+* `media-video/ffmpeg`
+* `media-libs/exiftool`
 
 ### Prepare the system
 
@@ -87,6 +90,12 @@ If you do not plan to make any modifications to your Pleroma instance, cloning d
 
 Not only does this make it much easier to deploy changes you make, as you can commit and pull from upstream and all that good stuff from the comfort of your local machine then simply `git pull` on your instance server when you're ready to deploy, it also ensures you are compliant with the Affero General Public Licence that Pleroma is licenced under, which stipulates that all network services provided with modified AGPL code must publish their changes on a publicly available internet service and for free. It also makes it much easier to ask for help from and provide help to your fellow Pleroma admins if your public repo always reflects what you are running because it is part of your deployment procedure.
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+```shell
+# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool
+```
+
 ### Install PleromaBE
 
 * Add a new system user for the Pleroma service and set up default directories:

docs/installation/netbsd_en.md

@@ -10,7 +10,7 @@ Pleroma uses.
 
 The `mksh` shell is needed to run the Elixir `mix` script.
 
-`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo`
+`# pkgin install acmesh elixir git-base git-docs mksh nginx postgresql11-server postgresql11-client postgresql11-contrib sudo ffmpeg4 ImageMagick`
 
 You can also build these packages using pkgsrc:
 ```
@@ -44,6 +44,10 @@ pgsql=YES
 
 First, run `# /etc/rc.d/pgsql start`. Then, `$ sudo -Hu pgsql -g pgsql createdb`.
 
+### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
+
+`# pkgin install ImageMagick ffmpeg4 p5-Image-ExifTool`
+
 ## Configuring Pleroma
 
 Create a user for Pleroma:

docs/installation/openbsd_en.md

@@ -10,20 +10,34 @@ The following packages need to be installed:
 
   * elixir
   * gmake
-  * ImageMagick
   * git
   * postgresql-server
   * postgresql-contrib
   * cmake
+  * ffmpeg
+  * ImageMagick
 
 To install them, run the following command (with doas or as root):
 
 ```
-pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib cmake
+pkg_add elixir gmake git postgresql-server postgresql-contrib cmake ffmpeg ImageMagick
 ```
 
 Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt.
 
+#### Optional software
+
+Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+To install the above:
+
+```
+pkg_add ImageMagick ffmpeg p5-Image-ExifTool
+```
+
 #### Creating the pleroma user
 Pleroma will be run by a dedicated user, \_pleroma. Before creating it, insert the following lines in login.conf:
 ```

docs/installation/openbsd_fi.md

@@ -16,7 +16,18 @@ Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua
 
 Asenna tarvittava ohjelmisto:
 
-`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake`
+`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake ffmpeg ImageMagick`
+
+#### Optional software
+
+[`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+Asenna tarvittava ohjelmisto:
+
+`# pkg_add ImageMagick ffmpeg p5-Image-ExifTool`
 
 Luo postgresql-tietokanta:
 

docs/installation/optional/media_graphics_packages.md

@@ -0,0 +1,32 @@
+# Optional software packages needed for specific functionality
+
+For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required:
+  * `ImageMagic`
+  * `ffmpeg`
+  * `exiftool`
+  
+Please refer to documentation in `docs/installation` on how to install them on specific OS.
+  
+Note: the packages are not required with the current default settings of Pleroma.
+
+## `ImageMagick`
+
+`ImageMagick` is a set of tools to create, edit, compose, or convert bitmap images.
+
+It is required for the following Pleroma features:
+  * `Pleroma.Upload.Filters.Mogrify`, `Pleroma.Upload.Filters.Mogrifun` upload filters (related config: `Plaroma.Upload/filters` in `config/config.exs`)
+  * Media preview proxy for still images (related config: `media_preview_proxy/enabled` in `config/config.exs`)
+  
+## `ffmpeg`
+
+`ffmpeg` is software to record, convert and stream audio and video.
+
+It is required for the following Pleroma features:
+  * Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`)
+
+## `exiftool`
+
+`exiftool` is media files metadata reader/writer.
+
+It is required for the following Pleroma features:
+  * `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)

docs/installation/otp_en.md

@@ -40,6 +40,25 @@ Other than things bundled in the OTP release Pleroma depends on:
     apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
     ```
 
+### Installing optional packages
+
+Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+  * ImageMagick
+  * ffmpeg
+  * exiftool
+
+=== "Alpine"
+    ```
+    echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
+    apk update
+    apk add imagemagick ffmpeg exiftool
+    ```
+
+=== "Debian/Ubuntu"
+    ```
+    apt install imagemagick ffmpeg libimage-exiftool-perl
+    ```
+
 ## Setup
 ### Configuring PostgreSQL
 #### (Optional) Installing RUM indexes

installation/pleroma.nginx

@@ -9,6 +9,12 @@
 proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m max_size=10g
                  inactive=720m use_temp_path=off;
 
+# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+# and `localhost.` resolves to [::0] on some systems: see issue #930
+upstream phoenix {
+    server 127.0.0.1:4000 max_fails=5 fail_timeout=60s;
+}
+
 server {
     server_name    example.tld;
 
@@ -63,19 +69,16 @@ server {
 
     # the nginx default is 1m, not enough for large media uploads
     client_max_body_size 16m;
+    ignore_invalid_headers off;
 
-    location / {
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
     proxy_set_header Host $http_host;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
-        # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+    location / {
-        # and `localhost.` resolves to [::0] on some systems: see issue #930
+        proxy_pass http://phoenix;
-        proxy_pass http://127.0.0.1:4000;
-
-        client_max_body_size 16m;
     }
 
     location ~ ^/(media|proxy) {
@@ -83,12 +86,16 @@ server {
         slice              1m;
         proxy_cache_key    $host$uri$is_args$args$slice_range;
         proxy_set_header   Range $slice_range;
-        proxy_http_version 1.1;
         proxy_cache_valid  200 206 301 304 1h;
         proxy_cache_lock   on;
         proxy_ignore_client_abort on;
         proxy_buffering    on;
         chunked_transfer_encoding on;
-        proxy_pass         http://127.0.0.1:4000;
+        proxy_pass         http://phoenix;
+    }
+
+    location /api/fedsocket/v1 {
+        proxy_request_buffering off;
+        proxy_pass http://phoenix/api/fedsocket/v1;
     }
 }

installation/pleroma.vcl

@@ -1,3 +1,4 @@
+# Recommended varnishncsa logging format: '%h %l %u %t "%m %{X-Forwarded-Proto}i://%{Host}i%U%q %H" %s %b "%{Referer}i" "%{User-agent}i"'
 vcl 4.1;
 import std;
 
@@ -14,8 +15,11 @@ acl purge {
 sub vcl_recv {
     # Redirect HTTP to HTTPS
     if (std.port(server.ip) != 443) {
+      set req.http.X-Forwarded-Proto = "http";
       set req.http.x-redir = "https://" + req.http.host + req.url;
       return (synth(750, ""));
+    } else {
+      set req.http.X-Forwarded-Proto = "https";
     }
 
     # CHUNKED SUPPORT
@@ -105,7 +109,7 @@ sub vcl_hash {
 
 sub vcl_backend_fetch {
     # Be more lenient for slow servers on the fediverse
-    if bereq.url ~ "^/proxy/" {
+    if (bereq.url ~ "^/proxy/") {
       set bereq.first_byte_timeout = 300s;
     }
 

lib/mix/tasks/pleroma/config.ex

@@ -32,7 +32,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    if Pleroma.Config.get([:configurable_from_database]) do
+    with true <- Pleroma.Config.get([:configurable_from_database]),
+         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -46,7 +47,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
       do_migrate_to_db(config_file)
     else
-      migration_error()
+      :error -> deprecation_error()
+      _ -> migration_error()
     end
   end
 
@@ -120,6 +122,10 @@ defmodule Mix.Tasks.Pleroma.Config do
     )
   end
 
+  defp deprecation_error do
+    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
+  end
+
   if Code.ensure_loaded?(Config.Reader) do
     defp config_header, do: "import Config\r\n\r\n"
     defp read_file(config_file), do: Config.Reader.read_imports!(config_file)

lib/mix/tasks/pleroma/database.ex

@@ -99,7 +99,7 @@ defmodule Mix.Tasks.Pleroma.Database do
       where: fragment("(?)->>'likes' is not null", object.data),
       select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
     )
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn objects ->
       ids =
         objects
@@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> where(local: true)
     |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
     |> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity ->
         expires_at =

lib/mix/tasks/pleroma/email.ex

@@ -2,11 +2,11 @@ defmodule Mix.Tasks.Pleroma.Email do
   use Mix.Task
   import Mix.Pleroma
 
-  @shortdoc "Simple Email test"
+  @shortdoc "Email administrative tasks"
   @moduledoc File.read!("docs/administration/CLI_tasks/email.md")
 
   def run(["test" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
 
     {options, [], []} =
       OptionParser.parse(
@@ -21,4 +21,20 @@ defmodule Mix.Tasks.Pleroma.Email do
 
     shell_info("Test email has been sent to #{inspect(email.to)} from #{inspect(email.from)}")
   end
+
+  def run(["resend_confirmation_emails"]) do
+    start_pleroma()
+
+    shell_info("Sending emails to all unconfirmed users")
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      confirmation_pending: true,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500)
+    |> Stream.each(&Pleroma.User.try_send_confirmation_email(&1))
+    |> Stream.run()
+  end
 end

lib/mix/tasks/pleroma/relay.ex

@@ -21,10 +21,19 @@ defmodule Mix.Tasks.Pleroma.Relay do
     end
   end
 
-  def run(["unfollow", target]) do
+  def run(["unfollow", target | rest]) do
     start_pleroma()
 
-    with {:ok, _activity} <- Relay.unfollow(target) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [force: :boolean],
+        aliases: [f: :force]
+      )
+
+    force = Keyword.get(options, :force, false)
+
+    with {:ok, _activity} <- Relay.unfollow(target, %{force: force}) do
       # put this task to sleep to allow the genserver to push out the messages
       :timer.sleep(500)
     else

lib/mix/tasks/pleroma/user.ex

@@ -179,7 +179,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -196,17 +196,24 @@ defmodule Mix.Tasks.Pleroma.User do
       OptionParser.parse(
         rest,
         strict: [
-          moderator: :boolean,
           admin: :boolean,
-          locked: :boolean
+          confirmed: :boolean,
+          locked: :boolean,
+          moderator: :boolean
         ]
       )
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       user =
-        case Keyword.get(options, :moderator) do
+        case Keyword.get(options, :admin) do
           nil -> user
-          value -> set_moderator(user, value)
+          value -> set_admin(user, value)
+        end
+
+      user =
+        case Keyword.get(options, :confirmed) do
+          nil -> user
+          value -> set_confirmed(user, value)
         end
 
       user =
@@ -216,9 +223,9 @@ defmodule Mix.Tasks.Pleroma.User do
         end
 
       _user =
-        case Keyword.get(options, :admin) do
+        case Keyword.get(options, :moderator) do
           nil -> user
-          value -> set_admin(user, value)
+          value -> set_moderator(user, value)
         end
     else
       _ ->
@@ -386,6 +393,42 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["confirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, false) end)
+    end)
+    |> Stream.run()
+  end
+
+  def run(["unconfirm_all"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{
+      local: true,
+      deactivated: false,
+      is_moderator: false,
+      is_admin: false,
+      invisible: false
+    })
+    |> Pleroma.Repo.chunk_stream(500, :batches)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user -> User.need_confirmation(user, true) end)
+    end)
+    |> Stream.run()
+  end
+
   def run(["sign_out", nickname]) do
     start_pleroma()
 
@@ -403,7 +446,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{local: true})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -443,4 +486,11 @@ defmodule Mix.Tasks.Pleroma.User do
     shell_info("Locked status of #{user.nickname}: #{user.locked}")
     user
   end
+
+  defp set_confirmed(user, value) do
+    {:ok, user} = User.need_confirmation(user, !value)
+
+    shell_info("Confirmation pending status of #{user.nickname}: #{user.confirmation_pending}")
+    user
+  end
 end

lib/pleroma/application.ex

@@ -56,7 +56,6 @@ defmodule Pleroma.Application do
     Pleroma.ApplicationRequirements.verify!()
     setup_instrumenters()
     load_custom_modules()
-    check_system_commands()
     Pleroma.Docs.JSON.compile()
 
     adapter = Application.get_env(:tesla, :adapter)
@@ -99,7 +98,7 @@ defmodule Pleroma.Application do
           {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
-        streamer_child(@env) ++
+        dont_run_in_test(@env) ++
         chat_child(@env, chat_enabled?()) ++
         [
           Pleroma.Web.Endpoint,
@@ -188,16 +187,17 @@ defmodule Pleroma.Application do
 
   defp chat_enabled?, do: Config.get([:chat, :enabled])
 
-  defp streamer_child(env) when env in [:test, :benchmark], do: []
+  defp dont_run_in_test(env) when env in [:test, :benchmark], do: []
 
-  defp streamer_child(_) do
+  defp dont_run_in_test(_) do
     [
       {Registry,
        [
          name: Pleroma.Web.Streamer.registry(),
          keys: :duplicate,
          partitions: System.schedulers_online()
-       ]}
+       ]},
+      Pleroma.Web.FedSockets.Supervisor
     ]
   end
 
@@ -259,21 +259,4 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(_, _), do: []
-
-  defp check_system_commands do
-    filters = Config.get([Pleroma.Upload, :filters])
-
-    check_filter = fn filter, command_required ->
-      with true <- filter in filters,
-           false <- Pleroma.Utils.command_available?(command_required) do
-        Logger.error(
-          "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found"
-        )
-      end
-    end
-
-    check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool")
-    check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify")
-    check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify")
-  end
 end

lib/pleroma/application_requirements.ex

@@ -9,6 +9,9 @@ defmodule Pleroma.ApplicationRequirements do
 
   defmodule VerifyError, do: defexception([:message])
 
+  alias Pleroma.Config
+  alias Pleroma.Helpers.MediaHelper
+
   import Ecto.Query
 
   require Logger
@@ -16,7 +19,8 @@ defmodule Pleroma.ApplicationRequirements do
   @spec verify!() :: :ok | VerifyError.t()
   def verify! do
     :ok
-    |> check_confirmation_accounts!
+    |> check_system_commands!()
+    |> check_confirmation_accounts!()
     |> check_migrations_applied!()
     |> check_welcome_message_config!()
     |> check_rum!()
@@ -48,7 +52,9 @@ defmodule Pleroma.ApplicationRequirements do
     if Pleroma.Config.get([:instance, :account_activation_required]) &&
          not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
       Logger.error(
-        "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer."
+        "Account activation enabled, but no Mailer settings enabled.\n" <>
+          "Please set config :pleroma, :instance, account_activation_required: false\n" <>
+          "Otherwise setup and enable Mailer."
       )
 
       {:error,
@@ -81,7 +87,9 @@ defmodule Pleroma.ApplicationRequirements do
               Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
 
             Logger.error(
-              "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
+              "The following migrations were not applied:\n#{down_migrations_text}" <>
+                "If you want to start Pleroma anyway, set\n" <>
+                "config :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
             )
 
             {:error, "Unapplied Migrations detected"}
@@ -124,14 +132,22 @@ defmodule Pleroma.ApplicationRequirements do
     case {setting, migrate} do
       {true, false} ->
         Logger.error(
-          "Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+          "Use `RUM` index is enabled, but were not applied migrations for it.\n" <>
+            "If you want to start Pleroma anyway, set\n" <>
+            "config :pleroma, :database, rum_enabled: false\n" <>
+            "Otherwise apply the following migrations:\n" <>
+            "`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
         )
 
         {:error, "Unapplied RUM Migrations detected"}
 
       {false, true} ->
         Logger.error(
-          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\n" <>
+            "If you want to use `RUM`, set\n" <>
+            "config :pleroma, :database, rum_enabled: true\n" <>
+            "Otherwise roll `RUM` migrations back.\n" <>
+            "`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
         )
 
         {:error, "RUM Migrations detected"}
@@ -140,4 +156,50 @@ defmodule Pleroma.ApplicationRequirements do
         :ok
     end
   end
+
+  defp check_system_commands!(:ok) do
+    filter_commands_statuses = [
+      check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"),
+      check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"),
+      check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+    ]
+
+    preview_proxy_commands_status =
+      if !Config.get([:media_preview_proxy, :enabled]) or
+           MediaHelper.missing_dependencies() == [] do
+        true
+      else
+        Logger.error(
+          "The following dependencies required by Media preview proxy " <>
+            "(which is currently enabled) are not installed: " <>
+            inspect(MediaHelper.missing_dependencies())
+        )
+
+        false
+      end
+
+    if Enum.all?([preview_proxy_commands_status | filter_commands_statuses], & &1) do
+      :ok
+    else
+      {:error,
+       "System commands missing. Check logs and see `docs/installation` for more details."}
+    end
+  end
+
+  defp check_system_commands!(result), do: result
+
+  defp check_filter(filter, command_required) do
+    filters = Config.get([Pleroma.Upload, :filters])
+
+    if filter in filters and not Pleroma.Utils.command_available?(command_required) do
+      Logger.error(
+        "#{filter} is specified in list of Pleroma.Upload filters, but the " <>
+          "#{command_required} command is not found"
+      )
+
+      false
+    else
+      true
+    end
+  end
 end

lib/pleroma/chat.ex

@@ -6,7 +6,9 @@ defmodule Pleroma.Chat do
   use Ecto.Schema
 
   import Ecto.Changeset
+  import Ecto.Query
 
+  alias Pleroma.Chat
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -16,6 +18,7 @@ defmodule Pleroma.Chat do
   It is a helper only, to make it easy to display a list of chats with other people, ordered by last bump. The actual messages are retrieved by querying the recipients of the ChatMessages.
   """
 
+  @type t :: %__MODULE__{}
   @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
   schema "chats" do
@@ -39,16 +42,28 @@ defmodule Pleroma.Chat do
     |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
   end
 
+  @spec get_by_user_and_id(User.t(), FlakeId.Ecto.CompatType.t()) ::
+          {:ok, t()} | {:error, :not_found}
+  def get_by_user_and_id(%User{id: user_id}, id) do
+    from(c in __MODULE__,
+      where: c.id == ^id,
+      where: c.user_id == ^user_id
+    )
+    |> Repo.find_resource()
+  end
+
+  @spec get_by_id(FlakeId.Ecto.CompatType.t()) :: t() | nil
   def get_by_id(id) do
-    __MODULE__
+    Repo.get(__MODULE__, id)
-    |> Repo.get(id)
   end
 
+  @spec get(FlakeId.Ecto.CompatType.t(), String.t()) :: t() | nil
   def get(user_id, recipient) do
-    __MODULE__
+    Repo.get_by(__MODULE__, user_id: user_id, recipient: recipient)
-    |> Repo.get_by(user_id: user_id, recipient: recipient)
   end
 
+  @spec get_or_create(FlakeId.Ecto.CompatType.t(), String.t()) ::
+          {:ok, t()} | {:error, Ecto.Changeset.t()}
   def get_or_create(user_id, recipient) do
     %__MODULE__{}
     |> changeset(%{user_id: user_id, recipient: recipient})
@@ -60,6 +75,8 @@ defmodule Pleroma.Chat do
     )
   end
 
+  @spec bump_or_create(FlakeId.Ecto.CompatType.t(), String.t()) ::
+          {:ok, t()} | {:error, Ecto.Changeset.t()}
   def bump_or_create(user_id, recipient) do
     %__MODULE__{}
     |> changeset(%{user_id: user_id, recipient: recipient})
@@ -69,4 +86,12 @@ defmodule Pleroma.Chat do
       conflict_target: [:user_id, :recipient]
     )
   end
+
+  @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t()
+  def for_user_query(user_id) do
+    from(c in Chat,
+      where: c.user_id == ^user_id,
+      order_by: [desc: c.updated_at]
+    )
+  end
 end

lib/pleroma/config/deprecation_warnings.ex

@@ -26,38 +26,25 @@ defmodule Pleroma.Config.DeprecationWarnings do
       !!!DEPRECATION WARNING!!!
       You are using the old configuration mechanism for the hellthread filter. Please check config.md.
       """)
-    end
-  end
 
-  def mrf_user_allowlist do
+      :error
-    config = Config.get(:mrf_user_allowlist)
+    else
-
+      :ok
-    if config && Enum.any?(config, fn {k, _} -> is_atom(k) end) do
-      rewritten =
-        Enum.reduce(Config.get(:mrf_user_allowlist), Map.new(), fn {k, v}, acc ->
-          Map.put(acc, to_string(k), v)
-        end)
-
-      Config.put(:mrf_user_allowlist, rewritten)
-
-      Logger.error("""
-      !!!DEPRECATION WARNING!!!
-      As of Pleroma 2.0.7, the `mrf_user_allowlist` setting changed of format.
-      Pleroma 2.1 will remove support for the old format. Please change your configuration to match this:
-
-      config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
-      """)
     end
   end
 
   def warn do
-    check_hellthread_threshold()
+    with :ok <- check_hellthread_threshold(),
-    mrf_user_allowlist()
+         :ok <- check_old_mrf_config(),
-    check_old_mrf_config()
+         :ok <- check_media_proxy_whitelist_config(),
-    check_media_proxy_whitelist_config()
+         :ok <- check_welcome_message_config(),
-    check_welcome_message_config()
+         :ok <- check_gun_pool_options(),
-    check_gun_pool_options()
+         :ok <- check_activity_expiration_config() do
-    check_activity_expiration_config()
+      :ok
+    else
+      _ ->
+        :error
+    end
   end
 
   def check_welcome_message_config do
@@ -70,10 +57,14 @@ defmodule Pleroma.Config.DeprecationWarnings do
     if use_old_config do
       Logger.error("""
       !!!DEPRECATION WARNING!!!
-      Your config is using the old namespace for Welcome messages configuration. You need to change to the new namespace:
+      Your config is using the old namespace for Welcome messages configuration. You need to convert to the new namespace. e.g.,
-      \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname`
+      \n* `config :pleroma, :instance, welcome_user_nickname` and `config :pleroma, :instance, welcome_message` are now equal to:
-      \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message`
+      \n* `config :pleroma, :welcome, direct_message: [enabled: true, sender_nickname: "NICKNAME", message: "Your welcome message"]`"
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -101,8 +92,11 @@ defmodule Pleroma.Config.DeprecationWarnings do
           end
       end)
 
-    if warning != "" do
+    if warning == "" do
+      :ok
+    else
       Logger.warn(warning_preface <> warning)
+      :error
     end
   end
 
@@ -115,6 +109,10 @@ defmodule Pleroma.Config.DeprecationWarnings do
       !!!DEPRECATION WARNING!!!
       Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -124,7 +122,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     if timeout = pool_config[:await_up_timeout] do
       Logger.warn("""
       !!!DEPRECATION WARNING!!!
-      Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+      Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`. Please change to `config :pleroma, :connections_pool, connect_timeout` to ensure compatibility with future releases.
       """)
 
       Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout))
@@ -157,6 +155,9 @@ defmodule Pleroma.Config.DeprecationWarnings do
       Logger.warn(Enum.join([warning_preface | pool_warnings]))
 
       Config.put(:pools, updated_config)
+      :error
+    else
+      :ok
     end
   end
 

lib/pleroma/emails/mailer.ex

@@ -35,6 +35,11 @@ defmodule Pleroma.Emails.Mailer do
   def deliver(email, config \\ [])
 
   def deliver(email, config) do
+    # temporary hackney fix until hackney max_connections bug is fixed
+    # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+    email =
+      Swoosh.Email.put_private(email, :hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+
     case enabled?() do
       true -> Swoosh.Mailer.deliver(email, parse_config(config))
       false -> {:error, :deliveries_disabled}

lib/pleroma/emoji.ex

@@ -56,6 +56,9 @@ defmodule Pleroma.Emoji do
     end
   end
 
+  @spec exist?(String.t()) :: boolean()
+  def exist?(name), do: not is_nil(get(name))
+
   @doc "Returns all the emojos!!"
   @spec get_all() :: list({String.t(), String.t(), String.t()})
   def get_all do

lib/pleroma/emoji/pack.ex

@@ -17,6 +17,7 @@ defmodule Pleroma.Emoji.Pack do
         }
 
   alias Pleroma.Emoji
+  alias Pleroma.Emoji.Pack
 
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
@@ -64,24 +65,93 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
+  @spec unpack_zip_emojies(list(tuple())) :: list(map())
-          {:ok, t()} | {:error, File.posix() | atom()}
+  defp unpack_zip_emojies(zip_files) do
-  def add_file(name, shortcode, filename, file) do
+    Enum.reduce(zip_files, [], fn
-    with :ok <- validate_not_empty([name, shortcode, filename]),
+      {_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
+        with(
+          filename <- Path.basename(path),
+          shortcode <- Path.basename(filename, Path.extname(filename)),
+          false <- Emoji.exist?(shortcode)
+        ) do
+          [%{path: path, filename: path, shortcode: shortcode} | acc]
+        else
+          _ -> acc
+        end
+
+      _, acc ->
+        acc
+    end)
+  end
+
+  @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
+          {:ok, t()}
+          | {:error, File.posix() | atom()}
+  def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
+    with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
+         [_ | _] = emojies <- unpack_zip_emojies(zip_files),
+         {:ok, tmp_dir} <- Pleroma.Utils.tmp_dir("emoji") do
+      try do
+        {:ok, _emoji_files} =
+          :zip.unzip(
+            to_charlist(file.path),
+            [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, tmp_dir}]
+          )
+
+        {_, updated_pack} =
+          Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
+            emoji_file = %Plug.Upload{
+              filename: item[:filename],
+              path: Path.join(tmp_dir, item[:path])
+            }
+
+            {:ok, updated_pack} =
+              do_add_file(
+                emoji_pack,
+                item[:shortcode],
+                to_string(item[:filename]),
+                emoji_file
+              )
+
+            {item, updated_pack}
+          end)
+
+        Emoji.reload()
+
+        {:ok, updated_pack}
+      after
+        File.rm_rf(tmp_dir)
+      end
+    else
+      {:error, _} = error ->
+        error
+
+      _ ->
+        {:ok, pack}
+    end
+  end
+
+  def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
+    with :ok <- validate_not_empty([shortcode, filename]),
          :ok <- validate_emoji_not_exists(shortcode),
-         {:ok, pack} <- load_pack(name),
+         {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
-         :ok <- save_file(file, pack, filename),
-         {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do
       Emoji.reload()
       {:ok, updated_pack}
     end
   end
 
-  @spec delete_file(String.t(), String.t()) ::
+  defp do_add_file(pack, shortcode, filename, file) do
+    with :ok <- save_file(file, pack, filename) do
+      pack
+      |> put_emoji(shortcode, filename)
+      |> save_pack()
+    end
+  end
+
+  @spec delete_file(t(), String.t()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def delete_file(name, shortcode) do
+  def delete_file(%Pack{} = pack, shortcode) do
-    with :ok <- validate_not_empty([name, shortcode]),
+    with :ok <- validate_not_empty([shortcode]),
-         {:ok, pack} <- load_pack(name),
          :ok <- remove_file(pack, shortcode),
          {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
       Emoji.reload()
@@ -89,11 +159,10 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
+  @spec update_file(t(), String.t(), String.t(), String.t(), boolean()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def update_file(name, shortcode, new_shortcode, new_filename, force) do
+  def update_file(%Pack{} = pack, shortcode, new_shortcode, new_filename, force) do
-    with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]),
+    with :ok <- validate_not_empty([shortcode, new_shortcode, new_filename]),
-         {:ok, pack} <- load_pack(name),
          {:ok, filename} <- get_filename(pack, shortcode),
          :ok <- validate_emoji_not_exists(new_shortcode, force),
          :ok <- rename_file(pack, filename, new_filename),
@@ -243,9 +312,10 @@ defmodule Pleroma.Emoji.Pack do
   defp validate_emoji_not_exists(_shortcode, true), do: :ok
 
   defp validate_emoji_not_exists(shortcode, _) do
-    case Emoji.get(shortcode) do
+    if Emoji.exist?(shortcode) do
-      nil -> :ok
+      {:error, :already_exists}
-      _ -> {:error, :already_exists}
+    else
+      :ok
     end
   end
 
@@ -386,25 +456,18 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  defp save_file(file, pack, filename) do
+  defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
     file_path = Path.join(pack.path, filename)
     create_subdirs(file_path)
 
-    case file do
+    with {:ok, _} <- File.copy(upload_path, file_path) do
-      %Plug.Upload{path: upload_path} ->
+      :ok
-        # Copy the uploaded file from the temporary directory
-        with {:ok, _} <- File.copy(upload_path, file_path), do: :ok
-
-      url when is_binary(url) ->
-        # Download and write the file
-        file_contents = Tesla.get!(url).body
-        File.write(file_path, file_contents)
     end
   end
 
   defp put_emoji(pack, shortcode, filename) do
     files = Map.put(pack.files, shortcode, filename)
-    %{pack | files: files}
+    %{pack | files: files, files_count: length(Map.keys(files))}
   end
 
   defp delete_emoji(pack, shortcode) do

lib/pleroma/gun/connection_pool/worker.ex

@@ -93,12 +93,8 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
       end)
 
     {ref, state} = pop_in(state.client_monitors[client_pid])
-    # DOWN message can receive right after `remove_client` call and cause worker to terminate
+
-    state =
+    Process.demonitor(ref, [:flush])
-      if is_nil(ref) do
-        state
-      else
-        Process.demonitor(ref)
 
     timer =
       if used_by == [] do
@@ -108,10 +104,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
         nil
       end
 
-        %{state | timer: timer}
+    {:reply, :ok, %{state | timer: timer}, :hibernate}
-      end
-
-    {:reply, :ok, state, :hibernate}
   end
 
   @impl true

lib/pleroma/helpers/media_helper.ex

@@ -0,0 +1,162 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.MediaHelper do
+  @moduledoc """
+  Handles common media-related operations.
+  """
+
+  alias Pleroma.HTTP
+
+  require Logger
+
+  def missing_dependencies do
+    Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
+      if Pleroma.Utils.command_available?(executable) do
+        acc
+      else
+        [sym | acc]
+      end
+    end)
+  end
+
+  def image_resize(url, options) do
+    with executable when is_binary(executable) <- System.find_executable("convert"),
+         {:ok, args} <- prepare_image_resize_args(options),
+         {:ok, env} <- HTTP.get(url, [], pool: :media),
+         {:ok, fifo_path} <- mkfifo() do
+      args = List.flatten([fifo_path, args])
+      run_fifo(fifo_path, env, executable, args)
+    else
+      nil -> {:error, {:convert, :command_not_found}}
+      {:error, _} = error -> error
+    end
+  end
+
+  defp prepare_image_resize_args(
+         %{max_width: max_width, max_height: max_height, format: "png"} = options
+       ) do
+    quality = options[:quality] || 85
+    resize = Enum.join([max_width, "x", max_height, ">"])
+
+    args = [
+      "-resize",
+      resize,
+      "-quality",
+      to_string(quality),
+      "png:-"
+    ]
+
+    {:ok, args}
+  end
+
+  defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
+    quality = options[:quality] || 85
+    resize = Enum.join([max_width, "x", max_height, ">"])
+
+    args = [
+      "-interlace",
+      "Plane",
+      "-resize",
+      resize,
+      "-quality",
+      to_string(quality),
+      "jpg:-"
+    ]
+
+    {:ok, args}
+  end
+
+  defp prepare_image_resize_args(_), do: {:error, :missing_options}
+
+  # Note: video thumbnail is intentionally not resized (always has original dimensions)
+  def video_framegrab(url) do
+    with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
+         {:ok, env} <- HTTP.get(url, [], pool: :media),
+         {:ok, fifo_path} <- mkfifo(),
+         args = [
+           "-y",
+           "-i",
+           fifo_path,
+           "-vframes",
+           "1",
+           "-f",
+           "mjpeg",
+           "-loglevel",
+           "error",
+           "-"
+         ] do
+      run_fifo(fifo_path, env, executable, args)
+    else
+      nil -> {:error, {:ffmpeg, :command_not_found}}
+      {:error, _} = error -> error
+    end
+  end
+
+  defp run_fifo(fifo_path, env, executable, args) do
+    pid =
+      Port.open({:spawn_executable, executable}, [
+        :use_stdio,
+        :stream,
+        :exit_status,
+        :binary,
+        args: args
+      ])
+
+    fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
+    fix = Pleroma.Helpers.QtFastStart.fix(env.body)
+    true = Port.command(fifo, fix)
+    :erlang.port_close(fifo)
+    loop_recv(pid)
+  after
+    File.rm(fifo_path)
+  end
+
+  defp mkfifo do
+    path = Path.join(System.tmp_dir!(), "pleroma-media-preview-pipe-#{Ecto.UUID.generate()}")
+
+    case System.cmd("mkfifo", [path]) do
+      {_, 0} ->
+        spawn(fifo_guard(path))
+        {:ok, path}
+
+      {_, err} ->
+        {:error, {:fifo_failed, err}}
+    end
+  end
+
+  defp fifo_guard(path) do
+    pid = self()
+
+    fn ->
+      ref = Process.monitor(pid)
+
+      receive do
+        {:DOWN, ^ref, :process, ^pid, _} ->
+          File.rm(path)
+      end
+    end
+  end
+
+  defp loop_recv(pid) do
+    loop_recv(pid, <<>>)
+  end
+
+  defp loop_recv(pid, acc) do
+    receive do
+      {^pid, {:data, data}} ->
+        loop_recv(pid, acc <> data)
+
+      {^pid, {:exit_status, 0}} ->
+        {:ok, acc}
+
+      {^pid, {:exit_status, status}} ->
+        {:error, status}
+    after
+      5000 ->
+        :erlang.port_close(pid)
+        {:error, :timeout}
+    end
+  end
+end

lib/pleroma/helpers/qt_fast_start.ex

@@ -0,0 +1,131 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.QtFastStart do
+  @moduledoc """
+  (WIP) Converts a "slow start" (data before metadatas) mov/mp4 file to a "fast start" one (metadatas before data).
+  """
+
+  # TODO: Cleanup and optimizations
+  # Inspirations: https://www.ffmpeg.org/doxygen/3.4/qt-faststart_8c_source.html
+  #               https://github.com/danielgtaylor/qtfaststart/blob/master/qtfaststart/processor.py
+  #               ISO/IEC 14496-12:2015, ISO/IEC 15444-12:2015
+  #               Paracetamol
+
+  def fix(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::bits>> = binary) do
+    index = fix(binary, 0, nil, nil, [])
+
+    case index do
+      :abort -> binary
+      [{"ftyp", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
+      [{"ftyp", _, _, _, _}, {"free", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
+      _ -> binary
+    end
+  end
+
+  def fix(binary) do
+    binary
+  end
+
+  # MOOV have been seen before MDAT- abort
+  defp fix(<<_::bits>>, _, true, false, _) do
+    :abort
+  end
+
+  defp fix(
+         <<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
+         pos,
+         got_moov,
+         got_mdat,
+         acc
+       ) do
+    full_size = (size - 8) * 8
+    <<data::bits-size(full_size), rest::bits>> = rest
+
+    acc = [
+      {fourcc, pos, pos + size, size,
+       <<size::integer-big-size(32), fourcc::bits-size(32), data::bits>>}
+      | acc
+    ]
+
+    fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc)
+  end
+
+  defp fix(<<>>, _pos, _, _, acc) do
+    :lists.reverse(acc)
+  end
+
+  defp faststart(index) do
+    {{_ftyp, _, _, _, ftyp}, index} = List.keytake(index, "ftyp", 0)
+
+    # Skip re-writing the free fourcc as it's kind of useless.
+    # Why stream useless bytes when you can do without?
+    {free_size, index} =
+      case List.keytake(index, "free", 0) do
+        {{_, _, _, size, _}, index} -> {size, index}
+        _ -> {0, index}
+      end
+
+    {{_moov, _, _, moov_size, moov}, index} = List.keytake(index, "moov", 0)
+    offset = -free_size + moov_size
+    rest = for {_, _, _, _, data} <- index, do: data, into: []
+    <<moov_head::bits-size(64), moov_data::bits>> = moov
+    [ftyp, moov_head, fix_moov(moov_data, offset, []), rest]
+  end
+
+  defp fix_moov(
+         <<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
+         offset,
+         acc
+       ) do
+    full_size = (size - 8) * 8
+    <<data::bits-size(full_size), rest::bits>> = rest
+
+    data =
+      cond do
+        fourcc in ["trak", "mdia", "minf", "stbl"] ->
+          # Theses contains sto or co64 part
+          [<<size::integer-big-size(32), fourcc::bits-size(32)>>, fix_moov(data, offset, [])]
+
+        fourcc in ["stco", "co64"] ->
+          # fix the damn thing
+          <<version::integer-big-size(32), count::integer-big-size(32), rest::bits>> = data
+
+          entry_size =
+            case fourcc do
+              "stco" -> 32
+              "co64" -> 64
+            end
+
+          [
+            <<size::integer-big-size(32), fourcc::bits-size(32), version::integer-big-size(32),
+              count::integer-big-size(32)>>,
+            rewrite_entries(entry_size, offset, rest, [])
+          ]
+
+        true ->
+          [<<size::integer-big-size(32), fourcc::bits-size(32)>>, data]
+      end
+
+    acc = [acc | data]
+    fix_moov(rest, offset, acc)
+  end
+
+  defp fix_moov(<<>>, _, acc), do: acc
+
+  for size <- [32, 64] do
+    defp rewrite_entries(
+           unquote(size),
+           offset,
+           <<pos::integer-big-size(unquote(size)), rest::bits>>,
+           acc
+         ) do
+      rewrite_entries(unquote(size), offset, rest, [
+        acc | <<pos + offset::integer-big-size(unquote(size))>>
+      ])
+    end
+  end
+
+  defp rewrite_entries(_, _, <<>>, acc), do: acc
+end

lib/pleroma/helpers/uri_helper.ex

@@ -3,18 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Helpers.UriHelper do
-  def append_uri_params(uri, appended_params) do
+  def modify_uri_params(uri, overridden_params, deleted_params \\ []) do
     uri = URI.parse(uri)
-    appended_params = for {k, v} <- appended_params, into: %{}, do: {to_string(k), v}
+
-    existing_params = URI.query_decoder(uri.query || "") |> Enum.into(%{})
+    existing_params = URI.query_decoder(uri.query || "") |> Map.new()
-    updated_params_keys = Enum.uniq(Map.keys(existing_params) ++ Map.keys(appended_params))
+    overridden_params = Map.new(overridden_params, fn {k, v} -> {to_string(k), v} end)
+    deleted_params = Enum.map(deleted_params, &to_string/1)
 
     updated_params =
-      for k <- updated_params_keys, do: {k, appended_params[k] || existing_params[k]}
+      existing_params
+      |> Map.merge(overridden_params)
+      |> Map.drop(deleted_params)
 
     uri
     |> Map.put(:query, URI.encode_query(updated_params))
     |> URI.to_string()
+    |> String.replace_suffix("?", "")
   end
 
   def maybe_add_base("/" <> uri, base), do: Path.join([base, uri])

lib/pleroma/http/web_push.ex

@@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.WebPush do
+  @moduledoc false
+
+  def post(url, payload, headers) do
+    list_headers = Map.to_list(headers)
+    Pleroma.HTTP.post(url, payload, list_headers)
+  end
+end

lib/pleroma/instances/instance.ex

@@ -156,9 +156,7 @@ defmodule Pleroma.Instances.Instance do
   defp scrape_favicon(%URI{} = instance_uri) do
     try do
       with {:ok, %Tesla.Env{body: html}} <-
-             Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
+             Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
-               adapter: [pool: :media]
-             ),
            {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
              {:parse,
               html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},

lib/pleroma/migration_helper/notification_backfill.ex

@@ -19,13 +19,13 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
     query
     |> Repo.chunk_stream(100)
     |> Enum.each(fn notification ->
-      type =
+      if notification.activity do
-        notification.activity
+        type = type_from_activity(notification.activity)
-        |> type_from_activity()
 
         notification
         |> Ecto.Changeset.change(%{type: type})
         |> Repo.update()
+      end
     end)
   end
 
@@ -72,8 +72,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
         "pleroma:emoji_reaction"
 
       "Create" ->
-        activity
+        type_from_activity_object(activity)
-        |> type_from_activity_object()
 
       t ->
         raise "No notification type for activity type #{t}"

lib/pleroma/moderation_log.ex

@@ -320,6 +320,19 @@ defmodule Pleroma.ModerationLog do
     |> insert_log_entry_with_message()
   end
 
+  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
+          {:ok, ModerationLog} | {:error, any}
+  def insert_log(%{actor: %User{} = actor, action: "chat_message_delete", subject_id: subject_id}) do
+    %ModerationLog{
+      data: %{
+        "actor" => %{"nickname" => actor.nickname},
+        "action" => "chat_message_delete",
+        "subject_id" => subject_id
+      }
+    }
+    |> insert_log_entry_with_message()
+  end
+
   @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any}
   defp insert_log_entry_with_message(entry) do
     entry.data["message"]
@@ -627,6 +640,17 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "chat_message_delete",
+          "subject_id" => subject_id
+        }
+      }) do
+    "@#{actor_nickname} deleted chat message ##{subject_id}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/object/fetcher.ex

@@ -12,6 +12,7 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
+  alias Pleroma.Web.FedSockets
 
   require Logger
   require Pleroma.Constants
@@ -98,8 +99,8 @@ defmodule Pleroma.Object.Fetcher do
       {:containment, _} ->
         {:error, "Object containment failed."}
 
-      {:transmogrifier, {:error, {:reject, nil}}} ->
+      {:transmogrifier, {:error, {:reject, e}}} ->
-        {:reject, nil}
+        {:reject, e}
 
       {:transmogrifier, _} = e ->
         {:error, e}
@@ -182,27 +183,20 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
+  def fetch_and_contain_remote_object_from_id(prm, opts \\ [])
+
+  def fetch_and_contain_remote_object_from_id(%{"id" => id}, opts),
+    do: fetch_and_contain_remote_object_from_id(id, opts)
+
+  def fetch_and_contain_remote_object_from_id(id, opts) when is_binary(id) do
     Logger.debug("Fetching object #{id} via AP")
 
-    date = Pleroma.Signature.signed_date()
-
-    headers =
-      [{"accept", "application/activity+json"}]
-      |> maybe_date_fetch(date)
-      |> sign_fetch(id, date)
-
-    Logger.debug("Fetch headers: #{inspect(headers)}")
-
     with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
-         {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers),
+         {:ok, body} <- get_object(id, opts),
-         {:ok, data} <- Jason.decode(body),
+         {:ok, data} <- safe_json_decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
     else
-      {:ok, %{status: code}} when code in [404, 410] ->
-        {:error, "Object has been deleted"}
-
       {:scheme, _} ->
         {:error, "Unsupported URI scheme"}
 
@@ -214,8 +208,44 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(%{"id" => id}),
+  def fetch_and_contain_remote_object_from_id(_id, _opts),
-    do: fetch_and_contain_remote_object_from_id(id)
+    do: {:error, "id must be a string"}
 
-  def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"}
+  defp get_object(id, opts) do
+    with false <- Keyword.get(opts, :force_http, false),
+         {:ok, fedsocket} <- FedSockets.get_or_create_fed_socket(id) do
+      Logger.debug("fetching via fedsocket - #{inspect(id)}")
+      FedSockets.fetch(fedsocket, id)
+    else
+      _other ->
+        Logger.debug("fetching via http - #{inspect(id)}")
+        get_object_http(id)
+    end
+  end
+
+  defp get_object_http(id) do
+    date = Pleroma.Signature.signed_date()
+
+    headers =
+      [{"accept", "application/activity+json"}]
+      |> maybe_date_fetch(date)
+      |> sign_fetch(id, date)
+
+    case HTTP.get(id, headers) do
+      {:ok, %{body: body, status: code}} when code in 200..299 ->
+        {:ok, body}
+
+      {:ok, %{status: code}} when code in [404, 410] ->
+        {:error, "Object has been deleted"}
+
+      {:error, e} ->
+        {:error, e}
+
+      e ->
+        {:error, e}
+    end
+  end
+
+  defp safe_json_decode(nil), do: {:ok, nil}
+  defp safe_json_decode(json), do: Jason.decode(json)
 end

lib/pleroma/plugs/oauth_scopes_plug.ex

@@ -53,7 +53,7 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
     |> assign(:token, nil)
   end
 
-  @doc "Filters descendants of supported scopes"
+  @doc "Keeps those of `scopes` which are descendants of `supported_scopes`"
   def filter_descendants(scopes, supported_scopes) do
     Enum.filter(
       scopes,

lib/pleroma/repo.ex

@@ -49,7 +49,21 @@ defmodule Pleroma.Repo do
     end
   end
 
-  def chunk_stream(query, chunk_size) do
+  @doc """
+  Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+  `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+  ## Examples
+
+  # fetch records one-by-one
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+  # fetch records in bulk
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+  """
+  @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
+  def chunk_stream(query, chunk_size, returns_as \\ :one) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
     # have individual records be the elements of the stream, instead of
@@ -69,7 +83,12 @@ defmodule Pleroma.Repo do
 
             records ->
               last_id = List.last(records).id
+
+              if returns_as == :one do
                 {records, last_id}
+              else
+                {[records], last_id}
+              end
           end
       end,
       fn _ -> :ok end

lib/pleroma/repo_streamer.ex

@@ -1,34 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.RepoStreamer do
-  alias Pleroma.Repo
-  import Ecto.Query
-
-  def chunk_stream(query, chunk_size) do
-    Stream.unfold(0, fn
-      :halt ->
-        {[], :halt}
-
-      last_id ->
-        query
-        |> order_by(asc: :id)
-        |> where([r], r.id > ^last_id)
-        |> limit(^chunk_size)
-        |> Repo.all()
-        |> case do
-          [] ->
-            {[], :halt}
-
-          records ->
-            last_id = List.last(records).id
-            {records, last_id}
-        end
-    end)
-    |> Stream.take_while(fn
-      [] -> false
-      _ -> true
-    end)
-  end
-end

lib/pleroma/reverse_proxy/reverse_proxy.ex

@@ -17,6 +17,9 @@ defmodule Pleroma.ReverseProxy do
   @failed_request_ttl :timer.seconds(60)
   @methods ~w(GET HEAD)
 
+  def max_read_duration_default, do: @max_read_duration
+  def default_cache_control_header, do: @default_cache_control_header
+
   @moduledoc """
   A reverse proxy.
 
@@ -391,6 +394,8 @@ defmodule Pleroma.ReverseProxy do
 
   defp body_size_constraint(_, _), do: :ok
 
+  defp check_read_duration(nil = _duration, max), do: check_read_duration(@max_read_duration, max)
+
   defp check_read_duration(duration, max)
        when is_integer(duration) and is_integer(max) and max > 0 do
     if duration > max do

lib/pleroma/signature.ex

@@ -39,7 +39,7 @@ defmodule Pleroma.Signature do
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
       {:ok, public_key}
     else
       e ->
@@ -50,8 +50,8 @@ defmodule Pleroma.Signature do
   def refetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
+         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id, force_http: true),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
       {:ok, public_key}
     else
       e ->

lib/pleroma/user.ex

@@ -25,7 +25,6 @@ defmodule Pleroma.User do
   alias Pleroma.Object
   alias Pleroma.Registration
   alias Pleroma.Repo
-  alias Pleroma.RepoStreamer
   alias Pleroma.User
   alias Pleroma.UserRelationship
   alias Pleroma.Web
@@ -276,9 +275,9 @@ defmodule Pleroma.User do
   @spec account_status(User.t()) :: account_status()
   def account_status(%User{deactivated: true}), do: :deactivated
   def account_status(%User{password_reset_pending: true}), do: :password_reset_pending
-  def account_status(%User{approval_pending: true}), do: :approval_pending
+  def account_status(%User{local: true, approval_pending: true}), do: :approval_pending
 
-  def account_status(%User{confirmation_pending: true}) do
+  def account_status(%User{local: true, confirmation_pending: true}) do
     if Config.get([:instance, :account_activation_required]) do
       :confirmation_pending
     else
@@ -814,7 +813,8 @@ defmodule Pleroma.User do
   def send_welcome_email(_), do: {:ok, :noop}
 
   @spec try_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop}
-  def try_send_confirmation_email(%User{confirmation_pending: true} = user) do
+  def try_send_confirmation_email(%User{confirmation_pending: true, email: email} = user)
+      when is_binary(email) do
     if Config.get([:instance, :account_activation_required]) do
       send_confirmation_email(user)
       {:ok, :enqueued}
@@ -915,9 +915,7 @@ defmodule Pleroma.User do
         FollowingRelationship.unfollow(follower, followed)
         {:ok, followed} = update_follower_count(followed)
 
-        {:ok, follower} =
+        {:ok, follower} = update_following_count(follower)
-          follower
-          |> update_following_count()
 
         {:ok, follower, followed}
 
@@ -1686,42 +1684,6 @@ defmodule Pleroma.User do
 
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
 
-  @spec perform(atom(), User.t(), list()) :: list() | {:error, any()}
-  def perform(:blocks_import, %User{} = blocker, blocked_identifiers)
-      when is_list(blocked_identifiers) do
-    Enum.map(
-      blocked_identifiers,
-      fn blocked_identifier ->
-        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
-             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
-          blocked
-        else
-          err ->
-            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
-  def perform(:follow_import, %User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    Enum.map(
-      followed_identifiers,
-      fn followed_identifier ->
-        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
-             {:ok, follower} <- maybe_direct_follow(follower, followed),
-             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
-          followed
-        else
-          err ->
-            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
   @spec external_users_query() :: Ecto.Query.t()
   def external_users_query do
     User.Query.build(%{
@@ -1750,21 +1712,6 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
-    BackgroundWorker.enqueue("blocks_import", %{
-      "blocker_id" => blocker.id,
-      "blocked_identifiers" => blocked_identifiers
-    })
-  end
-
-  def follow_import(%User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    BackgroundWorker.enqueue("follow_import", %{
-      "follower_id" => follower.id,
-      "followed_identifiers" => followed_identifiers
-    })
-  end
-
   def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
     Notification
     |> join(:inner, [n], activity in assoc(n, :activity))
@@ -1775,7 +1722,7 @@ defmodule Pleroma.User do
   def delete_user_activities(%User{ap_id: ap_id} = user) do
     ap_id
     |> Activity.Queries.by_actor()
-    |> RepoStreamer.chunk_stream(50)
+    |> Repo.chunk_stream(50, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity -> delete_activity(activity, user) end)
     end)
@@ -1821,12 +1768,12 @@ defmodule Pleroma.User do
 
   def html_filter_policy(_), do: Config.get([:markup, :scrub_policy])
 
-  def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
+  def fetch_by_ap_id(ap_id, opts \\ []), do: ActivityPub.make_user_from_ap_id(ap_id, opts)
 
-  def get_or_fetch_by_ap_id(ap_id) do
+  def get_or_fetch_by_ap_id(ap_id, opts \\ []) do
     cached_user = get_cached_by_ap_id(ap_id)
 
-    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id, opts)
 
     case {cached_user, maybe_fetched_user} do
       {_, {:ok, %User{} = user}} ->
@@ -1899,8 +1846,8 @@ defmodule Pleroma.User do
 
   def public_key(_), do: {:error, "key not found"}
 
-  def get_public_key_for_ap_id(ap_id) do
+  def get_public_key_for_ap_id(ap_id, opts \\ []) do
-    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id, opts),
          {:ok, public_key} <- public_key(user) do
       {:ok, public_key}
     else
@@ -2123,6 +2070,13 @@ defmodule Pleroma.User do
     Enum.map(users, &toggle_confirmation/1)
   end
 
+  @spec need_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()}
+  def need_confirmation(%User{} = user, bool) do
+    user
+    |> confirmation_changeset(need_confirmation: bool)
+    |> update_and_set_cache()
+  end
+
   def get_mascot(%{mascot: %{} = mascot}) when not is_nil(mascot) do
     mascot
   end
@@ -2337,7 +2291,9 @@ defmodule Pleroma.User do
 
     # if pinned activity was scheduled for deletion, we reschedule it for deletion
     if data["expires_at"] do
-      {:ok, expires_at, _} = DateTime.from_iso8601(data["expires_at"])
+      # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation
+      {:ok, expires_at} =
+        data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast()
 
       Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
         activity_id: id,

lib/pleroma/user/import.ex

@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Import do
+  use Ecto.Schema
+
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Workers.BackgroundWorker
+
+  require Logger
+
+  @spec perform(atom(), User.t(), list()) :: :ok | list() | {:error, any()}
+  def perform(:mutes_import, %User{} = user, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = muted_user} <- User.get_or_fetch(identifier),
+             {:ok, _} <- User.mute(user, muted_user) do
+          muted_user
+        else
+          error -> handle_error(:mutes_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:blocks_import, %User{} = blocker, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = blocked} <- User.get_or_fetch(identifier),
+             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
+          blocked
+        else
+          error -> handle_error(:blocks_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:follow_import, %User{} = follower, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = followed} <- User.get_or_fetch(identifier),
+             {:ok, follower} <- User.maybe_direct_follow(follower, followed),
+             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
+          followed
+        else
+          error -> handle_error(:follow_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(_, _, _), do: :ok
+
+  defp handle_error(op, user_id, error) do
+    Logger.debug("#{op} failed for #{user_id} with: #{inspect(error)}")
+    error
+  end
+
+  def blocks_import(%User{} = blocker, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "blocks_import",
+      %{"user_id" => blocker.id, "identifiers" => identifiers}
+    )
+  end
+
+  def follow_import(%User{} = follower, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "follow_import",
+      %{"user_id" => follower.id, "identifiers" => identifiers}
+    )
+  end
+
+  def mutes_import(%User{} = user, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "mutes_import",
+      %{"user_id" => user.id, "identifiers" => identifiers}
+    )
+  end
+end

lib/pleroma/user/query.ex

@@ -47,6 +47,7 @@ defmodule Pleroma.User.Query do
             is_moderator: boolean(),
             super_users: boolean(),
             invisible: boolean(),
+            internal: boolean(),
             followers: User.t(),
             friends: User.t(),
             recipients_from_activity: [String.t()],
@@ -80,7 +81,9 @@ defmodule Pleroma.User.Query do
   end
 
   defp prepare_query(query, criteria) do
-    Enum.reduce(criteria, query, &compose_query/2)
+    criteria
+    |> Map.put_new(:internal, false)
+    |> Enum.reduce(query, &compose_query/2)
   end
 
   defp compose_query({key, value}, query)
@@ -107,12 +110,12 @@ defmodule Pleroma.User.Query do
     where(query, [u], fragment("? && ?", u.tags, ^tags))
   end
 
-  defp compose_query({:is_admin, _}, query) do
+  defp compose_query({:is_admin, bool}, query) do
-    where(query, [u], u.is_admin)
+    where(query, [u], u.is_admin == ^bool)
   end
 
-  defp compose_query({:is_moderator, _}, query) do
+  defp compose_query({:is_moderator, bool}, query) do
-    where(query, [u], u.is_moderator)
+    where(query, [u], u.is_moderator == ^bool)
   end
 
   defp compose_query({:super_users, _}, query) do
@@ -129,14 +132,12 @@ defmodule Pleroma.User.Query do
 
   defp compose_query({:active, _}, query) do
     User.restrict_deactivated(query)
-    |> where([u], not is_nil(u.nickname))
     |> where([u], u.approval_pending == false)
   end
 
   defp compose_query({:legacy_active, _}, query) do
     query
     |> where([u], fragment("not (?->'deactivated' @> 'true')", u.info))
-    |> where([u], not is_nil(u.nickname))
   end
 
   defp compose_query({:deactivated, false}, query) do
@@ -145,7 +146,10 @@ defmodule Pleroma.User.Query do
 
   defp compose_query({:deactivated, true}, query) do
     where(query, [u], u.deactivated == ^true)
-    |> where([u], not is_nil(u.nickname))
+  end
+
+  defp compose_query({:confirmation_pending, bool}, query) do
+    where(query, [u], u.confirmation_pending == ^bool)
   end
 
   defp compose_query({:need_approval, _}, query) do
@@ -199,10 +203,15 @@ defmodule Pleroma.User.Query do
     limit(query, ^limit)
   end
 
+  defp compose_query({:internal, false}, query) do
+    query
+    |> where([u], not is_nil(u.nickname))
+    |> where([u], not like(u.nickname, "internal.%"))
+  end
+
   defp compose_query(_unsupported_param, query), do: query
 
   defp location_query(query, local) do
     where(query, [u], u.local == ^local)
-    |> where([u], not is_nil(u.nickname))
   end
 end

lib/pleroma/user/search.ex

@@ -3,8 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.User.Search do
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.Uri, as: UriType
   alias Pleroma.Pagination
   alias Pleroma.User
+
   import Ecto.Query
 
   @limit 20
@@ -19,16 +21,47 @@ defmodule Pleroma.User.Search do
 
     query_string = format_query(query_string)
 
-    maybe_resolve(resolve, for_user, query_string)
+    # If this returns anything, it should bounce to the top
+    maybe_resolved = maybe_resolve(resolve, for_user, query_string)
+
+    top_user_ids =
+      []
+      |> maybe_add_resolved(maybe_resolved)
+      |> maybe_add_ap_id_match(query_string)
+      |> maybe_add_uri_match(query_string)
 
     results =
       query_string
-      |> search_query(for_user, following)
+      |> search_query(for_user, following, top_user_ids)
       |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
 
     results
   end
 
+  defp maybe_add_resolved(list, {:ok, %User{} = user}) do
+    [user.id | list]
+  end
+
+  defp maybe_add_resolved(list, _), do: list
+
+  defp maybe_add_ap_id_match(list, query) do
+    if user = User.get_cached_by_ap_id(query) do
+      [user.id | list]
+    else
+      list
+    end
+  end
+
+  defp maybe_add_uri_match(list, query) do
+    with {:ok, query} <- UriType.cast(query),
+         q = from(u in User, where: u.uri == ^query, select: u.id),
+         users = Pleroma.Repo.all(q) do
+      users ++ list
+    else
+      _ -> list
+    end
+  end
+
   defp format_query(query_string) do
     # Strip the beginning @ off if there is a query
     query_string = String.trim_leading(query_string, "@")
@@ -47,21 +80,29 @@ defmodule Pleroma.User.Search do
     end
   end
 
-  defp search_query(query_string, for_user, following) do
+  defp search_query(query_string, for_user, following, top_user_ids) do
     for_user
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
+    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
+    |> select_top_users(top_user_ids)
     |> trigram_rank(query_string)
-    |> boost_search_rank(for_user)
+    |> boost_search_rank(for_user, top_user_ids)
     |> subquery()
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
   end
 
+  defp select_top_users(query, top_user_ids) do
+    from(u in query,
+      or_where: u.id in ^top_user_ids
+    )
+  end
+
   defp fts_search(query, query_string) do
     query_string = to_tsquery(query_string)
 
@@ -122,6 +163,10 @@ defmodule Pleroma.User.Search do
     from(q in query, where: q.invisible == false)
   end
 
+  defp filter_discoverable_users(query) do
+    from(q in query, where: q.discoverable == true)
+  end
+
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end
@@ -175,7 +220,7 @@ defmodule Pleroma.User.Search do
 
   defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
 
-  defp boost_search_rank(query, %User{} = for_user) do
+  defp boost_search_rank(query, %User{} = for_user, top_user_ids) do
     friends_ids = User.get_friends_ids(for_user)
     followers_ids = User.get_followers_ids(for_user)
 
@@ -187,6 +232,7 @@ defmodule Pleroma.User.Search do
              CASE WHEN (?) THEN (?) * 1.5
              WHEN (?) THEN (?) * 1.3
              WHEN (?) THEN (?) * 1.1
+             WHEN (?) THEN 9001
              ELSE (?) END
             """,
             u.id in ^friends_ids and u.id in ^followers_ids,
@@ -195,11 +241,26 @@ defmodule Pleroma.User.Search do
             u.search_rank,
             u.id in ^followers_ids,
             u.search_rank,
+            u.id in ^top_user_ids,
             u.search_rank
           )
       }
     )
   end
 
-  defp boost_search_rank(query, _for_user), do: query
+  defp boost_search_rank(query, _for_user, top_user_ids) do
+    from(u in subquery(query),
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+             CASE WHEN (?) THEN 9001
+             ELSE (?) END
+            """,
+            u.id in ^top_user_ids,
+            u.search_rank
+          )
+      }
+    )
+  end
 end

lib/pleroma/utils.ex

@@ -24,4 +24,24 @@ defmodule Pleroma.Utils do
   def command_available?(command) do
     match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
   end
+
+  @doc "creates the uniq temporary directory"
+  @spec tmp_dir(String.t()) :: {:ok, String.t()} | {:error, :file.posix()}
+  def tmp_dir(prefix \\ "") do
+    sub_dir =
+      [
+        prefix,
+        Timex.to_unix(Timex.now()),
+        :os.getpid(),
+        String.downcase(Integer.to_string(:rand.uniform(0x100000000), 36))
+      ]
+      |> Enum.join("-")
+
+    tmp_dir = Path.join(System.tmp_dir!(), sub_dir)
+
+    case File.mkdir(tmp_dir) do
+      :ok -> {:ok, tmp_dir}
+      error -> error
+    end
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -84,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Event]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
@@ -154,8 +154,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       {:remote_limit_pass, _} ->
         {:error, :remote_limit}
 
-      {:reject, reason} ->
+      {:reject, _} = e ->
-        {:error, reason}
+        {:error, e}
     end
   end
 
@@ -841,7 +841,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     from(
       [activity, object: o] in query,
       where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids),
-      where: fragment("not (? && ?)", activity.recipients, ^blocked_ap_ids),
+      where:
+        fragment(
+          "((not (? && ?)) or ? = ?)",
+          activity.recipients,
+          ^blocked_ap_ids,
+          activity.actor,
+          ^user.ap_id
+        ),
       where:
         fragment(
           "recipients_contain_blocked_domains(?, ?) = false",
@@ -1270,10 +1277,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
+           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address,
+             force_http: true
+           ),
          {:ok, hide_follows} <- collection_private(following_data),
          {:ok, followers_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
+           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address, force_http: true),
          {:ok, hide_followers} <- collection_private(followers_data) do
       {:ok,
        %{
@@ -1347,8 +1356,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def fetch_and_prepare_user_from_ap_id(ap_id) do
+  def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id, opts),
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
@@ -1390,13 +1399,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def make_user_from_ap_id(ap_id) do
+  def make_user_from_ap_id(ap_id, opts \\ []) do
     user = User.get_cached_by_ap_id(ap_id)
 
     if user && !User.ap_enabled?(user) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
-      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
+      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, opts) do
         if user do
           user
           |> User.remote_user_changeset(data)

lib/pleroma/web/activity_pub/mrf.ex

@@ -5,16 +5,34 @@
 defmodule Pleroma.Web.ActivityPub.MRF do
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
 
-  def filter(policies, %{} = object) do
+  def filter(policies, %{} = message) do
     policies
-    |> Enum.reduce({:ok, object}, fn
+    |> Enum.reduce({:ok, message}, fn
-      policy, {:ok, object} -> policy.filter(object)
+      policy, {:ok, message} -> policy.filter(message)
       _, error -> error
     end)
   end
 
   def filter(%{} = object), do: get_policies() |> filter(object)
 
+  def pipeline_filter(%{} = message, meta) do
+    object = meta[:object_data]
+    ap_id = message["object"]
+
+    if object && ap_id do
+      with {:ok, message} <- filter(Map.put(message, "object", object)) do
+        meta = Keyword.put(meta, :object_data, message["object"])
+        {:ok, Map.put(message, "object", ap_id), meta}
+      else
+        {err, message} -> {err, message, meta}
+      end
+    else
+      {err, message} = filter(message)
+
+      {err, message, meta}
+    end
+  end
+
   def get_policies do
     Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
   end

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -20,9 +20,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     String.match?(string, pattern)
   end
 
-  defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+  defp object_payload(%{} = object) do
+    [object["content"], object["summary"], object["name"]]
+    |> Enum.filter(& &1)
+    |> Enum.join("\n")
+  end
+
+  defp check_reject(%{"object" => %{} = object} = message) do
+    payload = object_payload(object)
+
     if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
-         string_matches?(content, pattern) or string_matches?(summary, pattern)
+         string_matches?(payload, pattern)
        end) do
       {:reject, "[KeywordPolicy] Matches with rejected keyword"}
     else
@@ -30,12 +38,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     end
   end
 
-  defp check_ftl_removal(
+  defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
-         %{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
+    payload = object_payload(object)
-       ) do
+
     if Pleroma.Constants.as_public() in to and
          Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
-           string_matches?(content, pattern) or string_matches?(summary, pattern)
+           string_matches?(payload, pattern)
          end) do
       to = List.delete(to, Pleroma.Constants.as_public())
       cc = [Pleroma.Constants.as_public() | message["cc"] || []]
@@ -51,35 +59,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     end
   end
 
-  defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+  defp check_replace(%{"object" => %{} = object} = message) do
-    content =
+    object =
-      if is_binary(content) do
+      ["content", "name", "summary"]
-        content
+      |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
-      else
+      |> Enum.reduce(object, fn field, object ->
-        ""
+        data =
-      end
-
-    summary =
-      if is_binary(summary) do
-        summary
-      else
-        ""
-      end
-
-    {content, summary} =
           Enum.reduce(
             Pleroma.Config.get([:mrf_keyword, :replace]),
-        {content, summary},
+            object[field],
-        fn {pattern, replacement}, {content_acc, summary_acc} ->
+            fn {pat, repl}, acc -> String.replace(acc, pat, repl) end
-          {String.replace(content_acc, pattern, replacement),
-           String.replace(summary_acc, pattern, replacement)}
-        end
           )
 
-    {:ok,
+        Map.put(object, field, data)
-     message
+      end)
-     |> put_in(["object", "content"], content)
+
-     |> put_in(["object", "summary"], summary)}
+    message = Map.put(message, "object", object)
+
+    {:ok, message}
   end
 
   @impl true

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -12,17 +12,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   require Logger
 
-  @options [
+  @adapter_options [
     pool: :media,
     recv_timeout: 10_000
   ]
 
   def perform(:prefetch, url) do
-    Logger.debug("Prefetching #{inspect(url)}")
+    # Fetching only proxiable resources
+    if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
+      # If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
+      prefetch_url = MediaProxy.preview_url(url)
 
-    url
+      Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
-    |> MediaProxy.url()
+
-    |> HTTP.get([], @options)
+      HTTP.get(prefetch_url, [], @adapter_options)
+    end
   end
 
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

lib/pleroma/web/activity_pub/mrf/subchain_policy.ex

@@ -28,8 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do
         }"
       )
 
-      subchain
+      MRF.filter(subchain, message)
-      |> MRF.filter(message)
     else
       _e -> {:ok, message}
     end

lib/pleroma/web/activity_pub/object_validator.ex

@@ -12,11 +12,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
@@ -149,10 +151,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
-  def validate(%{"type" => "Audio"} = object, meta) do
+  def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do
     with {:ok, object} <-
            object
-           |> AudioValidator.cast_and_validate()
+           |> AudioVideoValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Article"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> ArticleNoteValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
@@ -198,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio Event] do
+      when objtype in ~w[Question Answer Audio Video Event Article] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -232,14 +244,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     AnswerValidator.cast_and_apply(object)
   end
 
-  def cast_and_apply(%{"type" => "Audio"} = object) do
+  def cast_and_apply(%{"type" => type} = object) when type in ~w[Audio Video] do
-    AudioValidator.cast_and_apply(object)
+    AudioVideoValidator.cast_and_apply(object)
   end
 
   def cast_and_apply(%{"type" => "Event"} = object) do
     EventValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Article"} = object) do
+    ArticleNoteValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
   # is_struct/1 isn't present in Elixir 1.8.x
@@ -262,7 +278,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   def stringify_keys(object), do: object
 
   def fetch_actor(object) do
-    with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do
+    with actor <- Containment.get_actor(object),
+         {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do
       User.get_or_fetch_by_ap_id(actor)
     end
   end

lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
@@ -25,14 +25,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     # TODO: Write type
     field(:tag, {:array, :map}, default: [])
     field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
     field(:content, :string)
+
     field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
 
     # TODO: Remove actor on objects
     field(:actor, ObjectValidators.ObjectID)
 
     field(:attributedTo, ObjectValidators.ObjectID)
-    field(:summary, :string)
     field(:published, ObjectValidators.DateTime)
     field(:emoji, ObjectValidators.Emoji, default: %{})
     field(:sensitive, :boolean, default: false)
@@ -40,13 +45,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     field(:replies_count, :integer, default: 0)
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, :string)
+    field(:inReplyTo, ObjectValidators.ObjectID)
     field(:url, ObjectValidators.Uri)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
 
-    field(:likes, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
   end
 
   def cast_and_apply(data) do
@@ -62,19 +65,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
   end
 
   def cast_data(data) do
+    data = fix(data)
+
     %__MODULE__{}
     |> changeset(data)
   end
 
-  defp fix_url(%{"url" => url} = data) when is_list(url) do
+  defp fix_url(%{"url" => url} = data) when is_map(url) do
-    attachment =
+    Map.put(data, "url", url["href"])
-      Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end)
-
-    link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
-
-    data
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
   end
 
   defp fix_url(data), do: data
@@ -83,8 +81,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     data
     |> CommonFixes.fix_defaults()
     |> CommonFixes.fix_attribution()
-    |> Transmogrifier.fix_emoji()
+    |> CommonFixes.fix_actor()
     |> fix_url()
+    |> Transmogrifier.fix_emoji()
   end
 
   def changeset(struct, data) do
@@ -97,8 +96,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
 
   def validate_data(data_cng) do
     data_cng
-    |> validate_inclusion(:type, ["Audio"])
+    |> validate_inclusion(:type, ["Article", "Note"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
 
   import Ecto.Changeset
@@ -15,7 +16,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:mediaType, :string, default: "application/octet-stream")
     field(:name, :string)
 
-    embeds_many(:url, UrlObjectValidator)
+    embeds_many :url, UrlObjectValidator, primary_key: false do
+      field(:type, :string)
+      field(:href, ObjectValidators.Uri)
+      field(:mediaType, :string, default: "application/octet-stream")
+    end
   end
 
   def cast_and_validate(data) do
@@ -37,7 +42,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
 
     struct
     |> cast(data, [:type, :mediaType, :name])
-    |> cast_embed(:url, required: true)
+    |> cast_embed(:url, with: &url_changeset/2)
+    |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
+    |> validate_required([:type, :mediaType, :url])
+  end
+
+  def url_changeset(struct, data) do
+    data = fix_media_type(data)
+
+    struct
+    |> cast(data, [:type, :href, :mediaType])
+    |> validate_inclusion(:type, ["Link"])
+    |> validate_required([:type, :href, :mediaType])
   end
 
   def fix_media_type(data) do
@@ -75,6 +91,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
 
   def validate_data(cng) do
     cng
+    |> validate_inclusion(:type, ~w[Document Audio Image Video])
     |> validate_required([:mediaType, :url, :type])
   end
 end

lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex

@@ -0,0 +1,134 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EarmarkRenderer
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:bto, ObjectValidators.Recipients, default: [])
+    field(:bcc, ObjectValidators.Recipients, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
+    field(:content, :string)
+
+    field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:published, ObjectValidators.DateTime)
+    field(:emoji, ObjectValidators.Emoji, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, ObjectValidators.ObjectID)
+    field(:url, ObjectValidators.Uri)
+
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix_url(%{"url" => url} = data) when is_list(url) do
+    attachment =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and String.starts_with?(mime_type, ["video/", "audio/"])
+      end)
+
+    link_element =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and mime_type == "text/html"
+      end)
+
+    data
+    |> Map.put("attachment", [attachment])
+    |> Map.put("url", link_element["href"])
+  end
+
+  defp fix_url(data), do: data
+
+  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
+       when is_binary(content) do
+    content =
+      content
+      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
+      |> Pleroma.HTML.filter_tags()
+
+    Map.put(data, "content", content)
+  end
+
+  defp fix_content(data), do: data
+
+  defp fix(data) do
+    data
+    |> CommonFixes.fix_defaults()
+    |> CommonFixes.fix_attribution()
+    |> CommonFixes.fix_actor()
+    |> Transmogrifier.fix_emoji()
+    |> fix_url()
+    |> fix_content()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast_embed(:attachment)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Audio", "Video"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end

lib/pleroma/web/activity_pub/object_validators/common_fixes.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
+  alias Pleroma.Object.Containment
   alias Pleroma.Web.ActivityPub.Utils
 
   # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
@@ -19,4 +20,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
     data
     |> Map.put_new("actor", data["attributedTo"])
   end
+
+  def fix_actor(data) do
+    actor = Containment.get_actor(data)
+
+    data
+    |> Map.put("actor", actor)
+    |> Map.put("attributedTo", actor)
+  end
 end

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -10,9 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   import Ecto.Changeset
-  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   @primary_key false
 
@@ -75,14 +76,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     data
     |> fix_context(meta)
     |> fix_addressing(meta)
+    |> CommonFixes.fix_actor()
   end
 
   def validate_data(cng, meta \\ []) do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])
-    |> validate_actor_presence()
+    |> CommonValidations.validate_actor_presence()
-    |> validate_any_presence([:to, :cc])
+    |> CommonValidations.validate_any_presence([:to, :cc])
     |> validate_actors_match(meta)
     |> validate_context_match(meta)
     |> validate_object_nonexistence()

lib/pleroma/web/activity_pub/object_validators/note_validator.ex

@@ -1,73 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.Transmogrifier
-
-  import Ecto.Changeset
-
-  @primary_key false
-
-  embedded_schema do
-    field(:id, ObjectValidators.ObjectID, primary_key: true)
-    field(:to, ObjectValidators.Recipients, default: [])
-    field(:cc, ObjectValidators.Recipients, default: [])
-    field(:bto, ObjectValidators.Recipients, default: [])
-    field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
-    field(:tag, {:array, :map}, default: [])
-    field(:type, :string)
-
-    field(:name, :string)
-    field(:summary, :string)
-    field(:content, :string)
-
-    field(:context, :string)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
-
-    field(:actor, ObjectValidators.ObjectID)
-    field(:attributedTo, ObjectValidators.ObjectID)
-    field(:published, ObjectValidators.DateTime)
-    field(:emoji, ObjectValidators.Emoji, default: %{})
-    field(:sensitive, :boolean, default: false)
-    # TODO: Write type
-    field(:attachment, {:array, :map}, default: [])
-    field(:replies_count, :integer, default: 0)
-    field(:like_count, :integer, default: 0)
-    field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, ObjectValidators.ObjectID)
-    field(:url, ObjectValidators.Uri)
-
-    field(:likes, {:array, :string}, default: [])
-    field(:announcements, {:array, :string}, default: [])
-  end
-
-  def cast_and_validate(data) do
-    data
-    |> cast_data()
-    |> validate_data()
-  end
-
-  defp fix(data) do
-    data
-    |> Transmogrifier.fix_emoji()
-  end
-
-  def cast_data(data) do
-    data = fix(data)
-
-    %__MODULE__{}
-    |> cast(data, __schema__(:fields))
-  end
-
-  def validate_data(data_cng) do
-    data_cng
-    |> validate_inclusion(:type, ["Note"])
-    |> validate_required([:id, :actor, :to, :cc, :type, :content, :context])
-  end
-end

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -47,8 +47,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     # short identifier for PleromaFE to group statuses by context
     field(:context_id, :integer)
 
-    field(:likes, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
 
     field(:closed, ObjectValidators.DateTime)
     field(:voters, {:array, ObjectValidators.ObjectID}, default: [])

lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex

@@ -1,24 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-
-  import Ecto.Changeset
-  @primary_key false
-
-  embedded_schema do
-    field(:type, :string)
-    field(:href, ObjectValidators.Uri)
-    field(:mediaType, :string, default: "application/octet-stream")
-  end
-
-  def changeset(struct, data) do
-    struct
-    |> cast(data, __schema__(:fields))
-    |> validate_required([:type, :href, :mediaType])
-  end
-end

lib/pleroma/web/activity_pub/pipeline.ex

@@ -26,13 +26,17 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
 
       {:error, e} ->
         {:error, e}
+
+      {:reject, e} ->
+        {:reject, e}
     end
   end
 
   def do_common_pipeline(object, meta) do
     with {_, {:ok, validated_object, meta}} <-
            {:validate_object, ObjectValidator.validate(object, meta)},
-         {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
+         {_, {:ok, mrfd_object, meta}} <-
+           {:mrf_object, MRF.pipeline_filter(validated_object, meta)},
          {_, {:ok, activity, meta}} <-
            {:persist_object, ActivityPub.persist(mrfd_object, meta)},
          {_, {:ok, activity, meta}} <-
@@ -40,7 +44,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
          {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
       {:ok, activity, meta}
     else
-      {:mrf_object, {:reject, _}} -> {:ok, nil, meta}
+      {:mrf_object, {:reject, message, _}} -> {:reject, message}
       e -> {:error, e}
     end
   end

lib/pleroma/web/activity_pub/publisher.ex

@@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.FedSockets
 
   require Pleroma.Constants
 
@@ -50,15 +51,35 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
     Logger.debug("Federating #{id} to #{inbox}")
 
-    uri = URI.parse(inbox)
+    case FedSockets.get_or_create_fed_socket(inbox) do
+      {:ok, fedsocket} ->
+        Logger.debug("publishing via fedsockets - #{inspect(inbox)}")
+        FedSockets.publish(fedsocket, json)
 
+      _ ->
+        Logger.debug("publishing via http - #{inspect(inbox)}")
+        http_publish(inbox, actor, json, params)
+    end
+  end
+
+  def publish_one(%{actor_id: actor_id} = params) do
+    actor = User.get_cached_by_id(actor_id)
+
+    params
+    |> Map.delete(:actor_id)
+    |> Map.put(:actor, actor)
+    |> publish_one()
+  end
+
+  defp http_publish(inbox, actor, json, params) do
+    uri = %{path: path} = URI.parse(inbox)
     digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
 
     date = Pleroma.Signature.signed_date()
 
     signature =
       Pleroma.Signature.sign(actor, %{
-        "(request-target)": "post #{uri.path}",
+        "(request-target)": "post #{path}",
         host: signature_host(uri),
         "content-length": byte_size(json),
         digest: digest,
@@ -89,15 +110,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     end
   end
 
-  def publish_one(%{actor_id: actor_id} = params) do
-    actor = User.get_cached_by_id(actor_id)
-
-    params
-    |> Map.delete(:actor_id)
-    |> Map.put(:actor, actor)
-    |> publish_one()
-  end
-
   defp signature_host(%URI{port: port, scheme: scheme, host: host}) do
     if port == URI.default_port(scheme) do
       host

lib/pleroma/web/activity_pub/relay.ex

@@ -30,12 +30,16 @@ defmodule Pleroma.Web.ActivityPub.Relay do
     end
   end
 
-  @spec unfollow(String.t()) :: {:ok, Activity.t()} | {:error, any()}
+  @spec unfollow(String.t(), map()) :: {:ok, Activity.t()} | {:error, any()}
-  def unfollow(target_instance) do
+  def unfollow(target_instance, opts \\ %{}) do
     with %User{} = local_user <- get_actor(),
-         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, target_user} <- fetch_target_user(target_instance, opts),
          {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do
-      User.unfollow(local_user, target_user)
+      case target_user.id do
+        nil -> User.update_following_count(local_user)
+        _ -> User.unfollow(local_user, target_user)
+      end
+
       Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}")
       {:ok, activity}
     else
@@ -43,6 +47,14 @@ defmodule Pleroma.Web.ActivityPub.Relay do
     end
   end
 
+  defp fetch_target_user(ap_id, opts) do
+    case {opts[:force], User.get_or_fetch_by_ap_id(ap_id)} do
+      {_, {:ok, %User{} = user}} -> {:ok, user}
+      {true, _} -> {:ok, %User{ap_id: ap_id}}
+      {_, error} -> error
+    end
+  end
+
   @spec publish(any()) :: {:ok, Activity.t()} | {:error, any()}
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),

lib/pleroma/web/activity_pub/side_effects.ex

@@ -336,7 +336,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Question Event] do
+      when objtype in ~w[Audio Video Question Event Article] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   A module to handle coding from internal to wire ActivityPub and back.
   """
   alias Pleroma.Activity
-  alias Pleroma.EarmarkRenderer
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Maps
   alias Pleroma.Object
@@ -45,7 +44,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
-    |> fix_content
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -274,24 +272,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     Map.put(object, "url", url["href"])
   end
 
-  def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
+  def fix_url(%{"url" => url} = object) when is_list(url) do
-    attachment =
-      Enum.find(url, fn x ->
-        media_type = x["mediaType"] || x["mimeType"] || ""
-
-        is_map(x) and String.starts_with?(media_type, "video/")
-      end)
-
-    link_element =
-      Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end)
-
-    object
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
-  end
-
-  def fix_url(%{"type" => object_type, "url" => url} = object)
-      when object_type != "Video" and is_list(url) do
     first_element = Enum.at(url, 0)
 
     url_string =
@@ -371,18 +352,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_type(object, _), do: object
 
-  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object)
-       when is_binary(content) do
-    html_content =
-      content
-      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
-      |> Pleroma.HTML.filter_tags()
-
-    Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"})
-  end
-
-  defp fix_content(object), do: object
-
   # Reduce the object list to find the reported user.
   defp get_reported(objects) do
     Enum.reduce_while(objects, nil, fn ap_id, _ ->
@@ -455,7 +424,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ~w{Article Note Video Page} do
+      when objtype in ~w{Note Page} do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -546,13 +515,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => objtype}} = data,
+        %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data,
         _options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio Event} do
+      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
+    data = Map.put(data, "object", strip_internal_fields(data["object"]))
+
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
+         nil <- Activity.get_create_by_object_ap_id(obj_id),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
+    else
+      %Activity{} = activity -> {:ok, activity}
+      e -> e
     end
   end
 
@@ -1029,7 +1004,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def upgrade_user_from_ap_id(ap_id) do
     with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),
-         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),
+         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id, force_http: true),
          {:ok, user} <- update_user(user, data) do
       TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
       {:ok, user}

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -23,8 +23,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router
 
-  require Logger
-
   @users_page_size 50
 
   plug(
@@ -68,6 +66,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [:list_user_statuses, :list_instance_statuses]
   )
 
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:chats"], admin: true}
+    when action in [:list_user_chats]
+  )
+
   plug(
     OAuthScopesPlug,
     %{scopes: ["read"], admin: true}
@@ -256,6 +260,20 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = _params) do
+    with %User{id: user_id} <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
+      chats =
+        Pleroma.Chat.for_user_query(user_id)
+        |> Pleroma.Repo.all()
+
+      conn
+      |> put_view(AdminAPI.ChatView)
+      |> render("index.json", chats: chats)
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
   def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     user = User.get_cached_by_nickname(nickname)
 

lib/pleroma/web/admin_api/controllers/chat_controller.ex

@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ChatController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Activity
+  alias Pleroma.Chat
+  alias Pleroma.Chat.MessageReference
+  alias Pleroma.ModerationLog
+  alias Pleroma.Pagination
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:chats"], admin: true} when action in [:show, :messages]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:chats"], admin: true} when action in [:delete_message]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ChatOperation
+
+  def delete_message(%{assigns: %{user: user}} = conn, %{
+        message_id: message_id,
+        id: chat_id
+      }) do
+    with %MessageReference{object: %{data: %{"id" => object_ap_id}}} = cm_ref <-
+           MessageReference.get_by_id(message_id),
+         ^chat_id <- to_string(cm_ref.chat_id),
+         %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object_ap_id),
+         {:ok, _} <- CommonAPI.delete(activity_id, user) do
+      ModerationLog.insert_log(%{
+        action: "chat_message_delete",
+        actor: user,
+        subject_id: message_id
+      })
+
+      conn
+      |> put_view(MessageReferenceView)
+      |> render("show.json", chat_message_reference: cm_ref)
+    else
+      _e ->
+        {:error, :could_not_delete}
+    end
+  end
+
+  def messages(conn, %{id: id} = params) do
+    with %Chat{} = chat <- Chat.get_by_id(id) do
+      cm_refs =
+        chat
+        |> MessageReference.for_chat_query()
+        |> Pagination.fetch_paginated(params)
+
+      conn
+      |> put_view(MessageReferenceView)
+      |> render("index.json", chat_message_references: cm_refs)
+    else
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(%{error: "not found"})
+    end
+  end
+
+  def show(conn, %{id: id}) do
+    with %Chat{} = chat <- Chat.get_by_id(id) do
+      conn
+      |> put_view(AdminAPI.ChatView)
+      |> render("show.json", chat: chat)
+    end
+  end
+end

lib/pleroma/web/admin_api/controllers/instance_document_controller.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.InstanceStatic
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.InstanceDocument
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation
+
+  plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :show)
+  plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action in [:update, :delete])
+
+  def show(conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.get(document_name),
+         {:ok, content} <- File.read(InstanceStatic.file_path(url)) do
+      conn
+      |> put_resp_content_type("text/html")
+      |> send_resp(200, content)
+    end
+  end
+
+  def update(%{body_params: %{file: file}} = conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.put(document_name, file.path) do
+      json(conn, %{"url" => url})
+    end
+  end
+
+  def delete(conn, %{name: document_name}) do
+    with :ok <- InstanceDocument.delete(document_name) do
+      json(conn, %{})
+    end
+  end
+end

lib/pleroma/web/admin_api/controllers/relay_controller.ex

@@ -33,11 +33,7 @@ defmodule Pleroma.Web.AdminAPI.RelayController do
 
   def follow(%{assigns: %{user: admin}, body_params: %{relay_url: target}} = conn, _) do
     with {:ok, _message} <- Relay.follow(target) do
-      ModerationLog.insert_log(%{
+      ModerationLog.insert_log(%{action: "relay_follow", actor: admin, target: target})
-        action: "relay_follow",
-        actor: admin,
-        target: target
-      })
 
       json(conn, %{actor: target, followed_back: target in Relay.following()})
     else
@@ -48,13 +44,9 @@ defmodule Pleroma.Web.AdminAPI.RelayController do
     end
   end
 
-  def unfollow(%{assigns: %{user: admin}, body_params: %{relay_url: target}} = conn, _) do
+  def unfollow(%{assigns: %{user: admin}, body_params: %{relay_url: target} = params} = conn, _) do
-    with {:ok, _message} <- Relay.unfollow(target) do
+    with {:ok, _message} <- Relay.unfollow(target, %{force: params[:force]}) do
-      ModerationLog.insert_log(%{
+      ModerationLog.insert_log(%{action: "relay_unfollow", actor: admin, target: target})
-        action: "relay_unfollow",
-        actor: admin,
-        target: target
-      })
 
       json(conn, target)
     else

lib/pleroma/web/admin_api/views/chat_view.ex

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ChatView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Chat
+  alias Pleroma.User
+  alias Pleroma.Web.MastodonAPI
+  alias Pleroma.Web.PleromaAPI
+
+  def render("index.json", %{chats: chats} = opts) do
+    render_many(chats, __MODULE__, "show.json", Map.delete(opts, :chats))
+  end
+
+  def render("show.json", %{chat: %Chat{user_id: user_id}} = opts) do
+    user = User.get_by_id(user_id)
+    sender = MastodonAPI.AccountView.render("show.json", user: user, skip_visibility_check: true)
+
+    serialized_chat = PleromaAPI.ChatView.render("show.json", opts)
+
+    serialized_chat
+    |> Map.put(:sender, sender)
+    |> Map.put(:receiver, serialized_chat[:account])
+    |> Map.delete(:account)
+  end
+
+  def render(view, opts), do: PleromaAPI.ChatView.render(view, opts)
+end

lib/pleroma/web/admin_api/views/status_view.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.AdminAPI.StatusView do
   require Pleroma.Constants
 
   alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI
 
   defdelegate merge_account_views(user), to: AdminAPI.AccountView
@@ -17,7 +18,7 @@ defmodule Pleroma.Web.AdminAPI.StatusView do
   end
 
   def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
-    user = MastodonAPI.StatusView.get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
 
     MastodonAPI.StatusView.render("show.json", opts)
     |> Map.merge(%{account: merge_account_views(user)})

lib/pleroma/web/api_spec.ex

@@ -13,10 +13,15 @@ defmodule Pleroma.Web.ApiSpec do
   @impl OpenApi
   def spec do
     %OpenApi{
-      servers: [
+      servers:
+        if Phoenix.Endpoint.server?(:pleroma, Endpoint) do
+          [
             # Populate the Server info from a phoenix endpoint
             OpenApiSpex.Server.from_endpoint(Endpoint)
-      ],
+          ]
+        else
+          []
+        end,
       info: %OpenApiSpex.Info{
         title: "Pleroma",
         description: Application.spec(:pleroma, :description) |> to_string(),

lib/pleroma/web/api_spec/helpers.ex

@@ -72,7 +72,11 @@ defmodule Pleroma.Web.ApiSpec.Helpers do
   end
 
   def empty_array_response do
-    Operation.response("Empty array", "application/json", %Schema{type: :array, example: []})
+    Operation.response("Empty array", "application/json", %Schema{
+      type: :array,
+      items: %Schema{type: :object, example: %{}},
+      example: []
+    })
   end
 
   def no_content_response do

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -372,6 +372,10 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
       tags: ["accounts"],
       summary: "Identity proofs",
       operationId: "AccountController.identity_proofs",
+      # Validators complains about unused path params otherwise
+      parameters: [
+        %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}
+      ],
       description: "Not implemented",
       responses: %{
         200 => empty_array_response()
@@ -469,7 +473,6 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
         identifier: %Schema{type: :string},
         message: %Schema{type: :string}
       },
-      required: [],
       # Note: example of successful registration with failed login response:
       # example: %{
       #   "identifier" => "missing_confirmed_email",
@@ -530,7 +533,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           nullable: true,
           oneOf: [
             %Schema{type: :array, items: attribute_field()},
-            %Schema{type: :object, additionalProperties: %Schema{type: attribute_field()}}
+            %Schema{type: :object, additionalProperties: attribute_field()}
           ]
         },
         # NOTE: `source` field is not supported

lib/pleroma/web/api_spec/operations/admin/chat_operation.ex

@@ -0,0 +1,96 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
+  alias OpenApiSpex.Operation
+  alias Pleroma.Web.ApiSpec.Schemas.Chat
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def delete_message_operation do
+    %Operation{
+      tags: ["admin", "chat"],
+      summary: "Delete an individual chat message",
+      operationId: "AdminAPI.ChatController.delete_message",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "The ID of the Chat"),
+        Operation.parameter(:message_id, :path, :string, "The ID of the message")
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The deleted ChatMessage",
+            "application/json",
+            ChatMessage
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["write:chats"]
+        }
+      ]
+    }
+  end
+
+  def messages_operation do
+    %Operation{
+      tags: ["admin", "chat"],
+      summary: "Get the most recent messages of the chat",
+      operationId: "AdminAPI.ChatController.messages",
+      parameters:
+        [Operation.parameter(:id, :path, :string, "The ID of the Chat")] ++
+          pagination_params(),
+      responses: %{
+        200 =>
+          Operation.response(
+            "The messages in the chat",
+            "application/json",
+            Pleroma.Web.ApiSpec.ChatOperation.chat_messages_response()
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["read:chats"]
+        }
+      ]
+    }
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Create a chat",
+      operationId: "AdminAPI.ChatController.show",
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "The id of the chat",
+          required: true,
+          example: "1234"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The existing chat",
+            "application/json",
+            Chat
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["read"]
+        }
+      ]
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex

@@ -0,0 +1,115 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Helpers
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.show",
+      security: [%{"oAuth" => ["read"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => document_content(),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Update the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.update",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: Helpers.request_body("Parameters", update_request()),
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      title: "UpdateRequest",
+      description: "POST body for uploading the file",
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          type: :string,
+          format: :binary,
+          description: "The file to be uploaded, using multipart form data."
+        }
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.delete",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp instance_document do
+    %Schema{
+      title: "InstanceDocument",
+      type: :object,
+      properties: %{
+        url: %Schema{type: :string}
+      },
+      example: %{
+        "url" => "https://example.com/static/terms-of-service.html"
+      }
+    }
+  end
+
+  defp document_content do
+    Operation.response("InstanceDocumentContent", "text/html", %Schema{
+      type: :string,
+      example: "<h1>Instance panel</h1>"
+    })
+  end
+end

lib/pleroma/web/api_spec/operations/admin/relay_operation.ex

@@ -56,7 +56,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do
       operationId: "AdminAPI.RelayController.unfollow",
       security: [%{"oAuth" => ["write:follows"]}],
       parameters: admin_api_params(),
-      requestBody: request_body("Parameters", relay_url()),
+      requestBody: request_body("Parameters", relay_unfollow()),
       responses: %{
         200 =>
           Operation.response("Status", "application/json", %Schema{
@@ -91,4 +91,14 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do
       }
     }
   end
+
+  defp relay_unfollow do
+    %Schema{
+      type: :object,
+      properties: %{
+        relay_url: %Schema{type: :string, format: :uri},
+        force: %Schema{type: :boolean, default: false}
+      }
+    }
+  end
 end

lib/pleroma/web/api_spec/operations/chat_operation.ex

@@ -158,7 +158,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
             "The messages in the chat",
             "application/json",
             chat_messages_response()
-          )
+          ),
+        404 => Operation.response("Not Found", "application/json", ApiError)
       },
       security: [
         %{
@@ -184,7 +185,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
             "application/json",
             ChatMessage
           ),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        422 => Operation.response("MRF Rejection", "application/json", ApiError)
       },
       security: [
         %{

lib/pleroma/web/api_spec/operations/custom_emoji_operation.ex

@@ -69,7 +69,7 @@ defmodule Pleroma.Web.ApiSpec.CustomEmojiOperation do
           type: :object,
           properties: %{
             category: %Schema{type: :string},
-            tags: %Schema{type: :array}
+            tags: %Schema{type: :array, items: %Schema{type: :string}}
           }
         }
       ],

lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex

@@ -23,7 +23,7 @@ defmodule Pleroma.Web.ApiSpec.EmojiReactionOperation do
       parameters: [
         Operation.parameter(:id, :path, FlakeID, "Status ID", required: true),
         Operation.parameter(:emoji, :path, :string, "Filter by a single unicode emoji",
-          required: false
+          required: nil
         )
       ],
       security: [%{"oAuth" => ["read:statuses"]}],

lib/pleroma/web/api_spec/operations/list_operation.ex

@@ -187,8 +187,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
         type: :object,
         properties: %{
           account_ids: %Schema{type: :array, description: "Array of account IDs", items: FlakeID}
-        },
+        }
-        required: required && [:account_ids]
       },
       required: required
     )

lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex

@@ -0,0 +1,139 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.add_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", create_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp create_request do
+    %Schema{
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          description:
+            "File needs to be uploaded with the multipart request or link to remote file",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string, format: :uri}
+          ]
+        },
+        shortcode: %Schema{
+          type: :string,
+          description:
+            "Shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename."
+        },
+        filename: %Schema{
+          type: :string,
+          description:
+            "New emoji file name. If not specified will be taken from original filename."
+        }
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.update_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      required: [:shortcode, :new_shortcode, :new_filename],
+      properties: %{
+        shortcode: %Schema{
+          type: :string,
+          description: "Emoji file shortcode"
+        },
+        new_shortcode: %Schema{
+          type: :string,
+          description: "New emoji file shortcode"
+        },
+        new_filename: %Schema{
+          type: :string,
+          description: "New filename for emoji file"
+        },
+        force: %Schema{
+          type: :boolean,
+          description: "With true value to overwrite existing emoji with new shortcode",
+          default: false
+        }
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Delete emoji file from pack",
+      operationId: "PleromaAPI.EmojiPackController.delete_file",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        name_param(),
+        Operation.parameter(:shortcode, :query, :string, "File shortcode",
+          example: "cofe",
+          required: true
+        )
+      ],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp name_param do
+    Operation.parameter(:name, :path, :string, "Pack Name", example: "cofe", required: true)
+  end
+
+  defp files_object do
+    %Schema{
+      type: :object,
+      additionalProperties: %Schema{type: :string},
+      description: "Object with emoji names as keys and filenames as values"
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

@@ -175,111 +175,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
     }
   end
 
-  def add_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Add new file to the pack",
-      operationId: "PleromaAPI.EmojiPackController.add_file",
-      security: [%{"oAuth" => ["write"]}],
-      requestBody: request_body("Parameters", add_file_request(), required: true),
-      parameters: [name_param()],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError),
-        409 => Operation.response("Conflict", "application/json", ApiError)
-      }
-    }
-  end
-
-  defp add_file_request do
-    %Schema{
-      type: :object,
-      required: [:file],
-      properties: %{
-        file: %Schema{
-          description:
-            "File needs to be uploaded with the multipart request or link to remote file",
-          anyOf: [
-            %Schema{type: :string, format: :binary},
-            %Schema{type: :string, format: :uri}
-          ]
-        },
-        shortcode: %Schema{
-          type: :string,
-          description:
-            "Shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename."
-        },
-        filename: %Schema{
-          type: :string,
-          description:
-            "New emoji file name. If not specified will be taken from original filename."
-        }
-      }
-    }
-  end
-
-  def update_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Add new file to the pack",
-      operationId: "PleromaAPI.EmojiPackController.update_file",
-      security: [%{"oAuth" => ["write"]}],
-      requestBody: request_body("Parameters", update_file_request(), required: true),
-      parameters: [name_param()],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError),
-        409 => Operation.response("Conflict", "application/json", ApiError)
-      }
-    }
-  end
-
-  defp update_file_request do
-    %Schema{
-      type: :object,
-      required: [:shortcode, :new_shortcode, :new_filename],
-      properties: %{
-        shortcode: %Schema{
-          type: :string,
-          description: "Emoji file shortcode"
-        },
-        new_shortcode: %Schema{
-          type: :string,
-          description: "New emoji file shortcode"
-        },
-        new_filename: %Schema{
-          type: :string,
-          description: "New filename for emoji file"
-        },
-        force: %Schema{
-          type: :boolean,
-          description: "With true value to overwrite existing emoji with new shortcode",
-          default: false
-        }
-      }
-    }
-  end
-
-  def delete_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Delete emoji file from pack",
-      operationId: "PleromaAPI.EmojiPackController.delete_file",
-      security: [%{"oAuth" => ["write"]}],
-      parameters: [
-        name_param(),
-        Operation.parameter(:shortcode, :query, :string, "File shortcode",
-          example: "cofe",
-          required: true
-        )
-      ],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
-      }
-    }
-  end
-
   def import_from_filesystem_operation do
     %Operation{
       tags: ["Emoji Packs"],

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -55,7 +55,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
             "application/json",
             %Schema{oneOf: [Status, ScheduledStatus]}
           ),
-        422 => Operation.response("Bad Request", "application/json", ApiError)
+        422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError)
       }
     }
   end

lib/pleroma/web/api_spec/operations/user_import_operation.ex

@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.UserImportOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  @spec open_api_operation(atom) :: Operation.t()
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def follow_operation do
+    %Operation{
+      tags: ["follow_import"],
+      summary: "Imports your follows.",
+      operationId: "UserImportController.follow",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:follow"]}]
+    }
+  end
+
+  def blocks_operation do
+    %Operation{
+      tags: ["blocks_import"],
+      summary: "Imports your blocks.",
+      operationId: "UserImportController.blocks",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:blocks"]}]
+    }
+  end
+
+  def mutes_operation do
+    %Operation{
+      tags: ["mutes_import"],
+      summary: "Imports your mutes.",
+      operationId: "UserImportController.mutes",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:mutes"]}]
+    }
+  end
+
+  defp import_request do
+    %Schema{
+      type: :object,
+      required: [:list],
+      properties: %{
+        list: %Schema{
+          description:
+            "STRING or FILE containing a whitespace-separated list of accounts to import.",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string}
+          ]
+        }
+      }
+    }
+  end
+
+  defp ok_response do
+    Operation.response("Ok", "application/json", %Schema{type: :string, example: "ok"})
+  end
+end

lib/pleroma/web/api_spec/schemas/chat_message.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
   alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Emoji
 
   require OpenApiSpex
 
@@ -18,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
       chat_id: %Schema{type: :string},
       content: %Schema{type: :string, nullable: true},
       created_at: %Schema{type: :string, format: :"date-time"},
-      emojis: %Schema{type: :array},
+      emojis: %Schema{type: :array, items: Emoji},
       attachment: %Schema{type: :object, nullable: true},
       card: %Schema{
         type: :object,

lib/pleroma/web/api_spec/schemas/scheduled_status.ex

@@ -27,9 +27,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ScheduledStatus do
           media_ids: %Schema{type: :array, nullable: true, items: %Schema{type: :string}},
           sensitive: %Schema{type: :boolean, nullable: true},
           spoiler_text: %Schema{type: :string, nullable: true},
-          visibility: %Schema{type: VisibilityScope, nullable: true},
+          visibility: %Schema{allOf: [VisibilityScope], nullable: true},
           scheduled_at: %Schema{type: :string, format: :"date-time", nullable: true},
-          poll: %Schema{type: Poll, nullable: true},
+          poll: %Schema{allOf: [Poll], nullable: true},
           in_reply_to_id: %Schema{type: :string, nullable: true}
         }
       }

lib/pleroma/web/common_api/common_api.ex

@@ -48,6 +48,9 @@ defmodule Pleroma.Web.CommonAPI do
               local: true
             )} do
       {:ok, activity}
+    else
+      {:common_pipeline, {:reject, _} = e} -> e
+      e -> e
     end
   end
 
@@ -550,4 +553,21 @@ defmodule Pleroma.Web.CommonAPI do
   def show_reblogs(%User{} = user, %User{} = target) do
     UserRelationship.delete_reblog_mute(user, target)
   end
+
+  def get_user(ap_id, fake_record_fallback \\ true) do
+    cond do
+      user = User.get_cached_by_ap_id(ap_id) ->
+        user
+
+      user = User.get_by_guessed_nickname(ap_id) ->
+        user
+
+      fake_record_fallback ->
+        # TODO: refactor (fake records is never a good idea)
+        User.error_user(ap_id)
+
+      true ->
+        nil
+    end
+  end
 end

lib/pleroma/web/controller_helper.ex

@@ -48,13 +48,13 @@ defmodule Pleroma.Web.ControllerHelper do
 
   defp param_to_integer(_, default), do: default
 
-  def add_link_headers(conn, activities, extra_params \\ %{})
+  def add_link_headers(conn, entries, extra_params \\ %{})
 
-  def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _activities, _extra_params),
+  def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _entries, _extra_params),
     do: conn
 
-  def add_link_headers(conn, activities, extra_params) do
+  def add_link_headers(conn, entries, extra_params) do
-    case get_pagination_fields(conn, activities, extra_params) do
+    case get_pagination_fields(conn, entries, extra_params) do
       %{"next" => next_url, "prev" => prev_url} ->
         put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"")
 
@@ -78,19 +78,15 @@ defmodule Pleroma.Web.ControllerHelper do
     }
   end
 
-  def get_pagination_fields(conn, activities, extra_params \\ %{}) do
+  def get_pagination_fields(conn, entries, extra_params \\ %{}) do
-    case List.last(activities) do
+    case List.last(entries) do
       %{pagination_id: max_id} when not is_nil(max_id) ->
-        %{pagination_id: min_id} =
+        %{pagination_id: min_id} = List.first(entries)
-          activities
-          |> List.first()
 
         build_pagination_fields(conn, min_id, max_id, extra_params)
 
       %{id: max_id} ->
-        %{id: min_id} =
+        %{id: min_id} = List.first(entries)
-          activities
-          |> List.first()
 
         build_pagination_fields(conn, min_id, max_id, extra_params)
 

lib/pleroma/web/fed_sockets/fed_registry.ex

@@ -0,0 +1,185 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.FedRegistry do
+  @moduledoc """
+  The FedRegistry stores the active FedSockets for quick retrieval.
+
+  The storage and retrieval portion of the FedRegistry is done in process through
+  elixir's `Registry` module for speed and its ability to monitor for terminated processes.
+
+  Dropped connections will be caught by `Registry` and deleted. Since the next
+  message will initiate a new connection there is no reason to try and reconnect at that point.
+
+  Normally outside modules should have no need to call or use the FedRegistry themselves.
+  """
+
+  alias Pleroma.Web.FedSockets.FedSocket
+  alias Pleroma.Web.FedSockets.SocketInfo
+
+  require Logger
+
+  @default_rejection_duration 15 * 60 * 1000
+  @rejections :fed_socket_rejections
+
+  @doc """
+  Retrieves a FedSocket from the Registry given it's origin.
+
+  The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
+
+  Will return:
+    * {:ok, fed_socket} for working FedSockets
+    * {:error, :rejected} for origins that have been tried and refused within the rejection duration interval
+    * {:error, some_reason} usually :missing for unknown origins
+  """
+  def get_fed_socket(origin) do
+    case get_registry_data(origin) do
+      {:error, reason} ->
+        {:error, reason}
+
+      {:ok, %{state: :connected} = socket_info} ->
+        {:ok, socket_info}
+    end
+  end
+
+  @doc """
+  Adds a connected FedSocket to the Registry.
+
+  Always returns {:ok, fed_socket}
+  """
+  def add_fed_socket(origin, pid \\ nil) do
+    origin
+    |> SocketInfo.build(pid)
+    |> SocketInfo.connect()
+    |> add_socket_info
+  end
+
+  defp add_socket_info(%{origin: origin, state: :connected} = socket_info) do
+    case Registry.register(FedSockets.Registry, origin, socket_info) do
+      {:ok, _owner} ->
+        clear_prior_rejection(origin)
+        Logger.debug("fedsocket added: #{inspect(origin)}")
+
+        {:ok, socket_info}
+
+      {:error, {:already_registered, _pid}} ->
+        FedSocket.close(socket_info)
+        existing_socket_info = Registry.lookup(FedSockets.Registry, origin)
+
+        {:ok, existing_socket_info}
+
+      _ ->
+        {:error, :error_adding_socket}
+    end
+  end
+
+  @doc """
+  Mark this origin as having rejected a connection attempt.
+  This will keep it from getting additional connection attempts
+  for a period of time specified in the config.
+
+  Always returns {:ok, new_reg_data}
+  """
+  def set_host_rejected(uri) do
+    new_reg_data =
+      uri
+      |> SocketInfo.origin()
+      |> get_or_create_registry_data()
+      |> set_to_rejected()
+      |> save_registry_data()
+
+    {:ok, new_reg_data}
+  end
+
+  @doc """
+  Retrieves the FedRegistryData from the Registry given it's origin.
+
+  The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
+
+  Will return:
+    * {:ok, fed_registry_data} for known origins
+    * {:error, :missing} for uniknown origins
+    * {:error, :cache_error} indicating some low level runtime issues
+  """
+  def get_registry_data(origin) do
+    case Registry.lookup(FedSockets.Registry, origin) do
+      [] ->
+        if is_rejected?(origin) do
+          Logger.debug("previously rejected fedsocket requested")
+          {:error, :rejected}
+        else
+          {:error, :missing}
+        end
+
+      [{_pid, %{state: :connected} = socket_info}] ->
+        {:ok, socket_info}
+
+      _ ->
+        {:error, :cache_error}
+    end
+  end
+
+  @doc """
+  Retrieves a map of all sockets from the Registry. The keys are the origins and the values are the corresponding SocketInfo
+  """
+  def list_all do
+    (list_all_connected() ++ list_all_rejected())
+    |> Enum.into(%{})
+  end
+
+  defp list_all_connected do
+    FedSockets.Registry
+    |> Registry.select([{{:"$1", :_, :"$3"}, [], [{{:"$1", :"$3"}}]}])
+  end
+
+  defp list_all_rejected do
+    {:ok, keys} = Cachex.keys(@rejections)
+
+    {:ok, registry_data} =
+      Cachex.execute(@rejections, fn worker ->
+        Enum.map(keys, fn k -> {k, Cachex.get!(worker, k)} end)
+      end)
+
+    registry_data
+  end
+
+  defp clear_prior_rejection(origin),
+    do: Cachex.del(@rejections, origin)
+
+  defp is_rejected?(origin) do
+    case Cachex.get(@rejections, origin) do
+      {:ok, nil} ->
+        false
+
+      {:ok, _} ->
+        true
+    end
+  end
+
+  defp get_or_create_registry_data(origin) do
+    case get_registry_data(origin) do
+      {:error, :missing} ->
+        %SocketInfo{origin: origin}
+
+      {:ok, socket_info} ->
+        socket_info
+    end
+  end
+
+  defp save_registry_data(%SocketInfo{origin: origin, state: :connected} = socket_info) do
+    {:ok, true} = Registry.update_value(FedSockets.Registry, origin, fn _ -> socket_info end)
+    socket_info
+  end
+
+  defp save_registry_data(%SocketInfo{origin: origin, state: :rejected} = socket_info) do
+    rejection_expiration =
+      Pleroma.Config.get([:fed_sockets, :rejection_duration], @default_rejection_duration)
+
+    {:ok, true} = Cachex.put(@rejections, origin, socket_info, ttl: rejection_expiration)
+    socket_info
+  end
+
+  defp set_to_rejected(%SocketInfo{} = socket_info),
+    do: %SocketInfo{socket_info | state: :rejected}
+end