From 4229c7f579e0feebe73f29a59ffa2c6d37af79fc Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Wed, 7 Dec 2022 23:20:45 -0800 Subject: [PATCH 1/9] first draft of the new data exporter --- .rubocop.yml | 3 + Gemfile | 2 + Gemfile.lock | 2 + config/locales/views.en.yml | 2 +- lib/exporter.rb | 264 ++++------------------ lib/use_case/data_export/answers.rb | 23 ++ lib/use_case/data_export/appendables.rb | 25 ++ lib/use_case/data_export/base.rb | 27 +++ lib/use_case/data_export/comments.rb | 23 ++ lib/use_case/data_export/inbox_entries.rb | 23 ++ lib/use_case/data_export/mute_rules.rb | 23 ++ lib/use_case/data_export/questions.rb | 27 +++ lib/use_case/data_export/relationships.rb | 29 +++ lib/use_case/data_export/theme.rb | 27 +++ lib/use_case/data_export/user.rb | 79 +++++++ 15 files changed, 355 insertions(+), 224 deletions(-) create mode 100644 lib/use_case/data_export/answers.rb create mode 100644 lib/use_case/data_export/appendables.rb create mode 100644 lib/use_case/data_export/base.rb create mode 100644 lib/use_case/data_export/comments.rb create mode 100644 lib/use_case/data_export/inbox_entries.rb create mode 100644 lib/use_case/data_export/mute_rules.rb create mode 100644 lib/use_case/data_export/questions.rb create mode 100644 lib/use_case/data_export/relationships.rb create mode 100644 lib/use_case/data_export/theme.rb create mode 100644 lib/use_case/data_export/user.rb diff --git a/.rubocop.yml b/.rubocop.yml index be73721b..6091bca9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -128,3 +128,6 @@ Style/RescueStandardError: Style/Encoding: Enabled: false + +Style/EndlessMethod: + EnforcedStyle: allow_always diff --git a/Gemfile b/Gemfile index c20b4ef2..e8cd20ab 100644 --- a/Gemfile +++ b/Gemfile @@ -111,3 +111,5 @@ gem "net-imap" gem "net-pop" gem "pundit", "~> 2.2" + +gem "rubyzip", "~> 2.3" diff --git a/Gemfile.lock b/Gemfile.lock index 7c2e6ae3..be5d01e4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -413,6 +413,7 @@ GEM ruby-progressbar (1.11.0) ruby-vips (2.1.4) ffi (~> 1.12) + rubyzip (2.3.2) sanitize (6.0.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -570,6 +571,7 @@ DEPENDENCIES rubocop (~> 1.39) rubocop-rails (~> 2.17) ruby-progressbar + rubyzip (~> 2.3) sanitize sassc-rails sentry-rails diff --git a/config/locales/views.en.yml b/config/locales/views.en.yml index 9f3b527b..cfe1c84d 100644 --- a/config/locales/views.en.yml +++ b/config/locales/views.en.yml @@ -393,7 +393,7 @@ en: title: "Export" heading: "Export your data" body_html: | -

The data is inside a .tar.gz archive and available in three formats: YAML, JSON, and XML. +

The data is inside a .zip archive that contains some JSON files. The archive also contains a copy of your profile picture and header picture in all sizes.

Please note that you can only export your data once a week. Exporting your data will take a while, so please be patient. You will receive a question once exporting diff --git a/lib/exporter.rb b/lib/exporter.rb index e08abd78..72601b65 100644 --- a/lib/exporter.rb +++ b/lib/exporter.rb @@ -1,260 +1,78 @@ # frozen_string_literal: true -require "json" -require "yaml" -require "httparty" +require "fileutils" +require "securerandom" +require "zip/filesystem" +require "use_case/data_export/answers" +require "use_case/data_export/appendables" +require "use_case/data_export/comments" +require "use_case/data_export/inbox_entries" +require "use_case/data_export/mute_rules" +require "use_case/data_export/questions" +require "use_case/data_export/relationships" +require "use_case/data_export/theme" +require "use_case/data_export/user" + +# the justask data exporter, now with 200% less shelling out to system tools! +# +# the data export can be easily extended by subclassing `UseCase::DataExport::Base` +# and `require`ing it above class Exporter - EXPORT_ROLES = %i[administrator moderator].freeze - def initialize(user) @user = user - @obj = {} - @export_dirname = Dir.mktmpdir("rs-export-") - @export_filename = File.basename(@export_dirname) + + @export_name = "export-#{@user.id}-#{SecureRandom.base36(32)}" + FileUtils.mkdir_p(Rails.public_path.join("export")) # ensure the public export path exists + export_zipfile_path = Rails.public_path.join("export", "#{@export_name}.zip") + @zipfile = Zip::File.open(export_zipfile_path, Zip::File::CREATE) end def export @user.export_processing = true @user.save validate: false - collect_user_info - collect_questions - collect_answers - collect_comments - collect_smiles - finalize + + prepare_zipfile + write_files publish rescue => e Sentry.capture_exception(e) @user.export_processing = false @user.save validate: false + raise # so that e.g. the sidekiq job fails ensure - FileUtils.remove_dir(@export_dirname) + @zipfile.close end private - def collect_user_info - %i[answered_count asked_count comment_smiled_count commented_count - confirmation_sent_at confirmed_at created_at profile_header profile_header_h profile_header_w profile_header_x profile_header_y - profile_picture_w profile_picture_h profile_picture_x profile_picture_y current_sign_in_at current_sign_in_ip - id last_sign_in_at last_sign_in_ip locale - privacy_allow_anonymous_questions privacy_allow_public_timeline privacy_allow_stranger_answers - privacy_show_in_search profile_header_file_name profile_picture_file_name - screen_name show_foreign_themes sign_in_count smiled_count updated_at].each do |f| - @obj[f] = @user.send f - end + # creates some directories we want to exist and sets a nice comment + def prepare_zipfile + @zipfile.mkdir(@export_name) + @zipfile.mkdir("#{@export_name}/pictures") - @obj[:profile] = {} - %i[display_name motivation_header website location description].each do |f| - @obj[:profile][f] = @user.profile.send f - end - - EXPORT_ROLES.each do |role| - @obj[role] = @user.has_role?(role) - end + @zipfile.comment = <<~COMMENT + #{APP_CONFIG.fetch(:site_name)} export done for #{@user.screen_name} on #{Time.now.utc.iso8601} + COMMENT end - def collect_questions - @obj[:questions] = [] - @user.questions.each do |q| - @obj[:questions] << process_question(q, include_user: false) - end - end - - def collect_answers - @obj[:answers] = [] - @user.answers.each do |a| - @obj[:answers] << process_answer(a, include_user: false) - end - end - - def collect_comments - @obj[:comments] = [] - @user.comments.each do |c| - @obj[:comments] << process_comment(c, include_user: false, include_answer: true) - end - end - - def collect_smiles - @obj[:smiles] = [] - @user.smiles.each do |s| - @obj[:smiles] << process_smile(s) - end - end - - def finalize - `mkdir -p "#{Rails.root.join "public", "export"}"` - `mkdir -p #{@export_dirname}/pictures` - - if @user.profile_picture_file_name - %i[large medium small original].each do |s| - url = @user.profile_picture.url(s) - target_file = "#{@export_dirname}/pictures/picture_#{s}_#{@user.profile_picture_file_name}" - File.open target_file, "wb" do |f| - f.binmode - data = if url.start_with?("/") - File.read(Rails.root.join("public", url.sub(%r{\A/+}, ""))) - else - HTTParty.get(url).parsed_response - end - f.write data + # writes the files to the zip file + def write_files + UseCase::DataExport::Base.descendants.each do |export_klass| + export_klass.call(user: @user).each do |file_name, contents| + @zipfile.file.open("#{@export_name}/#{file_name}", "wb".dup) do |file| # .dup because of %(can't modify frozen String: "wb") + file.write contents end end end - - if @user.profile_header_file_name - %i[web mobile retina original].each do |s| - url = @user.profile_header.url(s) - target_file = "#{@export_dirname}/pictures/header_#{s}_#{@user.profile_header_file_name}" - File.open target_file, "wb" do |f| - f.binmode - data = if url.start_with?("/") - File.read(Rails.root.join("public", url.sub(%r{\A/+}, ""))) - else - HTTParty.get(url).parsed_response - end - f.write data - end - end - end - - File.open "#{@export_dirname}/#{@export_filename}.json", "w" do |f| - f.puts @obj.to_json - end - - File.open "#{@export_dirname}/#{@export_filename}.yml", "w" do |f| - f.puts @obj.to_yaml - end - - File.open "#{@export_dirname}/#{@export_filename}.xml", "w" do |f| - f.puts @obj.to_xml - end end def publish - `tar czvf #{Rails.public_path.join "export", "#{@export_filename}.tar.gz"} #{@export_dirname}` - url = "#{APP_CONFIG['https'] ? 'https' : 'http'}://#{APP_CONFIG['hostname']}/export/#{@export_filename}.tar.gz" + url = "#{APP_CONFIG['https'] ? 'https' : 'http'}://#{APP_CONFIG['hostname']}/export/#{@export_name}.zip" @user.export_processing = false @user.export_url = url @user.export_created_at = Time.now.utc @user.save validate: false url end - - def process_question(question, options = {}) - opts = { - include_user: true, - include_answers: true - }.merge(options) - - qobj = {} - %i[answer_count author_is_anonymous content created_at id].each do |f| - qobj[f] = question.send f - end - - if opts[:include_answers] - qobj[:answers] = [] - question.answers.each do |a| - next if a.nil? - - qobj[:answers] << process_answer(a, include_question: false) - end - end - - if opts[:include_user] - qobj[:user] = question.author_is_anonymous ? nil : user_stub(question.user) - end - - qobj - end - - def process_answer(answer, options = {}) - opts = { - include_question: true, - include_user: true, - include_comments: true - }.merge(options) - - aobj = {} - %i[comment_count content created_at id smile_count].each do |f| - aobj[f] = answer.send f - end - - if opts[:include_user] && answer.user - aobj[:user] = user_stub(answer.user) - end - - if opts[:include_question] && answer.question - aobj[:question] = process_question(answer.question, include_user: true, include_answers: false) - end - - if opts[:include_comments] - aobj[:comments] = [] - answer.comments.each do |c| - next if c.nil? - - aobj[:comments] << process_comment(c, include_user: true, include_answer: false) - end - end - - aobj - end - - def process_comment(comment, options = {}) - opts = { - include_user: true, - include_answer: false - }.merge(options) - - cobj = {} - %i[content created_at id].each do |f| - cobj[f] = comment.send f - end - - if opts[:include_user] - cobj[:user] = user_stub(comment.user) - end - - if opts[:include_answer] && comment.answer - cobj[:answer] = process_answer(comment.answer, include_comments: false) - end - - cobj - end - - def process_smile(smile) - return unless smile.parent - - sobj = {} - - %i[id created_at].each do |f| - sobj[f] = smile.send f - end - - type = smile.parent.class.name.downcase - sobj[type.to_sym] = send(:"process_#{type}", smile.parent, include_comments: false, include_user: false) - - sobj - end - - def user_stub(user) - return nil if user.nil? - uobj = {} - %i[answered_count asked_count comment_smiled_count commented_count created_at - id permanently_banned? screen_name smiled_count].each do |f| - uobj[f] = user.send f - end - - uobj[:profile] = {} - %i[display_name motivation_header website location description].each do |f| - uobj[:profile][f] = user.profile.send f - end - - EXPORT_ROLES.each do |role| - uobj[role] = user.has_role?(role) - end - - uobj - end end diff --git a/lib/use_case/data_export/answers.rb b/lib/use_case/data_export/answers.rb new file mode 100644 index 00000000..1443ec6d --- /dev/null +++ b/lib/use_case/data_export/answers.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class Answers < UseCase::DataExport::Base + def files = { + "answers.json" => json_file!( + answers: user.answers.map(&method(:collect_answer)) + ) + } + + def collect_answer(answer) + {}.tap do |h| + column_names(::Answer).each do |field| + h[field] = answer[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/appendables.rb b/lib/use_case/data_export/appendables.rb new file mode 100644 index 00000000..0525431b --- /dev/null +++ b/lib/use_case/data_export/appendables.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class Appendables < UseCase::DataExport::Base + def files = { + "appendables.json" => json_file!( + appendables: [ + *user.smiles.map(&method(:collect_appendable)) + ] + ) + } + + def collect_appendable(appendable) + {}.tap do |h| + column_names(::Appendable).each do |field| + h[field] = appendable[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/base.rb b/lib/use_case/data_export/base.rb new file mode 100644 index 00000000..dc2ded83 --- /dev/null +++ b/lib/use_case/data_export/base.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "json" + +require "use_case/base" + +module UseCase + module DataExport + class Base < UseCase::Base + # the user that is being exported + option :user + + def call = files + + # returns a hash with `{ "file_name" => "file_contents\n" }` + def files = raise NotImplementedError + + # helper method that returns the column names of `model` as symbols + def column_names(model) = model.column_names.map(&:to_sym) + + # helper method that generates the content of a json file + # + # it ensures the final newline exists, as the exporter only uses File#write + def json_file!(**hash) = "#{JSON.pretty_generate(hash.as_json)}\n" + end + end +end diff --git a/lib/use_case/data_export/comments.rb b/lib/use_case/data_export/comments.rb new file mode 100644 index 00000000..6a6e820d --- /dev/null +++ b/lib/use_case/data_export/comments.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class Comments < UseCase::DataExport::Base + def files = { + "comments.json" => json_file!( + comments: user.comments.map(&method(:collect_comment)) + ) + } + + def collect_comment(comment) + {}.tap do |h| + column_names(::Comment).each do |field| + h[field] = comment[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/inbox_entries.rb b/lib/use_case/data_export/inbox_entries.rb new file mode 100644 index 00000000..04b2efb4 --- /dev/null +++ b/lib/use_case/data_export/inbox_entries.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class InboxEntries < UseCase::DataExport::Base + def files = { + "inbox_entries.json" => json_file!( + inbox_entries: user.inboxes.map(&method(:collect_inbox_entry)) + ) + } + + def collect_inbox_entry(inbox_entry) + {}.tap do |h| + column_names(::Inbox).each do |field| + h[field] = inbox_entry[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/mute_rules.rb b/lib/use_case/data_export/mute_rules.rb new file mode 100644 index 00000000..6b8755ba --- /dev/null +++ b/lib/use_case/data_export/mute_rules.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class MuteRules < UseCase::DataExport::Base + def files = { + "mute_rules.json" => json_file!( + mute_rules: user.mute_rules.map(&method(:collect_mute_rule)) + ) + } + + def collect_mute_rule(mute_rule) + {}.tap do |h| + column_names(::MuteRule).each do |field| + h[field] = mute_rule[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/questions.rb b/lib/use_case/data_export/questions.rb new file mode 100644 index 00000000..7536e7b6 --- /dev/null +++ b/lib/use_case/data_export/questions.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class Questions < UseCase::DataExport::Base + IGNORED_FIELDS = %i[ + author_identifier + ].freeze + + def files = { + "questions.json" => json_file!( + questions: user.questions.map(&method(:collect_question)) + ) + } + + def collect_question(question) + {}.tap do |h| + (column_names(::Question) - IGNORED_FIELDS).each do |field| + h[field] = question[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/relationships.rb b/lib/use_case/data_export/relationships.rb new file mode 100644 index 00000000..57fb8cb9 --- /dev/null +++ b/lib/use_case/data_export/relationships.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class Relationships < UseCase::DataExport::Base + def files = { + "relationships.json" => json_file!( + relationships: [ + # don't want to add the passive (block) relationships here as it + # would reveal who e.g. blocked the exported user, which is + # considered A Bad Idea™ + *user.active_follow_relationships.map(&method(:collect_relationship)), + *user.active_block_relationships.map(&method(:collect_relationship)) + ] + ) + } + + def collect_relationship(relationship) + {}.tap do |h| + column_names(::Relationship).each do |field| + h[field] = relationship[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/theme.rb b/lib/use_case/data_export/theme.rb new file mode 100644 index 00000000..135fc89e --- /dev/null +++ b/lib/use_case/data_export/theme.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "use_case/data_export/base" + +module UseCase + module DataExport + class Theme < UseCase::DataExport::Base + def files + return {} unless user.theme + + { + "theme.json" => json_file!( + theme: theme_data + ) + } + end + + def theme_data + {}.tap do |obj| + column_names(::Theme).each do |field| + obj[field] = user.theme[field] + end + end + end + end + end +end diff --git a/lib/use_case/data_export/user.rb b/lib/use_case/data_export/user.rb new file mode 100644 index 00000000..f626bc10 --- /dev/null +++ b/lib/use_case/data_export/user.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require "httparty" + +require "use_case/data_export/base" + +module UseCase + module DataExport + class User < UseCase::DataExport::Base + EXPORT_ROLES = %i[administrator moderator].freeze + + IGNORED_FIELDS_USERS = %i[ + confirmation_token + encrypted_password + otp_secret_key + reset_password_sent_at + reset_password_token + ].freeze + + IGNORED_FIELDS_PROFILES = %i[ + id + user_id + ].freeze + + def files = { + "user.json" => json_file!( + user: user_data, + profile: profile_data, + roles: roles_data + ), + **pictures + } + + def user_data + {}.tap do |obj| + (column_names(::User) - IGNORED_FIELDS_USERS).each do |field| + obj[field] = user[field] + end + end + end + + def profile_data + {}.tap do |profile| + (column_names(::Profile) - IGNORED_FIELDS_PROFILES).each do |field| + profile[field] = user.profile[field] + end + end + end + + def roles_data + {}.tap do |obj| + EXPORT_ROLES.each do |role| + obj[role] = user.has_role?(role) + end + end + end + + def pictures + {}.tap do |hash| + add_picture(user.profile_picture, to: hash) + add_picture(user.profile_header, to: hash) + end.compact + end + + def add_picture(picture, to:) + return if picture.blank? + + picture.versions.each do |version, file| + export_filename = "pictures/#{file.mounted_as}_#{version}_#{file.filename}" + to[export_filename] = if file.url.start_with?("/") + Rails.public_path.join(file.url.sub(%r{\A/+}, "")).read rescue "ceci n'est pas un image" # TODO: fix this + else + HTTParty.get(file.url).parsed_response + end + end + end + end + end +end From 2b3e7ab6093b04f98921f528a6553377bc52507a Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 10 Dec 2022 03:28:17 +0100 Subject: [PATCH 2/9] add specs for new exporters --- lib/exporter.rb | 7 +- lib/use_case/data_export/user.rb | 7 +- spec/lib/exporter_spec.rb | 366 ++++-------------- spec/lib/use_case/data_export/answers_spec.rb | 44 +++ .../use_case/data_export/appendables_spec.rb | 60 +++ .../lib/use_case/data_export/comments_spec.rb | 45 +++ .../data_export/inbox_entries_spec.rb | 54 +++ .../use_case/data_export/mute_rules_spec.rb | 51 +++ .../use_case/data_export/questions_spec.rb | 68 ++++ .../data_export/relationships_spec.rb | 64 +++ spec/lib/use_case/data_export/theme_spec.rb | 58 +++ spec/lib/use_case/data_export/user_spec.rb | 176 +++++++++ spec/shared_examples/data_export.rb | 17 + spec/support/example_exporter.rb | 24 ++ 14 files changed, 752 insertions(+), 289 deletions(-) create mode 100644 spec/lib/use_case/data_export/answers_spec.rb create mode 100644 spec/lib/use_case/data_export/appendables_spec.rb create mode 100644 spec/lib/use_case/data_export/comments_spec.rb create mode 100644 spec/lib/use_case/data_export/inbox_entries_spec.rb create mode 100644 spec/lib/use_case/data_export/mute_rules_spec.rb create mode 100644 spec/lib/use_case/data_export/questions_spec.rb create mode 100644 spec/lib/use_case/data_export/relationships_spec.rb create mode 100644 spec/lib/use_case/data_export/theme_spec.rb create mode 100644 spec/lib/use_case/data_export/user_spec.rb create mode 100644 spec/shared_examples/data_export.rb create mode 100644 spec/support/example_exporter.rb diff --git a/lib/exporter.rb b/lib/exporter.rb index 72601b65..eb5101d3 100644 --- a/lib/exporter.rb +++ b/lib/exporter.rb @@ -39,7 +39,12 @@ class Exporter Sentry.capture_exception(e) @user.export_processing = false @user.save validate: false - raise # so that e.g. the sidekiq job fails + + # delete zipfile on errors + @zipfile.close # so it gets written to disk first + File.delete(@zipfile.name) + + raise # let e.g. the sidekiq job fail so it's retryable later ensure @zipfile.close end diff --git a/lib/use_case/data_export/user.rb b/lib/use_case/data_export/user.rb index f626bc10..86fe7f6e 100644 --- a/lib/use_case/data_export/user.rb +++ b/lib/use_case/data_export/user.rb @@ -68,7 +68,12 @@ module UseCase picture.versions.each do |version, file| export_filename = "pictures/#{file.mounted_as}_#{version}_#{file.filename}" to[export_filename] = if file.url.start_with?("/") - Rails.public_path.join(file.url.sub(%r{\A/+}, "")).read rescue "ceci n'est pas un image" # TODO: fix this + begin + Rails.public_path.join(file.url.sub(%r{\A/+}, "")).read + rescue + # TODO: fix image handling in local development environments!!! see #822 + "ceci n'est pas un image\n" + end else HTTParty.get(file.url).parsed_response end diff --git a/spec/lib/exporter_spec.rb b/spec/lib/exporter_spec.rb index 526b2978..0ad327cb 100644 --- a/spec/lib/exporter_spec.rb +++ b/spec/lib/exporter_spec.rb @@ -1,326 +1,118 @@ # frozen_string_literal: true require "rails_helper" +require "support/example_exporter" +require "base64" + require "exporter" +# This only tests the exporter itself to make sure zip file creation works. RSpec.describe Exporter do include ActiveSupport::Testing::TimeHelpers - let(:user_params) do - { - answered_count: 144, - asked_count: 72, - comment_smiled_count: 15, - commented_count: 12, - confirmation_sent_at: 2.weeks.ago.utc, - confirmed_at: 2.weeks.ago.utc + 1.hour, - created_at: 2.weeks.ago.utc, - current_sign_in_at: 8.hours.ago.utc, - current_sign_in_ip: "198.51.100.220", - last_sign_in_at: 1.hour.ago, - last_sign_in_ip: "192.0.2.14", - locale: "en", - privacy_allow_anonymous_questions: true, - privacy_allow_public_timeline: false, - privacy_allow_stranger_answers: false, - privacy_show_in_search: true, - screen_name: "fizzyraccoon", - show_foreign_themes: true, - sign_in_count: 10, - smiled_count: 28, - profile: { - display_name: "Fizzy Raccoon", - description: "A small raccoon", - location: "Binland", - motivation_header: "", - website: "https://retrospring.net" - } - } - end - let(:user) { FactoryBot.create(:user, **user_params) } + let(:user) { FactoryBot.create(:user, screen_name: "fizzyraccoon", export_processing: true) } let(:instance) { described_class.new(user) } + let(:zipfile_deletion_expected) { false } before do stub_const("APP_CONFIG", { - "hostname" => "example.com", - "https" => true, - "items_per_page" => 5, - "fog" => {} - }) + "site_name" => "justask", + "hostname" => "example.com", + "https" => true, + "items_per_page" => 5, + "fog" => {} + }.with_indifferent_access) end after do - filename = instance.instance_variable_get(:@export_dirname) - FileUtils.rm_r(filename) if File.exist?(filename) - end - - describe "#collect_user_info" do - subject { instance.send(:collect_user_info) } - - it "collects user info" do - subject - expect(instance.instance_variable_get(:@obj)).to eq(user_params.merge({ - administrator: false, - moderator: false, - id: user.id, - updated_at: user.updated_at, - profile_header: user.profile_header, - profile_header_file_name: nil, - profile_header_h: nil, - profile_header_w: nil, - profile_header_x: nil, - profile_header_y: nil, - profile_picture_file_name: nil, - profile_picture_h: nil, - profile_picture_w: nil, - profile_picture_x: nil, - profile_picture_y: nil - })) + filename = instance.instance_variable_get(:@zipfile)&.name + unless File.exist?(filename) + warn "exporter_spec.rb: wanted to clean up #{filename.inspect} but it does not exist!" unless zipfile_deletion_expected + next end + FileUtils.rm_r(filename) end - describe "#collect_questions" do - subject { instance.send(:collect_questions) } + describe "#export" do + let(:export_name) { instance.instance_variable_get(:@export_name) } - context "exporting a user with several questions" do - let!(:questions) { FactoryBot.create_list(:question, 25, user: user) } - - it "collects questions" do - subject - expect(instance.instance_variable_get(:@obj)[:questions]).to eq(questions.map do |q| - { - answer_count: 0, - answers: [], - author_is_anonymous: q.author_is_anonymous, - content: q.content, - created_at: q.reload.created_at, - id: q.id - } - end) + subject do + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) do + instance.export end end - context "exporting a user with a question which has been answered" do - let!(:question) { FactoryBot.create(:question, user: user, author_is_anonymous: false) } - let!(:answers) { FactoryBot.create_list(:answer, 5, question: question, user: FactoryBot.create(:user)) } - - it "collects questions and answers" do - subject - expect(instance.instance_variable_get(:@obj)[:questions]).to eq([ - { - answer_count: 5, - answers: answers.map do |a| - { - comment_count: 0, - comments: [], - content: a.content, - created_at: a.reload.created_at, - id: a.id, - smile_count: a.smile_count, - user: instance.send(:user_stub, a.user) - } - end, - author_is_anonymous: false, - content: question.content, - created_at: question.reload.created_at, - id: question.id - } - ]) - end - end - end - - describe "#collect_answers" do - let!(:answers) { FactoryBot.create_list(:answer, 25, user: user) } - - subject { instance.send(:collect_answers) } - - it "collects answers" do - subject - expect(instance.instance_variable_get(:@obj)[:answers]).to eq(answers.map do |a| - { - comment_count: 0, - comments: [], - content: a.content, - created_at: a.reload.created_at, - id: a.id, - question: instance.send(:process_question, - a.question.reload, - include_user: true, - include_answers: false), - smile_count: 0 - } - end) - end - end - - describe "#collect_comments" do - let!(:comments) do - FactoryBot.create_list(:comment, - 25, - user: user, - answer: FactoryBot.create(:answer, user: FactoryBot.create(:user))) - end - - subject { instance.send(:collect_comments) } - - it "collects comments" do - subject - expect(instance.instance_variable_get(:@obj)[:comments]).to eq(comments.map do |c| - { - content: c.content, - created_at: c.reload.created_at, - id: c.id, - answer: instance.send(:process_answer, - c.answer, - include_comments: false) - } - end) - end - end - - describe "#collect_smiles" do - let!(:smiles) { FactoryBot.create_list(:smile, 25, user: user) } - - subject { instance.send(:collect_smiles) } - - it "collects reactions" do - subject - expect(instance.instance_variable_get(:@obj)[:smiles]).to eq(smiles.map do |s| - { - id: s.id, - created_at: s.reload.created_at, - answer: { - comment_count: s.parent.comment_count, - content: s.parent.content, - created_at: s.parent.reload.created_at, - id: s.parent.id, - question: { - answer_count: s.parent.question.answer_count, - author_is_anonymous: s.parent.question.author_is_anonymous, - content: s.parent.question.content, - created_at: s.parent.question.reload.created_at, - id: s.parent.question.id, - user: nil # we're not populating this in the factory - }, - smile_count: s.parent.smile_count - } - } - end) - end - end - - describe "#finalize" do - let(:fake_rails_root) { Pathname(Dir.mktmpdir) } - let(:dir) { instance.instance_variable_get(:@export_dirname) } - let(:name) { instance.instance_variable_get(:@export_filename) } - before do - instance.instance_variable_set(:@obj, { - some: { - sample: { - data: "Text" - } - } - }) - - Dir.mkdir("#{fake_rails_root}/public") - FileUtils.cp_r(Rails.root.join("public/images"), "#{fake_rails_root}/public/images") - allow(Rails).to receive(:root).and_return(fake_rails_root) + allow(UseCase::DataExport::Base) + .to receive(:descendants) + .and_return([ExampleExporter]) end - after do - FileUtils.rm_r(fake_rails_root) - end + it "creates a zip file with the expected contents" do + subject - subject { instance.send(:finalize) } + # check created zip file + zip_path = Rails.public_path.join("export/#{export_name}.zip") + expect(File.exist?(zip_path)).to be true - context "exporting a user without a profile picture or header" do - it "prepares files to be archived" do - subject - expect(File.directory?(fake_rails_root.join("public/export"))).to eq(true) - expect(File.directory?("#{dir}/pictures")).to eq(true) - end + Zip::File.open(zip_path) do |zip| + # check for zip comment + expect(zip.comment).to eq "justask export done for fizzyraccoon on 2022-12-10T13:37:42Z\n" - it "outputs JSON" do - subject - path = "#{dir}/#{name}.json" - expect(File.exist?(path)).to eq(true) - expect(JSON.load_file(path, symbolize_names: true)).to eq(instance.instance_variable_get(:@obj)) - end + # check if all files and directories are there + expect(zip.entries.map(&:name).sort).to eq([ + # basic dirs from exporter + "#{export_name}/", + "#{export_name}/pictures/", + # files added by the ExampleExporter + "#{export_name}/textfile.txt", + "#{export_name}/pictures/example.jpg", + "#{export_name}/some.json" + ].sort) - it "outputs YAML" do - subject - path = "#{dir}/#{name}.yml" - expect(File.exist?(path)).to eq(true) - expect(YAML.load_file(path)).to eq(instance.instance_variable_get(:@obj)) - end - - it "outputs XML" do - subject - path = "#{dir}/#{name}.xml" - expect(File.exist?(path)).to eq(true) + # check if the file contents match + expect(zip.file.read("#{export_name}/textfile.txt")).to eq("Sample Text\n") + expect(Base64.encode64(zip.file.read("#{export_name}/pictures/example.jpg"))) + .to eq(Base64.encode64(File.read(File.expand_path("../fixtures/files/banana_racc.jpg", __dir__)))) + expect(zip.file.read("#{export_name}/some.json")).to eq(<<~JSON) + { + "animals": [ + "raccoon", + "fox", + "hyena", + "deer", + "dog" + ], + "big_number": 3457812374589235798, + "booleans": { + "yes": true, + "no": false, + "file_not_found": null + } + } + JSON end end - context "exporting a user with a profile header" do + it "updates the export fields of the user" do + expect { subject }.to change { user.export_processing }.from(true).to(false) + expect(user.export_url).to eq("https://example.com/export/#{export_name}.zip") + expect(user.export_created_at).to eq(Time.utc(2022, 12, 10, 13, 37, 42)) + expect(user).to be_persisted + end + + context "when exporting fails" do + let(:zipfile_deletion_expected) { true } + before do - user.profile_header = Rack::Test::UploadedFile.new(File.open("#{file_fixture_path}/banana_racc.jpg")) - user.save! + allow_any_instance_of(ExampleExporter).to receive(:files).and_raise(ArgumentError.new("just testing")) end - it "exports the header image" do - subject - dirname = instance.instance_variable_get(:@export_dirname) - %i[web mobile retina original].each do |size| - expect(File.exist?("#{dirname}/pictures/header_#{size}_banana_racc.jpg")).to eq(true) - end - end - end + it "deletes the zip file" do + expect { subject }.to raise_error(ArgumentError, "just testing") - context "exporting a user with a profile picture" do - before do - user.profile_picture = Rack::Test::UploadedFile.new(File.open("#{file_fixture_path}/banana_racc.jpg")) - user.save! - end - - it "exports the header image" do - subject - dirname = instance.instance_variable_get(:@export_dirname) - %i[large medium small original].each do |size| - expect(File.exist?("#{dirname}/pictures/picture_#{size}_banana_racc.jpg")).to eq(true) - end - end - end - end - - describe "#publish" do - let(:fake_rails_root) { Pathname(Dir.mktmpdir) } - let(:fake_rails_public_path) { fake_rails_root.join('public') } - let(:name) { instance.instance_variable_get(:@export_filename) } - - before do - FileUtils.mkdir_p("#{fake_rails_root}/public/export") - allow(Rails).to receive(:root).and_return(fake_rails_root) - allow(Rails).to receive(:public_path).and_return(fake_rails_public_path) - - user.export_processing = true - user.save! - end - - after do - FileUtils.rm_r(fake_rails_root) - end - - subject { instance.send(:publish) } - - it "publishes an archive" do - freeze_time do - expect { subject }.to change { user.export_processing }.from(true).to(false) - expect(File.exist?("#{fake_rails_root}/public/export/#{name}.tar.gz")).to eq(true) - expect(user.export_url).to eq("https://example.com/export/#{name}.tar.gz") - expect(user.export_created_at).to eq(Time.now.utc) - expect(user).to be_persisted + zip_path = Rails.public_path.join("export/#{export_name}.zip") + expect(File.exist?(zip_path)).to be false end end end diff --git a/spec/lib/use_case/data_export/answers_spec.rb b/spec/lib/use_case/data_export/answers_spec.rb new file mode 100644 index 00000000..e61d74d0 --- /dev/null +++ b/spec/lib/use_case/data_export/answers_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/answers" + +describe UseCase::DataExport::Answers, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have any answers" do + it "returns an empty set of answers" do + expect(json_file("answers.json")).to eq( + { + answers: [] + } + ) + end + end + + context "when user has made some answer" do + let!(:answer) do + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { FactoryBot.create(:answer, user:, content: "Yay, data export!") } + end + + it "returns the answers as json" do + expect(json_file("answers.json")).to eq( + { + answers: [ + { + id: answer.id, + content: "Yay, data export!", + question_id: answer.question.id, + comment_count: 0, + user_id: user.id, + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z", + smile_count: 0 + } + ] + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/appendables_spec.rb b/spec/lib/use_case/data_export/appendables_spec.rb new file mode 100644 index 00000000..60523f45 --- /dev/null +++ b/spec/lib/use_case/data_export/appendables_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/appendables" + +describe UseCase::DataExport::Appendables, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have any appendable" do + it "returns an empty set of appendables" do + expect(json_file("appendables.json")).to eq( + { + appendables: [] + } + ) + end + end + + context "when user has smiled some things" do + let(:answer) { FactoryBot.create(:answer, user:) } + let(:comment) { FactoryBot.create(:comment, user:, answer:) } + + let!(:appendables) do + [ + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { FactoryBot.create(:comment_smile, user:, parent: comment) }, + travel_to(Time.utc(2022, 12, 10, 13, 39, 21)) { FactoryBot.create(:smile, user:, parent: answer) } + ] + end + + it "returns the appendables as json" do + expect(json_file("appendables.json")).to eq( + { + appendables: [ + { + id: appendables[0].id, + type: "Appendable::Reaction", + user_id: user.id, + parent_id: appendables[0].parent_id, + parent_type: "Comment", + content: "🙂", + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z" + }, + { + id: appendables[1].id, + type: "Appendable::Reaction", + user_id: user.id, + parent_id: appendables[1].parent_id, + parent_type: "Answer", + content: "🙂", + created_at: "2022-12-10T13:39:21.000Z", + updated_at: "2022-12-10T13:39:21.000Z" + } + ] + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/comments_spec.rb b/spec/lib/use_case/data_export/comments_spec.rb new file mode 100644 index 00000000..162d1798 --- /dev/null +++ b/spec/lib/use_case/data_export/comments_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/comments" + +describe UseCase::DataExport::Comments, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have any comments" do + it "returns an empty set of comments" do + expect(json_file("comments.json")).to eq( + { + comments: [] + } + ) + end + end + + context "when user has made some comment" do + let(:answer) { FactoryBot.create(:answer, user:) } + + let!(:comment) do + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { FactoryBot.create(:comment, user:, answer:, content: "Yay, data export!") } + end + + it "returns the comments as json" do + expect(json_file("comments.json")).to eq( + { + comments: [ + { + id: comment.id, + content: "Yay, data export!", + answer_id: answer.id, + user_id: user.id, + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z", + smile_count: 0 + } + ] + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/inbox_entries_spec.rb b/spec/lib/use_case/data_export/inbox_entries_spec.rb new file mode 100644 index 00000000..9682ab76 --- /dev/null +++ b/spec/lib/use_case/data_export/inbox_entries_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/inbox_entries" + +describe UseCase::DataExport::InboxEntries, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have anything in their inbox" do + it "returns an empty set of inbox entries" do + expect(json_file("inbox_entries.json")).to eq( + { + inbox_entries: [] + } + ) + end + end + + context "when user has some questions in their inbox" do + let!(:inbox_entries) do + [ + # using `Inbox.create` here as for some reason FactoryBot.create(:inbox) always sets `new` to `nil`??? + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { Inbox.create(user:, question: FactoryBot.create(:question), new: false) }, + travel_to(Time.utc(2022, 12, 10, 13, 39, 21)) { Inbox.create(user:, question: FactoryBot.create(:question), new: true) } + ] + end + + it "returns the inbox entries as json" do + expect(json_file("inbox_entries.json")).to eq( + { + inbox_entries: [ + { + id: inbox_entries[0].id, + user_id: user.id, + question_id: inbox_entries[0].question_id, + new: false, + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z" + }, + { + id: inbox_entries[1].id, + user_id: user.id, + question_id: inbox_entries[1].question_id, + new: true, + created_at: "2022-12-10T13:39:21.000Z", + updated_at: "2022-12-10T13:39:21.000Z" + } + ] + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/mute_rules_spec.rb b/spec/lib/use_case/data_export/mute_rules_spec.rb new file mode 100644 index 00000000..9e610b07 --- /dev/null +++ b/spec/lib/use_case/data_export/mute_rules_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/mute_rules" + +describe UseCase::DataExport::MuteRules, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have any mute rules" do + it "returns an empty set of mute rules" do + expect(json_file("mute_rules.json")).to eq( + { + mute_rules: [] + } + ) + end + end + + context "when user has some mute rules" do + let!(:mute_rules) do + [ + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { MuteRule.create(user:, muted_phrase: "test") }, + travel_to(Time.utc(2022, 12, 10, 13, 39, 21)) { MuteRule.create(user:, muted_phrase: "python") } + ] + end + + it "returns the mute rules as json" do + expect(json_file("mute_rules.json")).to eq( + { + mute_rules: [ + { + id: mute_rules[0].id, + user_id: user.id, + muted_phrase: "test", + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z" + }, + { + id: mute_rules[1].id, + user_id: user.id, + muted_phrase: "python", + created_at: "2022-12-10T13:39:21.000Z", + updated_at: "2022-12-10T13:39:21.000Z" + } + ] + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/questions_spec.rb b/spec/lib/use_case/data_export/questions_spec.rb new file mode 100644 index 00000000..dfb7b78f --- /dev/null +++ b/spec/lib/use_case/data_export/questions_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/questions" + +describe UseCase::DataExport::Questions, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have any questions" do + it "returns an empty set of questions" do + expect(json_file("questions.json")).to eq( + { + questions: [] + } + ) + end + end + + context "when user has made some questions" do + let!(:questions) do + [ + travel_to(Time.utc(2022, 12, 10, 13, 12, 0)) { FactoryBot.create(:question, user:, content: "Yay, data export 1", author_is_anonymous: false, direct: false, answer_count: 12) }, + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { FactoryBot.create(:question, user:, content: "Yay, data export 2", author_is_anonymous: false, direct: true, answer_count: 1) }, + travel_to(Time.utc(2022, 12, 10, 13, 39, 21)) { FactoryBot.create(:question, user:, content: "Yay, data export 3", author_is_anonymous: true, direct: true) } + ] + end + + it "returns the questions as json" do + expect(json_file("questions.json")).to eq( + { + questions: [ + { + id: questions[0].id, + content: "Yay, data export 1", + author_is_anonymous: false, + user_id: user.id, + created_at: "2022-12-10T13:12:00.000Z", + updated_at: "2022-12-10T13:12:00.000Z", + answer_count: 12, + direct: false + }, + { + id: questions[1].id, + content: "Yay, data export 2", + author_is_anonymous: false, + user_id: user.id, + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z", + answer_count: 1, + direct: true + }, + { + id: questions[2].id, + content: "Yay, data export 3", + author_is_anonymous: true, + user_id: user.id, + created_at: "2022-12-10T13:39:21.000Z", + updated_at: "2022-12-10T13:39:21.000Z", + answer_count: 0, + direct: true + } + ] + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/relationships_spec.rb b/spec/lib/use_case/data_export/relationships_spec.rb new file mode 100644 index 00000000..7af992f9 --- /dev/null +++ b/spec/lib/use_case/data_export/relationships_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/relationships" + +describe UseCase::DataExport::Relationships, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have any relationships" do + it "returns an empty set of relationships" do + expect(json_file("relationships.json")).to eq( + { + relationships: [] + } + ) + end + end + + context "when user has made some relationships" do + let(:other_user) { FactoryBot.create(:user) } + let(:blocked_user) { FactoryBot.create(:user) } + let(:blocking_user) { FactoryBot.create(:user) } + + let!(:relationships) do + { + # user <-> other_user follow each other + user_to_other: travel_to(Time.utc(2022, 12, 10, 13, 12, 0)) { user.follow(other_user) }, + other_to_user: travel_to(Time.utc(2022, 12, 10, 13, 12, 36)) { other_user.follow(user) }, + + # user blocked blocked_user + block: travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { user.block(blocked_user) }, + + # user is blocked by blocking_user + blocked_by: travel_to(Time.utc(2022, 12, 10, 13, 39, 21)) { blocking_user.block(user) } + } + end + + it "returns the relationships as json" do + expect(json_file("relationships.json")).to eq( + { + relationships: [ + { + id: relationships[:user_to_other].id, + source_id: user.id, + target_id: other_user.id, + created_at: "2022-12-10T13:12:00.000Z", + updated_at: "2022-12-10T13:12:00.000Z", + type: "Relationships::Follow" + }, + { + id: relationships[:block].id, + source_id: user.id, + target_id: blocked_user.id, + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z", + type: "Relationships::Block" + } + ] + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/theme_spec.rb b/spec/lib/use_case/data_export/theme_spec.rb new file mode 100644 index 00000000..8a073d89 --- /dev/null +++ b/spec/lib/use_case/data_export/theme_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/theme" + +describe UseCase::DataExport::Theme, :data_export do + include ActiveSupport::Testing::TimeHelpers + + context "when user doesn't have a theme" do + it "returns nothing" do + expect(subject).to eq({}) + end + end + + context "when user has a theme" do + let!(:theme) do + travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) do + FactoryBot.create(:theme, user:) + end + end + + it "returns the theme as json" do + expect(json_file("theme.json")).to eq( + { + theme: { + id: theme.id, + user_id: user.id, + primary_color: 9342168, + primary_text: 16777215, + danger_color: 14257035, + danger_text: 16777215, + success_color: 12573067, + success_text: 16777215, + warning_color: 14261899, + warning_text: 16777215, + info_color: 9165273, + info_text: 16777215, + dark_color: 6710886, + dark_text: 15658734, + raised_background: 16777215, + background_color: 13026795, + body_text: 3355443, + muted_text: 3355443, + created_at: "2022-12-10T13:37:42.000Z", + updated_at: "2022-12-10T13:37:42.000Z", + input_color: 15789556, + input_text: 6710886, + raised_accent: 16250871, + light_color: 16316922, + light_text: 0, + input_placeholder: 7107965 + } + } + ) + end + end +end diff --git a/spec/lib/use_case/data_export/user_spec.rb b/spec/lib/use_case/data_export/user_spec.rb new file mode 100644 index 00000000..d769605b --- /dev/null +++ b/spec/lib/use_case/data_export/user_spec.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +require "rails_helper" + +require "use_case/data_export/user" + +describe UseCase::DataExport::User, :data_export do + let(:user_params) do + { + email: "fizzyraccoon@bsnss.biz", + answered_count: 144, + asked_count: 72, + comment_smiled_count: 15, + commented_count: 12, + confirmation_sent_at: 2.weeks.ago.utc, + confirmed_at: 2.weeks.ago.utc + 1.hour, + created_at: 2.weeks.ago.utc, + current_sign_in_at: 8.hours.ago.utc, + current_sign_in_ip: "198.51.100.220", + last_sign_in_at: 1.hour.ago, + last_sign_in_ip: "192.0.2.14", + locale: "en", + privacy_allow_anonymous_questions: true, + privacy_allow_public_timeline: false, + privacy_allow_stranger_answers: false, + privacy_show_in_search: true, + screen_name: "fizzyraccoon", + show_foreign_themes: true, + sign_in_count: 10, + smiled_count: 28, + profile: { + display_name: "Fizzy Raccoon", + description: "A small raccoon", + location: "Binland", + motivation_header: "", + website: "https://retrospring.net" + } + } + end + + it "returns the user as json" do + expect(json_file("user.json")).to eq( + { + user: { + id: user.id, + email: "fizzyraccoon@bsnss.biz", + remember_created_at: nil, + sign_in_count: 10, + current_sign_in_at: user.current_sign_in_at.as_json, + last_sign_in_at: user.last_sign_in_at.as_json, + current_sign_in_ip: "198.51.100.220", + last_sign_in_ip: "192.0.2.14", + created_at: user.created_at.as_json, + updated_at: user.updated_at.as_json, + screen_name: "fizzyraccoon", + asked_count: 72, + answered_count: 144, + commented_count: 12, + smiled_count: 28, + profile_picture_file_name: nil, + profile_picture_processing: nil, + profile_picture_x: nil, + profile_picture_y: nil, + profile_picture_w: nil, + profile_picture_h: nil, + privacy_allow_anonymous_questions: true, + privacy_allow_public_timeline: false, + privacy_allow_stranger_answers: false, + privacy_show_in_search: true, + comment_smiled_count: 15, + profile_header_file_name: nil, + profile_header_processing: nil, + profile_header_x: nil, + profile_header_y: nil, + profile_header_w: nil, + profile_header_h: nil, + locale: "en", + confirmed_at: user.confirmed_at.as_json, + confirmation_sent_at: user.confirmation_sent_at.as_json, + unconfirmed_email: nil, + show_foreign_themes: true, + export_url: nil, + export_processing: false, + export_created_at: nil, + otp_module: "disabled", + privacy_lock_inbox: false, + privacy_require_user: false, + privacy_hide_social_graph: false, + privacy_noindex: false + }, + profile: { + display_name: "Fizzy Raccoon", + description: "A small raccoon", + location: "Binland", + website: "https://retrospring.net", + motivation_header: "", + created_at: user.profile.created_at.as_json, + updated_at: user.profile.updated_at.as_json, + anon_display_name: nil + }, + roles: { + administrator: false, + moderator: false + } + } + ) + end + + it "does not have any pictures attached" do + expect(subject.keys.select { _1.start_with?("pictures/") }).to be_empty + end + + context "when user has a profile picture" do + let(:user_params) do + super().merge( + process_profile_picture_upload: true, # force carrierwave_backgrounder to immediately process the image + profile_picture: Rack::Test::UploadedFile.new(Rails.root.join("spec/fixtures/files/banana_racc.jpg"), "image/jpeg"), + profile_picture_x: 571, + profile_picture_y: 353, + profile_picture_w: 474, + profile_picture_h: 474 + ) + end + + it "includes the pictures in the file list" do + expect(subject.keys.select { _1.start_with?("pictures/") }.sort).to eq([ + "pictures/profile_picture_original_banana_racc.jpg", + "pictures/profile_picture_large_banana_racc.jpg", + "pictures/profile_picture_medium_banana_racc.jpg", + "pictures/profile_picture_small_banana_racc.jpg" + ].sort) + end + + it "contains the profile picture info on the exported user" do + expect(json_file("user.json").fetch(:user)).to include( + profile_picture_file_name: "banana_racc.jpg", + profile_picture_x: 571, + profile_picture_y: 353, + profile_picture_w: 474, + profile_picture_h: 474 + ) + end + end + + context "when user has a profile header" do + let(:user_params) do + super().merge( + process_profile_header_upload: true, # force carrierwave_backgrounder to immediately process the image + profile_header: Rack::Test::UploadedFile.new(Rails.root.join("spec/fixtures/files/banana_racc.jpg"), "image/jpeg"), + profile_header_x: 0, + profile_header_y: 412, + profile_header_w: 1813, + profile_header_h: 423 + ) + end + + it "includes the pictures in the file list" do + expect(subject.keys.select { _1.start_with?("pictures/") }.sort).to eq([ + "pictures/profile_header_original_banana_racc.jpg", + "pictures/profile_header_web_banana_racc.jpg", + "pictures/profile_header_mobile_banana_racc.jpg", + "pictures/profile_header_retina_banana_racc.jpg" + ].sort) + end + + it "contains the profile header info on the exported user" do + expect(json_file("user.json").fetch(:user)).to include( + profile_header_file_name: "banana_racc.jpg", + profile_header_x: 0, + profile_header_y: 412, + profile_header_w: 1813, + profile_header_h: 423 + ) + end + end +end diff --git a/spec/shared_examples/data_export.rb b/spec/shared_examples/data_export.rb new file mode 100644 index 00000000..8cd6b84e --- /dev/null +++ b/spec/shared_examples/data_export.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require "json" + +RSpec.shared_context "DataExport" do + let(:user_params) { {} } + + let!(:user) { FactoryBot.create(:user, **user_params) } + + subject { described_class.call(user:) } + + def json_file(filename) = JSON.parse(subject[filename], symbolize_names: true) +end + +RSpec.configure do |c| + c.include_context "DataExport", data_export: true +end diff --git a/spec/support/example_exporter.rb b/spec/support/example_exporter.rb new file mode 100644 index 00000000..66508eab --- /dev/null +++ b/spec/support/example_exporter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +raise ArgumentError.new("This file should only be required in the 'test' environment! Current environment: #{Rails.env}") unless Rails.env.test? + +require "use_case/data_export/base" + +# an example exporter to be used for the tests of `Exporter` +# +# this only returning basic files, nothing user-specific. each exporter should be tested individually. +class ExampleExporter < UseCase::DataExport::Base + def files = { + "textfile.txt" => "Sample Text\n", + "pictures/example.jpg" => File.read(File.expand_path("../fixtures/files/banana_racc.jpg", __dir__)), + "some.json" => json_file!( + animals: %w[raccoon fox hyena deer dog], + big_number: 3457812374589235798, + booleans: { + yes: true, + no: false, + file_not_found: nil + } + ) + } +end From 17783fbf3820c873d57be95a44ee0144a46303ef Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 10 Dec 2022 03:28:37 +0100 Subject: [PATCH 3/9] update example config for use with radosgw --- config/justask.yml.example | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/justask.yml.example b/config/justask.yml.example index f4803e0c..1b19cff6 100644 --- a/config/justask.yml.example +++ b/config/justask.yml.example @@ -65,8 +65,11 @@ redis_url: "redis://localhost:6379" # aws_access_key_id: 'ACCESS KEY' # aws_secret_access_key: 'SECRET KEY' # region: 'space-pluto-1' - # URL host, comment out to use default, GENERALLY you don't want to define this + ## URL host, comment out to use default, GENERALLY you don't want to define this # host: 's3.wherever.com' + ## if you want to use Ceph RadosGW, use these options (region doesn't matter) + # endpoint: 'http://radosgw.example.com' + # path_style: true # bucket name, required # directory: 'retrospring' From e1bdb1324f43ac15991795fd1db55ba60b1e4ee1 Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 10 Dec 2022 04:21:37 +0100 Subject: [PATCH 4/9] make the export worker create a *real* notification and add specs for it --- app/models/notification/data_exported.rb | 5 ++++ app/models/user/data_export.rb | 5 ++++ .../notifications/type/_dataexport.html.haml | 8 +++++ app/workers/export_worker.rb | 20 +++++++++---- config/locales/views.en.yml | 6 +++- spec/workers/export_worker_spec.rb | 29 +++++++++++++++++++ 6 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 app/models/notification/data_exported.rb create mode 100644 app/models/user/data_export.rb create mode 100644 app/views/notifications/type/_dataexport.html.haml create mode 100644 spec/workers/export_worker_spec.rb diff --git a/app/models/notification/data_exported.rb b/app/models/notification/data_exported.rb new file mode 100644 index 00000000..dc9d1634 --- /dev/null +++ b/app/models/notification/data_exported.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +# NOTE: `target` is not really used for this notification, but it should be of type `User::DataExport` +class Notification::DataExported < Notification +end diff --git a/app/models/user/data_export.rb b/app/models/user/data_export.rb new file mode 100644 index 00000000..c8aee5e9 --- /dev/null +++ b/app/models/user/data_export.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +# stub model to nicely allow for data export notifications, do NOT use this directly! +class User::DataExport < User +end diff --git a/app/views/notifications/type/_dataexport.html.haml b/app/views/notifications/type/_dataexport.html.haml new file mode 100644 index 00000000..b39210fe --- /dev/null +++ b/app/views/notifications/type/_dataexport.html.haml @@ -0,0 +1,8 @@ +.media.notification + .notification__icon + %i.fa.fa-2x.fa-fw.fa-download + .media-body + %h6.media-heading.notification__user + = t(".heading") + .notification__text + = t(".text_html", settings_export: link_to(t(".settings_export"), settings_export_path)) diff --git a/app/workers/export_worker.rb b/app/workers/export_worker.rb index 420ee06e..411c5318 100644 --- a/app/workers/export_worker.rb +++ b/app/workers/export_worker.rb @@ -1,4 +1,7 @@ -require 'exporter' +# frozen_string_literal: true + +require "exporter" + class ExportWorker include Sidekiq::Worker @@ -6,11 +9,16 @@ class ExportWorker # @param user_id [Integer] the user id def perform(user_id) - exporter = Exporter.new User.find(user_id) + user = User.find(user_id) + + exporter = Exporter.new(user) exporter.export - question = Question.create(content: "Your #{APP_CONFIG['site_name']} data export is ready! You can download it " + - "from the settings page under the \"Export\" tab.", author_is_anonymous: true, - author_identifier: "retrospring_exporter") - Inbox.create(user_id: user_id, question_id: question.id, new: true) + + Notification::DataExported.create( + target_id: user.id, + target_type: "User::DataExport", + recipient: user, + new: true + ) end end diff --git a/config/locales/views.en.yml b/config/locales/views.en.yml index cfe1c84d..f419a71a 100644 --- a/config/locales/views.en.yml +++ b/config/locales/views.en.yml @@ -327,6 +327,10 @@ en: link_text: "their answer" other: link_text_html: "%{user}'s answer" + dataexport: + heading: "Your data export is ready" + text_html: "Head over to %{settings_export} to download it." + settings_export: "the settings page" reaction: heading_html: "%{user} smiled %{type} %{time} ago" answer: @@ -396,7 +400,7 @@ en:

The data is inside a .zip archive that contains some JSON files. The archive also contains a copy of your profile picture and header picture in all sizes.

Please note that you can only export your data once a week. Exporting your data - will take a while, so please be patient. You will receive a question once exporting + will take a while, so please be patient. You will receive a notification once exporting is done.

export: "Export" export_url: diff --git a/spec/workers/export_worker_spec.rb b/spec/workers/export_worker_spec.rb new file mode 100644 index 00000000..71935d95 --- /dev/null +++ b/spec/workers/export_worker_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe ExportWorker do + let(:user) { FactoryBot.create(:user) } + + describe "#perform" do + let(:exporter_double) { double("Exporter") } + + before do + # stub away the testing of the exporter itself since it is done in lib/exporter_spec + allow(exporter_double).to receive(:export) + allow(Exporter).to receive(:new).and_return(exporter_double) + end + + subject { described_class.new.perform(user.id) } + + it "creates an exported notification" do + expect { subject }.to change { Notification::DataExported.count }.by(1) + + notification = Notification::DataExported.last + expect(notification.target_id).to eq(user.id) + expect(notification.target_type).to eq("User::DataExport") + expect(notification.recipient).to eq(user) + expect(notification.new).to be true + end + end +end From 3b99e438762215c555cdc25fb829a536ca81aebe Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 10 Dec 2022 04:50:36 +0100 Subject: [PATCH 5/9] add openssl-3.0.1 to bundle --- Gemfile | 3 +++ Gemfile.lock | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Gemfile b/Gemfile index e8cd20ab..fff8cea1 100644 --- a/Gemfile +++ b/Gemfile @@ -113,3 +113,6 @@ gem "net-pop" gem "pundit", "~> 2.2" gem "rubyzip", "~> 2.3" + +# to solve https://github.com/jwt/ruby-jwt/issues/526 +gem "openssl", "~> 3.0", ">= 3.0.1" diff --git a/Gemfile.lock b/Gemfile.lock index be5d01e4..02c68b72 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -294,6 +294,7 @@ GEM omniauth-twitter (1.4.0) omniauth-oauth (~> 1.1) rack + openssl (3.0.1) orm_adapter (0.5.0) parallel (1.22.1) parser (3.1.2.1) @@ -550,6 +551,7 @@ DEPENDENCIES omniauth omniauth-rails_csrf_protection (~> 1.0) omniauth-twitter + openssl (~> 3.0, >= 3.0.1) pg pghero puma From 3e143954e3f5c7fbf587e000f1a90d2a6536f61f Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 10 Dec 2022 15:47:54 +0100 Subject: [PATCH 6/9] data_export: yeet inbox_entries --- lib/exporter.rb | 1 - lib/use_case/data_export/inbox_entries.rb | 23 -------- .../data_export/inbox_entries_spec.rb | 54 ------------------- 3 files changed, 78 deletions(-) delete mode 100644 lib/use_case/data_export/inbox_entries.rb delete mode 100644 spec/lib/use_case/data_export/inbox_entries_spec.rb diff --git a/lib/exporter.rb b/lib/exporter.rb index eb5101d3..236db967 100644 --- a/lib/exporter.rb +++ b/lib/exporter.rb @@ -7,7 +7,6 @@ require "zip/filesystem" require "use_case/data_export/answers" require "use_case/data_export/appendables" require "use_case/data_export/comments" -require "use_case/data_export/inbox_entries" require "use_case/data_export/mute_rules" require "use_case/data_export/questions" require "use_case/data_export/relationships" diff --git a/lib/use_case/data_export/inbox_entries.rb b/lib/use_case/data_export/inbox_entries.rb deleted file mode 100644 index 04b2efb4..00000000 --- a/lib/use_case/data_export/inbox_entries.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require "use_case/data_export/base" - -module UseCase - module DataExport - class InboxEntries < UseCase::DataExport::Base - def files = { - "inbox_entries.json" => json_file!( - inbox_entries: user.inboxes.map(&method(:collect_inbox_entry)) - ) - } - - def collect_inbox_entry(inbox_entry) - {}.tap do |h| - column_names(::Inbox).each do |field| - h[field] = inbox_entry[field] - end - end - end - end - end -end diff --git a/spec/lib/use_case/data_export/inbox_entries_spec.rb b/spec/lib/use_case/data_export/inbox_entries_spec.rb deleted file mode 100644 index 9682ab76..00000000 --- a/spec/lib/use_case/data_export/inbox_entries_spec.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -require "use_case/data_export/inbox_entries" - -describe UseCase::DataExport::InboxEntries, :data_export do - include ActiveSupport::Testing::TimeHelpers - - context "when user doesn't have anything in their inbox" do - it "returns an empty set of inbox entries" do - expect(json_file("inbox_entries.json")).to eq( - { - inbox_entries: [] - } - ) - end - end - - context "when user has some questions in their inbox" do - let!(:inbox_entries) do - [ - # using `Inbox.create` here as for some reason FactoryBot.create(:inbox) always sets `new` to `nil`??? - travel_to(Time.utc(2022, 12, 10, 13, 37, 42)) { Inbox.create(user:, question: FactoryBot.create(:question), new: false) }, - travel_to(Time.utc(2022, 12, 10, 13, 39, 21)) { Inbox.create(user:, question: FactoryBot.create(:question), new: true) } - ] - end - - it "returns the inbox entries as json" do - expect(json_file("inbox_entries.json")).to eq( - { - inbox_entries: [ - { - id: inbox_entries[0].id, - user_id: user.id, - question_id: inbox_entries[0].question_id, - new: false, - created_at: "2022-12-10T13:37:42.000Z", - updated_at: "2022-12-10T13:37:42.000Z" - }, - { - id: inbox_entries[1].id, - user_id: user.id, - question_id: inbox_entries[1].question_id, - new: true, - created_at: "2022-12-10T13:39:21.000Z", - updated_at: "2022-12-10T13:39:21.000Z" - } - ] - } - ) - end - end -end From d52529c84038440265d577e342042cc82ecdaf7a Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 10 Dec 2022 15:56:07 +0100 Subject: [PATCH 7/9] mark dataexported notifications as read when visiting export page --- app/controllers/settings/export_controller.rb | 9 +++++++++ .../settings/export_controller_spec.rb | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/app/controllers/settings/export_controller.rb b/app/controllers/settings/export_controller.rb index 5656e614..1a8bc0a2 100644 --- a/app/controllers/settings/export_controller.rb +++ b/app/controllers/settings/export_controller.rb @@ -2,6 +2,7 @@ class Settings::ExportController < ApplicationController before_action :authenticate_user! + before_action :mark_notifications_as_read, only: %i[index] def index flash[:info] = t(".info") if current_user.export_processing @@ -17,4 +18,12 @@ class Settings::ExportController < ApplicationController redirect_to settings_export_path end + + private + + def mark_notifications_as_read + Notification::DataExported + .where(recipient: current_user, new: true) + .update_all(new: false) # rubocop:disable Rails/SkipsModelValidations + end end diff --git a/spec/controllers/settings/export_controller_spec.rb b/spec/controllers/settings/export_controller_spec.rb index 97cdae63..0750ed6e 100644 --- a/spec/controllers/settings/export_controller_spec.rb +++ b/spec/controllers/settings/export_controller_spec.rb @@ -15,6 +15,21 @@ describe Settings::ExportController, type: :controller do subject expect(response).to render_template(:index) end + + context "when user has a new DataExported notification" do + let(:notification) do + Notification::DataExported.create( + target_id: user.id, + target_type: "User::DataExport", + recipient: user, + new: true + ) + end + + it "marks the notification as read" do + expect { subject }.to change { notification.reload.new }.from(true).to(false) + end + end end end From 5bad3dda3332b82a36f4dc1863985d20ba7bce2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 11 Dec 2022 18:57:55 +0000 Subject: [PATCH 8/9] Bump nokogiri from 1.13.9 to 1.13.10 Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.9 to 1.13.10. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.13.9...v1.13.10) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 02c68b72..a5fc7f47 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -276,7 +276,7 @@ GEM net-smtp (0.3.3) net-protocol nio4r (2.5.8) - nokogiri (1.13.9) + nokogiri (1.13.10) mini_portile2 (~> 2.8.0) racc (~> 1.4) oauth (0.5.8) From 870167a9ae5de91324bc5897b3cf81b2dd8eb9ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 09:02:03 +0000 Subject: [PATCH 9/9] Bump typescript from 4.9.3 to 4.9.4 Bumps [typescript](https://github.com/Microsoft/TypeScript) from 4.9.3 to 4.9.4. - [Release notes](https://github.com/Microsoft/TypeScript/releases) - [Commits](https://github.com/Microsoft/TypeScript/compare/v4.9.3...v4.9.4) --- updated-dependencies: - dependency-name: typescript dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 911b9d62..824d4d8e 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "regenerator-runtime": "^0.13.11", "sweetalert": "1.1.3", "toastify-js": "^1.12.0", - "typescript": "^4.9.3" + "typescript": "^4.9.4" }, "devDependencies": { "@babel/core": "^7.20.5", diff --git a/yarn.lock b/yarn.lock index 69bde41b..a5161b1d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7597,10 +7597,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@^4.9.3: - version "4.9.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.3.tgz#3aea307c1746b8c384435d8ac36b8a2e580d85db" - integrity sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA== +typescript@^4.9.4: + version "4.9.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78" + integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== unbox-primitive@^1.0.1: version "1.0.1"