Anjing Webshells

Website : rimsha.abasa.com
: / var / canvas / app / models /
Filename : content_migration.rb
back
# frozen_string_literal: true

#
# Copyright (C) 2011 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#

class ContentMigration < ActiveRecord::Base
  include Workflow
  include HtmlTextHelper
  include Rails.application.routes.url_helpers
  include CanvasOutcomesHelper

  belongs_to :context, polymorphic: [:course, :account, :group, { context_user: "User" }]
  validate :valid_date_shift_options
  belongs_to :user
  belongs_to :attachment
  belongs_to :overview_attachment, class_name: "Attachment"
  belongs_to :exported_attachment, class_name: "Attachment"
  belongs_to :asset_map_attachment, class_name: "Attachment", optional: true
  belongs_to :source_course, class_name: "Course"
  belongs_to :root_account, class_name: "Account"
  has_one :content_export
  has_many :migration_issues, dependent: :destroy
  has_many :quiz_migration_alerts, as: :migration, inverse_of: :migration, dependent: :destroy
  has_one :job_progress, class_name: "Progress", as: :context, inverse_of: :context
  serialize :migration_settings, yaml: { permitted_classes: [Symbol, Class] }
  cattr_accessor :export_file_path
  before_save :assign_quiz_migration_limitation_alert
  before_save :set_started_at_and_finished_at
  after_save :handle_import_in_progress_notice
  after_save :check_for_blocked_migration
  before_create :set_root_account_id

  DATE_FORMAT = "%m/%d/%Y"

  attr_accessor :outcome_to_id_map,
                :attachment_path_id_lookup,
                :last_module_position,
                :skipped_master_course_items,
                :copied_external_outcome_map
  attr_writer :imported_migration_items

  has_a_broadcast_policy

  workflow do
    state :created
    state :queued
    # The pre_process states can be used by individual plugins as needed
    state :pre_processing
    state :pre_processed
    state :pre_process_error
    state :exporting
    state :exported
    state :importing
    state :imported
    state :failed
  end

  def self.migration_plugins(exclude_hidden = false)
    plugins = Canvas::Plugin.all_for_tag(:export_system)
    exclude_hidden ? plugins.reject { |p| p.meta[:hide_from_users] } : plugins
  end

  set_policy do
    given do |user, session|
      context.grants_any_right?(user, session, *RoleOverride::GRANULAR_FILE_PERMISSIONS)
    end
    can :read
  end

  def trigger_live_events!
    # Trigger live events for the source course and migration
    Canvas::LiveEventsCallbacks.after_update(context, context.saved_changes)
    Canvas::LiveEventsCallbacks.after_update(self, saved_changes)

    # Trigger live event for new quizzes if needed
    if initiated_source == :new_quizzes
      Canvas::LiveEvents.quizzes_next_migration_urls_complete(
        {
          original_course_uuid: source_course.uuid,
          new_course_uuid: context.uuid,
          domain: context.root_account&.domain(ApplicationController.test_cluster_name),
          resource_map_url: asset_map_url(generate_if_needed: true),
          migrated_urls_content_migration_id: global_id
        }
      )
    end

    # Trigger live events for all updated/created records
    imported_migration_items.each do |imported_item|
      next unless LiveEventsObserver.observed_classes.include? imported_item.class
      next if started_at.blank? || imported_item.created_at.blank?

      if imported_item.created_at > started_at
        Canvas::LiveEventsCallbacks.after_create(imported_item)
      else
        Canvas::LiveEventsCallbacks.after_update(imported_item, imported_item.saved_changes)
      end
    end
  end

  def set_started_at_and_finished_at
    if workflow_state_changed?
      if pre_processing? || exporting? || importing?
        self.started_at ||= Time.now.utc
      end
      if failed? || imported?
        self.finished_at ||= Time.now.utc
      end
    end
  end

  def assign_quiz_migration_limitation_alert
    if workflow_state_changed? && imported? && quizzes_next_migration? &&
       NewQuizzesFeaturesHelper.new_quizzes_bank_migrations_enabled?(context)
      context.create_or_update_quiz_migration_alert(user_id, self)
    end
  end

  def quota_context
    context
  end

  def migration_settings
    read_or_initialize_attribute(:migration_settings, {}.with_indifferent_access)
  end

  # this is needed by Attachment#clone_for, which is used to allow a ContentExport to be directly imported
  def attachments
    Attachment.where(id: attachment_id)
  end

  def update_migration_settings(new_settings)
    new_settings.each do |key, val|
      migration_settings[key] = val
    end
  end

  def import_immediately?
    !!migration_settings[:import_immediately]
  end

  def content_export
    if persisted? && !association(:content_export).loaded? && source_course_id
      association(:content_export).target = Shard.shard_for(source_course_id).activate do
        ContentExport.where(content_migration_id: self).first ||
          # since reusing content_exports in multiple content_migrations breaks the nominal 1:1 association
          # find the content_export by source_course_id and attachment_id instead
          (attachment_id && ContentExport.where(context_type: "Course",
                                                context_id: source_course_id,
                                                attachment_id:).first)
      end
    end
    super
  end

  def converter_class=(c_class)
    migration_settings[:converter_class] = c_class
  end

  def converter_class
    migration_settings[:converter_class]
  end

  def strand=(s)
    migration_settings[:strand] = s
  end

  def strand
    migration_settings[:strand]
  end

  def initiated_source
    migration_settings[:initiated_source] || :manual
  end

  def initiated_source=(value)
    migration_settings[:initiated_source] = value
  end

  def n_strand
    account_identifier = root_account.try(:global_id) || "global"
    type = for_master_course_import? ? "master_course" : initiated_source
    ["migrations:import_content", "#{type}_#{account_identifier}"]
  end

  def migration_ids_to_import=(val)
    migration_settings[:migration_ids_to_import] = val
    set_date_shift_options val[:copy]
  end

  def zip_path=(val)
    migration_settings[:export_archive_path] = val
  end

  def zip_path
    (migration_settings || {})[:export_archive_path]
  end

  def question_bank_name=(name)
    if (name = name&.strip) != ""
      migration_settings[:question_bank_name] = name
    end
  end

  def question_bank_name
    migration_settings[:question_bank_name]
  end

  def question_bank_id=(bank_id)
    migration_settings[:question_bank_id] = bank_id
  end

  def question_bank_id
    migration_settings[:question_bank_id]
  end

  def course_archive_download_url=(url)
    migration_settings[:course_archive_download_url] = url
  end

  def skip_job_progress=(val)
    if val
      migration_settings[:skip_job_progress] = true
    else
      migration_settings.delete(:skip_job_progress)
    end
  end

  def skip_job_progress
    !!migration_settings[:skip_job_progress]
  end

  def root_account
    return super if root_account_id

    context.root_account rescue nil
  end

  def migration_type
    read_attribute(:migration_type) || migration_settings["migration_type"]
  end

  def plugin_type
    if (plugin = Canvas::Plugin.find(migration_type))
      plugin.metadata(:select_text) || plugin.name
    else
      t(:unknown, "Unknown")
    end
  end

  def canvas_import?
    migration_settings[:worker_class] == CC::Importer::Canvas::Converter.name
  end

  # add todo/error/warning issue to the import. user_message is what will be
  # displayed to the end user.
  # type must be one of: :todo, :warning, :error
  #
  # The possible opts keys are:
  #
  # error_message - an admin-only error message
  # exception - an exception object
  # error_report_id - the id to an error report
  # fix_issue_html_url - the url to send the user to to fix problem
  #
  ISSUE_TYPE_TO_ERROR_LEVEL_MAP = {
    todo: :info,
    warning: :warn,
    error: :error
  }.freeze

  def add_issue(user_message, type, opts = {})
    mi = migration_issues.build(issue_type: type.to_s, description: user_message)
    if opts[:error_report_id]
      mi.error_report_id = opts[:error_report_id]
    elsif opts[:exception]
      level = ISSUE_TYPE_TO_ERROR_LEVEL_MAP[type]
      er = Canvas::Errors.capture_exception(:content_migration, opts[:exception], level)[:error_report]
      mi.error_report_id = er
    end
    mi.error_message = opts[:error_message]
    mi.fix_issue_html_url = opts[:fix_issue_html_url]

    # prevent duplicates
    if migration_issues.where(mi.attributes.slice(
                                "issue_type", "description", "error_message", "fix_issue_html_url"
                              )).any?
      mi.delete
    else
      mi.save!
    end

    mi
  end

  def add_todo(user_message, opts = {})
    add_issue(user_message, :todo, opts)
  end

  def add_error(user_message, opts = {})
    level = opts.fetch(:issue_level, :error)
    add_issue(user_message, level, opts)
  end

  def add_warning(user_message, opts = {})
    Rails.logger.warn("Migration warning: #{user_message}: #{opts.inspect}")
    unless opts.is_a? Hash
      # convert deprecated behavior to new
      exception_or_info = opts
      opts = {}
      if exception_or_info.is_a?(Exception)
        opts[:exception] = exception_or_info
      else
        opts[:error_message] = exception_or_info
      end
    end
    add_issue(user_message, :warning, opts)
  end

  def add_unique_warning(key, warning, opts = {})
    @added_warnings ||= Set.new
    return if @added_warnings.include?(key) # only add it once

    @added_warnings << key
    add_warning(warning, opts)
  end

  def add_import_warning(item_type, item_name, warning)
    item_name = CanvasTextHelper.truncate_text(item_name || "", max_length: 150)
    add_warning(t("errors.import_error", "Import Error:") + " #{item_type} - \"#{item_name}\"", warning)
  end

  def fail_with_error!(exception_or_info, error_message: nil, issue_level: :error)
    opts = { issue_level: }
    if exception_or_info.is_a?(Exception)
      opts[:exception] = exception_or_info
    else
      opts[:error_message] = exception_or_info
    end
    message = error_message || t(:unexpected_error, "There was an unexpected error, please contact support.")
    add_error(message, opts)
    self.workflow_state = :failed
    job_progress.fail if job_progress && !skip_job_progress
    save
    update_master_migration("failed") if for_master_course_import?
    resolve_content_links! # don't leave placeholders
  end

  # deprecated warning format
  def old_warnings_format
    migration_issues.map do |mi|
      message = mi.error_report_id ? "ErrorReport:#{mi.error_report_id}" : mi.error_message
      [mi.description, message]
    end
  end

  def warnings
    old_warnings_format.map(&:first)
  end

  # This will be called by the files api after the attachment finishes uploading
  def file_upload_success_callback(att)
    if att.file_state == "available"
      self.attachment = att
      migration_issues.delete_all if migration_issues.any?
      self.workflow_state = :pre_processed
      save
      queue_migration
    else
      self.workflow_state = :pre_process_error
      add_warning(t("bad_attachment", "The file was not successfully uploaded."))
    end
  end

  def reset_job_progress(wf_state = :queued)
    return if skip_job_progress

    self.progress = 0
    if job_progress
      p = job_progress
    else
      p = shard.activate { Progress.new(context: self, tag: "content_migration") }
      self.job_progress = p
    end
    p.workflow_state = wf_state
    p.completion = 0
    p.user = user
    p.save!
    p
  end

  def queue_migration(plugin = nil, retry_count: 0, expires_at: nil, priority: Delayed::LOW_PRIORITY)
    reset_job_progress unless plugin && plugin.settings[:skip_initial_progress]

    expires_at ||= Setting.get("content_migration_job_expiration_hours", "48").to_i.hours.from_now
    return if blocked_by_current_migration?(plugin, retry_count, expires_at)

    migration_issues.delete_all
    set_default_settings

    plugin ||= Canvas::Plugin.find(migration_type)
    if plugin
      queue_opts = { priority:,
                     max_attempts: 1,
                     expires_at: }
      if strand
        queue_opts[:strand] = strand
      else
        queue_opts[:n_strand] = n_strand
      end

      if plugin.settings[:import_immediately] || (workflow_state == "exported" && !plugin.settings[:skip_conversion_step])
        # it's ready to be imported
        self.workflow_state = :importing
        save
        delay(**queue_opts.merge(on_permanent_failure: :fail_with_error!)).import_content
      else
        # find worker and queue for conversion
        begin
          worker_class = Canvas::Migration::Worker.const_get(plugin.settings["worker"])
          self.workflow_state = :exporting
          save
          self.class.connection.after_transaction_commit do
            Delayed::Job.enqueue(worker_class.new(id), **queue_opts)
          end
        rescue NameError
          self.workflow_state = "failed"
          message = "The migration plugin #{migration_type} doesn't have a worker."
          migration_settings[:last_error] = message
          Canvas::Errors.capture_exception(:content_migration, $ERROR_INFO)
          logger.error message
          save
        end
      end
    else
      self.workflow_state = "failed"
      message = "No migration plugin of type #{migration_type} found."
      migration_settings[:last_error] = message
      logger.error message
      save
    end
  end
  alias_method :export_content, :queue_migration

  def blocked_by_current_migration?(plugin, retry_count, expires_at)
    return false if migration_type == "zip_file_importer"

    running_cutoff = Setting.get("content_migration_job_block_hours", "4").to_i.hours.ago # at some point just let the jobs keep going

    if context && context.content_migrations
                         .where(workflow_state: %w[created queued pre_processing pre_processed exporting importing]).where(id: ...id)
                         .where("started_at > ?", running_cutoff).exists?

      # there's another job already going so punt

      if retry_count > 5
        fail_with_error!(I18n.t("Blocked by running migration"))
      else
        self.workflow_state = :queued
        save

        run_at = Setting.get("content_migration_requeue_delay_minutes", "60").to_i.minutes.from_now
        # if everything goes right, we'll queue it right away after the currently running one finishes
        # but if something goes catastropically wrong, then make sure we recheck it eventually
        job = delay(ignore_transaction: true, run_at:).queue_migration(
          plugin, retry_count: retry_count + 1, expires_at:
        )

        if job_progress
          job_progress.delayed_job_id = job.id
          job_progress.save!
        end
      end

      true
    else
      false
    end
  end

  def set_default_settings
    if context.respond_to?(:root_account) &&
       (account = context.root_account) &&
       (default_ms = account.settings[:default_migration_settings])
      self.migration_settings = default_ms.merge(migration_settings).with_indifferent_access
    end

    unless migration_settings.key?(:overwrite_quizzes)
      migration_settings[:overwrite_quizzes] = for_course_copy? || for_master_course_import? || (migration_type && migration_type == "canvas_cartridge_importer")
    end
    migration_settings.reverse_merge!(prefer_existing_tools: true) if migration_type == "common_cartridge_importer" # default to true

    check_quiz_id_prepender
  end

  def process_domain_substitutions(url)
    unless @domain_substitution_map
      @domain_substitution_map = {}
      (migration_settings[:domain_substitution_map] || {}).each do |k, v|
        @domain_substitution_map[k.to_s] = v.to_s # ensure strings
      end
    end

    @domain_substitution_map.each do |from_domain, to_domain|
      if url.start_with?(from_domain)
        return url.sub(from_domain, to_domain)
      end
    end

    url
  end

  def check_quiz_id_prepender
    return unless context.respond_to?(:assessment_questions)

    if !migration_settings[:id_prepender] && (!migration_settings[:overwrite_questions] || !migration_settings[:overwrite_quizzes])
      migration_settings[:id_prepender] = id
    end
  end

  def to_import(val)
    migration_settings[:migration_ids_to_import] && migration_settings[:migration_ids_to_import][:copy] && migration_settings[:migration_ids_to_import][:copy][val]
  end

  def import_everything?
    return true unless migration_settings[:migration_ids_to_import] && migration_settings[:migration_ids_to_import][:copy] && !migration_settings[:migration_ids_to_import][:copy].empty?
    return true if is_set?(to_import(:everything))
    return true if copy_options && is_set?(copy_options[:everything])

    false
  end

  def original_id_for(mig_id)
    return nil unless mig_id.is_a?(String)

    prefix = "#{migration_settings[:id_prepender]}_"
    return nil unless mig_id.start_with? prefix

    mig_id[prefix.length..]
  end

  def import_object?(asset_type, mig_id)
    return false unless mig_id
    return true if import_everything?

    return true if is_set?(to_import("all_#{asset_type}"))

    return false unless to_import(asset_type).present?

    return true if (orig_id = original_id_for(mig_id)) && is_set?(to_import(asset_type)[orig_id])

    is_set?(to_import(asset_type)[mig_id])
  end

  def import_object!(asset_type, mig_id)
    return if import_everything?

    migration_settings[:migration_ids_to_import][:copy][asset_type] ||= {}
    migration_settings[:migration_ids_to_import][:copy][asset_type][mig_id] = "1"
  end

  def is_set?(option)
    Canvas::Plugin.value_to_boolean option
  end

  def capture_job_id
    job = Delayed::Worker.current_job
    return false unless job

    migration_settings[:job_ids] ||= []
    return false if migration_settings[:job_ids].include?(job.id)

    migration_settings[:job_ids] << job.id
    true
  end

  def import_content
    reset_job_progress(:running) unless import_immediately?
    self.workflow_state = :importing
    capture_job_id
    save

    Lti::Asset.opaque_identifier_for(context)

    all_files_path = nil
    begin
      data = nil
      if for_master_course_import?
        master_course_subscription.load_tags! # load child content tags
        master_course_subscription.master_template.preload_restrictions!

        data = JSON.parse(exported_attachment.open, max_nesting: 50)
        data = prepare_data(data)

        # handle deletions before files are copied
        deletions = data["deletions"].presence
        process_master_deletions(deletions.except("LearningOutcome", "AssignmentGroup")) if deletions # wait until after the import to do LearningOutcomes and AssignmentGroups

        # copy the attachments
        source_export = ContentExport.find(migration_settings[:master_course_export_id])
        if source_export.selective_export?
          # load in existing attachments to path resolution map
          file_mig_ids = source_export.settings[:referenced_file_migration_ids]
          if file_mig_ids.present?
            # ripped from copy_attachments_from_course
            root_folder_name = Folder.root_folders(context).first.name + "/"
            context.attachments.where(migration_id: file_mig_ids).each do |file|
              add_attachment_path(file.full_display_path.gsub(/\A#{root_folder_name}/, ""), file.migration_id)
            end
          end
        end
        # sync the existing folders first in case someone did something weird like deleted and replaced a folder in the same sync
        MasterCourses::FolderHelper.update_folder_names_and_states(context, source_export)
        context.copy_attachments_from_course(source_export.context, content_export: source_export, content_migration: self)
        MasterCourses::FolderHelper.recalculate_locked_folders(context)
      elsif for_course_template?
        data = JSON.parse(exported_attachment.open, max_nesting: 50)
        data = prepare_data(data)
      else
        @exported_data_zip = download_exported_data
        @zip_file = Zip::File.open(@exported_data_zip.path)
        @exported_data_zip.close
        data = JSON.parse(@zip_file.read("course_export.json"), max_nesting: 50)
        data = prepare_data(data)

        if @zip_file.find_entry("all_files.zip")
          # the file importer needs an actual file to process
          all_files_path = create_all_files_path(@exported_data_zip.path)
          @zip_file.extract("all_files.zip", all_files_path)
          data["all_files_export"]["file_path"] = all_files_path
        else
          data["all_files_export"]["file_path"] = nil
        end
        @zip_file.close
      end

      migration_settings[:migration_ids_to_import] ||= { copy: {} }

      import!(data)

      process_master_deletions(deletions.slice("LearningOutcome", "AssignmentGroup")) if deletions
      SmartSearch.copy_embeddings(self) if for_master_course_import?

      unless import_immediately?
        update_import_progress(100)
      end
      if for_master_course_import?
        update_master_migration("completed")
      end
    rescue => e
      self.workflow_state = :failed
      er_id = Canvas::Errors.capture_exception(:content_migration, e)[:error_report]
      migration_settings[:last_error] = "ErrorReport:#{er_id}"
      logger.error e
      save
      update_master_migration("failed") if for_master_course_import?
      raise e
    ensure
      File.delete(all_files_path) if all_files_path && File.exist?(all_files_path)
      clear_migration_data
    end
  end
  alias_method :import_content_without_send_later, :import_content

  def import!(data)
    return import_quizzes_next!(data) if cc_qti_migration? || quizzes_next_migration?

    Importers.content_importer_for(context_type)
             .import_content(
               context,
               data,
               migration_settings[:migration_ids_to_import],
               self
             )
  end

  def cc_qti_migration?
    context.instance_of?(Course) &&
      NewQuizzesFeaturesHelper.common_cartridge_qti_new_quizzes_import_enabled?(context) &&
      for_common_cartridge?
  end

  def import_quizzes_next?
    !!migration_settings[:import_quizzes_next]
  end

  def quizzes_next_migration?
    context.instance_of?(Course) &&
      context.feature_enabled?(:quizzes_next) &&
      import_quizzes_next?
  end

  def quizzes_next_banks_migration?
    quizzes_next_migration? && Account.site_admin.feature_enabled?(:new_quizzes_bank_migrations)
  end

  def import_quizzes_next!(data)
    quizzes2_importer =
      QuizzesNext::Importers::CourseContentImporter.new(data, self)
    quizzes2_importer.import_content(
      migration_settings[:migration_ids_to_import]
    )
  end

  def master_migration
    @master_migration ||= shard.activate { MasterCourses::MasterMigration.find(migration_settings[:master_migration_id]) }
  end

  def update_master_migration(state)
    master_migration.update_import_state!(self, state)
  end

  def master_course_subscription
    return unless for_master_course_import?

    @master_course_subscription ||= shard.activate { MasterCourses::ChildSubscription.find(child_subscription_id) }
  end

  def prepare_data(data)
    data = data.with_indifferent_access if data.is_a? Hash
    Utf8Cleaner.recursively_strip_invalid_utf8!(data, true)
    data["all_files_export"] ||= {}
    data
  end

  def copy_options
    migration_settings[:copy_options]
  end

  def copy_options=(options)
    migration_settings[:copy_options] = options
    set_date_shift_options options
  end

  def for_course_copy?
    migration_type == "course_copy_importer" || for_master_course_import?
  end

  def for_course_template?
    initiated_source == :course_template
  end

  def for_common_cartridge?
    ["common_cartridge_importer", "canvas_cartridge_importer"].include? migration_type
  end

  def should_skip_import?(content_importer)
    migration_settings[:importer_skips]&.include?(content_importer)
  end

  def for_master_course_import?
    migration_type == "master_course_import"
  end

  def add_skipped_item(item)
    @skipped_master_course_items ||= Set.new
    item = item.migration_id if item.is_a?(MasterCourses::ChildContentTag)
    @skipped_master_course_items << item
  end

  def process_master_deletions(deletions)
    deletions.each_key do |klass|
      next unless MasterCourses::CONTENT_TYPES_FOR_DELETIONS.include?(klass)

      mig_ids = deletions[klass]
      item_scope = case klass
                   when "Attachment"
                     context.attachments.not_deleted.where(migration_id: mig_ids)
                   when "CoursePace"
                     context.course_paces.where(migration_id: mig_ids)
                   else
                     klass.constantize.where(context_id: context, context_type: "Course", migration_id: mig_ids)
                          .where.not(workflow_state: "deleted")
                   end
      item_scope.each do |content|
        child_tag = master_course_subscription.content_tag_for(content)
        skip_item = child_tag.downstream_changes.any? && !content.editing_restricted?(:any)
        outcome, link = get_outcome_and_link(content, context)
        content_is_outcome = !outcome.nil? && !link.nil?
        if content.is_a?(AssignmentGroup) && !skip_item && content.assignments.active.exists?
          skip_item = true # don't delete an assignment group if an assignment is left (either they added one or changed one so it was skipped)
        end

        if skip_item
          Rails.logger.debug("skipping deletion sync for #{content.asset_string} due to downstream changes #{child_tag.downstream_changes}")
          add_skipped_item(child_tag)
        elsif content_is_outcome && outcome_has_results?(outcome, context)
          Rails.logger.debug { "skipping deletion sync for #{content.asset_string} due to there are Learning Outcomes Results" }
          add_skipped_item(child_tag)
        elsif content_is_outcome && outcome_has_active_alignments?(link, outcome, context)
          Rails.logger.debug { "skipping deletion sync for #{content.asset_string} due to there are active Alignments to Content" }
          add_skipped_item(child_tag)
        else
          Rails.logger.debug("syncing deletion of #{content.asset_string} from master course")
          content.skip_downstream_changes! if content.respond_to?(:skip_downstream_changes!)
          content.destroy
        end
      end
    end
  end

  def get_outcome_and_link(content, context)
    outcome = nil
    link = nil
    if content.is_a?(LearningOutcome)
      outcome = content
      context_type = context.is_a?(Course) ? "Course" : "Account"
      link = ContentTag.find_by(content_id: outcome.id, content_type: "LearningOutcome", associated_asset_type: "LearningOutcomeGroup", context_id: context.id, context_type:)
    elsif content.is_a?(ContentTag) && content.content_type == "LearningOutcome"
      link = content
      outcome = LearningOutcome.find_by(id: content.content_id, context_type: "Account")
    end
    [outcome, link]
  end

  def outcome_has_active_alignments?(link, outcome, context)
    !link.can_destroy? || outcome_has_alignments?(outcome, context)
  end

  def outcome_has_results?(outcome, context)
    return true if outcome.learning_outcome_results.where("workflow_state <> 'deleted' AND context_type='Course' AND context_code='course_#{context.id}'").count > 0

    outcome_has_authoritative_results?(outcome, context)
  end

  def check_cross_institution
    return unless context.is_a?(Course)

    data = context.full_migration_hash
    return unless data

    source_root_account_uuid = data[:course] && data[:course][:root_account_uuid]
    @cross_institution = source_root_account_uuid && source_root_account_uuid != context.root_account.uuid
  end

  def cross_institution?
    @cross_institution
  end

  def find_source_course_for_import
    return unless context.is_a?(Course)

    data = context.full_migration_hash[:context_info]
    return unless data.is_a?(Hash)

    course_id = data[:course_id]
    account_global_id = data[:root_account_id]
    account_uuid = data[:root_account_uuid]
    return unless course_id && account_global_id && account_uuid

    possible_root_account = Account.find_by(id: account_global_id)
    real_root_account = possible_root_account if possible_root_account&.uuid == account_uuid
    if Object.const_defined?(:AccountDomain) && !real_root_account
      domain = data[:canvas_domain]
      possible_root_account = domain && AccountDomain.find_cached(domain)&.account
      real_root_account = possible_root_account if possible_root_account&.uuid == account_uuid
    end

    if real_root_account
      self.source_course_id = Shard.global_id_for(course_id, real_root_account.shard)
    end

    source_course_id
  end

  def set_date_shift_options(opts)
    if opts && (Canvas::Plugin.value_to_boolean(opts[:shift_dates]) || Canvas::Plugin.value_to_boolean(opts[:remove_dates]))
      migration_settings[:date_shift_options] = opts.slice(:shift_dates, :remove_dates, :old_start_date, :old_end_date, :new_start_date, :new_end_date, :day_substitutions, :time_zone)
    end
  end

  def date_shift_options
    migration_settings[:date_shift_options]
  end

  def valid_date_shift_options
    if date_shift_options && Canvas::Plugin.value_to_boolean(date_shift_options[:shift_dates]) && Canvas::Plugin.value_to_boolean(date_shift_options[:remove_dates])
      errors.add(:date_shift_options, t("errors.cannot_shift_and_remove", "cannot specify shift_dates and remove_dates simultaneously"))
    end
  end

  scope :for_context, ->(context) { where(context_id: context, context_type: context.class.to_s) }

  scope :successful, -> { where(workflow_state: "imported") }
  scope :running, -> { where(workflow_state: ["exporting", "importing"]) }
  scope :waiting, -> { where(workflow_state: "exported") }
  scope :failed, -> { where(workflow_state: ["failed", "pre_process_error"]) }

  def complete?
    %w[imported failed pre_process_error].include?(workflow_state)
  end

  def download_exported_data
    raise "No exported data to import" unless exported_attachment

    config = ConfigFile.load("external_migration") || {}
    @exported_data_zip = exported_attachment.open(
      temp_folder: config[:data_folder]
    )
    @exported_data_zip
  end

  def create_all_files_path(temp_path)
    "#{temp_path}_all_files.zip"
  end

  def clear_migration_data
    @zip_file&.close
    @zip_file = nil
  end

  def finished_converting
    # TODO: finish progress if selective
  end

  # expects values between 0 and 100 for the conversion process
  def update_conversion_progress(prog)
    if import_immediately?
      fast_update_progress(prog * 0.5)
    else
      fast_update_progress(prog)
    end
  end

  # expects values between 0 and 100 for the import process
  def update_import_progress(prog)
    if import_immediately?
      fast_update_progress(50 + (prog * 0.5))
    else
      fast_update_progress(prog)
    end
  end

  def progress
    return nil if workflow_state == "created"

    mig_prog = read_attribute(:progress) || 0
    if for_course_copy?
      # this is for a course copy so it needs to combine the progress of the export and import
      # The export will count for 40% of progress
      # The importing step (so the value of progress on this object)will be 60%
      mig_prog *= 0.6

      if content_export
        export_prog = content_export.progress || 0
        mig_prog += export_prog * 0.4
      end
    end

    mig_prog
  end

  def fast_update_progress(val)
    reset_job_progress unless job_progress
    unless skip_job_progress
      if val == 100
        job_progress.completion = 100
        job_progress.workflow_state = "completed"
        job_progress.save!
      else
        job_progress.update_completion!(val)
      end
    end
    # Until this progress is phased out
    self.progress = val
    ContentMigration.where(id: self).update_all(progress: val)
  end

  def html_converter
    @html_converter ||= CanvasImportedHtmlConverter.new(self)
  end

  def convert_html(*args, **keyword_args)
    html_converter.convert(*args, **keyword_args)
  end

  def convert_text(text)
    format_message(text || "")[0]
  end

  delegate :resolve_content_links!, to: :html_converter

  def add_warning_for_missing_content_links(type, field, missing_links, fix_issue_url)
    add_warning(t(:missing_content_links_title, "Missing links found in imported content") + " - #{type} #{field}",
                { error_message: "#{type} #{field} - " + t(:missing_content_links_message,
                                                           "The following references could not be resolved:") + " " + missing_links.join(", "),
                  fix_issue_html_url: fix_issue_url })
  end

  UPLOAD_TIMEOUT = 1.hour
  def check_for_pre_processing_timeout
    if pre_processing? && (updated_at.utc + UPLOAD_TIMEOUT) < Time.now.utc
      add_error(t(:upload_timeout_error, "The file upload process timed out."))
      self.workflow_state = :failed
      job_progress.fail if job_progress && !skip_job_progress
      save
    end
  end

  # maps the key in the copy parameters hash to the asset string prefix
  # (usually it's just .singularize; weird names needing special casing go here :P)
  def self.asset_string_prefix(key)
    case key
    when "quizzes"
      "quizzes:quiz"
    when "announcements"
      "discussion_topic"
    else
      key.singularize
    end
  end

  def self.collection_name(key)
    key = key.to_s
    case key
    when "modules"
      "context_modules"
    when "module_items"
      "content_tags"
    when "pages"
      "wiki_pages"
    when "files"
      "attachments"
    else
      key
    end
  end

  def use_global_identifiers?
    if content_export
      content_export.global_identifiers?
    elsif source_course
      source_course.content_exports.temp_record.can_use_global_identifiers?
    else
      false
    end
  end

  # strips out the "id_" prepending the migration ids in the form
  # also converts arrays of migration ids (or real ids for course exports) into the old hash format
  def self.process_copy_params(hash, for_content_export: false, return_asset_strings: false, global_identifiers: false)
    return {} if hash.blank?

    process_key = if return_asset_strings
                    ->(asset_string) { asset_string }
                  else
                    ->(asset_string) { CC::CCHelper.create_key(asset_string, global: global_identifiers) }
                  end
    new_hash = {}

    hash.each do |key, value|
      key = collection_name(key)
      case value
      when Hash # e.g. second level in :copy => {:context_modules => {:id_100 => true, etc}}
        new_sub_hash = {}

        value.each do |sub_key, sub_value|
          if for_content_export
            new_sub_hash[process_key.call(sub_key)] = sub_value
          elsif sub_key.is_a?(String) && sub_key.start_with?("id_")
            new_sub_hash[sub_key.sub("id_", "")] = sub_value
          else
            new_sub_hash[sub_key] = sub_value
          end
        end

        new_hash[key] = new_sub_hash
      when Array
        # e.g. :select => {:context_modules => [100, 101]} for content exports
        # or :select => {:context_modules => [blahblahblah, blahblahblah2]} for normal migration ids
        sub_hash = {}
        if for_content_export
          asset_type = asset_string_prefix(key.to_s)
          value.each do |id|
            sub_hash[process_key.call("#{asset_type}_#{id}")] = "1"
          end
        else
          value.each do |id|
            sub_hash[id] = "1"
          end
        end
        new_hash[key] = sub_hash
      else
        new_hash[key] = value
      end
    end
    new_hash
  end

  def imported_migration_items
    @imported_migration_items_hash ||= {}
    @imported_migration_items_hash.values.map(&:values).flatten
  end

  def imported_migration_items_hash(klass = nil)
    @imported_migration_items_hash ||= {}
    return @imported_migration_items_hash unless klass

    @imported_migration_items_hash[klass.name] ||= {}
  end

  def imported_migration_items_by_class(klass)
    imported_migration_items_hash(klass).values
  end

  def imported_migration_items_for_insert_type
    import_type = migration_settings[:insert_into_module_type]
    if import_type.present?
      class_name = self.class.import_class_name(import_type)
      imported_migration_items_hash[class_name] ||= {}
      imported_migration_items_hash[class_name].values
    else
      imported_migration_items
    end
  end

  def self.import_class_name(import_type)
    prefix = asset_string_prefix(collection_name(import_type.pluralize))
    ActiveRecord::Base.convert_class_name(prefix)
  end

  def find_imported_migration_item(klass, migration_id)
    imported_migration_items_hash(klass)[migration_id]
  end

  def add_imported_item(item, key: item.migration_id)
    imported_migration_items_hash(item.class)[key] = item
  end

  def add_attachment_path(path, migration_id)
    self.attachment_path_id_lookup ||= {}
    self.attachment_path_id_lookup[path] = migration_id
  end

  def add_external_tool_translation(migration_id, target_tool, custom_fields)
    @external_tool_translation_map ||= {}
    @external_tool_translation_map[migration_id] = [target_tool.id, custom_fields]
  end

  def find_external_tool_translation(migration_id)
    @external_tool_translation_map && migration_id && @external_tool_translation_map[migration_id]
  end

  def handle_import_in_progress_notice
    return unless context.is_a?(Course) && is_set?(migration_settings[:import_in_progress_notice])

    if (just_created || (saved_change_to_workflow_state? && %w[created queued].include?(workflow_state_before_last_save))) &&
       %w[pre_processing pre_processed exporting importing].include?(workflow_state)
      context.add_content_notice(:import_in_progress, 4.hours)
    elsif saved_change_to_workflow_state? && %w[pre_process_error exported imported failed].include?(workflow_state)
      context.remove_content_notice(:import_in_progress)
    end
  end

  def check_for_blocked_migration
    if saved_change_to_workflow_state? &&
       %w[pre_process_error exported imported failed].include?(workflow_state) &&
       context &&
       (next_cm = context.content_migrations.where(workflow_state: "queued").order(:id).first) &&
       (job_id = next_cm.job_progress.try(:delayed_job_id)) &&
       (job = Delayed::Job.where(id: job_id, locked_at: nil).first)
      job.run_at = Time.now # it's okay to try it again now
      job.save
    end
  end

  def set_root_account_id
    self.root_account_id ||=
      case context
      when Course, Group
        context.root_account_id
      when Account
        context.resolved_root_account_id
      when User
        0 # root account id unknown, use dummy root account id
      end
    Account.ensure_dummy_root_account if root_account_id == 0
  end

  def notification_link_anchor
    "!/blueprint/blueprint_subscriptions/#{child_subscription_id}/#{id}"
  end

  ASSET_ID_MAP_TYPES = %w[Assignment Announcement Attachment ContentTag ContextModule DiscussionTopic Quizzes::Quiz WikiPage].freeze

  MIGRATION_DATA_FIELDS = {
    "WikiPage" => %i[url current_lookup_id],
    "Attachment" => %i[media_entry_id]
  }.freeze

  def migration_data_fields_for(asset_type)
    if asset_type == "Attachment" && context.root_account.feature_enabled?(:file_verifiers_for_quiz_links)
      return MIGRATION_DATA_FIELDS[asset_type].clone << :uuid
    end

    MIGRATION_DATA_FIELDS[asset_type] || []
  end

  def add_asset_pair_to_mapping(mapping, key, mig_id, src_asset_fields, dest_asset_fields)
    # mig_ids are md5 hashes (eg they have 32 digits), so there should be zero overlap with
    # the src_ids which are DB primary keys or global_ids and they can safely be stored on the same
    # hash.
    #
    src_asset_fields[:id] = src_asset_fields[:id].to_s if src_asset_fields[:id]
    dest_asset_fields[:id] = dest_asset_fields[:id].to_s

    src_id = src_asset_fields[:id]
    dest_id = dest_asset_fields[:id]

    mapping[key][src_id] = dest_id if src_id.present?

    return unless asset_map_v2?

    mapping[key][mig_id] = {
      source: src_asset_fields,
      destination: dest_asset_fields
    }
  end

  def asset_id_mapping
    return nil unless imported? || importing?

    mapping = {}
    master_template = migration_type == "master_course_import" &&
                      master_course_subscription&.master_template
    global_ids = master_template.present? || use_global_identifiers?

    ASSET_ID_MAP_TYPES.each do |asset_type|
      mig_id_to_dest_id = {}
      scope = nil
      klass = asset_type.constantize
      next unless klass.column_names.include? "migration_id"

      key = Context.api_type_name(klass)

      has_attached_assignment = klass.column_names.include?("assignment_id")
      fields = [*migration_data_fields_for(asset_type)]
      fields.push(:assignment_id) if has_attached_assignment
      context.shard.activate do
        scope = klass.select(:id, :migration_id, *fields)
        scope = scope.where(context:).where.not(migration_id: nil)
        scope = scope.only_discussion_topics if asset_type == "DiscussionTopic"
      end

      scope.each do |o|
        mig_id_to_dest_id[o.migration_id.to_s] = {}
        mig_id_to_dest_id[o.migration_id.to_s][:id] = o.id
        mig_id_to_dest_id[o.migration_id.to_s][:shell_id] = o.assignment_id if has_attached_assignment && o.assignment_id

        migration_data_fields_for(asset_type).each do |field|
          mig_id_to_dest_id[o.migration_id.to_s][field] = o.send(field)
        end
      end

      if key == "files" && context.root_account.feature_enabled?(:file_verifiers_for_quiz_links)
        dest_id_to_dest_uuid = {}
        scope.each do |file|
          dest_id_to_dest_uuid[file.id] = file.uuid
        end
        mapping["verifiers"] = dest_id_to_dest_uuid unless dest_id_to_dest_uuid.empty?
      end

      next if mig_id_to_dest_id.empty?

      mapping[key] ||= {}
      unless source_course.present?
        mig_id_to_dest_id.each do |mig_id, mig_fields|
          add_asset_pair_to_mapping(mapping, key, mig_id, {}, mig_fields)
        end
        next
      end

      if master_template
        # migration_ids are complicated in blueprint courses; fortunately, we have a stored mapping
        # between source id and migration_id in the MasterContentTags (except for ContentTags, which
        # fortunately _aren't_ complicated)
        if asset_type == "ContentTag"
          src_ids = source_course.context_module_tags.pluck(:id)
          src_ids.each do |src_id|
            global_asset_string = klass.asset_string(Shard.global_id_for(src_id, source_course.shard))
            mig_id = master_template.migration_id_for(global_asset_string)

            add_asset_pair_to_mapping(mapping, key, mig_id, { id: src_id }, { id: mig_id_to_dest_id[mig_id][:id] }) if mig_id_to_dest_id[mig_id]
          end
        else
          association_name = MasterCourses::MasterContentTag.polymorphic_assoc_for(klass)
          src_fields = [:content_id, :migration_id, *fields]

          master_template.master_content_tags
                         .where(migration_id: mig_id_to_dest_id.keys)
                         .joins(association_name)
                         .pluck(*src_fields)
                         .each do |src_results|
            src = src_fields.zip(src_results).to_h
            src[:id] = src[:content_id]
            mig_id = src[:migration_id]
            next unless mig_id_to_dest_id[mig_id]

            add_asset_pair_to_mapping(mapping, key, mig_id, src, mig_id_to_dest_id[mig_id]) if mig_id_to_dest_id[mig_id][:id]
            src_assignment_id = mig_id_to_dest_id[mig_id][:shell_id] && src[:assignment_id]
            next unless src_assignment_id

            add_asset_pair_to_mapping(mapping, "assignments", mig_id, { id: src_assignment_id }, { id: mig_id_to_dest_id[mig_id][:shell_id] })
          end
        end
      else
        src_fields = [:id, *migration_data_fields_for(asset_type)]
        # with course copy, there is no stored mapping between source id and migration_id,
        # so we will need to recompute migration_ids to discover the mapping
        source_course.shard.activate do
          srcs = klass.where(context: source_course).pluck(*src_fields).map do |field_values|
            src_fields.zip(Array.wrap(field_values)).to_h
          end
          srcs.each do |src|
            asset_string = klass.asset_string(src[:id])
            mig_id = CC::CCHelper.create_key(asset_string, global: global_ids)

            add_asset_pair_to_mapping(mapping, key, mig_id, src, mig_id_to_dest_id[mig_id]) if mig_id_to_dest_id[mig_id]
          end
        end
      end
    end

    mapping
  end

  def asset_map_url(generate_if_needed: false)
    generate_asset_map if !asset_map_attachment && generate_if_needed
    asset_map_attachment && file_download_url(
      asset_map_attachment,
      {
        verifier: asset_map_attachment.uuid,
        download: "1",
        download_frd: "1",
        host: context.root_account.domain(ApplicationController.test_cluster_name)
      }
    )
  end

  def asset_map_v2?
    Account.site_admin.feature_enabled?(:content_migration_asset_map_v2)
  end

  def generate_asset_map
    data = asset_id_mapping
    return if data.nil?

    payload = {
      "source_host" => source_course&.root_account&.domain(ApplicationController.test_cluster_name),
      "source_course" => source_course_id&.to_s,
      "contains_migration_ids" => Account.site_admin.feature_enabled?(:content_migration_asset_map_v2),
      "resource_mapping" => data,
      "migration_user_uuid" => user&.uuid
    }

    if asset_map_v2?
      payload["destination_course"] = context.id.to_s
      payload["destination_hosts"] = destination_hosts
      root_folder = Folder.root_folders(context).first
      payload["destination_root_folder"] = root_folder.name + "/" if root_folder
      payload["attachment_path_id_lookup"] = migration_settings[:attachment_path_id_lookup].presence || attachment_path_id_lookup
    end

    self.asset_map_attachment = Attachment.new(context: self, filename: "asset_map.json")
    Attachments::Storage.store_for_attachment(asset_map_attachment, StringIO.new(payload.to_json))
    asset_map_attachment.save!
    save!
  end

  # this is a much-simplified subset of asset_id_mapping that includes only ids,
  # for only the items imported in this migration
  def imported_asset_id_map
    return nil unless imported? && for_course_copy?

    mapping = {}
    master_template = migration_type == "master_course_import" &&
                      master_course_subscription&.master_template
    global_ids = master_template.present? || use_global_identifiers?

    migration_settings[:imported_assets].each do |asset_type, dest_ids|
      klass = asset_type.constantize
      next unless klass.column_names.include? "migration_id"

      dest_ids = dest_ids.split(",").map(&:to_i)
      mig_id_to_dest_id = context.shard.activate do
        scope = klass.where(context:, id: dest_ids)
        scope = scope.only_discussion_topics if asset_type == "DiscussionTopic"
        scope.where.not(migration_id: nil)
             .pluck(:migration_id, :id)
             .to_h
      end

      mapping[asset_type] ||= {}
      if master_template
        master_template.master_content_tags
                       .where(migration_id: mig_id_to_dest_id.keys)
                       .pluck(:content_id, :migration_id)
                       .each do |src_id, mig_id|
          mapping[asset_type][src_id] = mig_id_to_dest_id[mig_id]
        end
      else
        source_course.shard.activate do
          src_ids = klass.where(context: source_course).pluck(:id)
          src_ids.each do |src_id|
            asset_string = klass.asset_string(src_id)
            mig_id = CC::CCHelper.create_key(asset_string, global: global_ids)
            dest_id = mig_id_to_dest_id[mig_id]
            mapping[asset_type][src_id] = dest_id if dest_id
          end
        end
      end
    end

    mapping
  end

  def destination_hosts
    return [] unless context

    HostUrl.context_hosts(context.root_account).map { |h| h.split(":").first }
  end

  set_broadcast_policy do |p|
    p.dispatch :blueprint_content_added
    p.to { context.participating_admins }
    p.whenever do |record|
      record.changed_state_to(:imported) && record.for_master_course_import? &&
        record.master_migration && record.master_migration.send_notification?
    end
  end

  def self.expire_days
    Setting.get("content_migrations_expire_after_days", "30").to_i
  end

  def self.expire?
    ContentMigration.expire_days > 0
  end

  def expired?
    return false unless ContentMigration.expire?

    created_at < ContentMigration.expire_days.days.ago
  end

  scope :expired, lambda {
    if ContentMigration.expire?
      where(created_at: ...ContentMigration.expire_days.days.ago)
    else
      none
    end
  }

  def self.find_most_recent_by_course_ids(source_course_id, context_id)
    ContentMigration.where(source_course_id:, context_id:).order(finished_at: :desc).first
  end
end