prepare_untracked_uploads.rb 4.67 KB
Newer Older
1
2
module Gitlab
  module BackgroundMigration
3
    class PrepareUntrackedUploads
4
5
6
      # For bulk_queue_background_migration_jobs_by_range
      include Database::MigrationHelpers

7
      FILE_PATH_BATCH_SIZE = 500
8
9
      RELATIVE_UPLOAD_DIR = "uploads".freeze
      ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze
Michael Kozono's avatar
Michael Kozono committed
10
      FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
11
12
13
      START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/}
      EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
      EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
14

15
      class UntrackedFile < ActiveRecord::Base
16
17
        include EachBatch

18
        self.table_name = 'untracked_files_for_uploads'
19
20
21
      end

      def perform
22
        ensure_temporary_tracking_table_exists
Michael Kozono's avatar
Michael Kozono committed
23
24
25
26
27
28
29
30
31
32
33

        # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since
        # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be
        # slow, start with an empty table for Postgres < 9.5.
        # That way we can do bulk inserts at ~30x the speed of individual
        # inserts (~20 minutes worth of inserts at GitLab.com scale instead of
        # ~10 hours).
        # In all other cases, installations will get both bulk inserts and the
        # ability for these jobs to retry without having to clear and reinsert.
        clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates?

34
        store_untracked_file_paths
Michael Kozono's avatar
Michael Kozono committed
35

36
37
38
39
40
        schedule_populate_untracked_uploads_jobs
      end

      private

41
42
43
44
      def ensure_temporary_tracking_table_exists
        unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads)
          UntrackedFile.connection.create_table :untracked_files_for_uploads do |t|
            t.string :path, limit: 600, null: false
Michael Kozono's avatar
Michael Kozono committed
45
            t.index :path, unique: true
46
47
          end
        end
48
49
      end

Michael Kozono's avatar
Michael Kozono committed
50
51
52
53
      def clear_untracked_file_paths
        UntrackedFile.delete_all
      end

54
      def store_untracked_file_paths
55
        return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR)
56

57
        each_file_batch(ABSOLUTE_UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths|
58
          insert_file_paths(file_paths)
59
60
61
        end
      end

62
      def each_file_batch(search_dir, batch_size, &block)
63
        cmd = build_find_command(search_dir)
64

65
        Open3.popen2(*cmd) do |stdin, stdout, status_thread|
66
67
          yield_paths_in_batches(stdout, batch_size, &block)

68
69
70
71
          raise "Find command failed" unless status_thread.value.success?
        end
      end

72
73
74
75
      def yield_paths_in_batches(stdout, batch_size, &block)
        paths = []

        stdout.each_line("\0") do |line|
76
          paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '')
77
78
79
80
81
82
83
84
85
86

          if paths.size >= batch_size
            yield(paths)
            paths = []
          end
        end

        yield(paths)
      end

87
      def build_find_command(search_dir)
88
        cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0]
89

Michael Kozono's avatar
Michael Kozono committed
90
91
        ionice = which_ionice
        cmd = %W[#{ionice} -c Idle] + cmd if ionice
92

Michael Kozono's avatar
Michael Kozono committed
93
94
        Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\""

95
96
97
        cmd
      end

Michael Kozono's avatar
Michael Kozono committed
98
      def which_ionice
99
100
101
102
103
104
105
        Gitlab::Utils.which('ionice')
      rescue StandardError
        # In this case, returning false is relatively safe, even though it isn't very nice
        false
      end

      def insert_file_paths(file_paths)
Michael Kozono's avatar
Michael Kozono committed
106
107
108
109
110
111
112
        sql = if postgresql_pre_9_5?
                "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};"
              elsif postgresql?
                "INSERT INTO #{table_columns_and_values_for_insert(file_paths)} ON CONFLICT DO NOTHING;"
              else # MySQL
                "INSERT IGNORE INTO #{table_columns_and_values_for_insert(file_paths)};"
              end
113

Michael Kozono's avatar
Michael Kozono committed
114
115
        ActiveRecord::Base.connection.execute(sql)
      end
116

Michael Kozono's avatar
Michael Kozono committed
117
118
      def table_columns_and_values_for_insert(file_paths)
        values = file_paths.map do |file_path|
119
          ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend
Michael Kozono's avatar
Michael Kozono committed
120
        end.join(', ')
121

122
        "#{UntrackedFile.table_name} (path) VALUES #{values}"
123
124
      end

125
126
127
128
      def postgresql?
        @postgresql ||= Gitlab::Database.postgresql?
      end

Michael Kozono's avatar
Michael Kozono committed
129
130
131
132
      def can_bulk_insert_and_ignore_duplicates?
        !postgresql_pre_9_5?
      end

133
      def postgresql_pre_9_5?
Michael Kozono's avatar
Michael Kozono committed
134
        @postgresql_pre_9_5 ||= postgresql? && Gitlab::Database.version.to_f < 9.5
135
136
      end

137
      def schedule_populate_untracked_uploads_jobs
138
        bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION)
139
140
141
142
      end
    end
  end
end