Read about our upcoming Code of Conduct on this issue

prepare_untracked_uploads.rb 5.27 KB
Newer Older
Michael Kozono's avatar
Michael Kozono committed
1
2
# frozen_string_literal: true

3
4
module Gitlab
  module BackgroundMigration
Michael Kozono's avatar
Michael Kozono committed
5
6
7
    # This class finds all non-hashed uploaded file paths and saves them to a
    # `untracked_files_for_uploads` table.
    class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength
8
9
      # For bulk_queue_background_migration_jobs_by_range
      include Database::MigrationHelpers
10
      include ::Gitlab::Utils::StrongMemoize
11

Michael Kozono's avatar
Michael Kozono committed
12
      FIND_BATCH_SIZE = 500
13
14
      RELATIVE_UPLOAD_DIR = "uploads".freeze
      ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze
Michael Kozono's avatar
Michael Kozono committed
15
      FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
16
17
18
      START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/}
      EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
      EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
19

Michael Kozono's avatar
Michael Kozono committed
20
21
      # This class is used to iterate over batches of
      # `untracked_files_for_uploads` rows.
22
      class UntrackedFile < ActiveRecord::Base
23
24
        include EachBatch

25
        self.table_name = 'untracked_files_for_uploads'
26
27
28
      end

      def perform
29
        ensure_temporary_tracking_table_exists
Michael Kozono's avatar
Michael Kozono committed
30
31
32
33
34
35
36
37
38
39
40

        # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since
        # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be
        # slow, start with an empty table for Postgres < 9.5.
        # That way we can do bulk inserts at ~30x the speed of individual
        # inserts (~20 minutes worth of inserts at GitLab.com scale instead of
        # ~10 hours).
        # In all other cases, installations will get both bulk inserts and the
        # ability for these jobs to retry without having to clear and reinsert.
        clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates?

41
        store_untracked_file_paths
Michael Kozono's avatar
Michael Kozono committed
42

43
44
45
46
47
        schedule_populate_untracked_uploads_jobs
      end

      private

48
      def ensure_temporary_tracking_table_exists
Michael Kozono's avatar
Michael Kozono committed
49
50
51
        table_name = :untracked_files_for_uploads
        unless UntrackedFile.connection.table_exists?(table_name)
          UntrackedFile.connection.create_table table_name do |t|
52
            t.string :path, limit: 600, null: false
Michael Kozono's avatar
Michael Kozono committed
53
            t.index :path, unique: true
54
55
          end
        end
56
57
      end

Michael Kozono's avatar
Michael Kozono committed
58
59
60
61
      def clear_untracked_file_paths
        UntrackedFile.delete_all
      end

62
      def store_untracked_file_paths
63
        return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR)
64

Michael Kozono's avatar
Michael Kozono committed
65
        each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths|
66
          insert_file_paths(file_paths)
67
68
69
        end
      end

70
      def each_file_batch(search_dir, batch_size, &block)
71
        cmd = build_find_command(search_dir)
72

73
        Open3.popen2(*cmd) do |stdin, stdout, status_thread|
74
75
          yield_paths_in_batches(stdout, batch_size, &block)

76
77
78
79
          raise "Find command failed" unless status_thread.value.success?
        end
      end

80
81
82
83
      def yield_paths_in_batches(stdout, batch_size, &block)
        paths = []

        stdout.each_line("\0") do |line|
84
          paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '')
85
86
87
88
89
90
91
92
93
94

          if paths.size >= batch_size
            yield(paths)
            paths = []
          end
        end

        yield(paths)
      end

95
      def build_find_command(search_dir)
Michael Kozono's avatar
Michael Kozono committed
96
        cmd = %W[find -L #{search_dir}
Michael Kozono's avatar
Michael Kozono committed
97
98
99
100
                 -type f
                 ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune )
                 ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune )
                 -print0]
101

Michael Kozono's avatar
Michael Kozono committed
102
103
        ionice = which_ionice
        cmd = %W[#{ionice} -c Idle] + cmd if ionice
104

Michael Kozono's avatar
Michael Kozono committed
105
106
        log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\""
        Rails.logger.info log_msg
Michael Kozono's avatar
Michael Kozono committed
107

108
109
110
        cmd
      end

Michael Kozono's avatar
Michael Kozono committed
111
      def which_ionice
112
113
        Gitlab::Utils.which('ionice')
      rescue StandardError
Michael Kozono's avatar
Michael Kozono committed
114
115
        # In this case, returning false is relatively safe,
        # even though it isn't very nice
116
117
118
119
        false
      end

      def insert_file_paths(file_paths)
Michael Kozono's avatar
Michael Kozono committed
120
        sql = insert_sql(file_paths)
121

Michael Kozono's avatar
Michael Kozono committed
122
123
        ActiveRecord::Base.connection.execute(sql)
      end
124

Michael Kozono's avatar
Michael Kozono committed
125
126
127
128
129
130
131
132
133
134
135
136
      def insert_sql(file_paths)
        if postgresql_pre_9_5?
          "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};"
        elsif postgresql?
          "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\
            " ON CONFLICT DO NOTHING;"
        else # MySQL
          "INSERT IGNORE INTO"\
            " #{table_columns_and_values_for_insert(file_paths)};"
        end
      end

Michael Kozono's avatar
Michael Kozono committed
137
138
      def table_columns_and_values_for_insert(file_paths)
        values = file_paths.map do |file_path|
Michael Kozono's avatar
Michael Kozono committed
139
          ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength
Michael Kozono's avatar
Michael Kozono committed
140
        end.join(', ')
141

142
        "#{UntrackedFile.table_name} (path) VALUES #{values}"
143
144
      end

145
      def postgresql?
146
147
148
        strong_memoize(:postgresql) do
          Gitlab::Database.postgresql?
        end
149
150
      end

Michael Kozono's avatar
Michael Kozono committed
151
152
153
154
      def can_bulk_insert_and_ignore_duplicates?
        !postgresql_pre_9_5?
      end

155
      def postgresql_pre_9_5?
156
157
158
        strong_memoize(:postgresql_pre_9_5) do
          postgresql? && Gitlab::Database.version.to_f < 9.5
        end
159
160
      end

161
      def schedule_populate_untracked_uploads_jobs
Michael Kozono's avatar
Michael Kozono committed
162
163
        bulk_queue_background_migration_jobs_by_range(
          UntrackedFile, FOLLOW_UP_MIGRATION)
164
165
166
167
      end
    end
  end
end