prepare_untracked_uploads.rb 2.89 KB
Newer Older
1
2
module Gitlab
  module BackgroundMigration
3
    class PrepareUntrackedUploads
4
5
6
      # For bulk_queue_background_migration_jobs_by_range
      include Database::MigrationHelpers

7
      FILE_PATH_BATCH_SIZE = 500
Michael Kozono's avatar
Michael Kozono committed
8
9
      UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze
      FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
10

11
      class UntrackedFile < ActiveRecord::Base
12
13
        include EachBatch

14
        self.table_name = 'untracked_files_for_uploads'
15
16
17
18
19
      end

      def perform
        return unless migrate?

20
        store_untracked_file_paths
21
22
23
24
25
26
        schedule_populate_untracked_uploads_jobs
      end

      private

      def migrate?
27
        UntrackedFile.table_exists?
28
29
      end

30
      def store_untracked_file_paths
Michael Kozono's avatar
Michael Kozono committed
31
        return unless Dir.exist?(UPLOAD_DIR)
32

33
34
        each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths|
          insert_file_paths(file_paths)
35
36
37
        end
      end

38
      def each_file_batch(search_dir, batch_size, &block)
39
        cmd = build_find_command(search_dir)
40

41
        Open3.popen2(*cmd) do |stdin, stdout, status_thread|
42
43
          yield_paths_in_batches(stdout, batch_size, &block)

44
45
46
47
          raise "Find command failed" unless status_thread.value.success?
        end
      end

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
      def yield_paths_in_batches(stdout, batch_size, &block)
        paths = []

        stdout.each_line("\0") do |line|
          paths << line.chomp("\0")

          if paths.size >= batch_size
            yield(paths)
            paths = []
          end
        end

        yield(paths)
      end

63
      def build_find_command(search_dir)
64
65
66
        hashed_path = "#{UPLOAD_DIR}/@hashed/*"
        tmp_path = "#{UPLOAD_DIR}/tmp/*"
        cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0]
67

Michael Kozono's avatar
Michael Kozono committed
68
        cmd = %w[ionice -c Idle] + cmd if ionice_is_available?
69
70
71
72
73
74
75
76
77
78
79
80
81

        cmd
      end

      def ionice_is_available?
        Gitlab::Utils.which('ionice')
      rescue StandardError
        # In this case, returning false is relatively safe, even though it isn't very nice
        false
      end

      def insert_file_paths(file_paths)
        file_paths.each do |file_path|
82
          insert_file_path(file_path)
83
        end
84
85
      end

86
      def insert_file_path(file_path)
87
        table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)'
88
89
90
91
92
93
94
95
96
97
98
99

        sql = if Gitlab::Database.postgresql?
          "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;"
        else
          "INSERT IGNORE INTO #{table_columns_and_values};"
        end

        timestamp = Time.now.utc.iso8601
        sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp])
        ActiveRecord::Base.connection.execute(sql)
      end

100
      def schedule_populate_untracked_uploads_jobs
101
        bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION)
102
103
104
105
      end
    end
  end
end