Unverified Commit 3ef4f74b authored by Markus Koller's avatar Markus Koller
Browse files

Add more storage statistics

This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.

The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)

The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.
parent b0d885b53d26
......@@ -17,10 +17,10 @@
%ul.nav
%li
= link_to project_files_path(@project) do
Files (#{repository_size})
Files (#{storage_counter(@project.statistics.total_repository_size)})
%li
= link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do
#{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)})
#{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)})
%li
= link_to namespace_project_branches_path(@project.namespace, @project) do
#{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)})
......
......@@ -6,26 +6,27 @@ class ProjectCacheWorker
LEASE_TIMEOUT = 15.minutes.to_i
# project_id - The ID of the project for which to flush the cache.
# refresh - An Array containing extra types of data to refresh such as
# `:readme` to flush the README and `:changelog` to flush the
# CHANGELOG.
def perform(project_id, refresh = [])
# files - An Array containing extra types of files to refresh such as
# `:readme` to flush the README and `:changelog` to flush the
# CHANGELOG.
# statistics - An Array containing columns from ProjectStatistics to
# refresh, if empty all columns will be refreshed
def perform(project_id, files = [], statistics = [])
project = Project.find_by(id: project_id)
return unless project && project.repository.exists?
update_repository_size(project)
project.update_commit_count
update_statistics(project, statistics.map(&:to_sym))
project.repository.refresh_method_caches(refresh.map(&:to_sym))
project.repository.refresh_method_caches(files.map(&:to_sym))
end
def update_repository_size(project)
return unless try_obtain_lease_for(project.id, :update_repository_size)
def update_statistics(project, statistics = [])
return unless try_obtain_lease_for(project.id, :update_statistics)
Rails.logger.info("Updating repository size for project #{project.id}")
Rails.logger.info("Updating statistics for project #{project.id}")
project.update_repository_size
project.statistics.refresh!(only: statistics)
end
private
......
---
title: Add more storage statistics
merge_request: 7754
author: Markus Koller
......@@ -10,5 +10,5 @@
# end
#
ActiveSupport::Inflector.inflections do |inflect|
inflect.uncountable %w(award_emoji)
inflect.uncountable %w(award_emoji project_statistics)
end
class CreateProjectStatistics < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
# use bigint columns to support values >2GB
counter_column = { limit: 8, null: false, default: 0 }
create_table :project_statistics do |t|
t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
t.references :namespace, null: false, index: true
t.integer :commit_count, counter_column
t.integer :storage_size, counter_column
t.integer :repository_size, counter_column
t.integer :lfs_objects_size, counter_column
t.integer :build_artifacts_size, counter_column
end
end
end
class MigrateProjectStatistics < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = true
DOWNTIME_REASON = 'Removes two columns from the projects table'
def up
# convert repository_size in float (megabytes) to integer (bytes),
# initialize total storage_size with repository_size
execute <<-EOF
INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size)
SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects
EOF
remove_column :projects, :repository_size
remove_column :projects, :commit_count
end
def down
add_column_with_default :projects, :repository_size, :float, default: 0.0
add_column_with_default :projects, :commit_count, :integer, default: 0
end
end
......@@ -901,6 +901,19 @@
add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
create_table "project_statistics", force: :cascade do |t|
t.integer "project_id", null: false
t.integer "namespace_id", null: false
t.integer "commit_count", limit: 8, default: 0, null: false
t.integer "storage_size", limit: 8, default: 0, null: false
t.integer "repository_size", limit: 8, default: 0, null: false
t.integer "lfs_objects_size", limit: 8, default: 0, null: false
t.integer "build_artifacts_size", limit: 8, default: 0, null: false
end
add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
create_table "projects", force: :cascade do |t|
t.string "name"
t.string "path"
......@@ -915,11 +928,9 @@
t.boolean "archived", default: false, null: false
t.string "avatar"
t.string "import_status"
t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.integer "commit_count", default: 0
t.text "import_error"
t.integer "ci_id"
t.boolean "shared_runners_enabled", default: true, null: false
......@@ -1288,6 +1299,7 @@
add_foreign_key "personal_access_tokens", "users"
add_foreign_key "project_authorizations", "projects", on_delete: :cascade
add_foreign_key "project_authorizations", "users", on_delete: :cascade
add_foreign_key "project_statistics", "projects", on_delete: :cascade
add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
add_foreign_key "protected_branch_push_access_levels", "protected_branches"
add_foreign_key "subscriptions", "projects", on_delete: :cascade
......
......@@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo
[reconfigure gitlab]: restart_gitlab.md "How to restart GitLab"
[restart gitlab]: restart_gitlab.md "How to restart GitLab"
## Storage statistics
You can see the total storage used for build artifacts on groups and projects
in the administration area, as well as through the [groups](../api/groups.md)
and [projects APIs](../api/projects.md).
......@@ -13,6 +13,7 @@ Parameters:
| `search` | string | no | Return list of authorized groups matching the search criteria |
| `order_by` | string | no | Order groups by `name` or `path`. Default is `name` |
| `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` |
| `statistics` | boolean | no | Include group statistics (admins only) |
```
GET /groups
......@@ -31,7 +32,6 @@ GET /groups
You can search for groups by name or path, see below.
=======
## List owned groups
Get a list of groups which are owned by the authenticated user.
......@@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user.
GET /groups/owned
```
Parameters:
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `statistics` | boolean | no | Include group statistics |
## List a group's projects
Get a list of projects in this group.
......
......@@ -307,6 +307,8 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria |
| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
| `statistics` | boolean | no | Include project statistics |
### List starred projects
......@@ -325,6 +327,7 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria |
| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
### List ALL projects
......@@ -343,6 +346,7 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria |
| `statistics` | boolean | no | Include project statistics |
### Get single project
......
......@@ -40,6 +40,12 @@ In `config/gitlab.yml`:
storage_path: /mnt/storage/lfs-objects
```
## Storage statistics
You can see the total storage used for LFS objects on groups and projects
in the administration area, as well as through the [groups](../api/groups.md)
and [projects APIs](../api/projects.md).
## Known limitations
* Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets)
......@@ -47,3 +53,5 @@ In `config/gitlab.yml`:
* Currently, removing LFS objects from GitLab Git LFS storage is not supported
* LFS authentications via SSH was added with GitLab 8.12
* Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2.
* The storage statistics currently count each LFS object multiple times for
every project linking to it
......@@ -101,6 +101,16 @@ class Project < Grape::Entity
expose :only_allow_merge_if_build_succeeds
expose :request_access_enabled
expose :only_allow_merge_if_all_discussions_are_resolved
expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
end
class ProjectStatistics < Grape::Entity
expose :commit_count
expose :storage_size
expose :repository_size
expose :lfs_objects_size
expose :build_artifacts_size
end
class Member < UserBasic
......@@ -127,6 +137,15 @@ class Group < Grape::Entity
expose :avatar_url
expose :web_url
expose :request_access_enabled
expose :statistics, if: :statistics do
with_options format_with: -> (value) { value.to_i } do
expose :storage_size
expose :repository_size
expose :lfs_objects_size
expose :build_artifacts_size
end
end
end
class GroupDetail < Group
......
......@@ -11,6 +11,20 @@ class Groups < Grape::API
optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
end
params :statistics_params do
optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
end
def present_groups(groups, options = {})
options = options.reverse_merge(
with: Entities::Group,
current_user: current_user,
)
groups = groups.with_statistics if options[:statistics]
present paginate(groups), options
end
end
resource :groups do
......@@ -18,6 +32,7 @@ class Groups < Grape::API
success Entities::Group
end
params do
use :statistics_params
optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
optional :search, type: String, desc: 'Search for a specific group'
......@@ -38,7 +53,7 @@ class Groups < Grape::API
groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
groups = groups.reorder(params[:order_by] => params[:sort])
present paginate(groups), with: Entities::Group, current_user: current_user
present_groups groups, statistics: params[:statistics] && current_user.is_admin?
end
desc 'Get list of owned groups for authenticated user' do
......@@ -46,10 +61,10 @@ class Groups < Grape::API
end
params do
use :pagination
use :statistics_params
end
get '/owned' do
groups = current_user.owned_groups
present paginate(groups), with: Entities::Group, current_user: current_user
present_groups current_user.owned_groups, statistics: params[:statistics]
end
desc 'Create a group. Available only for users who can create groups.' do
......
......@@ -248,7 +248,7 @@ def handle_api_exception(exception)
rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
end
# Projects helpers
# project helpers
def filter_projects(projects)
if params[:search].present?
......
......@@ -40,6 +40,15 @@ def map_public_to_visibility_level(attrs)
resource :projects do
helpers do
params :collection_params do
use :sort_params
use :filter_params
use :pagination
optional :simple, type: Boolean, default: false,
desc: 'Return only the ID, URL, name, and path of each project'
end
params :sort_params do
optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
default: 'created_at', desc: 'Return projects ordered by field'
......@@ -52,97 +61,94 @@ def map_public_to_visibility_level(attrs)
optional :visibility, type: String, values: %w[public internal private],
desc: 'Limit by visibility'
optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
use :sort_params
end
params :statistics_params do
optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
end
params :create_params do
optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
optional :import_url, type: String, desc: 'URL from which the project is imported'
end
def present_projects(projects, options = {})
options = options.reverse_merge(
with: Entities::Project,
current_user: current_user,
simple: params[:simple],
)
projects = filter_projects(projects)
projects = projects.with_statistics if options[:statistics]
options[:with] = Entities::BasicProjectDetails if options[:simple]
present paginate(projects), options
end
end
desc 'Get a list of visible projects for authenticated user' do
success Entities::BasicProjectDetails
end
params do
optional :simple, type: Boolean, default: false,
desc: 'Return only the ID, URL, name, and path of each project'
use :filter_params
use :pagination
use :collection_params
end
get '/visible' do
projects = ProjectsFinder.new.execute(current_user)
projects = filter_projects(projects)
entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
present paginate(projects), with: entity, current_user: current_user
entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
present_projects ProjectsFinder.new.execute(current_user), with: entity
end
desc 'Get a projects list for authenticated user' do
success Entities::BasicProjectDetails
end
params do
optional :simple, type: Boolean, default: false,
desc: 'Return only the ID, URL, name, and path of each project'
use :filter_params
use :pagination
use :collection_params
end
get do
authenticate!
projects = current_user.authorized_projects
projects = filter_projects(projects)
entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
present paginate(projects), with: entity, current_user: current_user
present_projects current_user.authorized_projects,
with: Entities::ProjectWithAccess
end
desc 'Get an owned projects list for authenticated user' do
success Entities::BasicProjectDetails
end
params do
use :filter_params
use :pagination
use :collection_params
use :statistics_params
end
get '/owned' do
authenticate!
projects = current_user.owned_projects
projects = filter_projects(projects)
present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
present_projects current_user.owned_projects,
with: Entities::ProjectWithAccess,
statistics: params[:statistics]
end
desc 'Gets starred project for the authenticated user' do
success Entities::BasicProjectDetails
end
params do
use :filter_params
use :pagination
use :collection_params
end
get '/starred' do
authenticate!
projects = current_user.viewable_starred_projects
projects = filter_projects(projects)
present paginate(projects), with: Entities::Project, current_user: current_user
present_projects current_user.viewable_starred_projects
end
desc 'Get all projects for admin user' do
success Entities::BasicProjectDetails
end
params do
use :filter_params
use :pagination
use :collection_params
use :statistics_params
end
get '/all' do
authenticated_as_admin!
projects = Project.all
projects = filter_projects(projects)
present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
end
desc 'Search for projects the current user has access to' do
......
......@@ -63,8 +63,7 @@ namespace :gitlab do
if project.persisted?
puts " * Created #{project.name} (#{repo_path})".color(:green)
project.update_repository_size
project.update_commit_count
ProjectCacheWorker.perform(project.id)
else
puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
puts " Errors: #{project.errors.messages}".color(:red)
......
......@@ -2,7 +2,7 @@
FactoryGirl.define do
factory :lfs_object do
oid "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
sequence(:oid) { |n| "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a%05x" % n }
size 499013
end
......
FactoryGirl.define do
factory :project_statistics do
project { create :project }
namespace { project.namespace }
end
end
require 'spec_helper'
describe StorageHelper do
describe '#storage_counter' do
it 'formats bytes to one decimal place' do
expect(helper.storage_counter(1.23.megabytes)).to eq '1.2 MB'
end
it 'does not add decimals for sizes < 1 MB' do
expect(helper.storage_counter(23.5.kilobytes)).to eq '24 KB'
end
it 'does not add decimals for zeroes' do
expect(helper.storage_counter(2.megabytes)).to eq '2 MB'
end
it 'uses commas as thousands separator' do
expect(helper.storage_counter(100_000_000_000_000_000)).to eq '90,949.5 TB'
end
end
end
......@@ -192,6 +192,7 @@ project:
- authorized_users
- project_authorizations
- route
- statistics
award_emoji:
- awardable
- user
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment