This instance will be upgraded to Heptapod 0.31.0rc on 2022-05-19 at 11:00 UTC+2 (a few minutes of down time)

Commit ef8121c4 authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Make all legacy security reports to use raw format

- This introduces and uses `:raw` format for all legacy reports,
  the ones that do not have yet proper parsers on Backend
- Raw format is needed to make Frontend be able to parse reports,
  without the need of decompressing,
- This also extends fixtures to seed security reports with database,
  even though parser code is part of EE
parent 7f98a2b2001d
......@@ -27,11 +27,15 @@ class JobArtifact < ActiveRecord::Base
metadata: :gzip,
trace: :raw,
junit: :gzip,
codequality: :gzip,
sast: :gzip,
dependency_scanning: :gzip,
container_scanning: :gzip,
dast: :gzip
# All these file formats use `raw` as we need to store them uncompressed
# for Frontend to fetch the files and do analysis
# When they will be only used by backend, they can be `gzipped`.
codequality: :raw,
sast: :raw,
dependency_scanning: :raw,
container_scanning: :raw,
dast: :raw
}.freeze
belongs_to :project
......@@ -100,7 +104,8 @@ class JobArtifact < ActiveRecord::Base
}
FILE_FORMAT_ADAPTERS = {
gzip: Gitlab::Ci::Build::Artifacts::GzipFileAdapter
gzip: Gitlab::Ci::Build::Artifacts::Adapters::GzipStream,
raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream
}.freeze
def valid_file_format?
......
......@@ -30,12 +30,12 @@ def create_archive(artifacts)
def create_reports(reports, expire_in:)
return unless reports&.any?
reports.map do |k, v|
reports.map do |report_type, report_paths|
{
artifact_type: k.to_sym,
artifact_format: :gzip,
name: ::Ci::JobArtifact::DEFAULT_FILE_NAMES[k.to_sym],
paths: v,
artifact_type: report_type.to_sym,
artifact_format: ::Ci::JobArtifact::TYPE_AND_FORMAT_PAIRS.fetch(report_type.to_sym),
name: ::Ci::JobArtifact::DEFAULT_FILE_NAMES.fetch(report_type.to_sym),
paths: report_paths,
when: 'always',
expire_in: expire_in
}
......
---
title: Make all legacy security reports to use raw format
merge_request:
author:
type: changed
require './spec/support/sidekiq'
class Gitlab::Seeder::Pipelines
STAGES = %w[build test deploy notify]
STAGES = %w[build test security deploy notify]
BUILDS = [
# build stage
{ name: 'build:linux', stage: 'build', status: :success,
......@@ -31,6 +31,16 @@ class Gitlab::Seeder::Pipelines
{ name: 'spinach:osx', stage: 'test', status: :failed, allow_failure: true,
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
# security stage
{ name: 'dast', stage: 'security', status: :success,
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
{ name: 'sast', stage: 'security', status: :success,
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
{ name: 'dependency_scanning', stage: 'security', status: :success,
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
{ name: 'container_scanning', stage: 'security', status: :success,
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
# deploy stage
{ name: 'staging', stage: 'deploy', environment: 'staging', status_event: :success,
options: { environment: { action: 'start', on_stop: 'stop staging' } },
......@@ -108,6 +118,11 @@ def build_create!(pipeline, opts = {})
setup_artifacts(build)
setup_test_reports(build)
if build.ref == build.project.default_branch
setup_security_reports_file(build)
else
setup_security_reports_legacy_archive(build)
end
setup_build_log(build)
build.project.environments.
......@@ -143,6 +158,55 @@ def setup_test_reports(build)
end
end
def setup_security_reports_file(build)
return unless build.stage == "security"
# we have two sources: master and feature-branch
branch_name = build.ref == build.project.default_branch ?
'master' : 'feature-branch'
artifacts_cache_file(security_reports_path(branch_name, build.name)) do |file|
build.job_artifacts.build(
project: build.project,
file_type: build.name,
file_format: :raw,
file: file)
end
end
def setup_security_reports_legacy_archive(build)
return unless build.stage == "security"
# we have two sources: master and feature-branch
branch_name = build.ref == build.project.default_branch ?
'master' : 'feature-branch'
artifacts_cache_file(security_reports_archive_path(branch_name)) do |file|
build.job_artifacts.build(
project: build.project,
file_type: :archive,
file_format: :zip,
file: file)
end
# assign dummy metadata
artifacts_cache_file(artifacts_metadata_path) do |file|
build.job_artifacts.build(
project: build.project,
file_type: :metadata,
file_format: :gzip,
file: file)
end
build.options = {
artifacts: {
paths: [
Ci::JobArtifact::DEFAULT_FILE_NAMES.fetch(build.name.to_sym)
]
}
}
end
def setup_build_log(build)
if %w(running success failed).include?(build.status)
build.trace.set(FFaker::Lorem.paragraphs(6).join("\n\n"))
......@@ -190,6 +254,15 @@ def test_reports_failed_path
Rails.root + 'spec/fixtures/junit/junit.xml.gz'
end
def security_reports_archive_path(branch)
Rails.root.join('spec', 'fixtures', 'security-reports', branch + '.zip')
end
def security_reports_path(branch, name)
file_name = Ci::JobArtifact::DEFAULT_FILE_NAMES.fetch(name.to_sym)
Rails.root.join('spec', 'fixtures', 'security-reports', branch, file_name)
end
def artifacts_cache_file(file_path)
file = Tempfile.new("artifacts")
file.close
......
module Gitlab
module Ci
module Build
module Artifacts
module Adapters
class GzipStream
attr_reader :stream
InvalidStreamError = Class.new(StandardError)
def initialize(stream)
raise InvalidStreamError, "Stream is required" unless stream
@stream = stream
end
def each_blob
stream.seek(0)
until stream.eof?
gzip(stream) do |gz|
yield gz.read, gz.orig_name
unused = gz.unused&.length.to_i
# pos has already reached to EOF at the moment
# We rewind the pos to the top of unused files
# to read next gzip stream, to support multistream archives
# https://golang.org/src/compress/gzip/gunzip.go#L117
stream.seek(-unused, IO::SEEK_CUR)
end
end
end
private
def gzip(stream, &block)
gz = Zlib::GzipReader.new(stream)
yield(gz)
rescue Zlib::Error => e
raise InvalidStreamError, e.message
ensure
gz&.finish
end
end
end
end
end
end
end
module Gitlab
module Ci
module Build
module Artifacts
module Adapters
class RawStream
attr_reader :stream
InvalidStreamError = Class.new(StandardError)
def initialize(stream)
raise InvalidStreamError, "Stream is required" unless stream
@stream = stream
end
def each_blob
stream.seek(0)
yield(stream.read, 'raw') unless stream.eof?
end
end
end
end
end
end
end
module Gitlab
module Ci
module Build
module Artifacts
class GzipFileAdapter
attr_reader :stream
InvalidStreamError = Class.new(StandardError)
def initialize(stream)
raise InvalidStreamError, "Stream is required" unless stream
@stream = stream
end
def each_blob
stream.seek(0)
until stream.eof?
gzip(stream) do |gz|
yield gz.read, gz.orig_name
unused = gz.unused&.length.to_i
# pos has already reached to EOF at the moment
# We rewind the pos to the top of unused files
# to read next gzip stream, to support multistream archives
# https://golang.org/src/compress/gzip/gunzip.go#L117
stream.seek(-unused, IO::SEEK_CUR)
end
end
end
private
def gzip(stream, &block)
gz = Zlib::GzipReader.new(stream)
yield(gz)
rescue Zlib::Error => e
raise InvalidStreamError, e.message
ensure
gz&.finish
end
end
end
end
end
end
......@@ -119,11 +119,11 @@
trait :codequality do
file_type :codequality
file_format :gzip
file_format :raw
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/codequality/codequality.json.gz'), 'application/x-gzip')
Rails.root.join('spec/fixtures/codequality/codequality.json'), 'application/json')
end
end
......
{
"image": "registry.gitlab.com/bikebilly/auto-devops-10-6/feature-branch:e7315ba964febb11bac8f5cd6ec433db8a3a1583",
"unapproved": [
"CVE-2017-15650"
],
"vulnerabilities": [
{
"featurename": "musl",
"featureversion": "1.1.14-r15",
"vulnerability": "CVE-2017-15650",
"namespace": "alpine:v3.4",
"description": "",
"link": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15650",
"severity": "Medium",
"fixedby": "1.1.14-r16"
}
]
}
{
"site": {
"alerts": [
{
"sourceid": "3",
"wascid": "15",
"cweid": "16",
"reference": "<p>http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx</p><p>https://www.owasp.org/index.php/List_of_useful_HTTP_headers</p>",
"otherinfo": "<p>This issue still applies to error type pages (401, 403, 500, etc) as those pages are often still affected by injection issues, in which case there is still concern for browsers sniffing pages away from their actual content type.</p><p>At \"High\" threshold this scanner will not alert on client or server error responses.</p>",
"solution": "<p>Ensure that the application/web server sets the Content-Type header appropriately, and that it sets the X-Content-Type-Options header to 'nosniff' for all web pages.</p><p>If possible, ensure that the end user uses a standards-compliant and modern web browser that does not perform MIME-sniffing at all, or that can be directed by the web application/web server to not perform MIME-sniffing.</p>",
"count": "2",
"pluginid": "10021",
"alert": "X-Content-Type-Options Header Missing",
"name": "X-Content-Type-Options Header Missing",
"riskcode": "1",
"confidence": "2",
"riskdesc": "Low (Medium)",
"desc": "<p>The Anti-MIME-Sniffing header X-Content-Type-Options was not set to 'nosniff'. This allows older versions of Internet Explorer and Chrome to perform MIME-sniffing on the response body, potentially causing the response body to be interpreted and displayed as a content type other than the declared content type. Current (early 2014) and legacy versions of Firefox will use the declared content type (if one is set), rather than performing MIME-sniffing.</p>",
"instances": [
{
"param": "X-Content-Type-Options",
"method": "GET",
"uri": "http://bikebilly-spring-auto-devops-review-feature-br-3y2gpb.35.192.176.43.xip.io"
},
{
"param": "X-Content-Type-Options",
"method": "GET",
"uri": "http://bikebilly-spring-auto-devops-review-feature-br-3y2gpb.35.192.176.43.xip.io/"
}
]
}
],
"@ssl": "false",
"@port": "80",
"@host": "bikebilly-spring-auto-devops-review-feature-br-3y2gpb.35.192.176.43.xip.io",
"@name": "http://bikebilly-spring-auto-devops-review-feature-br-3y2gpb.35.192.176.43.xip.io"
},
"@generated": "Fri, 13 Apr 2018 09:22:01",
"@version": "2.7.0"
}
[
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2012-4387",
"url": "http://struts.apache.org/docs/s2-011.html",
"message": "Long parameter name DoS for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
],
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2013-1966",
"url": "http://struts.apache.org/docs/s2-014.html",
"message": "Remote command execution due to flaw in the includeParams attribute of URL and Anchor tags for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
],
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2013-2115",
"url": "http://struts.apache.org/docs/s2-014.html",
"message": "Remote command execution due to flaw in the includeParams attribute of URL and Anchor tags for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
],
"tool": "gemnasium"
},
{
"priority": "Unknown",
"file": "pom.xml",
"cve": "CVE-2013-2134",
"url": "http://struts.apache.org/docs/s2-015.html",
"message": "Arbitrary OGNL code execution via unsanitized wildcard matching for org.apache.struts/struts2-core",
"tools": [
"gemnasium"
],
"tool": "gemnasium"
}
]
{
"licenses": [
{
"count": 13,
"name": "MIT"
},
{
"count": 2,
"name": "New BSD"
},
{
"count": 1,
"name": "LGPL"
}
],
"dependencies": [
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "bundler",
"url": "http://bundler.io",
"description": "The best way to manage your application's dependencies",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "concurrent-ruby",
"url": "http://www.concurrent-ruby.com",
"description": "Modern concurrency tools for Ruby. Inspired by Erlang, Clojure, Scala, Haskell, F#, C#, Java, and classic concurrency patterns.",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "connection_pool",
"url": "https://github.com/mperham/connection_pool",
"description": "Generic connection pool for Ruby",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "mini_portile2",
"url": "http://github.com/flavorjones/mini_portile",
"description": "Simplistic port-like solution for developers",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "mustermann",
"url": "https://github.com/sinatra/mustermann",
"description": "Your personal string matching expert.",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "nokogiri",
"url": "http://nokogiri.org",
"description": "Nokogiri (鋸) is an HTML, XML, SAX, and Reader parser",
"pathes": [
"."
]
}
},
{
"license": {
"name": "New BSD",
"url": "http://opensource.org/licenses/BSD-3-Clause"
},
"dependency": {
"name": "pg",
"url": "https://bitbucket.org/ged/ruby-pg",
"description": "Pg is the Ruby interface to the {PostgreSQL RDBMS}[http://www.postgresql.org/]",
"pathes": [
"."
]
}
},
{
"license": {
"name": "New BSD",
"url": "http://opensource.org/licenses/BSD-3-Clause"
},
"dependency": {
"name": "puma",
"url": "http://puma.io",
"description": "Puma is a simple, fast, threaded, and highly concurrent HTTP 1.1 server for Ruby/Rack applications",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "rack",
"url": "https://rack.github.io/",
"description": "a modular Ruby webserver interface",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "rack-protection",
"url": "http://github.com/sinatra/sinatra/tree/master/rack-protection",
"description": "Protect against typical web attacks, works with all Rack apps, including Rails.",
"pathes": [
"."
]
}
},
{
"license": {
"name": "MIT",
"url": "http://opensource.org/licenses/mit-license"
},
"dependency": {
"name": "redis",
"url": "https://github.com/redis/redis-rb",
"description": "A Ruby client library for Redis",
"pathes": [
"."
]
}
},