Class: Gitlab::GithubImport::AttachmentsDownloader

Inherits:
Object
  • Object
show all
Includes:
BulkImports::FileDownloads::FilenameFetch, BulkImports::FileDownloads::Validations, ImportExport::CommandLineUtil
Defined in:
lib/gitlab/github_import/attachments_downloader.rb

Constant Summary collapse

DownloadError = 
Class.new(StandardError)
NotRetriableError = 
Class.new(StandardError)
FILENAME_SIZE_LIMIT =

chars before the extension

255
DEFAULT_FILE_SIZE_LIMIT = 
Gitlab::CurrentSettings.max_attachment_size.megabytes
TMP_DIR = 
File.join(Dir.tmpdir, 'github_attachments').freeze
SUPPORTED_VIDEO_MEDIA_TYPES = 
%w[mov mp4 webm].freeze
ALLOWED_FILENAME_CHARACTERS = 
/[^a-zA-Z0-9\-_.]/
REDIRECT_STATUS_CODES = 
[301, 302, 303, 307, 308].freeze
NON_RETRIABLE_ERROR_CODES = 
[403, 404, 410].freeze
SUCCESS_STATUS_CODE = 
200
RATE_LIMIT_STATUS_CODES =

GitHub sometimes returns rate limit responses as 403s

[403, 429].freeze
RATE_LIMIT_DEFAULT_RESET_IN = 
120

Constants included from BulkImports::FileDownloads::FilenameFetch

BulkImports::FileDownloads::FilenameFetch::REMOTE_FILENAME_PATTERN

Constants included from ImportExport::CommandLineUtil

ImportExport::CommandLineUtil::CLEAN_DIR_IGNORE_FILE_NAMES, ImportExport::CommandLineUtil::CommandLineUtilError, ImportExport::CommandLineUtil::DEFAULT_DIR_MODE, ImportExport::CommandLineUtil::FileOversizedError, ImportExport::CommandLineUtil::HardLinkError, ImportExport::CommandLineUtil::UNTAR_MASK

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from BulkImports::FileDownloads::Validations

#response_headers

Methods included from ImportExport::CommandLineUtil

#gunzip, #gzip, #gzip_with_options, #mkdir_p, #tar_cf, #tar_czf, #untar_xf, #untar_zxf

Constructor Details

#initialize(file_url, options: {}, file_size_limit: DEFAULT_FILE_SIZE_LIMIT, web_endpoint: ::Octokit::Default.web_endpoint) ⇒ AttachmentsDownloader

Returns a new instance of AttachmentsDownloader.



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 27

def initialize(
  file_url, options: {}, file_size_limit: DEFAULT_FILE_SIZE_LIMIT,
  web_endpoint: ::Octokit::Default.web_endpoint)
  @file_url = file_url
  @options = options
  @file_size_limit = file_size_limit
  @web_endpoint = web_endpoint

  filename = URI(file_url).path.split('/').last
  filename = CGI.unescape(filename) # Decode URL-encoded characters

  # Check for path traversal before sanitization
  Gitlab::PathTraversal.check_path_traversal!(File.join(TMP_DIR, filename))

  filename = sanitize_filename(filename)
  @filename = ensure_filename_size(filename)
end

Instance Attribute Details

#file_size_limit ⇒ Object (readonly)

Returns the value of attribute file_size_limit.



25
26
27
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 25

def file_size_limit
  @file_size_limit
end

#file_url ⇒ Object (readonly)

Returns the value of attribute file_url.



25
26
27
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 25

def file_url
  @file_url
end

#filename ⇒ Object (readonly)

Returns the value of attribute filename.



25
26
27
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 25

def filename
  @filename
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



25
26
27
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 25

def options
  @options
end

#web_endpoint ⇒ Object (readonly)

Returns the value of attribute web_endpoint.



25
26
27
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 25

def web_endpoint
  @web_endpoint
end

Instance Method Details

#delete ⇒ Object 



68
69
70
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 68

def delete
  FileUtils.rm_rf File.dirname(filepath)
end

#perform ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/gitlab/github_import/attachments_downloader.rb', line 45

def perform
  validate_filepath

  download_url = get_assets_download_redirection_url

  # skip download for GHE server urls that redirect to a login url
  return file_url if download_url.include?("login?return_to=")

  parsed_file_name = File.basename(URI.parse(download_url).path)
  parsed_file_name = sanitize_filename(parsed_file_name)

  # if the file has a video filetype extension, we update both the @filename and @filepath with the filetype ext.
  if parsed_file_name.end_with?(*SUPPORTED_VIDEO_MEDIA_TYPES.map { |ext| ".#{ext}" })
    @filename = ensure_filename_size(parsed_file_name)
    add_extension_to_file_path(filename)
  end

  file = download_from(download_url)

  validate_symlink
  file
end