From 99198f3e66fe4e949f50c037d9685b04438a86c7 Mon Sep 17 00:00:00 2001 From: katzy687 Date: Mon, 9 Jan 2023 00:13:33 +0200 Subject: [PATCH] added regex option for matching exact filename in gitlab --- .../cm/ansible/domain/filename_extractor.py | 18 ++++++------- .../cm/ansible/domain/playbook_downloader.py | 8 +++--- package/tests/test_filename_extractor.py | 25 +++++++++++++++---- package/version.txt | 2 +- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/package/cloudshell/cm/ansible/domain/filename_extractor.py b/package/cloudshell/cm/ansible/domain/filename_extractor.py index e5bd29a..c530236 100644 --- a/package/cloudshell/cm/ansible/domain/filename_extractor.py +++ b/package/cloudshell/cm/ansible/domain/filename_extractor.py @@ -8,25 +8,26 @@ class FilenameExtractor(object): def __init__(self): self._filename_pattern = r"(?P\s*([\w,\s-]*|^.*\.?[^/\\&\?]+)\.(yaml|yml|zip)\s*(?=([\?&].*$|$)))" self.filename_patterns = { - "content-disposition": "\s*((?i)inline|attachment|extension-token)\s*;\s*filename=" + self._filename_pattern, - "x-artifactory-filename": self._filename_pattern + "content-disposition": "\s*((?i)inline|attachment|extension-token)\s*;\s*filename=" + self._filename_pattern, + "x-artifactory-filename": self._filename_pattern, + "X-Gitlab-File-Name": self._filename_pattern } - def get_filename(self,response): + def get_filename(self, response): file_name = None; for header_value, pattern in self.filename_patterns.items(): - matching = re.match(pattern, response.headers.get(header_value,"")) + matching = re.match(pattern, response.headers.get(header_value, "")) if matching: file_name = matching.group('filename') break - #fallback, couldn't find file name from header, get it from url + # fallback, couldn't find file name from header, get it from url if not file_name: file_name_from_url = urllib.parse.unquote(response.url[response.url.rfind('/') + 1:]) matching = re.match(self._filename_pattern, file_name_from_url) if matching: - file_name = matching.group('filename') - - # fallback, couldn't find file name regular URL, check gitlab structure (filename in [-2] position) + file_name = matching.group('filename') + + # fallback, couldn't find file name regular URL, check gitlab structure (filename in [-2] position) if not file_name: file_name_from_url = urllib.parse.unquote(response.url.split('/')[-2]) matching = re.match(self._filename_pattern, file_name_from_url) @@ -36,4 +37,3 @@ def get_filename(self,response): if not file_name: raise AnsibleException("playbook file of supported types: '.yml', '.yaml', '.zip' was not found") return file_name.strip() - diff --git a/package/cloudshell/cm/ansible/domain/playbook_downloader.py b/package/cloudshell/cm/ansible/domain/playbook_downloader.py index ea3ffcc..81c7177 100644 --- a/package/cloudshell/cm/ansible/domain/playbook_downloader.py +++ b/package/cloudshell/cm/ansible/domain/playbook_downloader.py @@ -11,6 +11,7 @@ def __init__(self, username, password, token): self.password = password self.token = token + class PlaybookDownloader(object): CHUNK_SIZE = 1024 * 1024 @@ -58,19 +59,20 @@ def _download(self, url, auth, logger, cancel_sampler, verify_certificate): # assume repo is public, try to download without credentials logger.info('Starting download script as public... from \'%s\' ...'%url) response = self.http_request_service.get_response(url, auth, verify_certificate) - response_valid = self._is_response_valid(logger ,response, "public") + response_valid = self._is_response_valid(logger, response, "public") if response_valid: file_name = self.filename_extractor.get_filename(response) # if fails on public and no auth - no point carry on, user need to fix his URL or add credentials if not response_valid and auth is None: - raise Exception('Please make sure the URL is valid, and the credentials are correct and necessary.') + raise ValueError('Please make sure the URL is valid, and the credentials are correct and necessary.') # repo is private and token provided + # Gitlab Private will succeed with this header as well if not response_valid and auth.token is not None: logger.info("Token provided. Starting download script with Token...") - headers = {"Authorization": "Bearer %s" % auth.token } + headers = {"Authorization": "Bearer %s" % auth.token} response = self.http_request_service.get_response_with_headers(url, headers, verify_certificate) response_valid = self._is_response_valid(logger, response, "Token") diff --git a/package/tests/test_filename_extractor.py b/package/tests/test_filename_extractor.py index e8d4f23..5a9cc34 100644 --- a/package/tests/test_filename_extractor.py +++ b/package/tests/test_filename_extractor.py @@ -10,20 +10,19 @@ def setUp(self): self.filename_extractor = FilenameExtractor() self.response = Mock() - def test_filename_from_header_RFC_content_disposition_(self): filename = " my_file.yaml " header = {'content-disposition': 'attachment ; filename=' + filename} self.response.headers = header extracted_filename = self.filename_extractor.get_filename(self.response) - self.assertEqual(filename.strip(),extracted_filename) + self.assertEqual(filename.strip(), extracted_filename) def test_filename_from_header_RFC_content_disposition_no_spcaces(self): filename = "my_file.yaml" header = {'content-disposition': 'attachment ; filename=' + filename} self.response.headers = header extracted_filename = self.filename_extractor.get_filename(self.response) - self.assertEqual(filename,extracted_filename) + self.assertEqual(filename, extracted_filename) def test_filename_from_header_artifactory(self): filename = " my_file.yml " @@ -86,7 +85,8 @@ def test_unsupported_playbook_file(self): self.response.url = "http://www.template.myurl/a/b/c/" + filename with self.assertRaises(Exception) as unsupportedExc: self.filename_extractor.get_filename(self.response) - self.assertEqual(str(unsupportedExc.exception),"playbook file of supported types: '.yml', '.yaml', '.zip' was not found") + self.assertEqual(str(unsupportedExc.exception), + "playbook file of supported types: '.yml', '.yaml', '.zip' was not found") def test_unsupported_playbook_file_no_spaces(self): filename = "my_file.exe" @@ -95,4 +95,19 @@ def test_unsupported_playbook_file_no_spaces(self): self.response.url = "http://www.template.myurl/a/b/c/" + filename with self.assertRaises(Exception) as unsupportedExc: self.filename_extractor.get_filename(self.response) - self.assertEqual(str(unsupportedExc.exception),"playbook file of supported types: '.yml', '.yaml', '.zip' was not found") \ No newline at end of file + self.assertEqual(str(unsupportedExc.exception), + "playbook file of supported types: '.yml', '.yaml', '.zip' was not found") + + def test_filename_from_url_gitlab_headers(self): + """ took real headers from response.headers - X-Gitlab-File-Name is exact match""" + filename = "hello_world.yml" + header = { + 'Server': 'nginx', 'Date': 'Sun, 08 Jan 2023 21:23:47 GMT', + 'Content-Disposition': 'inline; filename="hello_world.yml"; filename*=UTF-8\'\'hello_world.yml', + 'X-Gitlab-File-Name': 'hello_world.yml', + 'X-Gitlab-File-Path': 'ansibletest/hello_world.yml' + } + self.response.headers = header + self.response.url = "http://192.168.85.26/api/v4/projects/2/repository/files/ansibletest%2Fhello_world.yml/raw?ref=test-branch" + extracted_filename = self.filename_extractor.get_filename(self.response) + self.assertEqual(filename, extracted_filename) diff --git a/package/version.txt b/package/version.txt index e3a4f19..fae692e 100644 --- a/package/version.txt +++ b/package/version.txt @@ -1 +1 @@ -2.2.0 \ No newline at end of file +2.2.1 \ No newline at end of file