Add support for downloading packages from private sources

This changeset provides support for downloading autobuild packages from
restricted sources such as private repositories using GitHub release
artifacts or GitLab generic packages.

The way this works is by the addition of an optional installable
parameter: `creds`, which may be specified when adding or editing
dependencies:

```
autobuild installables edit my-pkg creds=github url=...
```

This property is used to inform autobuild that it should set a HTTP
authorization header specific to the provider (github, gitlab) when
downloading the respective artifact.

In the future, this could be enhanced to include more providers or
authorization schemes. The `autobuild install` command could also be
enhanced with equivilent kwargs `--github-token` and `--gitlab-token`.

README.md

@@ -25,6 +25,8 @@ For more information, see [Autobuild's wiki page][wiki].
 | AUTOBUILD_BUILD_ID | - | Build identifier |
 | AUTOBUILD_CONFIGURATION | - | Target build configuration |
 | AUTOBUILD_CONFIG_FILE | autobuild.xml | Autobuild configuration filename |
+| AUTOBUILD_GITHUB_TOKEN | - | GitHub HTTP authorization token to use during package download |
+| AUTOBUILD_GITLAB_TOKEN | - | GitLab HTTP authorization token to use during package download |
 | AUTOBUILD_INSTALLABLE_CACHE | - | Location of local download cache |
 | AUTOBUILD_LOGLEVEL | WARNING | Log level |
 | AUTOBUILD_PLATFORM | - | Target platform |

autobuild/autobuild_tool_install.py

@@ -8,6 +8,7 @@
 """
 
 import errno
+import http.client
 import logging
 import os
 import pprint
@@ -23,9 +24,20 @@
 
 logger = logging.getLogger('autobuild.install')
 
+CREDENTIAL_ENVVARS = {
+    'github': 'AUTOBUILD_GITHUB_TOKEN',
+    'gitlab': 'AUTOBUILD_GITLAB_TOKEN',
+}
+
+
 class InstallError(common.AutobuildError):
     pass
 
+
+class CredentialsNotFoundError(common.AutobuildError):
+    pass
+
+
 __help = """\
 This autobuild command fetches and installs package archives.
 
@@ -179,7 +191,31 @@ def package_cache_path(package):
     """
     return os.path.join(common.get_install_cache_dir(), os.path.basename(package))
 
-def get_package_file(package_name, package_url, hash_algorithm='md5', expected_hash=None):
+
+def download_package(package_url: str, timeout=120, creds=None, package_name="") -> http.client.HTTPResponse:
+    headers = {}
+    if creds:
+
+        try:
+            token_var = CREDENTIAL_ENVVARS[creds]
+        except KeyError:
+            logger.warning(f"Unrecognized creds={creds} value")
+
+        token = os.environ.get(token_var)
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        else:
+            raise CredentialsNotFoundError(
+                f"Package {package_name} is set to use '{creds}' credentials type but no {token_var} "
+                "environment variable is set"
+            )
+
+    req = urllib.request.Request(package_url, headers=headers)
+
+    return urllib.request.urlopen(req, data=None, timeout=timeout)
+
+
+def get_package_file(package_name, package_url, hash_algorithm='md5', expected_hash=None, creds=None):
     """
     Get the package file in the cache, downloading if needed.
     Validate the cache file using the hash (removing it if needed)
@@ -209,11 +245,14 @@ def get_package_file(package_name, package_url, hash_algorithm='md5', expected_h
             # Attempt to download the remote file
             logger.info("downloading %s:\n  %s\n     to %s" % (package_name, package_url, cache_file))
             try:
-                package_response = urllib.request.urlopen(package_url, None, download_timeout_seconds)
+                package_response = download_package(package_url, timeout=download_timeout_seconds, creds=creds, package_name=package_name)
             except urllib.error.URLError as err:
                 logger.error("error: %s\n  downloading package %s" % (err, package_url))
                 package_response = None
                 cache_file = None
+            except CredentialsNotFoundError as err:
+                logger.error(err)
+                return None
 
             if package_response is not None:
                 with open(cache_file, 'wb') as cache:

autobuild/autobuild_tool_installables.py

@@ -74,7 +74,7 @@ def run(self, args):
 
 
 _PACKAGE_ATTRIBUTES = ['description', 'copyright', 'license', 'license_file', 'version']
-_ARCHIVE_ATTRIBUTES = ['hash', 'hash_algorithm', 'url']
+_ARCHIVE_ATTRIBUTES = ['hash', 'hash_algorithm', 'url', 'creds']
 
 
 def _dict_from_key_value_arguments(arguments):
@@ -108,7 +108,8 @@ def _get_new_metadata(config, args_name, args_archive, arguments):
             archive_url = 'file://'+config.absolute_path(archive_path)
         archive_file = get_package_file(args_name, archive_url,
                                         hash_algorithm=key_values.get('hash_algorithm','md5'),
-                                        expected_hash=key_values.get('hash',None))
+                                        expected_hash=key_values.get('hash'),
+                                        creds=key_values.get('creds'))
         if archive_file:
             metadata = get_metadata_from_package(archive_file)
             metadata.archive = configfile.ArchiveDescription()

autobuild/common.py

@@ -25,7 +25,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 class AutobuildError(RuntimeError):
     pass
 

tests/test_install.py

@@ -10,8 +10,10 @@
 from string import Template
 from threading import Thread
 from unittest import TestCase
+from unittest.mock import MagicMock, patch
 
 from autobuild import autobuild_tool_install, autobuild_tool_uninstall, common
+from autobuild.autobuild_tool_install import CredentialsNotFoundError
 from tests.basetest import *
 
 # ****************************************************************************
@@ -739,3 +741,37 @@ def test_install_all(self):
             self.options.list_archives=True
             autobuild_tool_install.AutobuildTool().run(self.options)
         self.assertEqual(set_from_stream(stream), set(("argparse", "bogus")))
+
+# -------------------------------------  -------------------------------------
+class TestDownloadPackage(unittest.TestCase):
+    @patch("urllib.request.urlopen")
+    def test_download(self, mock_urlopen: MagicMock):
+        mock_urlopen.return_value = None
+        with envvar("AUTOBUILD_GITHUB_TOKEN", None):
+            autobuild_tool_install.download_package("https://example.org/foo.tar.bz2")
+            mock_urlopen.assert_called()
+
+    @patch("urllib.request.urlopen")
+    def test_download_github(self, mock_urlopen: MagicMock):
+        mock_urlopen.return_value = None
+        with envvar("AUTOBUILD_GITHUB_TOKEN", "token-123"):
+            autobuild_tool_install.download_package("https://example.org/foo.tar.bz2", creds="github")
+            mock_urlopen.assert_called()
+            got_req = mock_urlopen.mock_calls[0].args[0]
+            self.assertEqual(got_req.headers["Authorization"], "Bearer token-123")
+
+    @patch("urllib.request.urlopen")
+    def test_download_gitlab(self, mock_urlopen: MagicMock):
+        mock_urlopen.return_value = None
+        with envvar("AUTOBUILD_GITLAB_TOKEN", "token-123"):
+            autobuild_tool_install.download_package("https://example.org/foo.tar.bz2", creds="gitlab")
+            mock_urlopen.assert_called()
+            got_req = mock_urlopen.mock_calls[0].args[0]
+            self.assertEqual(got_req.headers["Authorization"], "Bearer token-123")
+
+    @patch("urllib.request.urlopen")
+    def test_download_github_without_creds(self, mock_urlopen: MagicMock):
+        mock_urlopen.return_value = None
+        with envvar("AUTOBUILD_GITHUB_TOKEN", None):
+            with self.assertRaises(CredentialsNotFoundError):
+                autobuild_tool_install.download_package("https://example.org/foo.tar.bz2", creds="github")