Add chromium_revision tag to SKP CIPD package

Bug: skia:8680
Change-Id: Ic84f33bc9c864005369e94a9d0039f92eb1bc17d
Reviewed-on: https://skia-review.googlesource.com/c/184922
Commit-Queue: Eric Boren <[email protected]>
Reviewed-by: Ravi Mistry <[email protected]>

infra/bots/assets/asset_utils.py

@@ -117,17 +117,21 @@ def get_available_versions(self, name):
     versions.sort()
     return versions
 
-  def upload(self, name, version, target_dir):
+  def upload(self, name, version, target_dir, extra_tags=None):
     """Create a CIPD package."""
-    self._run([
+    cmd = [
         'create',
         '--name', CIPD_PACKAGE_NAME_TMPL % name,
         '--in', target_dir,
         '--tag', TAG_PROJECT_SKIA,
         '--tag', TAG_VERSION_TMPL % version,
         '--compression-level', '1',
         '-verification-timeout', '30m0s',
-    ])
+    ]
+    if extra_tags:
+      for tag in extra_tags:
+        cmd.extend(['--tag', tag])
+    self._run(cmd)
 
   def download(self, name, version, target_dir):
     """Download a CIPD package."""
@@ -182,7 +186,8 @@ def get_available_versions(self, name):
     versions.sort()
     return versions
 
-  def upload(self, name, version, target_dir):
+  # pylint: disable=unused-argument
+  def upload(self, name, version, target_dir, extra_tags=None):
     """Upload to GS."""
     target_dir = os.path.abspath(target_dir)
     with utils.tmp_dir():
@@ -224,9 +229,9 @@ def __init__(self, cipd_url=DEFAULT_CIPD_SERVICE_URL,
   def get_available_versions(self, name):
     return self._cipd.get_available_versions(name)
 
-  def upload(self, name, version, target_dir):
-    self._cipd.upload(name, version, target_dir)
-    self._gs.upload(name, version, target_dir)
+  def upload(self, name, version, target_dir, extra_tags=None):
+    self._cipd.upload(name, version, target_dir, extra_tags=extra_tags)
+    self._gs.upload(name, version, target_dir, extra_tags=extra_tags)
 
   def download(self, name, version, target_dir):
     self._gs.download(name, version, target_dir)
@@ -279,10 +284,10 @@ def download_current_version(self, target_dir):
     v = self.get_current_version()
     self.download_version(v, target_dir)
 
-  def upload_new_version(self, target_dir, commit=False):
+  def upload_new_version(self, target_dir, commit=False, extra_tags=None):
     """Upload a new version and update the version file for the asset."""
     version = self.get_next_version()
-    self._store.upload(self._name, version, target_dir)
+    self._store.upload(self._name, version, target_dir, extra_tags=extra_tags)
 
     def _write_version():
       with open(self.version_file, 'w') as f:

infra/bots/assets/asset_utils_test.py

@@ -60,7 +60,7 @@ def get_available_versions(self, name):
     contents = os.listdir(os.path.join(self.dir, name))
     return sorted([int(d) for d in contents])
 
-  def upload(self, name, version, target_dir):
+  def upload(self, name, version, target_dir, extra_tags=None):
     shutil.copytree(target_dir, os.path.join(self.dir, name, str(version)))
 
   def download(self, name, version, target_dir):

infra/bots/assets/assets.py

@@ -46,7 +46,8 @@ def upload(args):
   """Upload a new version of the asset."""
   asset = asset_utils.Asset(args.asset_name,
                             asset_utils.MultiStore(gsutil=args.gsutil))
-  asset.upload_new_version(args.target_dir, commit=args.commit)
+  asset.upload_new_version(args.target_dir, commit=args.commit,
+                           extra_tags=args.extra_tags)
 
 
 def main(argv):
@@ -77,6 +78,9 @@ def main(argv):
   prs_upload.add_argument('--target_dir', '-t', required=True)
   prs_upload.add_argument('--gsutil')
   prs_upload.add_argument('--commit', action='store_true')
+  prs_upload.add_argument(
+    '--extra_tags', nargs='+',
+    help='Additional tags for the CIPD package, "key:value"')
 
   args = parser.parse_args(argv)
   args.func(args)

infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json

@@ -208,7 +208,9 @@
       "python",
       "[START_DIR]/cache/work/skia/infra/bots/upload_skps.py",
       "--target_dir",
-      "[START_DIR]/skp_output"
+      "[START_DIR]/skp_output",
+      "--chromium_path",
+      "[START_DIR]/cache/work/src"
     ],
     "cwd": "[START_DIR]/cache/work/skia",
     "env": {

infra/bots/recipes/recreate_skps.expected/failed_upload.json

@@ -208,7 +208,9 @@
       "python",
       "[START_DIR]/cache/work/skia/infra/bots/upload_skps.py",
       "--target_dir",
-      "[START_DIR]/skp_output"
+      "[START_DIR]/skp_output",
+      "--chromium_path",
+      "[START_DIR]/cache/work/src"
     ],
     "cwd": "[START_DIR]/cache/work/skia",
     "env": {

infra/bots/recipes/recreate_skps.py

@@ -86,7 +86,8 @@ def RunSteps(api):
   if 'Canary' not in api.properties['buildername']:
     cmd = ['python',
            skia_dir.join('infra', 'bots', 'upload_skps.py'),
-           '--target_dir', output_dir]
+           '--target_dir', output_dir,
+           '--chromium_path', src_dir]
     with api.context(cwd=skia_dir, env=api.infra.go_env):
       api.run(api.step, 'Upload SKPs', cmd=cmd)
 

infra/bots/upload_skps.py

@@ -25,11 +25,19 @@
 
 
 def main(target_dir):
+  # We're going to sync a new, clean Skia checkout to upload the CL to update
+  # the SKPs. However, we want to use the scripts from the current checkout,
+  # in order to facilitate running this as a try job.
+  infrabots_dir = os.path.dirname(os.path.realpath(__file__))
+  skp_dir = os.path.join(infrabots_dir, 'assets', 'skp')
+  upload_py = os.path.join(skp_dir, 'upload.py')
+
   with git_utils.NewGitCheckout(repository=SKIA_REPO):
     # First verify that there are no gen_tasks diffs.
-    gen_tasks = os.path.join(os.getcwd(), 'infra', 'bots', 'gen_tasks.go')
+    tmp_infrabots_dir = os.path.join(os.getcwd(), 'infra', 'bots')
+    tmp_gen_tasks = os.path.join(tmp_infrabots_dir, 'gen_tasks.go')
     try:
-      subprocess.check_call(['go', 'run', gen_tasks, '--test'])
+      subprocess.check_call(['go', 'run', tmp_gen_tasks, '--test'])
     except subprocess.CalledProcessError as e:
       print >> sys.stderr, (
          'gen_tasks.go failed, not uploading SKP update:\n\n%s' % e.output)
@@ -39,16 +47,28 @@ def main(target_dir):
     with git_utils.GitBranch(branch_name='update_skp_version',
                              commit_msg=COMMIT_MSG,
                              commit_queue=True):
-      upload_script = os.path.join(
-          os.getcwd(), 'infra', 'bots', 'assets', 'skp', 'upload.py')
-      subprocess.check_call(['python', upload_script, '-t', target_dir])
-      subprocess.check_call(['go', 'run', gen_tasks])
+      upload_cmd = ['python', upload_py, '-t', target_dir]
+      if args.chromium_path:
+        chromium_revision = subprocess.check_output(
+            ['git', 'rev-parse', 'HEAD'], cwd=args.chromium_path).rstrip()
+        upload_cmd.extend([
+            '--extra_tags', 'chromium_revision:%s' % chromium_revision])
+      subprocess.check_call(upload_cmd)
+      # We used upload.py from the repo that this script lives in, NOT the temp
+      # repo we've created. Therefore, the VERSION file was written in that repo
+      # so we need to copy it to the temp repo in order to commit it.
+      src = os.path.join(skp_dir, 'VERSION')
+      dst = os.path.join(
+          os.getcwd(), 'infra', 'bots', 'assets', 'skp', 'VERSION')
+      subprocess.check_call(['cp', src, dst])
+      subprocess.check_call(['go', 'run', tmp_gen_tasks])
       subprocess.check_call([
           'git', 'add', os.path.join('infra', 'bots', 'tasks.json')])
 
 
 if '__main__' == __name__:
   parser = argparse.ArgumentParser()
   parser.add_argument("--target_dir")
+  parser.add_argument("--chromium_path")
   args = parser.parse_args()
   main(args.target_dir)