diff --git a/.travis.yml b/.travis.yml index 3f5ea786..f20e8a24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,16 +3,15 @@ "language": "node_js", "node_js": ["5"], "before_install": [ - "travis_retry pip install -r test-infra/requirements.txt --user", + # Remove ./node_modules/.bin from PATH so node-which doesn't replace Unix which and cause RVM to barf. See https://github.com/travis-ci/travis-ci/issues/5092 + "export PATH=$(python -c 'from sys import argv;from collections import OrderedDict as od;print(':'.join(od((p,None) for p in argv[1].split(':') if p.startswith('/')).keys()))' '$PATH')", "rvm install 2.2", - "rvm use 2.2 --fuzzy", - "export GEMDIR=$(rvm gemdir)", - "echo \"ruby=$(basename $GEMDIR) jekyll=$JEKYLL_VERSION\" > pseudo_Gemfile.lock" + "rvm use 2.2 --fuzzy" ], "install": [ "npm install -g grunt-cli", - "./test-infra/s3_cache.py download npm-modules", - "./test-infra/s3_cache.py download rubygems" + "npm install", + "bundle install --without development --deployment --jobs=3" ], "before_script": [ "git clone --depth 1 https://github.com/twbs/bootstrap.git", @@ -26,16 +25,12 @@ "diff test/fixtures/cli/x-ua-compatible-missing.output.txt x-ua-compatible-missing.output.actual.txt" ], "after_script": [ - "npm run coveralls", - "./test-infra/s3_cache.py upload npm-modules", - "./test-infra/s3_cache.py upload rubygems" + "npm run coveralls" ], - "env": { - "global": [ - {"JEKYLL_VERSION": "3.1.2"}, - {"secure": "dRdiG/5UykFAVW6GWWcaNHGSPy16PiCeF9XySPDdCSbw+pI2zqE8VNyXgn1kORhLFsKjdIQaLnmFWR1Xw7sP59zpnIRUkZ77spw7hKNf1RlAv3uckE8LFxO1FkMFNOlSHgmCXnyseUNGMDL/lIBMCLfsTOlc6KvbQir7pz+TDwM="}, - {"secure": "Ej9x2sBilYq9Wr86j7NujcPN0qAMHgUzjB9tX0vN90nKOPfAmWWcO9omFzwkuH0VZkZmauK/YQlEqKhx99quVwRHctu0LhZRnBgaCnsGp6CnRorFW4IKZwJSM38BF9XXvCCnQtz6PyItpE+ycbpFI0MHvs0H3BXFmwifvY/Q4/Y="}, - {"secure": "Oq9mxJduZiy0oZmKlg3kkFFWlBTuCynRpZoz0SyG3hHt2uipWfXlm0AlpcjU6jB6g8kNtGF4ZA5uGiMUePSOfK0RveYQa95ixUY6VrbuAIzdDtz/rbMajtGCVpMBgOf5o33rEfocwgADE/i1lE1JJeNjdUBsO1Iyron1ZiT52mE="} + "cache": { + "directories": [ + "node_modules", + "vendor/bundle" ] } } diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..719b61a0 --- /dev/null +++ b/Gemfile @@ -0,0 +1,5 @@ +source 'https://rubygems.org' + +group :development, :test do + gem 'jekyll', '~> 3.1.2' +end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 00000000..f496d72a --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,39 @@ +GEM + remote: https://rubygems.org/ + specs: + colorator (0.1) + ffi (1.9.10-x64-mingw32) + jekyll (3.1.2) + colorator (~> 0.1) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 1.1) + kramdown (~> 1.3) + liquid (~> 3.0) + mercenary (~> 0.3.3) + rouge (~> 1.7) + safe_yaml (~> 1.0) + jekyll-sass-converter (1.4.0) + sass (~> 3.4) + jekyll-watch (1.3.1) + listen (~> 3.0) + kramdown (1.10.0) + liquid (3.0.6) + listen (3.0.6) + rb-fsevent (>= 0.9.3) + rb-inotify (>= 0.9.7) + mercenary (0.3.5) + rb-fsevent (0.9.7) + rb-inotify (0.9.7) + ffi (>= 0.5.0) + rouge (1.10.1) + safe_yaml (1.0.4) + sass (3.4.21) + +PLATFORMS + x64-mingw32 + +DEPENDENCIES + jekyll (~> 3.1.2) + +BUNDLED WITH + 1.11.2 diff --git a/test-infra/S3Cachefile.json b/test-infra/S3Cachefile.json deleted file mode 100644 index 5335b286..00000000 --- a/test-infra/S3Cachefile.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "npm-modules": { - "key": "./npm-shrinkwrap.json", - "cache": "../node_modules", - "generate": "./uncached-npm-install.sh" - }, - "rubygems": { - "key": "../pseudo_Gemfile.lock", - "cache": "$GEMDIR", - "generate": "gem install -N jekyll -v $JEKYLL_VERSION" - } -} diff --git a/test-infra/requirements.txt b/test-infra/requirements.txt deleted file mode 100644 index fe44343d..00000000 --- a/test-infra/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -boto==2.25.0 diff --git a/test-infra/s3_cache.py b/test-infra/s3_cache.py deleted file mode 100755 index 9d46b817..00000000 --- a/test-infra/s3_cache.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python2.7 -# pylint: disable=C0301 -# -# Copyright 2011-2014 Twitter, Inc. -# Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - -from __future__ import absolute_import, unicode_literals, print_function, division - -from sys import argv -from os import environ, stat, chdir, remove as _delete_file -from os.path import dirname, basename, abspath, realpath, expandvars -from hashlib import sha256 -from subprocess import check_call as run -from json import load, dump as save -from contextlib import contextmanager -from datetime import datetime - -from boto.s3.connection import S3Connection -from boto.s3.key import Key -from boto.exception import S3ResponseError - - -CONFIG_FILE = './S3Cachefile.json' -UPLOAD_TODO_FILE = './S3CacheTodo.json' -BYTES_PER_MB = 1024 * 1024 - - -@contextmanager -def timer(): - start = datetime.utcnow() - yield - end = datetime.utcnow() - elapsed = end - start - print("\tDone. Took", int(elapsed.total_seconds()), "second(s).") - - -@contextmanager -def todo_file(writeback=True): - try: - with open(UPLOAD_TODO_FILE, 'rt') as json_file: - todo = load(json_file) - except (IOError, OSError, ValueError): - todo = {} - - yield todo - - if writeback: - try: - with open(UPLOAD_TODO_FILE, 'wt') as json_file: - save(todo, json_file) - except (OSError, IOError) as save_err: - print("Error saving {}:".format(UPLOAD_TODO_FILE), save_err) - - -def _sha256_of_file(filename): - hasher = sha256() - with open(filename, 'rb') as input_file: - hasher.update(input_file.read()) - file_hash = hasher.hexdigest() - print('sha256({}) = {}'.format(filename, file_hash)) - return file_hash - - -def _delete_file_quietly(filename): - try: - _delete_file(filename) - except (OSError, IOError): - pass - - -def mark_needs_uploading(cache_name): - with todo_file() as todo: - todo[cache_name] = True - - -def mark_uploaded(cache_name): - with todo_file() as todo: - todo.pop(cache_name, None) - - -def need_to_upload(cache_name): - with todo_file(writeback=False) as todo: - return todo.get(cache_name, False) - - -def _tarball_size(directory): - kib = stat(_tarball_filename_for(directory)).st_size // BYTES_PER_MB - return "{} MiB".format(kib) - - -def _tarball_filename_for(directory): - return abspath('./{}.tar.gz'.format(basename(directory))) - - -def _create_tarball(directory): - print("Creating tarball of {}...".format(directory)) - with timer(): - run(['tar', '-czf', _tarball_filename_for(directory), '-C', dirname(directory), basename(directory)]) - - -def _extract_tarball(directory): - print("Extracting tarball of {}...".format(directory)) - with timer(): - run(['tar', '-xzf', _tarball_filename_for(directory), '-C', dirname(directory)]) - - -def download(directory): - mark_uploaded(cache_name) # reset - try: - print("Downloading {} tarball from S3...".format(cache_name)) - with timer(): - key.get_contents_to_filename(_tarball_filename_for(directory)) - except S3ResponseError as err: - mark_needs_uploading(cache_name) - raise SystemExit("Cached {} download failed!".format(cache_name)) - print("Downloaded {}.".format(_tarball_size(directory))) - _extract_tarball(directory) - print("{} successfully installed from cache.".format(cache_name)) - - -def upload(directory): - _create_tarball(directory) - print("Uploading {} tarball to S3... ({})".format(cache_name, _tarball_size(directory))) - with timer(): - key.set_contents_from_filename(_tarball_filename_for(directory)) - print("{} cache successfully updated.".format(cache_name)) - mark_uploaded(cache_name) - - -if __name__ == '__main__': - # Uses environment variables: - # AWS_ACCESS_KEY_ID -- AWS Access Key ID - # AWS_SECRET_ACCESS_KEY -- AWS Secret Access Key - argv.pop(0) - if len(argv) != 2: - raise SystemExit("USAGE: s3_cache.py ") - mode, cache_name = argv - script_dir = dirname(realpath(__file__)) - chdir(script_dir) - try: - with open(CONFIG_FILE, 'rt') as config_file: - config = load(config_file) - except (IOError, OSError, ValueError) as config_err: - print(config_err) - raise SystemExit("Error when trying to load config from JSON file!") - - try: - cache_info = config[cache_name] - key_file = expandvars(cache_info["key"]) - fallback_cmd = cache_info["generate"] - directory = expandvars(cache_info["cache"]) - except (TypeError, KeyError) as load_err: - print(load_err) - raise SystemExit("Config for cache named {!r} is missing or malformed!".format(cache_name)) - - try: - try: - BUCKET_NAME = environ['TWBS_S3_BUCKET'] - except KeyError: - raise SystemExit("TWBS_S3_BUCKET environment variable not set!") - - conn = S3Connection() - bucket = conn.lookup(BUCKET_NAME) - if bucket is None: - raise SystemExit("Could not access bucket!") - - key_file_hash = _sha256_of_file(key_file) - - key = Key(bucket, key_file_hash) - key.storage_class = 'REDUCED_REDUNDANCY' - - if mode == 'download': - download(directory) - elif mode == 'upload': - if need_to_upload(cache_name): - upload(directory) - else: - print("No need to upload anything.") - else: - raise SystemExit("Unrecognized mode {!r}".format(mode)) - except BaseException as exc: - if mode != 'download': - raise - print("Error!:", exc) - print("Unable to download from cache.") - print("Running fallback command to generate cache directory {!r}: {}".format(directory, fallback_cmd)) - with timer(): - run(fallback_cmd, shell=True) diff --git a/test-infra/uncached-npm-install.sh b/test-infra/uncached-npm-install.sh deleted file mode 100755 index ab3f8569..00000000 --- a/test-infra/uncached-npm-install.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright 2011-2014 Twitter, Inc. -# Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) -set -e -cd .. # /bootlint/ -cp test-infra/npm-shrinkwrap.json npm-shrinkwrap.json -# npm is flaky, so try multiple times -MAXTRIES=3 -TRIES=1 -while ! npm install; do - if [ $TRIES -ge $MAXTRIES ]; then - exit 1 - fi - TRIES=$(($TRIES + 1)) - echo "Retrying npm install (Try $TRIES of $MAXTRIES)..." -done -rm npm-shrinkwrap.json