Add CD for client templates and components.

- Client templates and components will now be built on every commit and uploaded
  to cloud storage for OS X and linux (appveyor handles windows).
- Fix a objectfilter bug

.travis.yml

@@ -1,28 +1,36 @@
-language: python
-sudo: false
+python: 2.7
+# From https://goo.gl/XuEMpi GCP CD with travis
+env: PROTOC="${HOME}/protobuf/bin/protoc" PATH=$PATH:${HOME}/google-cloud-sdk/bin CLOUDSDK_CORE_DISABLE_PROMPTS=1
+matrix:
+  include:
+  - os: linux
+    language: python
+    python: 2.7
+    sudo: false
+  - os: osx
+    # psutil fails to install on the default beta-xcode6.1
+    osx_image: beta-xcode6.2
+    language: generic
+    sudo: required
 cache:
   directories:
-    - $HOME/.cache/pip
-    - $HOME/protobuf
-python:
-  - "2.7"
-before_install: /bin/bash travis/install_protobuf.sh
-env:
-  - PROTOC="${HOME}/protobuf/bin/protoc"
+  - $HOME/.cache/pip
+  - $HOME/protobuf
+before_install:
+- /bin/bash travis/install_protobuf.sh
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update && brew install libffi && git clone https://github.com/MacPython/terryfy && source terryfy/travis_tools.sh; get_python_environment macpython 2.7.11; fi
+# We upgrade pip inside the virtualenv to avoid sudo here.
+- pip install --upgrade virtualenv
+- virtualenv ${HOME}/INSTALL
 install:
-  - pip install --upgrade pip wheel setuptools
-  - pip install -e .
-  - pip install -e grr/config/grr-response-server/
-  - pip install -e grr/config/grr-response-test/
-  - python makefile.py
-  - cd grr/artifacts && python makefile.py && cd -
-script: grr_run_tests --processes=2
+- /bin/bash travis/install.sh
+script:
+  - grr_run_tests --processes=2
+  - /bin/bash travis/deploy_to_gcs.sh
 addons:
   apt:
     packages:
     - libffi-dev
     - libssl-dev
     - python-dev
     - prelink
-
-

grr/lib/aff4.py

@@ -2933,15 +2933,6 @@ class SchemaCls(AFF4MemoryStreamBase.SchemaCls):
                         versioned=False)
 
 
-# Still needed for backwards compatibility.
-# TODO(user): Remove when current flows (Jan 14) have finished processing.
-class AFF4ObjectCache(utils.FastStore):
-  """A cache which closes its objects when they expire."""
-
-  def KillObject(self, obj):
-    obj.Close(sync=True)
-
-
 class ChunkCache(utils.FastStore):
   """A cache which closes its objects when they expire."""
 

grr/lib/artifact_registry.py

@@ -608,7 +608,7 @@ def ReduceDict(in_dict):
     if sources_dict:
       artifact_dict["sources"] = [ReduceDict(c) for c in sources_dict]
     # Do some clunky stuff to put the name and doc first in the YAML.
-    # Unfortunatley PYYaml makes doing this difficult in other ways.
+    # Unfortunately PYYaml makes doing this difficult in other ways.
     name = artifact_dict.pop("name")
     doc = artifact_dict.pop("doc")
     doc_str = yaml.safe_dump({"doc": doc}, allow_unicode=True, width=80)[1:-2]

grr/lib/lexer.py

@@ -309,7 +309,7 @@ def Compile(self, filter_implemention):
       raise ParseError("Invalid binary operator %s" % operator)
 
     args = [x.Compile(filter_implemention) for x in self.args]
-    return getattr(filter_implemention, method)(*args)
+    return filter_implemention.GetFilter(method)(*args)
 
 
 class IdentityExpression(Expression):

grr/tools/config_updater.py

@@ -385,6 +385,12 @@ def ShowUser(username, token=None):
     parents=[],
     help="Sign and upload all client components.")
 
+parser_upload_components.add_argument(
+    "--overwrite_component",
+    default=False,
+    action="store_true",
+    help="Allow overwriting of the component path.")
+
 subparsers.add_parser(
     "download_missing_rekall_profiles",
     parents=[],

travis/deploy_to_gcs.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Build client templates and upload them to cloud storage.
+#
+# This script must be run inside travis, as it relies on some travis specific
+# environment variables. Ideally it should run in the travis deploy stage rather
+# than the script phase (which will run for every pull request), but here is a
+# litany of reasons why it doesn't.
+#
+# There is already a travis gcs deployer but it isn't usable:
+# https://github.com/travis-ci/dpl/issues/476
+#
+# When using the experimental script deployer there were differences between the
+# virtual env in the deploy and install stages that caused the client build to
+# fail (previously installed grr-* packages would somehow be missing from the
+# virtualenv). I'm still attempting to make a minimal reproducible case for a
+# good bug report there.
+#
+# and it's also annoying to debug:
+# https://github.com/travis-ci/dpl/issues/477
+#
+# We could use after_success but it doesn't exit on error:
+# https://github.com/travis-ci/travis-ci/issues/758
+
+set -e
+
+source "${HOME}/INSTALL/bin/activate"
+
+# Temporary workaround until we migrate off PackageMaker.
+if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  cd /Applications
+  wget -q https://storage.googleapis.com/grr-osx-buildtools/packagemaker.tar.gz
+  tar zxf packagemaker.tar.gz
+  cd -
+fi
+
+grr_client_build build --output built_templates
+grr_client_build build_components --output built_templates
+
+# If we don't have the sdk, go get it. While we could cache the cloud sdk
+# directory it may contain authentication tokens after the authorization step
+# below, so we don't.
+gcloud version || ( wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-116.0.0-linux-x86_64.tar.gz && tar zxf google-cloud-sdk-116.0.0-linux-x86_64.tar.gz -C "${HOME}" )
+
+# See https://docs.travis-ci.com/user/encrypting-files/
+openssl aes-256-cbc -K "$encrypted_db009a5a71c6_key" \
+  -iv "$encrypted_db009a5a71c6_iv" \
+  -in travis/travis_uploader_service_account.json.enc \
+  -out travis/travis_uploader_service_account.json -d
+
+gcloud auth activate-service-account --key-file travis/travis_uploader_service_account.json
+echo Uploading templates to "gs://autobuilds.grr-response.com/${TRAVIS_JOB_NUMBER}"
+gsutil -m cp built_templates/* "gs://autobuilds.grr-response.com/${TRAVIS_JOB_NUMBER}/"
+
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  shred -u travis/travis_uploader_service_account.json
+fi
+if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  srm -sz travis/travis_uploader_service_account.json
+fi

travis/install.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Install grr into a virtualenv
+
+set -e
+
+source "${HOME}/INSTALL/bin/activate"
+pip install --upgrade pip wheel setuptools
+pip install -e .
+pip install -e grr/config/grr-response-test/
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  pip install -e grr/config/grr-response-server/
+fi
+pip install -e grr/config/grr-response-client/
+
+python makefile.py
+cd grr/artifacts && python makefile.py && cd -