Improved configuration

.dockerignore

@@ -0,0 +1,3 @@
+.DS_Store
+**/.DS_Store
+_env*

.gitignore

@@ -8,3 +8,6 @@ manta.pub
 
 # temp
 python-manta/
+
+# macos frustration
+.DS_Store

Dockerfile

@@ -2,11 +2,11 @@ FROM percona:5.6
 
 RUN apt-get update \
     && apt-get install -y \
-    python \
-    python-dev \
-    gcc \
-    curl \
-    percona-xtrabackup \
+        python \
+        python-dev \
+        gcc \
+        curl \
+        percona-xtrabackup \
     && rm -rf /var/lib/apt/lists/*
 
 # get Python drivers MySQL, Consul, and Manta
@@ -17,22 +17,25 @@ RUN curl -Ls -o get-pip.py https://bootstrap.pypa.io/get-pip.py && \
         python-Consul==0.4.7 \
         manta==2.5.0
 
-# get Containerpilot release
-RUN export CP_VERSION=2.0.1 &&\
-   curl -Lo /tmp/containerpilot.tar.gz \
-   https://github.com/joyent/containerpilot/releases/download/${CP_VERSION}/containerpilot-${CP_VERSION}.tar.gz && \
-   tar -xzf /tmp/containerpilot.tar.gz && \
-   mv /containerpilot /bin/
+# Add ContainerPilot and set its configuration file path
+ENV CONTAINERPILOT_VER 2.0.1
+ENV CONTAINERPILOT file:///etc/containerpilot.json
+RUN export CONTAINERPILOT_CHECKSUM=a4dd6bc001c82210b5c33ec2aa82d7ce83245154 \
+    && curl -Lso /tmp/containerpilot.tar.gz \
+        "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \
+    && echo "${CONTAINERPILOT_CHECKSUM}  /tmp/containerpilot.tar.gz" | sha1sum -c \
+    && tar zxf /tmp/containerpilot.tar.gz -C /usr/local/bin \
+    && rm /tmp/containerpilot.tar.gz
 
-# configure Containerpilot and MySQL
-COPY bin/* /bin/
+# configure ContainerPilot and MySQL
 COPY etc/* /etc/
+COPY bin/* /usr/local/bin/
 
 # override the parent entrypoint
 ENTRYPOINT []
 
 # use --console to get error logs to stderr
-CMD [ "/bin/containerpilot", \
+CMD [ "containerpilot", \
       "mysqld", \
       "--console", \
       "--log-bin=mysql-bin", \

README.md

@@ -14,21 +14,23 @@ MySQL designed for automated operation using the [Autopilot Pattern](http://auto
 A running cluster includes the following components:
 
 - [MySQL](https://dev.mysql.com/): we're using MySQL5.6 via [Percona Server](https://www.percona.com/software/mysql-database/percona-server), and [`xtrabackup`](https://www.percona.com/software/mysql-database/percona-xtrabackup) for running hot snapshots.
-- [Consul](https://www.consul.io/): used to coordinate replication and failover
+- [ContainerPilot](https://www.joyent.com/containerpilot): included in our MySQL containers to orchestrate bootstrap behavior and coordinate replication using keys and checks stored in Consul in the `preStart`, `health`, and `onChange` handlers.
+- [Consul](https://www.consul.io/): is our service catalog that works with ContainerPilot and helps coordinate service discovery, replication, and failover
 - [Manta](https://www.joyent.com/object-storage): the Joyent object store, for securely and durably storing our MySQL snapshots.
-- [ContainerPilot](http://containerpilot.io): included in our MySQL containers orchestrate bootstrap behavior and coordinate replication using keys and checks stored in Consul in the `onStart`, `health`, and `onChange` handlers.
 - `triton-mysql.py`: a small Python application that ContainerPilot will call into to do the heavy lifting of bootstrapping MySQL.
 
-When a new MySQL node is started, ContainerPilot's `onStart` handler will call into `triton-mysql.py`.
+When a new MySQL node is started, ContainerPilot's `preStart` handler will call into `triton-mysql.py`.
 
 
-### Bootstrapping via `onStart` handler
+### Bootstrapping via `preStart` handler
+
+`preStart` (formerly `onStart`) runs and must exit cleanly before the main application is started.
 
 The first thing the `triton-mysql.py` application does is to ask Consul whether a primary node exists. If not, the application will atomically mark the node as primary in Consul and then bootstrap the node as a new primary. Bootstrapping a primary involves setting up users (root, default, and replication), and creating a default schema. Once the primary bootstrap process is complete, it will use `xtrabackup` to create a backup and upload it to Manta. The application then writes a TTL key to Consul which will tell us when next to run a backup, and a non-expiring key that records the path on Manta where the most recent backup was stored.
 
 If a primary already exists, then the application will ask Consul for the path to the most recent backup snapshot and download it and the most recent binlog. The application will then ask Consul for the IP address of the primary and set up replication from that primary before allowing the new replica to join the cluster.
 
-Replication in this architecture uses [Global Transaction Idenitifers (GTID)](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids.html) so that replicas can autoconfigure their position within the binlog.
+Replication in this architecture uses [Global Transaction Identifiers (GTID)](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids.html) so that replicas can autoconfigure their position within the binlog.
 
 ### Maintenance via `health` handler
 
@@ -56,29 +58,29 @@ By default, the primary performs the backup snapshots. For deployments with high
 
 ## Running the cluster
 
-Starting a new cluster is easy. Just run `docker-compose up -d` and in a few moments you'll have a running MySQL primary. Both the primary and replicas are described as a single `docker-compose` service. During startup, [ContainerPilot](http://containerpilot.io) will ask Consul if an existing primary has been created. If not, the node will initialize as a new primary and all future nodes will self-configure replication with the primary in their `onStart` handler.
+Starting a new cluster is easy once you have [your `_env` file set with the configuration details](#configuration), **just run `docker-compose up -d` and in a few moments you'll have a running MySQL primary**. Both the primary and replicas are described as a single `docker-compose` service. During startup, [ContainerPilot](http://containerpilot.io) will ask Consul if an existing primary has been created. If not, the node will initialize as a new primary and all future nodes will self-configure replication with the primary in their `preStart` handler.
 
-Run `docker-compose scale mysql=2` to add a replica (or more than one!). The replicas will automatically configure themselves to to replicate from the primary and will register themselves in Consul as replicas once they're ready.
+**Run `docker-compose scale mysql=2` to add a replica (or more than one!)**. The replicas will automatically configure themselves to to replicate from the primary and will register themselves in Consul as replicas once they're ready.
 
 ### Configuration
 
-Pass these variables in your environment or via an `_env` file.
+Pass these variables via an `_env` file. The included `setup.sh` can be used to test your Docker and Triton environment, and to encode the Manta SSH key in the `_env` file.
 
 - `MYSQL_USER`: this user will be set up as the default non-root user on the node
 - `MYSQL_PASSWORD`: this user will be set up as the default non-root user on the node
+- `MANTA_URL`: the full Manta endpoint URL. (ex. `https://us-east.manta.joyent.com`)
+- `MANTA_USER`: the Manta account name.
+- `MANTA_SUBUSER`: the Manta subuser account name, if any.
+- `MANTA_ROLE`: the Manta role name, if any.
+- `MANTA_KEY_ID`: the MD5-format ssh key id for the Manta account/subuser (ex. `1a:b8:30:2e:57:ce:59:1d:16:f6:19:97:f2:60:2b:3d`); the included `setup.sh` will encode this automatically
+- `MANTA_PRIVATE_KEY`: the private ssh key for the Manta account/subuser; the included `setup.sh` will encode this automatically
+- `MANTA_BUCKET`: the path on Manta where backups will be stored. (ex. `/myaccount/stor/triton-mysql`); the bucket must already exist and be writeable by the `MANTA_USER`/`MANTA_PRIVATE_KEY`
 
 These variables are optional but you most likely want them:
 
 - `MYSQL_REPL_USER`: this user will be used on all instances to set up MySQL replication. If not set, then replication will not be set up on the replicas.
 - `MYSQL_REPL_PASSWORD`: this password will be used on all instances to set up MySQL replication. If not set, then replication will not be set up on the replicas.
 - `MYSQL_DATABASE`: create this database on startup if it doesn't already exist. The `MYSQL_USER` user will be granted superuser access to that DB.
-- `MANTA_URL`: the full Manta endpoint URL. (ex. `https://us-east.manta.joyent.com`)
-- `MANTA_USER`: the Manta account name.
-- `MANTA_SUBUSER`: the Manta subuser account name, if any.
-- `MANTA_ROLE`: the Manta role name, if any.
-- `MANTA_KEY_ID`: the MD5-format ssh key id for the Manta account/subuser (ex. `1a:b8:30:2e:57:ce:59:1d:16:f6:19:97:f2:60:2b:3d`).
-- `MANTA_PRIVATE_KEY`: the private ssh key for the Manta account/subuser.
-- `MANTA_BUCKET`: the path on Manta where backups will be stored. (ex. `/myaccount/stor/triton-mysql`)
 - `LOG_LEVEL`: will set the logging level of the `triton-mysql.py` application. It defaults to `DEBUG` and uses the Python stdlib [log levels](https://docs.python.org/2/library/logging.html#levels). In production you'll want this to be at `INFO` or above.
 - `TRITON_MYSQL_CONSUL` is the hostname for the Consul instance(s). Defaults to `consul`.
 - `USE_STANDBY` tells the `triton-mysql.py` application to use a separate standby MySQL node to run backups. This might be useful if you have a very high write throughput on the primary node. Defaults to `no` (turn on with `yes` or `on`).
@@ -107,7 +109,9 @@ These variables will be written to `/etc/my.cnf`.
 
 ### Where to store data
 
-On Triton there's not need to use data volumes because the performance hit you normally take with overlay file systems in Linux doesn't happen with ZFS.
+This pattern automates the data management and makes container effectively stateless to the Docker daemon and schedulers. This is designed to maximize convenience and reliability by minimizing the external coordination needed to manage the database. The use of external volumes (`--volumes-from`, `-v`, etc.) is not recommended.
+
+On Triton, there's no need to use data volumes because the performance hit you normally take with overlay file systems in Linux doesn't happen with ZFS.
 
 ### Using an existing database
 

bin/triton-mysql.py

@@ -32,7 +32,7 @@
 
 log = logging.getLogger('triton-mysql')
 
-consul = pyconsul.Consul(host=os.environ.get('TRITON_MYSQL_CONSUL', 'consul'))
+consul = pyconsul.Consul(host=os.environ.get('CONSUL', 'consul'))
 config = None
 
 # consts for node state
@@ -175,7 +175,7 @@ def __init__(self):
         self.user = os.environ.get('MANTA_SUBUSER', None)
         self.role = os.environ.get('MANTA_ROLE', None)
         self.key_id = os.environ.get('MANTA_KEY_ID', None)
-        self.private_key = os.environ.get('MANTA_PRIVATE_KEY')
+        self.private_key = os.environ.get('MANTA_PRIVATE_KEY').replace('#', '\n')
         self.url = os.environ.get('MANTA_URL',
                                   'https://us-east.manta.joyent.com')
         self.bucket = os.environ.get('MANTA_BUCKET',
@@ -203,15 +203,15 @@ def put_backup(self, backup_id, infile):
 
 # ---------------------------------------------------------
 
-class Containerpilot(object):
+class ContainerPilot(object):
     """
-    Containerpilot config is where we rewrite Containerpilot's own config
+    ContainerPilot config is where we rewrite ContainerPilot's own config
     so that we can dynamically alter what service we advertise
     """
 
     def __init__(self, node):
         # TODO: we should make sure we can support JSON-in-env-var
-        # the same as Containerpilot itself
+        # the same as ContainerPilot itself
         self.node = node
         self.path = os.environ.get('CONTAINERPILOT').replace('file://', '')
         with open(self.path, 'r') as f:
@@ -231,12 +231,12 @@ def render(self):
             f.write(new_config)
 
     def reload(self):
-        """ force Containerpilot to reload its configuration """
-        log.info('Reloading Containerpilot configuration.')
+        """ force ContainerPilot to reload its configuration """
+        log.info('Reloading ContainerPilot configuration.')
         os.kill(1, signal.SIGHUP)
 
 # ---------------------------------------------------------
-# Top-level functions called by Containerpilot or forked by this program
+# Top-level functions called by ContainerPilot or forked by this program
 
 def on_start():
     """
@@ -257,24 +257,24 @@ def on_start():
 def health():
     """
     Run a simple health check. Also acts as a check for whether the
-    Containerpilot configuration needs to be reloaded (if it's been
+    ContainerPilot configuration needs to be reloaded (if it's been
     changed externally), or if we need to make a backup because the
     backup TTL has expired.
     """
     log.debug('health check fired.')
     try:
         node = MySQLNode()
-        cb = Containerpilot(node)
-        if cb.update():
-            cb.reload()
+        cp = ContainerPilot(node)
+        if cp.update():
+            cp.reload()
             return
 
-        # cb.reload() will exit early so no need to setup
+        # cp.reload() will exit early so no need to setup
         # connection until this point
         ctx = dict(user=config.repl_user,
                    password=config.repl_password,
                    database=config.mysql_db,
-                   timeout=cb.config['services'][0]['ttl'])
+                   timeout=cp.config['services'][0]['ttl'])
         node.conn = wait_for_connection(**ctx)
 
         # Update our lock on being the primary/standby.
@@ -312,15 +312,15 @@ def on_change():
     log.debug('on_change check fired.')
     try:
         node = MySQLNode()
-        cb = Containerpilot(node)
-        cb.update() # this will populate MySQLNode state correctly
+        cp = ContainerPilot(node)
+        cp.update() # this will populate MySQLNode state correctly
         if node.is_primary():
             return
 
         ctx = dict(user=config.repl_user,
                    password=config.repl_password,
                    database=config.mysql_db,
-                   timeout=cb.config['services'][0]['ttl'])
+                   timeout=cp.config['services'][0]['ttl'])
         node.conn = wait_for_connection(**ctx)
 
         # need to stop replication whether we're the new primary or not
@@ -341,8 +341,8 @@ def on_change():
                 session_id = get_session(no_cache=True)
                 if mark_with_session(PRIMARY_KEY, node.hostname, session_id):
                     node.state = PRIMARY
-                    if cb.update():
-                        cb.reload()
+                    if cp.update():
+                        cp.reload()
                     return
                 else:
                     # we lost the race to lock the session for ourselves
@@ -354,8 +354,8 @@ def on_change():
             # if it's not healthy, we'll throw an exception and start over.
             ip = get_primary_host(primary=primary)
             if ip == node.ip:
-                if cb.update():
-                    cb.reload()
+                if cp.update():
+                    cp.reload()
                 return
 
             set_primary_for_replica(node.conn)
@@ -791,9 +791,9 @@ def write_snapshot(conn):
     # we set the BACKUP_TTL before we run the backup so that we don't
     # have multiple health checks running concurrently. We then fork the
     # create_snapshot call and return. The snapshot process will be
-    # re-parented to Containerpilot
+    # re-parented to ContainerPilot
     set_backup_ttl()
-    subprocess.Popen(['python', '/bin/triton-mysql.py', 'create_snapshot'])
+    subprocess.Popen(['python', '/usr/local/bin/triton-mysql.py', 'create_snapshot'])
 
 def set_backup_ttl():
     """

docker-compose.yml

@@ -1,11 +1,32 @@
 mysql:
-    extends:
-      file: common-compose.yml
-      service: mysql
-    links:
-      - consul:consul
+    image: autopilotpattern/mysql:latest
+    mem_limit: 4g
+    restart: always
+    # expose for linking, but each container gets a private IP for
+    # internal use as well
+    expose:
+      - 3306
+    labels:
+      - triton.cns.services=mysql
+    env_file: _env
+    environment:
+      - CONTAINERPILOT=file:///etc/containerpilot.json
 
 consul:
-    extends:
-      file: common-compose.yml
-      service: consul
+    image: progrium/consul:latest
+    command: -server -bootstrap -ui-dir /ui
+    restart: always
+    mem_limit: 128m
+    ports:
+      - 8500
+    expose:
+      - 53
+      - 8300
+      - 8301
+      - 8302
+      - 8400
+      - 8500
+    dns:
+      - 127.0.0.1
+    labels:
+      - triton.cns.services=consul

etc/containerbuddy.json → etc/containerpilot.json

@@ -1,11 +1,11 @@
 {
   "consul": "{{ .CONSUL }}:8500",
-  "onStart": "python /bin/triton-mysql.py",
+  "preStart": "python /usr/local/bin/triton-mysql.py",
   "services": [
     {
       "name": "mysql",
       "port": 3306,
-      "health": "python /bin/triton-mysql.py health",
+      "health": "python /usr/local/bin/triton-mysql.py health",
       "poll": 5,
       "ttl": 25
     }
@@ -14,7 +14,7 @@
     {
       "name": "mysql-primary",
       "poll": 10,
-      "onChange": "python /bin/triton-mysql.py on_change"
+      "onChange": "python /usr/local/bin/triton-mysql.py on_change"
     }
   ]
 }