From 29372a03246b789c944da970e2cd5079b27d2469 Mon Sep 17 00:00:00 2001 From: Christoph Wiechert Date: Thu, 16 Apr 2015 23:45:14 +0200 Subject: [PATCH] init --- .gitignore | 3 + LICENSE | 21 ++++ README.md | 101 +++++++++++++++++++ app.js | 89 +++++++++++++++++ fleet-unit-files/skydns.service | 43 ++++++++ lib/Docker.js | 169 ++++++++++++++++++++++++++++++++ lib/SkydnsBE.js | 151 ++++++++++++++++++++++++++++ lib/etcd.js | 46 +++++++++ lib/service.js | 73 ++++++++++++++ package.json | 36 +++++++ 10 files changed, 732 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 app.js create mode 100644 fleet-unit-files/skydns.service create mode 100644 lib/Docker.js create mode 100644 lib/SkydnsBE.js create mode 100644 lib/etcd.js create mode 100644 lib/service.js create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6a6020 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +tmp +.idea +node_modules \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8f64e01 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Christoph Wiechert + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..1250362 --- /dev/null +++ b/README.md @@ -0,0 +1,101 @@ +# docker-etcd-registrator + +Docker service registrator for etcd and skydns (and CoreOS). +The very end of `sidekick.service` + +* Startup synchronization: bring etcd up to date + * Add already running containers + * Remove stopped but registred container +* Realtime: Listening for docker events +* Registers all ports + * defined via `EXPOSE` in the `Dockerfile` + * exposed via `-p` commandline argument +* Supports secured etcd +* Service config using ENV +* Written in Javascript +* for (but not limited to) CoreOS, see [fleet-unit-files](https://github.com/psi-4ward/docker-etcd-registrator/tree/master/fleet-unit-files) + +(thanks to [gliderlabs/registrator](https://github.com/gliderlabs/registrator) for the some ideas)* + +### TODO / Planned + +* [Vulcanproxy](vulcanproxy.com) support +* Some general info logging to stdout +* Configuration using commandline arguments +* Support for publicIPs and `--net=host` +* Improve docu + + +## Install & Config + +* For now its only possible to configure docker-etcd-registrator using environment variables +* Make sure the app can read/write to `DOCKER_HOST` (default: `/var/run/docker.sock`) + +```shell +git clone https://github.com/psi-4ward/docker-etcd-registrator.git +npm install +ETCD_ENDPOINTS=http://10.1.0.1:4001 node app.js +``` + +### Config parameters + +All params are optional + +* `HOSTNAME`: Hostname of the system +* `SKYDNS_ETCD_PREFIX`: `/skydns/local/skydns` +
+* `DOCKER_HOST`: `/var/run/docker.sock` or `tcp://localhost:2376` +* `DOCKER_TLS_VERIFY` from docker-modem +* `DOCKER_CERT_PATH`: Directory containing `ca.pem`, `cert.pem`, `key.pem` (filenames hardcoded) +
+* `ETCD_ENDPOINTS`: `http://127.0.0.1:4001` +* `ETCD_CAFILE` +* `ETCD_CERTFILE` +* `ETCD_KEYFILE` + +### Debug +Enable debugging using `DEBUG` env var: `DEBUG=docker,skydns,service node app.js` + +flag | description +---------|----------------------------- + * | print every debug message | + docker | docker related messages | + service | container-inspect => service transformation | + skydns | skydns etcd data population | + + +## Service Discovery Configration + +* Use env vars to configure a specific container / service +* Everything is optional +* Name is received from `SERVICE_NAME` or `--name` or the container ID +* Services with `SERVICE_IGNORE` are not observed + +``` +$ docker run -d --name mariadb \ + -e "SERVICE_NAME=mysql" \ + -e "SERVICE_TAGS=database,customers" \ + mariadb +``` + +### Multiple Services per Container + +You can specify a service identified by a given port `SERVICE__`: +``` +$ docker run -p 80:80 -p 443:443 -p 9000:9000 \ + -e "SERVICE_80_NAME=http-proxy" \ + -e "SERVICE_443_NAME=https-proxy" \ + -e "SERVICE_9000_IGNORE=yes" \ + docker/image +``` + + +## Authors + +* Christoph Wiechert + + + +## License + + [MIT](LICENSE) \ No newline at end of file diff --git a/app.js b/app.js new file mode 100644 index 0000000..54e306f --- /dev/null +++ b/app.js @@ -0,0 +1,89 @@ +var _ = require('lodash'); +var util = require('util'); +var execSync = require('child_process').execSync; +var serviceFactory = require('./lib/service'); +var Docker = require('./lib/Docker'); +var SkydnsBE = require('./lib/SkydnsBE'); + +/*******************/ +/* Config defaults */ +/*******************/ + +_.defaults(process.env, { + HOSTNAME: execSync('hostname').toString().trim(), + + //DOCKER_HOST: '/var/run/docker.sock', + //DOCKER_HOST: 'tcp://localhost:2376', + //DOCKER_TLS_VERIFY + //DOCKER_CERT_PATH + + //ETCD_ENDPOINTS: 'http://127.0.0.1:4001', + //ETCD_CAFILE: undefined, + //ETCD_CERTFILE: undefined, + //ETCD_KEYFILE: undefined, + + //SKYDNS_ETCD_PREFIX: '/skydns/local/skydns' +}); + + +/**********/ +/* Docker */ +/**********/ + +var docker = new Docker(); + +// Docker socket gone? +docker.on('eventstream_close', function() { + console.error('Error: Lost connection to docker daemon'); + process.exit(2); +}); + +// Error speaking with docker daemon +docker.on('error', function(err) { + if(err.code === 'ECONNREFUSED') { + console.error('Error: Connection to ' + process.env.DOCKER_HOST + ' refused!'); + process.exit(2); + } + console.error(util.inspect(err, {showHidden: false, depth: 4})); +}); + + +/**********/ +/* SkyDNS */ +/**********/ + +var skydnsBE = new SkydnsBE(); + +function err(method, err) { + if(!err) return; + console.error('Error: ' + method); + console.error(util.inspect(err, {showHidden: false, depth: 4})); +} + +// Docker started a container +docker.on('start', function(data) { + var service = serviceFactory(data); + if(!service) return; + + service.byPorts().forEach(function(portService) { + skydnsBE.addService(portService, err.bind('addService')); + }); +}); + +// A container died (kill, crash, stop, ...) +docker.on('die', function(cid) { + skydnsBE.removeServiceByCid(cid); +}); + +// Remove services which dont have running containers +// Add services for container already running +docker.getRunning(function(err, data) { + var portServices = _(data) + .map(serviceFactory) + .compact() + .invoke('byPorts') + .flatten() + .value(); + + skydnsBE.sync(portServices); +}); diff --git a/fleet-unit-files/skydns.service b/fleet-unit-files/skydns.service new file mode 100644 index 0000000..0d2b289 --- /dev/null +++ b/fleet-unit-files/skydns.service @@ -0,0 +1,43 @@ +[Unit] +Description=infra-skydns +Requires=docker.service etcd.service +After=docker.service etcd.service + +[Service] +Restart=always +RestartSec=5s +TimeoutStartSec=120 +TimeoutStopSec=25 + +EnvironmentFile=/etc/environment + +# remove old container +ExecStartPre=/bin/sh -c "docker ps -a | grep %p 1>/dev/null && docker rm %p || true" + +# initial write config (only if /skydns/config does not exist +ExecStartPre=/bin/sh -c "\ + /opt/bin/etcdctl2 ls /skydns/config &> /dev/null \ + || \ + /opt/bin/etcdctl2 set /skydns/config \ + '{\"domain\":\"local.\",\"hostmaster\":\"admin@example.com\",\"nameservers\":[\"8.8.8.8:53\",\"8.8.4.4:53\"],\"ttl\":60}' \ + " + +# Start the container +ExecStart=/bin/sh -c "\ + /usr/bin/docker run \ + --rm \ + --name=%p \ + -p 0.0.0.0:53:53 -p 0.0.0.0:53:53/udp \ + --env ETCD_MACHINES=https://${COREOS_PUBLIC_IPV4}:4001 \ + --env ETCD_TLSKEY=/etc/ssl/etcd/node.key \ + --env ETCD_TLSPEM=/etc/ssl/etcd/node.crt \ + --env ETCD_CACERT=/etc/ssl/etcd/ca.crt \ + --env SKYDNS_ADDR=0.0.0.0:53 \ + -v /etc/ssl/etcd:/etc/ssl/etcd \ + skynetservices/skydns" + +ExecStop=/usr/bin/docker stop %p + + +[X-Fleet] +Global=true diff --git a/lib/Docker.js b/lib/Docker.js new file mode 100644 index 0000000..46f33f4 --- /dev/null +++ b/lib/Docker.js @@ -0,0 +1,169 @@ +var util = require('util'); +var EventEmitter = require('events').EventEmitter; +var Modem = require('docker-modem'); +var debug = require('debug')('docker'); +var _ = require('lodash'); +var async = require('async'); + + +/** + * Constructor + * + * @emits: start, die, eventstream_close, error + * + * @param {object} opts + * @returns {Docker} + * @constructor + */ +function Docker(opts) { + if(! this instanceof Docker) return new Docker(opts); + + this.Modem = new Modem(); + if(!opts) opts = {}; + if(opts.timeout) this.Modem.timeout = opts.timeout; + this.maxCidLength = opts.maxCidLength || 16; + + this.opts = opts; + + this._init(); + +} +util.inherits(Docker, EventEmitter); + + +/** + * Init the event-stream + */ +Docker.prototype._init = function init() { + var self = this; + debug('Init Docker event listener'); + + this.Modem.dial({ + path: '/events', + method: 'GET', + isStream: true, + statusCodes: { + 200: true, + 500: "server error" + } + }, function(err, stream) { + if(err) return self.emit('error', err); + + stream + .on('data', function(buff) { + var obj = JSON.parse(buff); + obj.id = obj.id.substr(0, self.maxCidLength); + switch(obj.status) { + case 'die': + debug('Emit "die" event CID:' + obj.id); + self.emit('die', obj.id); + break; + + case 'start': + self.inspect(obj.id, function(err, data) { + if(err) return self.emit('error', err); + debug('Emit "start" event CID:' + obj.id); + self.emit('start', data); + }); + break; + + default: + debug('Ignore "' + obj.status + '" event CID:' + obj.id); + } + }) + .on('error', function(err) { + self.emit('error', err); + }) + .on('close', function() { + self.emit('eventstream_close') + }); + }); +}; + + +/** + * Inspect a running container and return the parsed dara + * @param {string} id + * @param cb + */ +Docker.prototype.inspect = function inspect(id, cb) { + var self = this; + + debug('Inspect container ' + id.substr(0, this.maxCidLength)); + this.Modem.dial({ + path: '/containers/' + id + '/json', + method: 'GET', + statusCodes: { + 200: true, + 404: "no such container", + 500: "server error" + } + }, function(err, obj) { + if(err) return cb(err); + cb(null, self._parseInspect(obj)); + }); +}; + + +/** + * Helper method to pick the interesting data from insect-json + * @param {obj} inspect-data + * @returns {{id: *, name: string, image: *, ports: Array, env: {}, networkMode: *}} + */ +Docker.prototype._parseInspect = function parseInspect(obj) { + var data = { + id: obj.Id.substr(0, this.maxCidLength), + name: obj.Name.substr(1), + image: obj.Image, + ports: [], + env: {}, + networkMode: obj.HostConfig.NetworkMode + }; + + _.forEach(obj.NetworkSettings.Ports, function(cfg, portProto) { + var port = {}; + portProto = portProto.split('/'); + port.port = portProto[0]; + port.protocol = portProto[1]; + port.containerIP = obj.NetworkSettings.IPAddress; + port.hostIP = cfg && cfg[0].HostIp; + port.hostPort = cfg && cfg[0].HostPort; + + data.ports.push(port); + }); + + obj.Config.Env.forEach(function(val) { + val = val.split('='); + data.env[val.shift()] = val.join('='); + }); + + return data; +}; + + +/** + * Return the parsed data for all running containers + * @param cb + */ +Docker.prototype.getRunning = function getRunning(cb) { + var self = this; + debug('Fetch all running containers'); + this.Modem.dial({ + path: '/containers/json', + method: 'GET', + statusCodes: { + 200: true, + 400: "bad parameter", + 500: "server error" + } + }, function(err, obj) { + if(err) return cb(err); + async.map( + _.pluck(obj, 'Id'), + self.inspect.bind(self), + cb + ); + }); +}; + +module.exports = Docker; \ No newline at end of file diff --git a/lib/SkydnsBE.js b/lib/SkydnsBE.js new file mode 100644 index 0000000..d24707c --- /dev/null +++ b/lib/SkydnsBE.js @@ -0,0 +1,151 @@ +var util = require('util'); +var _ = require('lodash'); +var debug = require('debug')('skydns'); +var etcd = require('./etcd.js'); + + +/** + * Create a new Skydns backend + * @constructor + */ +function SkydnsBE() { + this.prefix = process.env.SKYDNS_ETCD_PREFIX || '/skydns/local/skydns'; + this.cidCache = {}; +} + + +/** + * Add a DNS by portService + * @param {Object} portService + * @param cb + */ +SkydnsBE.prototype.addService = function addService(portService, cb) { + var self = this; + var url = this._buildUrl(portService); + + var text = portService.protocol; + if(_.isArray(portService.attribs.TAGS)) text += ',' + portService.attribs.TAGS.join(','); + + var val = { + host: portService.ip, + port: parseInt(portService.port, 10), + priority: portService.attribs.SKYDNS_PRIORITY || 1, + weight: portService.attribs.SKYDNS_WEIGHT || 1, + text: text + }; + + debug('Add service: ' + url + ' => ' + portService.ip + ':' + portService.port); + etcd.set(url, JSON.stringify(val), function(err) { + if(err) return cb(err); + if(!self.cidCache[portService.cid]) self.cidCache[portService.cid] = []; + self.cidCache[portService.cid].push(url); + cb(); + }); +}; + + +/** + * Remove a service by container id + * @param {String} cid + */ +SkydnsBE.prototype.removeServiceByCid = function removeServiceByCid(cid) { + var self = this; + var urls = this.cidCache[cid]; + if(urls) { + this.removeByUrls(urls); + delete this.cidCache[cid]; + } else { + // cid not in cache, search it + this.findUrlsByCid(cid, function(err, urls) { + if(err) return console.error('Could fetch keys ' + err.error.cause + ': ' + err.error.message); + self.removeByUrls(urls); + }); + } +}; + + +/** + * Remove services by etcd-keys + * @param {Array} urls + */ +SkydnsBE.prototype.removeByUrls = function removeByUrls(urls) { + urls.forEach(function(url) { + debug('Remove service: ' + url); + etcd.del(url, function(err) { + if(err) return console.error('Error: Could not delete ' + err.error.cause + ': ' + err.error.message); + }); + }); + // TODO: remove empty directory also ??? +}; + + +/** + * Sync the etcd-services to the given + * @param {Object} activeServicesByPort + */ +SkydnsBE.prototype.sync = function (activeServicesByPort) { + var self = this; + + var runningMap = {}; + activeServicesByPort.forEach(function(portService) { + runningMap[self._buildUrl(portService)] = portService; + }); + + // Fetch current etcd-services + etcd.get(this.prefix, {recursive: true}, function(err, obj) { + if(err) { + console.error('Error: etcd get ' + self.prefix); + console.error(util.inspect(err, {showHidden: false, depth: 4})); + return; + } + + // recursive find keys beginning with our HOSTNAME + var inEtcdUrls = []; + if(obj.node.nodes) { + inEtcdUrls = etcd.deepFindKeys(obj.node, new RegExp('/' + process.env.HOSTNAME + '-[^/]*')); + } + + // remove not running + var runningUrls = _.keys(runningMap); + var toDelete = _.difference(inEtcdUrls, runningUrls); + if(toDelete.length) { + debug('Remove ' + toDelete.length + ' obsolete services'); + self.removeByUrls(toDelete); + } + + // add not registred + var toAdd = _.difference(runningUrls, inEtcdUrls); + if(toAdd.length) { + debug('Adding ' + toAdd.length + ' already running services'); + toAdd.forEach(function(url) { + self.addService(runningMap[url], function(err) { + if(err) return console.error('Error: Could add service ' + err.error.cause + ': ' + err.error.message); + }); + }); + } + }); + +}; + + +/** + * Find all services matching a given container-id + * @param {String} cid + * @param cb + */ +SkydnsBE.prototype.findUrlsByCid = function(cid, cb) { + etcd.get(this.prefix, {recursive:true}, function(err, obj) { + if(err) return cb(err); + if(!obj.node.nodes) return; + + cb(null, etcd.deepFindKeys(obj.node, new RegExp('-'+cid+'-'))); + }); +}; + + +SkydnsBE.prototype._buildUrl = function _buildUrl(portService) { + return this.prefix + '/' + portService.name + '/' + process.env.HOSTNAME + '-' + portService.cid + '-' + portService.port; +}; + + +module.exports = SkydnsBE; \ No newline at end of file diff --git a/lib/etcd.js b/lib/etcd.js new file mode 100644 index 0000000..a644822 --- /dev/null +++ b/lib/etcd.js @@ -0,0 +1,46 @@ +fs = require('fs'); +var _ = require('lodash'); +Etcd = require('node-etcd'); + +var endpoints = []; +var sslopts = false; +if(!process.env.ETCD_ENDPOINTS) { + endpoints.push('127.0.0.1:4001'); +} else { + var ssl = false; + endpoints = process.env.ETCD_ENDPOINTS.split(',').map(function(ep) { + + if(ep.substr(0,8) === 'https://') ssl = true; + return ep.replace(/^https?:\/\//, '', ep); + }); + + if(ssl) { + sslopts = { + ca: fs.readFileSync(process.env.ETCD_CAFILE), + cert: fs.readFileSync(process.env.ETCD_CERTFILE), + key: fs.readFileSync(process.env.ETCD_KEYFILE), + securityOptions: 'SSL_OP_NO_SSLv3' + }; + } +} + +var etcd = new Etcd(endpoints, sslopts); + +// mixin some useful methods + +function deepFindKeys(node, regexp) { + var hits = []; + if(regexp.test(node.key)) { + hits.push(node.key); + return hits; + } + if(node.nodes && _.isArray(node.nodes)) { + node.nodes.forEach(function(node) { + hits = hits.concat(deepFindKeys(node, regexp)); + }); + } + return hits; +} +etcd.deepFindKeys = deepFindKeys; + +module.exports = etcd; diff --git a/lib/service.js b/lib/service.js new file mode 100644 index 0000000..f0b66f6 --- /dev/null +++ b/lib/service.js @@ -0,0 +1,73 @@ +var debug = require('debug')('service'); +var _ = require('lodash'); + +/** + * Service definition + * @param {obj} dockerData + * @constructor + */ +function Service(dockerData) { + var self = this; + + // parse attribs from env-vars + this.attribs = {common: {}}; + _.forEach(dockerData.env, function(val, key) { + var m = key.match(/^SERVICE_([0-9]+_)?(.*)/); + if(!m) return; + var port = m[1] || 'common'; + var attrib = m[2]; + if(!self.attribs[port]) self.attribs[port] = {}; + if(attrib.toLowerCase() === 'tags') val = val.split(','); + self.attribs[port][attrib] = val; + }); + + + this.name = this.attribs['common'].NAME || dockerData.name; + this.ports = dockerData.ports; + this.cid = dockerData.id; + this.image = dockerData.image; + this.networkMode = dockerData.networkMode; +} + +/** + * Return the service splitted by ports + * @returns {array} PortServices + */ +Service.prototype.byPorts = function() { + var self = this; + + return _(this.ports) + .map(function(portCfg) { + var attribs = self.attribs[portCfg.port]; + if(attribs.IGNORE) return false; + + return { + name: attribs.NAME || self.name, + ident: self.hostname + '-' + (attribs.NAME || self.name) + '-' + portCfg.port, + protocol: portCfg.protocol, + port: portCfg.port, // TODO: any logic to use the HostIP? + ip: portCfg.containerIP, // TODO: any logic to use the HostIP? + cid: self.cid, + image: self.image, + attribs: attribs || self.attribs['common'] || {} + } + }) + .compact() + .value(); +}; + + +module.exports = function serviceFactory(data) { + var s = new Service(data); + + if(s.networkMode !== 'bridge') { + debug('Omit container ' + s.cid + ' cause --net=host'); + return false; + } + if(s.attribs['common'].IGNORE) { + debug('Omit container ' + s.cid + ' SERVICE_IGNORE'); + return false; + } + + return s; +}; \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..c8dc57b --- /dev/null +++ b/package.json @@ -0,0 +1,36 @@ +{ + "name": "docker-etcd-registrator", + "version": "1.0.0", + "description": "Docker service registrator for etcd and skydns", + "main": "app.js", + "dependencies": { + "debug": "^2.1.3", + "async": "^0.9.0", + "docker-modem": "^0.2.1", + "lodash": "^3.6.0", + "node-etcd": "^4.0.1" + }, + "devDependencies": {}, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "https://github.com/psi-4ward/docker-etcd-registrator" + }, + "keywords": [ + "docker", + "etcd", + "coreos", + "skydns", + "service", + "discovery", + "dns" + ], + "author": "Christoph Wiechert ", + "license": "MIT", + "bugs": { + "url": "https://github.com/psi-4ward/docker-etcd-registrator/issues" + }, + "homepage": "https://github.com/psi-4ward/docker-etcd-registrator" +}