Map generation, ACLs & refactor

config.yml

@@ -8,7 +8,48 @@ rproxy:
     - rproxy1.local
     - rproxy2.local
   auth:
-    username: root,
+    username: root
     privateKey: /root/.ssh/id_rsa
-  confPath: /etc/haproxy/haproxy.cfg
+  confDir: /etc/haproxy
-  reloadCmd: service haproxy reload
+  reloadCmd: service haproxy reload
+acls:
+  public:
+    ips:
+    - 0.0.0.0/0
+    zones:
+    - public
+  private:
+    ips:
+    - 10.0.0.0/8
+    - 172.16.0.0/12
+    - 192.168.0.0/16
+    zones:
+    - private
+  it:
+    ips:
+    - 10.5.2.0/24
+    zones:
+    - it
+  sysadmin:
+    ips:
+    - 10.5.1.0/24
+    zones:
+    - dmz
+    - it
+defaults:
+  https: true
+  zone: public
+domains:
+  domain.local:
+    foo:
+      rules:
+      - foo
+      - domain:
+        - foo1
+        - foo2
+        path:
+        - /foo
+        - /bar
+      https: false
+      zone: dmz
+    bar:

index.js

@@ -1,19 +1,20 @@
 require('require-yaml');
-let Docker = require('dockerode');
+const Docker = require('dockerode');
-let handlebars = require('handlebars');
+const handlebars = require('handlebars');
-let ssh = require('node-ssh');
+const ssh = require('node-ssh');
-let fs = require('fs');
+const fs = require('fs-extra');
-let shajs = require('sha.js');
+const shajs = require('sha.js');
-let conf = require('./config.yml');
+const conf = require('./config.yml');
-let package = require('./package.json');
+const package = require('./package.json');
 
 let docker;
 let template;
 let lastInfoHash;
-let appName = package.name;
+const appName = package.name;
-let isProduction = process.env.NODE_ENV === 'production';
+const isProduction = process.env.NODE_ENV === 'production';
-let tmpDir = isProduction ? `/tmp/${appName}` : `${__dirname}/tmp`;
+const tmpDir = isProduction ? `/tmp/${appName}` : `${__dirname}/tmp`;
-let hashFile = `${tmpDir}/config.hash`;
+const hashFile = `${tmpDir}/config.hash`;
+const confDir = conf.rproxy.confDir;
 
 async function updateProxy() {
     console.log('Updating reverse proxy configuration.');
@@ -35,45 +36,52 @@ async function updateProxy() {
         return a.name > b.name ? 1 : a.name < b.name ? -1 : 0;
     }
 
-    let nodes = [];
+    const nodes = [];
-    for (let node of info.nodes) {
+    for (const node of info.nodes) {
-        let address = node.ManagerStatus
+        const address = node.ManagerStatus
             ? node.ManagerStatus.Addr.split(':')[0]
             : node.Status.Addr;
+        const role = node.Spec && node.Spec.Role
 
         nodes.push({
             name: node.Description.Hostname,
-            address
+            address,
+            isWorker: role == 'worker'
         });
     }
-    nodes = nodes.sort(sortFn);
+    nodes.sort(sortFn);
 
-    let services = [];
+    const services = [];
-    for (let service of info.services) {
+    for (const service of info.services) {
-        let ports = service.Endpoint.Ports;
+        const ports = service.Endpoint.Ports;
         if (!Array.isArray(ports) || !ports.length) continue;
 
         let name = service.Spec.Name;
-        let match = name.match(/^(.+)_main$/);
+        const match = name.match(/^(.+)_main$/);
         if (match) name = match[1];
 
-        let port = ports[0];
+        const port = ports[0];
-        let protocol = port.Protocol;
+        const protocol = port.Protocol;
 
         services.push({
             name,
             port: port.PublishedPort,
             protocol,
-            tcp: protocol == 'tcp'
+            isTcp: protocol == 'tcp'
         });
     }
-    services = services.sort(sortFn);
+    services.sort(sortFn);
 
-    let configString = template({services, nodes, info});
+    const configString = template({
+        services,
+        nodes,
+        info,
+        acls: conf.acls
+    });
 
     // Cheking settings hash
 
-    let infoHash = shajs('sha256')
+    const infoHash = shajs('sha256')
         .update(configString)
         .digest('hex');
     console.log('Settings hash:', infoHash);
@@ -85,11 +93,13 @@ async function updateProxy() {
 
     // Creating configuration file
 
-    let tmpConf = `${tmpDir}/config.cfg`;
+    const tmpConf = `${tmpDir}/config.cfg`;
     fs.writeFileSync(tmpConf, configString);
 
     if (conf.debug) {
-        let delimiter = '#'.repeat(80);
+        const delimiter = '#' + '='.repeat(79);
+        console.log(delimiter);
+        console.log(`# ${confDir}`);
         console.log(delimiter);
         console.log(configString);
         console.log(delimiter);
@@ -97,18 +107,23 @@ async function updateProxy() {
 
     // Updating reverse proxies
 
-    let files = {
+    const files = {
         local: tmpConf,
-        remote: conf.rproxy.confPath
+        remote: `${confDir}/haproxy.cfg`
     };
 
-    for (let host of conf.rproxy.hosts) {
+    for (const host of conf.rproxy.hosts) {
         console.log(`Updating host: ${host}`);
         if (!isProduction) continue;
 
-        let sshClient = new ssh();
+        const sshClient = new ssh();
         await sshClient.connect(Object.assign({host}, conf.rproxy.auth));
         await sshClient.putFiles([files]);
+        await sshClient.putDirectory(
+            `${tmpDir}/maps`,
+            `${confDir}/maps`,
+            {recursive: true}
+        );
         if (conf.rproxy.reloadCmd)
             await sshClient.exec(conf.rproxy.reloadCmd);
         await sshClient.dispose();
@@ -139,6 +154,154 @@ async function updateProxy() {
         console.log('Saved settings hash:', lastInfoHash);
     }
 
+    const hostMap = [];
+    const baseMap = [];
+    const https = [];
+    const zoneMap = [];
+    const aclMap = [];
+    const accessMap = [];
+
+    for (const domain in conf.domains) {
+        const domainConf = conf.domains[domain];
+        for (const service in domainConf)
+            addService(service, domainConf[service], domain);
+    }
+
+    function addService(service, serviceConf, mainDomain) {
+        let rules;
+        if (typeof serviceConf == 'string') {
+            rules = serviceConf;
+            serviceConf = undefined;
+        }
+
+        serviceConf = Object.assign({},
+            conf.defaults,
+            serviceConf
+        );
+
+        if (serviceConf.https)
+            https.push(service);
+        if (serviceConf.zone)
+            zoneMap.push([service,  serviceConf.zone]);
+
+        rules = rules || serviceConf.rules;
+
+        if (!rules)
+            rules = service;
+        if (!Array.isArray(rules))
+            rules = [rules];
+
+        for (let rule of rules) {
+            if (typeof rule == 'string')
+                rule = {domain: rule};
+
+            let domains = rule.domain;
+            let paths = rule.path;
+            if (!Array.isArray(domains))
+                domains = [domains];
+            if (!Array.isArray(paths))
+                paths = [paths];
+
+            for (const domain of domains) {
+            for (const path of paths) {
+                const fullDomain = domain && domain !== '$'
+                    ? `${domain}.${mainDomain}`
+                    : mainDomain;
+                if (!path)
+                    hostMap.push([fullDomain, service]);
+                else
+                    baseMap.push([fullDomain + path, service]);
+            }}
+        }
+    }
+
+    const acls = [];
+
+    for (const acl in conf.acls) {
+        const aclConf = conf.acls[acl];
+        const ips = [];
+        for (const ip of aclConf.ips) {
+            aclMap.push([ip, acl]);
+            ips.push(parseNet(ip));
+        }
+        acls.push({
+            name: acl,
+            ips,
+            zones: new Set(aclConf.zones)
+        });
+    }
+
+    function parseNet(net) {
+        const netSplit = net.split('/');
+        const mask = parseInt(netSplit[1]);
+        const ip = netSplit[0].split('.')
+            .reduce((ipInt, octet) => (ipInt<<8) + parseInt(octet, 10), 0) >>> 0;
+        return {ip, mask};
+    }
+
+    for (const aAcl of acls) {
+    for (const aNet of aAcl.ips) {
+    for (const bAcl of acls) {
+        if (aAcl === bAcl) continue;
+        let match = false;
+        for (const bNet of bAcl.ips) {
+            match = bNet.mask === 0;
+            if (bNet.mask > 0) {
+                const netMask = (~0) << (32 - bNet.mask);
+                const aSubnet = aNet.ip & netMask;
+                const bSubnet = bNet.ip & netMask;
+                match = aSubnet === bSubnet;
+            }
+            if (match) break;
+        }
+        if (match) {
+            for (const zone of bAcl.zones)
+                aAcl.zones.add(zone);
+        }
+    }}}
+
+    for (const acl of acls)
+    for (const zone of acl.zones)
+        accessMap.push(`${acl.name}/${zone}`);
+
+    const files = {
+        host: hostMap,
+        base: baseMap,
+        zone: zoneMap,
+        acl: aclMap,
+        access: accessMap,
+        https: https
+    };
+
+    function strSortFn(a, b) {
+        return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
+    }
+    function netSortFn(a, b) {
+        const aMask = parseInt(a[0].split('/')[1], 10);
+        const bMask = parseInt(b[0].split('/')[1], 10);
+        return bMask - aMask;
+    }
+
+    let mapDir = `${tmpDir}/maps`;
+    if (!await fs.pathExists(mapDir))
+            await fs.mkdir(mapDir);
+
+    for (const file in files) {
+        files[file].sort(file == 'acl'
+            ? netSortFn
+            : strSortFn
+        );
+
+        const fd = await fs.open(`${mapDir}/${file}.map`, 'w+');
+        files[file].forEach(map => {
+            if (Array.isArray(map))
+                fs.write(fd, `${map[0]} ${map[1]}\n`);
+            else
+                fs.write(fd, `${map}\n`);
+        });
+        await fs.close(fd);
+    }
+
     await updateProxy();
 
     console.log('Listening for events.')

package-lock.json

@@ -156,6 +156,21 @@
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
+    "fs-extra": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
+      "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
+      "requires": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      }
+    },
+    "graceful-fs": {
+      "version": "4.2.8",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz",
+      "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg=="
+    },
     "handlebars": {
       "version": "4.7.2",
       "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.2.tgz",
@@ -191,6 +206,15 @@
         "esprima": "^4.0.0"
       }
     },
+    "jsonfile": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
+      "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
+      "requires": {
+        "graceful-fs": "^4.1.6",
+        "universalify": "^2.0.0"
+      }
+    },
     "jsonparse": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
@@ -422,6 +446,11 @@
         "source-map": "~0.6.1"
       }
     },
+    "universalify": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
+      "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

package.json

@@ -1,6 +1,7 @@
 {
   "name": "docker-discover",
   "description": "Docker service autodiscovery",
+  "version": "1.0.0",
   "license": "GPL-3.0",
   "repository": {
     "type": "git",
@@ -8,6 +9,7 @@
   },
   "dependencies": {
     "dockerode": "^3.0.2",
+    "fs-extra": "^10.0.0",
     "handlebars": "^4.7.2",
     "node-ssh": "^7.0.0",
     "require-yaml": "0.0.1",

rproxy.handlebars

@@ -1,11 +1,50 @@
+global
+	balance roundrobin
+
+frontend http
+	bind :80
+	bind :443 ssl crt /etc/haproxy/cert.pem
+	option forwardfor
+
+	# XXX: To test configuration
+	#http-request set-header Host domain.local
+
+	# Set environment
+
+	http-request set-var(req.backend)     req.hdr(host),map_str(/etc/haproxy/maps/host.map)
+	http-request set-var(req.backend)     base,map_beg(/etc/haproxy/maps/base.map)
+	http-request set-var(req.acl)         src,map_ip(/etc/haproxy/maps/acl.map)
+	http-request set-var(req.zone)        var(req.backend),map_str(/etc/haproxy/maps/zone.map)
+	http-request set-var(req.aclZone)     var(req.acl),concat(/,req.zone)
+
+	# XXX: Debugging
+	#log-format "%[var(txn.test)]"
+
+	# ACL check
+
+	acl allow var(req.aclZone) -f /etc/haproxy/maps/access.map
+	http-request deny if !allow
+
+	# HTTPS redirect
+
+	acl https var(req.backend) -f /etc/haproxy/maps/https.map
+	http-request add-header X-Forwarded-Proto https if { ssl_fc }
+	redirect scheme https if !{ ssl_fc } https
+
+	# Backend
+
+	default_backend not-found
+	use_backend %[var(req.backend)]
 
 # Auto-generated backends
 
 {{#each services}}
-{{#if tcp}}
+{{#if isTcp}}
 backend {{name}}
 	{{#each ../nodes}}
+	{{#if isWorker}}
 	server {{name}}:{{../port}} {{address}}:{{../port}} check
+	{{/if}}
 	{{/each}}
 {{/if}}
 {{/each}}

test.json

@@ -37,16 +37,28 @@
     ],
     "nodes": [
         {
-            "Description": {"Hostname": "fooNode"},
+            "Description": {"Hostname": "mgr1"},
             "ManagerStatus": {"Addr": "10.0.0.1:2377"},
+            "Spec": {"Role": "manager"},
             "Status": {"Addr": "0.0.0.0"}
         }, {
-            "Description": {"Hostname": "barNode"},
+            "Description": {"Hostname": "mgr2"},
             "ManagerStatus": {"Addr": "10.0.0.2:2377"},
+            "Spec": {"Role": "manager"},
             "Status": {"Addr": "10.0.0.2"}
         }, {
-            "Description": {"Hostname": "bazNode"},
+            "Description": {"Hostname": "worker1"},
-            "Status": {"Addr": "10.0.0.3"}
+            "ManagerStatus": {"Addr": "10.0.0.2:2377"},
+            "Spec": {"Role": "worker"},
+            "Status": {"Addr": "10.0.1.1"}
+        }, {
+            "Description": {"Hostname": "worker2"},
+            "ManagerStatus": {"Addr": "10.0.0.2:2377"},
+            "Spec": {"Role": "worker"},
+            "Status": {"Addr": "10.0.1.2"}
+        }, {
+            "Description": {"Hostname": "worker3"},
+            "Status": {"Addr": "10.0.1.3"}
         }
     ]
 }