From bca931a4cd7b6b47ee8a8de8c2f04d55bc0fdd64 Mon Sep 17 00:00:00 2001 From: CoryGH Date: Wed, 19 May 2021 14:36:51 -0400 Subject: [PATCH 01/11] connection router to accomodate multithreading connection router hook added to accommodate multithreaded servers --- lib/server.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/server.js b/lib/server.js index 512c679..f50c7dd 100644 --- a/lib/server.js +++ b/lib/server.js @@ -313,7 +313,7 @@ function Server (options) { return c } - function newConnection (conn) { + self.newConnection = function (conn) { setupConnection(conn) log.trace('new connection from %s', conn.ldap.id) @@ -438,9 +438,9 @@ function Server (options) { this.routes = {} if ((options.cert || options.certificate) && options.key) { options.cert = options.cert || options.certificate - this.server = tls.createServer(options, newConnection) + this.server = tls.createServer(options, options.connectionRouter ? options.connectionRouter : self.newConnection) } else { - this.server = net.createServer(newConnection) + this.server = net.createServer(options.connectionRouter ? options.connectionRouter : self.newConnection) } this.server.log = options.log this.server.ldap = { From 6cce4f60cc2c0dc80dd0cf8c45c72cc8475b2dbf Mon Sep 17 00:00:00 2001 From: CoryGH Date: Thu, 20 May 2021 07:21:52 -0400 Subject: [PATCH 02/11] removing hook while keeping newConnection change removal of hook while keeping `newConnection` change from private to public for use in multithreaded applications --- lib/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/server.js b/lib/server.js index f50c7dd..903cf1b 100644 --- a/lib/server.js +++ b/lib/server.js @@ -438,9 +438,9 @@ function Server (options) { this.routes = {} if ((options.cert || options.certificate) && options.key) { options.cert = options.cert || options.certificate - this.server = tls.createServer(options, options.connectionRouter ? options.connectionRouter : self.newConnection) + this.server = tls.createServer(options, self.newConnection) } else { - this.server = net.createServer(options.connectionRouter ? options.connectionRouter : self.newConnection) + this.server = net.createServer(self.newConnection) } this.server.log = options.log this.server.ldap = { From 534e3dd542c943e3f1353d319718d28cff0c45d0 Mon Sep 17 00:00:00 2001 From: CoryGH Date: Thu, 20 May 2021 10:33:46 -0400 Subject: [PATCH 03/11] test for multithreading support tests for multithreading support via non-ldapjs-server and via hook into ldapjs server --- test/server.test.js | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/server.test.js b/test/server.test.js index 949cc27..f608e78 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -331,3 +331,49 @@ tap.test('close passes error to callback', function (t) { t.end() }) }) + +tap.test('multithreading support via external server', function (t) { + let serverOptions = { } + const server = ldap.createServer(serverOptions) + const fauxServer = net.createServer(serverOptions, (connection) => { + server.newConnection(connection) + }) + fauxServer.log = serverOptions.log + fauxServer.ldap = { + config: serverOptions + } + t.ok(server) + fauxServer.listen(5555, 'localhost', function () { + t.ok(true, 'server listening on ' + server.url) + + t.ok(fauxServer) + const client = ldap.createClient({ url: 'ldap://127.0.0.1:5555' }) + client.on('connect', function () { + t.ok(client) + client.unbind() + fauxServer.close(() => t.end()) + }) + }) +}) + +tap.test('multithreading support via hook', function (t) { + let serverOptions = { + connectionRouter: (connection) => { + server.newConnection(connection) + } + } + const server = ldap.createServer(serverOptions) + const fauxServer = ldap.createServer(serverOptions) + t.ok(server) + fauxServer.listen(0, 'localhost', function () { + t.ok(true, 'server listening on ' + server.url) + + t.ok(fauxServer) + const client = ldap.createClient({ url: fauxServer.url }) + client.on('connect', function () { + t.ok(client) + client.unbind() + fauxServer.close(() => t.end()) + }) + }) +}) From 992be5a5570fb9eb0f88f3c92c3a57bc813d3798 Mon Sep 17 00:00:00 2001 From: CoryGH Date: Thu, 20 May 2021 10:35:13 -0400 Subject: [PATCH 04/11] adding connectionRouter hook adding connectionRouter hook back in --- lib/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/server.js b/lib/server.js index 903cf1b..f50c7dd 100644 --- a/lib/server.js +++ b/lib/server.js @@ -438,9 +438,9 @@ function Server (options) { this.routes = {} if ((options.cert || options.certificate) && options.key) { options.cert = options.cert || options.certificate - this.server = tls.createServer(options, self.newConnection) + this.server = tls.createServer(options, options.connectionRouter ? options.connectionRouter : self.newConnection) } else { - this.server = net.createServer(self.newConnection) + this.server = net.createServer(options.connectionRouter ? options.connectionRouter : self.newConnection) } this.server.log = options.log this.server.ldap = { From 8a791d4bf055809e36cb6390908345c8b7c45faf Mon Sep 17 00:00:00 2001 From: CoryGH Date: Thu, 20 May 2021 10:40:37 -0400 Subject: [PATCH 05/11] net dependency net dependency --- test/server.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/server.test.js b/test/server.test.js index f608e78..b868c30 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -1,5 +1,6 @@ 'use strict' +const net = require('net') const tap = require('tap') const vasync = require('vasync') const { getSock } = require('./utils') From 91cac2fdaa4def44622af4a69cf8ca0190b9bb6f Mon Sep 17 00:00:00 2001 From: CoryGH Date: Thu, 20 May 2021 12:27:10 -0400 Subject: [PATCH 06/11] linting typos linting typos fixed --- test/server.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/server.test.js b/test/server.test.js index b868c30..459cfa3 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -334,7 +334,7 @@ tap.test('close passes error to callback', function (t) { }) tap.test('multithreading support via external server', function (t) { - let serverOptions = { } + const serverOptions = { } const server = ldap.createServer(serverOptions) const fauxServer = net.createServer(serverOptions, (connection) => { server.newConnection(connection) @@ -358,7 +358,7 @@ tap.test('multithreading support via external server', function (t) { }) tap.test('multithreading support via hook', function (t) { - let serverOptions = { + const serverOptions = { connectionRouter: (connection) => { server.newConnection(connection) } From c97f1becff4ddcb0a301e2be474770a6e2e868e7 Mon Sep 17 00:00:00 2001 From: CoryGH Date: Fri, 21 May 2021 04:53:57 -0400 Subject: [PATCH 07/11] adding example for cluster-based multithreading working example of cluster-based multithreading utilizing connectionRouter override. --- examples/cluster-threading.js | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 examples/cluster-threading.js diff --git a/examples/cluster-threading.js b/examples/cluster-threading.js new file mode 100644 index 0000000..b631c44 --- /dev/null +++ b/examples/cluster-threading.js @@ -0,0 +1,66 @@ +const cluster = require('cluster'); +const ldap = require('ldapjs'); +const os = require('os'); + +const threads = []; +threads.getNext = function () { + return (Math.floor(Math.random() * this.length)); +}; + +const serverOptions = { + connectionRouter: (socket) => { + socket.pause(); + console.log('ldapjs client requesting connection'); + let routeTo = threads.getNext(); + threads[routeTo].send({ type: 'connection' }, socket); + } +}; + +const server = ldap.createServer(serverOptions); + +if (cluster.isMaster) { + for (let i = 0; i < os.cpus().length; i++) { + let thread = cluster.fork({ + 'id': i + }); + thread.id = i; + thread.on('message', function (msg) { + + }); + threads.push(thread); + } + + server.listen(1389, function () { + console.log('ldapjs listening at ' + server.url); + }); +} else { + let threadId = process.env.id; + serverOptions.connectionRouter = (connection) => { + console.log('should not be hit'); + }; + + process.on('message', (msg, socket) => { + switch (msg.type) { + case 'connection': + server.newConnection(socket); + socket.resume(); + console.log('ldapjs client connection accepted on ' + threadId.toString()); + } + }); + + server.search('dc=example', function (req, res, next) { + console.log('ldapjs search initiated on ' + threadId.toString()); + var obj = { + dn: req.dn.toString(), + attributes: { + objectclass: ['organization', 'top'], + o: 'example' + } + }; + + if (req.filter.matches(obj.attributes)) + res.send(obj); + + res.end(); + }); +} From dede2e61652d05e9ae1dea221e778aa03e9a47a9 Mon Sep 17 00:00:00 2001 From: CoryGH Date: Fri, 21 May 2021 05:01:31 -0400 Subject: [PATCH 08/11] addition of example for threading via net server Addition of working example demonstrating threading via clustering and a net server forwarding socket connections to ldapjs. --- examples/cluster-threading-net-server.js | 66 ++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 examples/cluster-threading-net-server.js diff --git a/examples/cluster-threading-net-server.js b/examples/cluster-threading-net-server.js new file mode 100644 index 0000000..da9a2d1 --- /dev/null +++ b/examples/cluster-threading-net-server.js @@ -0,0 +1,66 @@ +const cluster = require('cluster'); +const ldap = require('ldapjs'); +const net = require('net'); +const os = require('os'); + +const threads = []; +threads.getNext = function () { + return (Math.floor(Math.random() * this.length)); +}; + +const serverOptions = { + port: 1389 +}; + +if (cluster.isMaster) { + const server = net.createServer(serverOptions, (socket) => { + socket.pause(); + console.log('ldapjs client requesting connection'); + let routeTo = threads.getNext(); + threads[routeTo].send({ type: 'connection' }, socket); + }); + + for (let i = 0; i < os.cpus().length; i++) { + let thread = cluster.fork({ + 'id': i + }); + thread.id = i; + thread.on('message', function (msg) { + + }); + threads.push(thread); + } + + server.listen(serverOptions.port, function () { + console.log('ldapjs listening at ldap://127.0.0.1:' + serverOptions.port); + }); +} else { + const server = ldap.createServer(serverOptions); + + let threadId = process.env.id; + + process.on('message', (msg, socket) => { + switch (msg.type) { + case 'connection': + server.newConnection(socket); + socket.resume(); + console.log('ldapjs client connection accepted on ' + threadId.toString()); + } + }); + + server.search('dc=example', function (req, res, next) { + console.log('ldapjs search initiated on ' + threadId.toString()); + var obj = { + dn: req.dn.toString(), + attributes: { + objectclass: ['organization', 'top'], + o: 'example' + } + }; + + if (req.filter.matches(obj.attributes)) + res.send(obj); + + res.end(); + }); +} From b860932b9b406e6271e4a1dcd72906bd4962fe00 Mon Sep 17 00:00:00 2001 From: CoryGH Date: Fri, 21 May 2021 05:08:51 -0400 Subject: [PATCH 09/11] updated documentation Updated examples documentation. --- docs/examples.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/examples.md b/docs/examples.md index 9fa652a..ccbafb5 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -550,3 +550,76 @@ To test out this example, try: $ ldapsearch -H ldap://localhost:389 -x -D cn=demo,dc=example,dc=com \ -w demo -b "dc=example,dc=com" objectclass=* ``` + +# Multi-threaded Server + +This example demonstrates multi-threading via the `cluster` module utilizing a `net` server for initial socket receipt. An alternate example demonstrating use of the `connectionRouter` `serverOptions` hook is available in the `examples` directory. + +``` +const cluster = require('cluster'); +const ldap = require('ldapjs'); +const net = require('net'); +const os = require('os'); + +const threads = []; +threads.getNext = function () { + return (Math.floor(Math.random() * this.length)); +}; + +const serverOptions = { + port: 1389 +}; + +if (cluster.isMaster) { + const server = net.createServer(serverOptions, (socket) => { + socket.pause(); + console.log('ldapjs client requesting connection'); + let routeTo = threads.getNext(); + threads[routeTo].send({ type: 'connection' }, socket); + }); + + for (let i = 0; i < os.cpus().length; i++) { + let thread = cluster.fork({ + 'id': i + }); + thread.id = i; + thread.on('message', function (msg) { + + }); + threads.push(thread); + } + + server.listen(serverOptions.port, function () { + console.log('ldapjs listening at ldap://127.0.0.1:' + serverOptions.port); + }); +} else { + const server = ldap.createServer(serverOptions); + + let threadId = process.env.id; + + process.on('message', (msg, socket) => { + switch (msg.type) { + case 'connection': + server.newConnection(socket); + socket.resume(); + console.log('ldapjs client connection accepted on ' + threadId.toString()); + } + }); + + server.search('dc=example', function (req, res, next) { + console.log('ldapjs search initiated on ' + threadId.toString()); + var obj = { + dn: req.dn.toString(), + attributes: { + objectclass: ['organization', 'top'], + o: 'example' + } + }; + + if (req.filter.matches(obj.attributes)) + res.send(obj); + + res.end(); + }); +} +``` From c2494eca0054ef3fdf334f2e5ef0373685dfc929 Mon Sep 17 00:00:00 2001 From: James Sumners Date: Fri, 21 May 2021 08:04:41 -0400 Subject: [PATCH 10/11] Apply suggestions from code review --- docs/examples.md | 2 +- lib/server.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/examples.md b/docs/examples.md index ccbafb5..6c6b21c 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -555,7 +555,7 @@ $ ldapsearch -H ldap://localhost:389 -x -D cn=demo,dc=example,dc=com \ This example demonstrates multi-threading via the `cluster` module utilizing a `net` server for initial socket receipt. An alternate example demonstrating use of the `connectionRouter` `serverOptions` hook is available in the `examples` directory. -``` +```js const cluster = require('cluster'); const ldap = require('ldapjs'); const net = require('net'); diff --git a/lib/server.js b/lib/server.js index f50c7dd..74f2a67 100644 --- a/lib/server.js +++ b/lib/server.js @@ -314,6 +314,8 @@ function Server (options) { } self.newConnection = function (conn) { + // TODO: make `newConnection` available on the `Server` prototype + // https://github.com/ldapjs/node-ldapjs/pull/727/files#r636572294 setupConnection(conn) log.trace('new connection from %s', conn.ldap.id) From 6d1f23a8318c4be29db69ee4963afe406363c223 Mon Sep 17 00:00:00 2001 From: Tony Brix Date: Thu, 27 May 2021 14:16:05 -0500 Subject: [PATCH 11/11] lint --- examples/cluster-threading-net-server.js | 107 +++++++++++------------ examples/cluster-threading.js | 107 +++++++++++------------ 2 files changed, 106 insertions(+), 108 deletions(-) diff --git a/examples/cluster-threading-net-server.js b/examples/cluster-threading-net-server.js index da9a2d1..4d49089 100644 --- a/examples/cluster-threading-net-server.js +++ b/examples/cluster-threading-net-server.js @@ -1,66 +1,65 @@ -const cluster = require('cluster'); -const ldap = require('ldapjs'); -const net = require('net'); -const os = require('os'); +const cluster = require('cluster') +const ldap = require('ldapjs') +const net = require('net') +const os = require('os') -const threads = []; +const threads = [] threads.getNext = function () { - return (Math.floor(Math.random() * this.length)); -}; + return (Math.floor(Math.random() * this.length)) +} const serverOptions = { - port: 1389 -}; + port: 1389 +} if (cluster.isMaster) { - const server = net.createServer(serverOptions, (socket) => { - socket.pause(); - console.log('ldapjs client requesting connection'); - let routeTo = threads.getNext(); - threads[routeTo].send({ type: 'connection' }, socket); - }); + const server = net.createServer(serverOptions, (socket) => { + socket.pause() + console.log('ldapjs client requesting connection') + const routeTo = threads.getNext() + threads[routeTo].send({ type: 'connection' }, socket) + }) - for (let i = 0; i < os.cpus().length; i++) { - let thread = cluster.fork({ - 'id': i - }); - thread.id = i; - thread.on('message', function (msg) { + for (let i = 0; i < os.cpus().length; i++) { + const thread = cluster.fork({ + id: i + }) + thread.id = i + thread.on('message', function () { - }); - threads.push(thread); + }) + threads.push(thread) + } + + server.listen(serverOptions.port, function () { + console.log('ldapjs listening at ldap://127.0.0.1:' + serverOptions.port) + }) +} else { + const server = ldap.createServer(serverOptions) + + const threadId = process.env.id + + process.on('message', (msg, socket) => { + switch (msg.type) { + case 'connection': + server.newConnection(socket) + socket.resume() + console.log('ldapjs client connection accepted on ' + threadId.toString()) + } + }) + + server.search('dc=example', function (req, res) { + console.log('ldapjs search initiated on ' + threadId.toString()) + const obj = { + dn: req.dn.toString(), + attributes: { + objectclass: ['organization', 'top'], + o: 'example' + } } - server.listen(serverOptions.port, function () { - console.log('ldapjs listening at ldap://127.0.0.1:' + serverOptions.port); - }); -} else { - const server = ldap.createServer(serverOptions); + if (req.filter.matches(obj.attributes)) { res.send(obj) } - let threadId = process.env.id; - - process.on('message', (msg, socket) => { - switch (msg.type) { - case 'connection': - server.newConnection(socket); - socket.resume(); - console.log('ldapjs client connection accepted on ' + threadId.toString()); - } - }); - - server.search('dc=example', function (req, res, next) { - console.log('ldapjs search initiated on ' + threadId.toString()); - var obj = { - dn: req.dn.toString(), - attributes: { - objectclass: ['organization', 'top'], - o: 'example' - } - }; - - if (req.filter.matches(obj.attributes)) - res.send(obj); - - res.end(); - }); + res.end() + }) } diff --git a/examples/cluster-threading.js b/examples/cluster-threading.js index b631c44..8def06a 100644 --- a/examples/cluster-threading.js +++ b/examples/cluster-threading.js @@ -1,66 +1,65 @@ -const cluster = require('cluster'); -const ldap = require('ldapjs'); -const os = require('os'); +const cluster = require('cluster') +const ldap = require('ldapjs') +const os = require('os') -const threads = []; +const threads = [] threads.getNext = function () { - return (Math.floor(Math.random() * this.length)); -}; + return (Math.floor(Math.random() * this.length)) +} const serverOptions = { - connectionRouter: (socket) => { - socket.pause(); - console.log('ldapjs client requesting connection'); - let routeTo = threads.getNext(); - threads[routeTo].send({ type: 'connection' }, socket); - } -}; + connectionRouter: (socket) => { + socket.pause() + console.log('ldapjs client requesting connection') + const routeTo = threads.getNext() + threads[routeTo].send({ type: 'connection' }, socket) + } +} -const server = ldap.createServer(serverOptions); +const server = ldap.createServer(serverOptions) if (cluster.isMaster) { - for (let i = 0; i < os.cpus().length; i++) { - let thread = cluster.fork({ - 'id': i - }); - thread.id = i; - thread.on('message', function (msg) { + for (let i = 0; i < os.cpus().length; i++) { + const thread = cluster.fork({ + id: i + }) + thread.id = i + thread.on('message', function () { - }); - threads.push(thread); + }) + threads.push(thread) + } + + server.listen(1389, function () { + console.log('ldapjs listening at ' + server.url) + }) +} else { + const threadId = process.env.id + serverOptions.connectionRouter = () => { + console.log('should not be hit') + } + + process.on('message', (msg, socket) => { + switch (msg.type) { + case 'connection': + server.newConnection(socket) + socket.resume() + console.log('ldapjs client connection accepted on ' + threadId.toString()) + } + }) + + server.search('dc=example', function (req, res) { + console.log('ldapjs search initiated on ' + threadId.toString()) + const obj = { + dn: req.dn.toString(), + attributes: { + objectclass: ['organization', 'top'], + o: 'example' + } } - server.listen(1389, function () { - console.log('ldapjs listening at ' + server.url); - }); -} else { - let threadId = process.env.id; - serverOptions.connectionRouter = (connection) => { - console.log('should not be hit'); - }; + if (req.filter.matches(obj.attributes)) { res.send(obj) } - process.on('message', (msg, socket) => { - switch (msg.type) { - case 'connection': - server.newConnection(socket); - socket.resume(); - console.log('ldapjs client connection accepted on ' + threadId.toString()); - } - }); - - server.search('dc=example', function (req, res, next) { - console.log('ldapjs search initiated on ' + threadId.toString()); - var obj = { - dn: req.dn.toString(), - attributes: { - objectclass: ['organization', 'top'], - o: 'example' - } - }; - - if (req.filter.matches(obj.attributes)) - res.send(obj); - - res.end(); - }); + res.end() + }) }