diff --git a/config/logs.yml b/config/logs.yml index c1b5052..cddf5b2 100644 --- a/config/logs.yml +++ b/config/logs.yml @@ -1,10 +1,14 @@ upperCaseTable: true -userField: - - editorFk +userField: editorFk +rowExcludeField: logExclude +excludeRegex: '__$' showFields: - name - description - nickname +excludeFields: + - created + - updated castTypes: tinyint: boolean logs: diff --git a/lib/model-loader.js b/lib/model-loader.js index 3c2f4b2..5873a84 100644 --- a/lib/model-loader.js +++ b/lib/model-loader.js @@ -65,23 +65,13 @@ module.exports = class ModelLoader { ? toUpperCamelCase(table.name) : table.name; } - - const { - showField, - relation, - idName - } = tableConf; - + Object.assign(tableInfo, { conf: tableConf, - exclude: new Set(tableConf.exclude), modelName, - showField, - relation, - idName, - userField: tableConf.userField || conf.userField + relation: tableConf.relation }); - + return tableInfo; } } @@ -89,10 +79,31 @@ module.exports = class ModelLoader { async loadSchema() { const {db, schemaMap} = this.logger; const {conf} = this; + + const excludeFields = new Set(conf.excludeFields); + const excludeRegex = conf.excludeRegex + ? new RegExp(conf.excludeRegex) : null; + + const localProps = [ + 'idName' + ]; + const globalProps = [ + 'showField', + 'userField', + 'rowExcludeField' + ]; for (const [schema, table, tableInfo] of schemaMap) { const tableConf = tableInfo.conf; - + + for (const prop of localProps) + tableInfo[prop] = tableConf[prop]; + + for (const prop of globalProps) + tableInfo[prop] = tableConf[prop] !== undefined + ? tableConf[prop] + : conf[prop]; + // Fetch columns & types Object.assign (tableInfo, { @@ -113,9 +124,17 @@ module.exports = class ModelLoader { WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?`, [table, schema] ); + + const exclude = new Set(tableConf.exclude); + exclude.add(tableInfo.userField); for (const {col, type, def} of dbCols) { - if (!tableInfo.exclude.has(col) && col != tableInfo.userField) + const isExcluded = + excludeFields.has(col) + || (excludeRegex && excludeRegex.test(col)) + || exclude.has(col); + + if (!isExcluded) tableInfo.columns.set(col, {type, def}); const castType = conf.castTypes[type]; @@ -125,7 +144,7 @@ module.exports = class ModelLoader { // Fetch primary key - if (!tableConf.idName) { + if (!tableInfo.idName) { const [dbPks] = await db.query( `SELECT COLUMN_NAME idName FROM information_schema.KEY_COLUMN_USAGE @@ -146,9 +165,9 @@ module.exports = class ModelLoader { // Get show field - if (!tableInfo.isMain) { - const {showField} = tableInfo; - if (!showField) { + const {showField} = tableInfo; + if (showField !== null && !tableInfo.isMain) { + if (showField === undefined) { for (const field of conf.showFields) { if (tableInfo.columns.has(field)) { tableInfo.showField = field; @@ -159,8 +178,7 @@ module.exports = class ModelLoader { const match = showField.match(/(^.*)\$$/); if (match) tableInfo.showRelation = match[1]; } - } else - tableInfo.showField = null; + } } // Fetch relation to main table diff --git a/lib/show-db.js b/lib/show-db.js index b02e120..29ef57c 100644 --- a/lib/show-db.js +++ b/lib/show-db.js @@ -9,19 +9,20 @@ module.exports = class ShowDb { logger, conf: logger.conf.showCache, tables: new MultiMap(), - valueDb: new MultiMap() + cache: new MultiMap() }); } checkDb() { - const {conf, valueDb} = this; + const {conf, cache} = this; + const now = Date.now(); const dbOutdated = this.loops % conf.maxLoops == 0 - || this.lastFlush > Date.now() + conf.life * 1000 + || this.lastFlush > now + conf.life * 1000 if (dbOutdated) { - valueDb.clear(); + cache.clear(); this.loops = 0; - this.lastFlush = Date.now(); + this.lastFlush = now; } this.loops++; } @@ -134,8 +135,8 @@ module.exports = class ShowDb { } async getValues(db, ops) { - const {tables, valueDb} = this; - const showIds = new MultiMap(); + const {tables, cache} = this; + const fetchMap = new MultiMap(); this.checkDb(); @@ -164,11 +165,11 @@ module.exports = class ShowDb { const {schema, table} = relation; const id = row[col]; - let ids = valueDb.get(schema, table); + let ids = cache.get(schema, table); if (ids && ids.has(id)) continue; - ids = showIds.get(schema, table); - if (!ids) showIds.set(schema, table, ids = new Set()); + ids = fetchMap.get(schema, table); + if (!ids) fetchMap.set(schema, table, ids = new Set()); ids.add(id); } } @@ -176,18 +177,18 @@ module.exports = class ShowDb { // Query show values to database - for (const [schema, table, ids] of showIds) { + for (const [schema, table, fetchIds] of fetchMap) { const tableInfo = tables.get(schema, table); const [res] = await db.query( tableInfo.selectStmt, - [Array.from(ids.keys())] + [Array.from(fetchIds.keys())] ); - let cacheIds = valueDb.get(schema, table); - if (!cacheIds) valueDb.set(schema, table, cacheIds = new Map()); + let ids = cache.get(schema, table); + if (!ids) cache.set(schema, table, ids = new Map()); for (const row of res) - cacheIds.set(row.id, row.val); + ids.set(row.id, row.val); } // Fill rows with show values @@ -225,7 +226,7 @@ module.exports = class ShowDb { function getValue(relation, row, col) { const {schema, table} = relation; - const ids = valueDb.get(schema, table); + const ids = cache.get(schema, table); return ids && ids.get(row[col]) } } diff --git a/mylogger.js b/mylogger.js index 4588b8e..22ee769 100644 --- a/mylogger.js +++ b/mylogger.js @@ -227,8 +227,8 @@ module.exports = class MyLogger { await new Promise(resolve => { zongji.ctrlConnection.query('KILL ?', [zongji.connection.threadId], err => { - if (err && err.code !== 'ER_NO_SUCH_THREAD') - logError(err); + // if (err && err.code !== 'ER_NO_SUCH_THREAD'); + // console.error(err); resolve(); }); }); @@ -313,8 +313,15 @@ module.exports = class MyLogger { if (!tableInfo) return; const action = actions[eventName]; - const columns = tableInfo.columns; - let changes = []; + const { + columns, + rowExcludeField + } = tableInfo; + const changes = []; + + function isExcluded(row) { + return rowExcludeField && row[rowExcludeField]; + } function castValue(col, value) { switch(tableInfo.castTypes.get(col)) { @@ -348,11 +355,13 @@ module.exports = class MyLogger { if (action == 'update') { for (const row of evt.rows) { - let nColsChanged = 0; - const before = row.before; const after = row.after; + if (isExcluded(after)) continue; + + const before = row.before; const oldI = {}; const newI = {}; + let nColsChanged = 0; for (const col in before) { if (columns.has(col) @@ -370,6 +379,8 @@ module.exports = class MyLogger { const cols = columns.keys(); for (const row of evt.rows) { + if (isExcluded(row)) continue; + const instance = {}; for (const col of cols) { if (row[col] !== null) diff --git a/package-lock.json b/package-lock.json index 63856e4..5e47340 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mylogger", - "version": "0.1.24", + "version": "1.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mylogger", - "version": "0.1.24", + "version": "1.0.0", "license": "GPL-3.0", "dependencies": { "colors": "^1.4.0", diff --git a/package.json b/package.json index 9c13ff1..c5c60c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mylogger", - "version": "0.1.24", + "version": "1.0.0", "author": "Verdnatura Levante SL", "description": "MySQL and MariaDB logger using binary log", "license": "GPL-3.0",