From ab775da88a689ee1c423c304b648157873062bf9 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Thu, 8 Jun 2023 18:29:08 +0200 Subject: [PATCH] refs #5563 config check fixes, options added New options: logId, logRelation, showRelation and logFields --- config/logs.yml | 5 ++++ lib/model-loader.js | 72 +++++++++++++++++++++++++++------------------ lib/show-db.js | 28 ++++++++++++++---- mylogger.js | 34 ++++++++++++--------- package-lock.json | 4 +-- package.json | 2 +- 6 files changed, 94 insertions(+), 51 deletions(-) diff --git a/config/logs.yml b/config/logs.yml index cddf5b2..d7cc8c9 100644 --- a/config/logs.yml +++ b/config/logs.yml @@ -1,3 +1,6 @@ +logId: true +logRelation: true +logMainShowField: false upperCaseTable: true userField: editorFk rowExcludeField: logExclude @@ -19,5 +22,7 @@ logs: - itemTag - name: item showField: name + logFields: + - size exclude: - image diff --git a/lib/model-loader.js b/lib/model-loader.js index 5873a84..371e338 100644 --- a/lib/model-loader.js +++ b/lib/model-loader.js @@ -2,6 +2,9 @@ const path = require('path'); const {loadConfig, toUpperCamelCase} = require('./util'); const MultiMap = require('./multi-map'); +/** + * Loads model configuration. + */ module.exports = class ModelLoader { init(logger) { const configDir = path.join(__dirname, '..'); @@ -54,7 +57,7 @@ module.exports = class ModelLoader { if (!tableInfo) { tableInfo = { conf: tableConf, - log: logInfo + logInfo }; schemaMap.set(table.schema, table.name, tableInfo); } @@ -85,10 +88,11 @@ module.exports = class ModelLoader { ? new RegExp(conf.excludeRegex) : null; const localProps = [ - 'idName' + 'idName', + 'showField', + 'logFields' ]; const globalProps = [ - 'showField', 'userField', 'rowExcludeField' ]; @@ -106,9 +110,10 @@ module.exports = class ModelLoader { // Fetch columns & types + const columns = new Set(); Object.assign (tableInfo, { castTypes: new Map(), - columns: new Map() + columns }); if (tableConf.types) @@ -118,8 +123,7 @@ module.exports = class ModelLoader { const [dbCols] = await db.query( `SELECT COLUMN_NAME \`col\`, - DATA_TYPE \`type\`, - COLUMN_DEFAULT \`def\` + DATA_TYPE \`type\` FROM information_schema.\`COLUMNS\` WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?`, [table, schema] @@ -128,14 +132,14 @@ module.exports = class ModelLoader { const exclude = new Set(tableConf.exclude); exclude.add(tableInfo.userField); - for (const {col, type, def} of dbCols) { + for (const {col, type} of dbCols) { const isExcluded = excludeFields.has(col) || (excludeRegex && excludeRegex.test(col)) || exclude.has(col); if (!isExcluded) - tableInfo.columns.set(col, {type, def}); + columns.add(col); const castType = conf.castTypes[type]; if (castType && !tableInfo.castTypes.has(col)) @@ -144,9 +148,10 @@ module.exports = class ModelLoader { // Fetch primary key - if (!tableInfo.idName) { - const [dbPks] = await db.query( - `SELECT COLUMN_NAME idName + const {idName} = tableInfo; + if (!idName) { + const [pks] = await db.query( + `SELECT COLUMN_NAME pk FROM information_schema.KEY_COLUMN_USAGE WHERE CONSTRAINT_NAME = 'PRIMARY' AND TABLE_NAME = ? @@ -154,38 +159,39 @@ module.exports = class ModelLoader { [table, schema] ); - if (!dbPks.length) - throw new Error(`Primary not found for table: ${schema}.${table}`); - if (dbPks.length > 1) + if (!pks.length) + throw new Error(`Primary key not found: ${schema}.${table}`); + if (pks.length > 1) throw new Error(`Only one column primary is supported: ${schema}.${table}`); - - for (const {idName} of dbPks) - tableInfo.idName = idName; - } - + + const [{pk}] = pks; + tableInfo.idName = pk; + } else if (!columns.has(idName)) + throw new Error(`Primary column not found: ${schema}.${table}.${idName}`); + // Get show field const {showField} = tableInfo; - if (showField !== null && !tableInfo.isMain) { + if (showField !== null) { if (showField === undefined) { + if (!tableInfo.isMain || conf.logMainShowField) for (const field of conf.showFields) { - if (tableInfo.columns.has(field)) { + if (columns.has(field)) { tableInfo.showField = field; break; } } - } else { - const match = showField.match(/(^.*)\$$/); - if (match) tableInfo.showRelation = match[1]; - } + } else if (!columns.has(showField)) + throw new Error(`Show column not found: ${schema}.${table}.${showField}`); } } - // Fetch relation to main table - for (const [schema, table, tableInfo] of schemaMap) { + + // Fetch relation to main table + if (!tableInfo.conf.relation && !tableInfo.isMain) { - const mainTable = tableInfo.log.mainTable; + const mainTable = tableInfo.logInfo.mainTable; const mainInfo = schemaMap.get(mainTable.schema, mainTable.name); const [mainRelations] = await db.query( @@ -213,6 +219,16 @@ module.exports = class ModelLoader { for (const {relation} of mainRelations) tableInfo.relation = relation; } + + // Set instance columns + + const {columns} = tableInfo; + const cols = new Set(columns); + if (!conf.logId) + cols.delete(tableInfo.idName); + if (!conf.logRelation) + cols.delete(tableInfo.relation); + tableInfo.instanceColumns = [...cols.keys()]; } } } diff --git a/lib/show-db.js b/lib/show-db.js index 29ef57c..6819d5d 100644 --- a/lib/show-db.js +++ b/lib/show-db.js @@ -1,7 +1,7 @@ const MultiMap = require("./multi-map"); /** - * TODO: #5563 Fetch relations and show values in fronted + * Caches and sets user friendly values for instaces and relations. */ module.exports = class ShowDb { init(logger) { @@ -107,7 +107,7 @@ module.exports = class ShowDb { if (save) tableInfo.showField = col; } - // Clean tables and relations without required information + // Remove tables without showable columns and prepare SELECT statements for (const [schema, table] of relatedList) { const tableInfo = tables.get(schema, table); @@ -127,10 +127,26 @@ module.exports = class ShowDb { WHERE ${sqlIdName} IN (?)`; } - for (const tableInfo of schemaMap.values()) - for (const [col, relation] of tableInfo.relations) { - if (!tables.has(relation.schema, relation.table)) - tableInfo.relations.delete(col); + // Remove relations without showable tables and set show configuration + + for (const [schema, table, tableInfo] of schemaMap) { + const {relations} = tableInfo; + + for (const [col, relation] of relations) { + if (!tables.has(relation.schema, relation.table)) + relations.delete(col); + } + + const {showRelation} = tableInfo.conf; + if (showRelation) { + if (!relations.has(showRelation)) + throw new Error(`Relation not found: ${schema}.${table}.${showRelation}`); + + Object.assign(tableInfo, { + showField: showRelation +'$', + showRelation + }); + } } } diff --git a/mylogger.js b/mylogger.js index 22ee769..ccee904 100644 --- a/mylogger.js +++ b/mylogger.js @@ -313,10 +313,7 @@ module.exports = class MyLogger { if (!tableInfo) return; const action = actions[eventName]; - const { - columns, - rowExcludeField - } = tableInfo; + const {rowExcludeField} = tableInfo; const changes = []; function isExcluded(row) { @@ -354,6 +351,8 @@ module.exports = class MyLogger { } if (action == 'update') { + const cols = tableInfo.columns; + for (const row of evt.rows) { const after = row.after; if (isExcluded(after)) continue; @@ -364,10 +363,9 @@ module.exports = class MyLogger { let nColsChanged = 0; for (const col in before) { - if (columns.has(col) + if (cols.has(col) && !equals(after[col], before[col])) { - if (before[col] !== null) - oldI[col] = castValue(col, before[col]); + oldI[col] = castValue(col, before[col]); newI[col] = castValue(col, after[col]); nColsChanged++; } @@ -376,7 +374,7 @@ module.exports = class MyLogger { changes.push({row: after, oldI, newI}); } } else { - const cols = columns.keys(); + const cols = tableInfo.instanceColumns; for (const row of evt.rows) { if (isExcluded(row)) continue; @@ -474,13 +472,16 @@ module.exports = class MyLogger { evt, changes } = op; + const { + logInfo, + isMain, + relation, + modelName, + logFields + } = tableInfo; - const logInfo = tableInfo.log; const isDelete = action == 'delete'; const isUpdate = action == 'update'; - const isSecondary = !tableInfo.isMain; - const relation = tableInfo.relation; - const modelName = tableInfo.modelName; const created = new Date(evt.timestamp); for (const change of changes) { @@ -491,6 +492,11 @@ module.exports = class MyLogger { case 'update': newI = change.newI; oldI = change.oldI; + if (logFields) { + for (const field of logFields) + if (newI[field] === undefined) + newI[field] = row[field]; + } break; case 'insert': newI = change.instance; @@ -503,8 +509,8 @@ module.exports = class MyLogger { const modelId = row[tableInfo.idName]; const modelValue = change.modelValue ?? null; const oldInstance = oldI ? JSON.stringify(oldI) : null; - const originFk = isSecondary ? row[relation] : modelId; - const originChanged = isUpdate && isSecondary + const originFk = !isMain ? row[relation] : modelId; + const originChanged = isUpdate && !isMain && newI[relation] !== undefined; let deleteRow; diff --git a/package-lock.json b/package-lock.json index 5e47340..c94e1a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mylogger", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mylogger", - "version": "1.0.0", + "version": "1.1.0", "license": "GPL-3.0", "dependencies": { "colors": "^1.4.0", diff --git a/package.json b/package.json index c5c60c1..7cbb78e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mylogger", - "version": "1.0.0", + "version": "1.1.0", "author": "Verdnatura Levante SL", "description": "MySQL and MariaDB logger using binary log", "license": "GPL-3.0",