refs #5563 rowExcludeField, excludeRegex, excludeFields

config/logs.yml

@@ -1,10 +1,14 @@
 upperCaseTable: true
-userField:
-  - editorFk
+userField: editorFk
+rowExcludeField: logExclude
+excludeRegex: '__$'
 showFields:
   - name
   - description
   - nickname
+excludeFields:
+  - created
+  - updated
 castTypes:
   tinyint: boolean
 logs:

lib/model-loader.js

@@ -65,23 +65,13 @@ module.exports = class ModelLoader {
           ? toUpperCamelCase(table.name)
           : table.name;
       }
-  
-      const {
-        showField,
-        relation,
-        idName
-      } = tableConf;
-  
+
       Object.assign(tableInfo, {
         conf: tableConf,
-        exclude: new Set(tableConf.exclude),
         modelName,
-        showField,
-        relation,
-        idName,
-        userField: tableConf.userField || conf.userField
+        relation: tableConf.relation
       });
-  
+
       return tableInfo;
     }
   }
@@ -89,10 +79,31 @@ module.exports = class ModelLoader {
   async loadSchema() {
     const {db, schemaMap} = this.logger;
     const {conf} = this;
+
+    const excludeFields = new Set(conf.excludeFields);
+    const excludeRegex = conf.excludeRegex
+      ? new RegExp(conf.excludeRegex) : null;
+  
+    const localProps = [
+      'idName'
+    ];
+    const globalProps = [
+      'showField',
+      'userField',
+      'rowExcludeField'
+    ];
   
     for (const [schema, table, tableInfo] of schemaMap) {
       const tableConf = tableInfo.conf;
-  
+
+      for (const prop of localProps)
+        tableInfo[prop] = tableConf[prop];
+
+      for (const prop of globalProps)
+        tableInfo[prop] = tableConf[prop] !== undefined
+          ? tableConf[prop]
+          : conf[prop];
+
       // Fetch columns & types
   
       Object.assign (tableInfo, {
@@ -113,9 +124,17 @@ module.exports = class ModelLoader {
           WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?`,
         [table, schema]
       );
+
+      const exclude = new Set(tableConf.exclude);
+      exclude.add(tableInfo.userField);
   
       for (const {col, type, def} of dbCols) {
-        if (!tableInfo.exclude.has(col) && col != tableInfo.userField)
+        const isExcluded =
+             excludeFields.has(col)
+          || (excludeRegex && excludeRegex.test(col))
+          || exclude.has(col);
+
+        if (!isExcluded)
           tableInfo.columns.set(col, {type, def});
   
         const castType = conf.castTypes[type];
@@ -125,7 +144,7 @@ module.exports = class ModelLoader {
   
       // Fetch primary key
   
-      if (!tableConf.idName) {
+      if (!tableInfo.idName) {
         const [dbPks] = await db.query(
           `SELECT COLUMN_NAME idName
             FROM information_schema.KEY_COLUMN_USAGE
@@ -146,9 +165,9 @@ module.exports = class ModelLoader {
   
       // Get show field
   
-      if (!tableInfo.isMain) {
-        const {showField} = tableInfo;
-        if (!showField) {
+      const {showField} = tableInfo;
+      if (showField !== null && !tableInfo.isMain) {
+        if (showField === undefined) {
           for (const field of conf.showFields) {
             if (tableInfo.columns.has(field)) {
               tableInfo.showField = field;
@@ -159,8 +178,7 @@ module.exports = class ModelLoader {
           const match = showField.match(/(^.*)\$$/);
           if (match) tableInfo.showRelation = match[1];
         }
-      } else
-        tableInfo.showField = null;
+      }
     }
 
     // Fetch relation to main table

lib/show-db.js

@@ -9,19 +9,20 @@ module.exports = class ShowDb {
       logger,
       conf: logger.conf.showCache,
       tables: new MultiMap(),
-      valueDb: new MultiMap()
+      cache: new MultiMap()
     });
   }
 
   checkDb() {
-    const {conf, valueDb} = this;
+    const {conf, cache} = this;
+    const now = Date.now();
     const dbOutdated = this.loops % conf.maxLoops == 0
-      || this.lastFlush > Date.now() + conf.life * 1000
+      || this.lastFlush > now + conf.life * 1000
 
     if (dbOutdated) {
-      valueDb.clear();
+      cache.clear();
       this.loops = 0;
-      this.lastFlush = Date.now();
+      this.lastFlush = now;
     }
     this.loops++;
   }
@@ -134,8 +135,8 @@ module.exports = class ShowDb {
   }
 
   async getValues(db, ops) {
-    const {tables, valueDb} = this;
-    const showIds = new MultiMap();
+    const {tables, cache} = this;
+    const fetchMap = new MultiMap();
 
     this.checkDb();
 
@@ -164,11 +165,11 @@ module.exports = class ShowDb {
           const {schema, table} = relation;
           const id = row[col];
 
-          let ids = valueDb.get(schema, table);
+          let ids = cache.get(schema, table);
           if (ids && ids.has(id)) continue;
 
-          ids = showIds.get(schema, table);
-          if (!ids) showIds.set(schema, table, ids = new Set());
+          ids = fetchMap.get(schema, table);
+          if (!ids) fetchMap.set(schema, table, ids = new Set());
           ids.add(id);
         }
       }
@@ -176,18 +177,18 @@ module.exports = class ShowDb {
 
     // Query show values to database
 
-    for (const [schema, table, ids] of showIds) {
+    for (const [schema, table, fetchIds] of fetchMap) {
       const tableInfo = tables.get(schema, table);
       const [res] = await db.query(
         tableInfo.selectStmt,
-        [Array.from(ids.keys())]
+        [Array.from(fetchIds.keys())]
       );
 
-      let cacheIds = valueDb.get(schema, table);
-      if (!cacheIds) valueDb.set(schema, table, cacheIds = new Map());
+      let ids = cache.get(schema, table);
+      if (!ids) cache.set(schema, table, ids = new Map());
 
       for (const row of res)
-        cacheIds.set(row.id, row.val);
+        ids.set(row.id, row.val);
     }
 
     // Fill rows with show values
@@ -225,7 +226,7 @@ module.exports = class ShowDb {
 
     function getValue(relation, row, col) {
       const {schema, table} = relation;
-      const ids = valueDb.get(schema, table);
+      const ids = cache.get(schema, table);
       return ids && ids.get(row[col])
     }
   }

mylogger.js

@@ -227,8 +227,8 @@ module.exports = class MyLogger {
     await new Promise(resolve => {
       zongji.ctrlConnection.query('KILL ?', [zongji.connection.threadId],
       err => {
-        if (err && err.code !== 'ER_NO_SUCH_THREAD')
-          logError(err);
+        // if (err && err.code !== 'ER_NO_SUCH_THREAD');
+        //   console.error(err);
         resolve();
       });
     });
@@ -313,8 +313,15 @@ module.exports = class MyLogger {
     if (!tableInfo) return;
 
     const action = actions[eventName];
-    const columns = tableInfo.columns;
-    let changes = [];
+    const {
+      columns,
+      rowExcludeField
+    } = tableInfo;
+    const changes = [];
+
+    function isExcluded(row) {
+      return rowExcludeField && row[rowExcludeField];
+    }
 
     function castValue(col, value) {
       switch(tableInfo.castTypes.get(col)) {
@@ -348,11 +355,13 @@ module.exports = class MyLogger {
 
     if (action == 'update') {
       for (const row of evt.rows) {
-        let nColsChanged = 0;
-        const before = row.before;
         const after = row.after;
+        if (isExcluded(after)) continue;
+
+        const before = row.before;
         const oldI = {};
         const newI = {};
+        let nColsChanged = 0;
 
         for (const col in before) {
           if (columns.has(col)
@@ -370,6 +379,8 @@ module.exports = class MyLogger {
       const cols = columns.keys();
 
       for (const row of evt.rows) {
+        if (isExcluded(row)) continue;
+
         const instance = {};
         for (const col of cols) {
           if (row[col] !== null)

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mylogger",
-  "version": "0.1.24",
+  "version": "1.0.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "mylogger",
-      "version": "0.1.24",
+      "version": "1.0.0",
       "license": "GPL-3.0",
       "dependencies": {
         "colors": "^1.4.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "mylogger",
-  "version": "0.1.24",
+  "version": "1.0.0",
   "author": "Verdnatura Levante SL",
   "description": "MySQL and MariaDB logger using binary log",
   "license": "GPL-3.0",