diff --git a/nbactions.xml b/nbactions.xml index c1e221c..3aeffe1 100644 --- a/nbactions.xml +++ b/nbactions.xml @@ -1,40 +1,40 @@ - - - - run - - process-classes - org.codehaus.mojo:exec-maven-plugin:1.2:exec - - - -classpath %classpath net.certifi.audittablegen.AuditTableGen -d capturagp -s localhost --driver postgresql url jdbc:postgresql://localhost:5432/capturagp?searchpath=public -u postgres -p -xxx --schema public - java - runtime - - - - debug - - process-classes - org.codehaus.mojo:exec-maven-plugin:1.2:exec - - - -Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath net.certifi.audittablegen.AuditTableGen -d capturagp -s localhost --driver postgresql url jdbc:postgresql://localhost:5432/capturagp?searchpath=public -u postgres -p -xxx --schema public - java - runtime - true - - - - profile - - process-classes - org.codehaus.mojo:exec-maven-plugin:1.2:exec - - - -classpath %classpath net.certifi.audittablegen.AuditTableGen -d capturagp -s localhost --driver postgresql url jdbc:postgresql://localhost:5432/capturagp?searchpath=public -u postgres -p -xxx --schema public - ${profiler.java} - profile - - - + + + + run + + process-classes + org.codehaus.mojo:exec-maven-plugin:1.2:exec + + + -classpath %classpath net.certifi.audittablegen.AuditTableGen -d postgres -s localhost --driver postgresql url jdbc:postgresql://localhost:5432/postgres?searchpath=public -u cservin -p -xxx --schema audit_table + java + runtime + + + + debug + + process-classes + org.codehaus.mojo:exec-maven-plugin:1.2:exec + + + -Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath net.certifi.audittablegen.AuditTableGen -d postgres -s localhost --driver postgresql url jdbc:postgresql://localhost:5432/postgres?searchpath=public -u cservin -p -xxx --schema audit_table + java + runtime + true + + + + profile + + process-classes + org.codehaus.mojo:exec-maven-plugin:1.2:exec + + + -classpath %classpath net.certifi.audittablegen.AuditTableGen -d postgres -s localhost --driver postgresql url jdbc:postgresql://localhost:5432/postgres?searchpath=public -u cservin -p -xxx --schema audit_table + ${profiler.java} + profile + + + diff --git a/pom.xml b/pom.xml index 3c5ba73..43c8822 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.certifi AuditTableGen - 0.1.0-SNAPSHOT + 0.1.4 jar AuditTableGen diff --git a/src/main/java/net/certifi/audittablegen/ChangeSourceFactory.java b/src/main/java/net/certifi/audittablegen/ChangeSourceFactory.java index bbb9fa0..a2aea56 100644 --- a/src/main/java/net/certifi/audittablegen/ChangeSourceFactory.java +++ b/src/main/java/net/certifi/audittablegen/ChangeSourceFactory.java @@ -19,6 +19,8 @@ package net.certifi.audittablegen; import java.util.*; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.lang3.exception.ContextedRuntimeException; import org.slf4j.Logger; @@ -77,6 +79,7 @@ public class ChangeSourceFactory { break; case iddatatype: auditIdTypeName = attrib.getValue(); + break; case userdatatype: auditUserTypeName = attrib.getValue(); break; @@ -133,10 +136,20 @@ Boolean isPatternMatch(String str, String pattern) { //TODO: this is where regexp or wildcard pattern matching should go if (pattern.isEmpty() || pattern.equals("*") - || pattern.toLowerCase().equals(str.toLowerCase())) { + || pattern.toLowerCase().equals(str.toLowerCase()) ) { return Boolean.TRUE; } + try { + Pattern p = Pattern.compile( pattern, Pattern.CASE_INSENSITIVE|Pattern.UNICODE_CASE ); + if ( p != null && p.matcher( str ).matches() ) return Boolean.TRUE; + + } catch( IllegalArgumentException x) { + + logger.warn( "Invalid Regexp " + x.getMessage() ); + } + + return Boolean.FALSE; } @@ -295,7 +308,7 @@ List getDBChangeList(TableDef baseTableDef){ workUnit.setColumnName(auditActionColumn); workUnit.setTableName(auditTableName); workUnit.setTypeName(auditActionTypeName); //insert, update, or delete - workUnit.setSize(6); + workUnit.setSize(1); workUnit.setDecimalSize(0); tableChangeUnits.add(workUnit); @@ -330,14 +343,69 @@ List getDBChangeList(TableDef baseTableDef){ //end of table tableChangeUnits.add(new DBChangeUnit(DBChangeType.end)); + + // populate the table + tableChangeUnits.add( new DBChangeUnit(DBChangeType.begin) ); + workUnit = new DBChangeUnit(DBChangeType.fillAuditTable); + workUnit.setAuditTableName( auditTableName ); + workUnit.setTypeName( baseTableName ); + tableChangeUnits.add(workUnit); + + // fill all columns on the base table + for (ColumnDef baseColumn : baseTableDef.getColumns()) { + workUnit = new DBChangeUnit(DBChangeType.addColumn); + workUnit.setColumnName(baseColumn.getName()); + workUnit.setTableName(auditTableName); + workUnit.setTypeName(baseColumn.getTypeName()); + workUnit.setSize(baseColumn.getSize()); + workUnit.setDecimalSize(baseColumn.getDecimalSize()); + tableChangeUnits.add(workUnit); + } + + //action + workUnit = new DBChangeUnit(DBChangeType.addTriggerAction); + workUnit.setColumnName(auditActionColumn); + workUnit.setTableName(baseTableName); + workUnit.setAuditTableName(auditTableName); + tableChangeUnits.add(workUnit); + + //user + workUnit = new DBChangeUnit(DBChangeType.addTriggerUser); + workUnit.setColumnName(auditUserColumn); + workUnit.setTableName(baseTableName); + workUnit.setAuditTableName(auditTableName); + tableChangeUnits.add(workUnit); + + //timestamp + workUnit = new DBChangeUnit(DBChangeType.addTriggerTimeStamp); + workUnit.setColumnName(auditTimeStampColumn); + workUnit.setTableName(baseTableName); + workUnit.setAuditTableName(auditTableName); + tableChangeUnits.add(workUnit); + + //sessionuser + if (!sessionUserSQL.isEmpty()) { + workUnit = new DBChangeUnit(DBChangeType.addTriggerSessionUser); + workUnit.setColumnName(sessionUserColumn); + workUnit.setTableName(baseTableName); + workUnit.setAuditTableName(auditTableName); + tableChangeUnits.add(workUnit); } - else { + + workUnit = new DBChangeUnit(DBChangeType.end); + workUnit.setTableName(baseTableName); + workUnit.setAuditTableName(auditTableName); + tableChangeUnits.add(workUnit); + + } else { //alter table //there might not be any changes, so store up any changes in //a temporary list, and evaluate. alterTableChangeUnits.add(new DBChangeUnit(DBChangeType.begin)); workUnit = new DBChangeUnit(DBChangeType.alterTable); - workUnit.setTableName(auditTableName); + workUnit.setTableName(baseTableName); + workUnit.setAuditTableName( auditTableName ); + workUnit.setTableDef( baseTableDef ); alterTableChangeUnits.add(workUnit); //to make this a little easier, get a map for the column list @@ -391,6 +459,7 @@ List getDBChangeList(TableDef baseTableDef){ workUnit = new DBChangeUnit(DBChangeType.addColumn); workUnit.setColumnName(sessionUserColumn); workUnit.setTableName(auditTableName); + workUnit.setAuditTableName( auditTableName ); workUnit.setTypeName(sessionUserTypeName); workUnit.setSize(sessionUserDataSize); workUnit.setDecimalSize(0); @@ -460,6 +529,7 @@ else if (auditColumn.getTypeName().equalsIgnoreCase(baseColumn.getTypeName()) //new column workUnit = new DBChangeUnit(DBChangeType.addColumn); workUnit.setTableName(auditTableName); + workUnit.setAuditTableName( auditTableName ); workUnit.setColumnName(baseColumn.getName()); workUnit.setTypeName(baseColumn.getTypeName()); workUnit.setSize(baseColumn.getSize()); @@ -469,7 +539,11 @@ else if (auditColumn.getTypeName().equalsIgnoreCase(baseColumn.getTypeName()) } //end of table - alterTableChangeUnits.add(new DBChangeUnit(DBChangeType.end)); + workUnit = new DBChangeUnit( DBChangeType.end ); + workUnit.setTableName( baseTableName ); + workUnit.setAuditTableName( auditTableName ); + workUnit.setTableDef( baseTableDef ); + alterTableChangeUnits.add( workUnit ); //add the workUnits to the return value if (!renameColumnChangeUnits.isEmpty()){ diff --git a/src/main/java/net/certifi/audittablegen/ColumnDef.java b/src/main/java/net/certifi/audittablegen/ColumnDef.java index f0d8ac1..c2f0daf 100644 --- a/src/main/java/net/certifi/audittablegen/ColumnDef.java +++ b/src/main/java/net/certifi/audittablegen/ColumnDef.java @@ -95,4 +95,9 @@ public void setDataTypeDef(DataTypeDef dataTypeDef) { this.dataTypeDef = dataTypeDef; } + @Override + public String toString() { + return "ColumnDef{" + "name=" + name + ", typeName=" + typeName + ", sqlType=" + sqlType + ", size=" + size + ", decimalSize=" + decimalSize + ", dataTypeDef=" + dataTypeDef + ", sourceMeta=" + sourceMeta + '}'; + } + } diff --git a/src/main/java/net/certifi/audittablegen/DBChangeType.java b/src/main/java/net/certifi/audittablegen/DBChangeType.java index cec5573..7ff0e20 100644 --- a/src/main/java/net/certifi/audittablegen/DBChangeType.java +++ b/src/main/java/net/certifi/audittablegen/DBChangeType.java @@ -41,5 +41,6 @@ public enum DBChangeType { addTriggerUser, addTriggerTimeStamp, addTriggerSessionUser, + fillAuditTable, notSet; } diff --git a/src/main/java/net/certifi/audittablegen/DBChangeUnit.java b/src/main/java/net/certifi/audittablegen/DBChangeUnit.java index 0e2edbd..9bf4745 100644 --- a/src/main/java/net/certifi/audittablegen/DBChangeUnit.java +++ b/src/main/java/net/certifi/audittablegen/DBChangeUnit.java @@ -41,8 +41,17 @@ public class DBChangeUnit { int size = 0; int decimalSize = 0; Boolean identity = Boolean.FALSE; + TableDef tableDef = null; //DataTypeDef dataTypeDef = null; + public TableDef getTableDef() { + return tableDef; + } + + public void setTableDef(TableDef tableDef) { + this.tableDef = tableDef; + } + public DBChangeUnit( DBChangeType changeType) { this.changeType = changeType; } @@ -290,6 +299,7 @@ else if ( workListTag == DBChangeType.notSet){ case alterTable: case createTriggers: case dropTriggers: + case fillAuditTable: if (!beginTag){ //begin tag missing. //could be implied, but treat as error condition. @@ -312,7 +322,8 @@ else if (workListTag != DBChangeType.notSet){ valid = false; } else if (workListTag != DBChangeType.createTable - && workListTag != DBChangeType.alterTable) { + && workListTag != DBChangeType.alterTable + && workListTag != DBChangeType.fillAuditTable ) { logger.info ("improperly formed List. Unit{%s} not of valid for {%s} at element %d", unit.getChangeType().toString(), workListTag.toString(), i); valid = false; @@ -340,6 +351,18 @@ else if (workListTag != DBChangeType.createTable case fireOnInsert: case fireOnUpdate: case fireOnDelete: + if (beginTag == false){ + logger.info ("improperly formed List. Missing [begin] before element %d", i); + } + if (workListTag != DBChangeType.createTriggers){ + throw new RuntimeException ("improperly formed List. Unit{" + + unit.getChangeType().toString() + "} not of valid for {" + + workListTag.toString() +"}"); + } + else { + valid = validateUnit(unit, parentUnit); + } + break; case addTriggerAction: case addTriggerTimeStamp: case addTriggerUser: @@ -347,7 +370,8 @@ else if (workListTag != DBChangeType.createTable if (beginTag == false){ logger.info ("improperly formed List. Missing [begin] before element %d", i); } - if (workListTag != DBChangeType.createTriggers){ + if (workListTag != DBChangeType.createTriggers + && workListTag != DBChangeType.fillAuditTable ){ throw new RuntimeException ("improperly formed List. Unit{" + unit.getChangeType().toString() + "} not of valid for {" + workListTag.toString() +"}"); diff --git a/src/main/java/net/certifi/audittablegen/GenericDMR.java b/src/main/java/net/certifi/audittablegen/GenericDMR.java index 1a2ef6b..b52b4aa 100644 --- a/src/main/java/net/certifi/audittablegen/GenericDMR.java +++ b/src/main/java/net/certifi/audittablegen/GenericDMR.java @@ -405,6 +405,7 @@ public Map getDataTypes (){ } + conn.close(); } catch (SQLException e) { throw Throwables.propagate(e); } @@ -550,6 +551,7 @@ public void readDBChangeList(List units) { case alterTable: case createTriggers: case dropTriggers: + case fillAuditTable: workListType = unit.getChangeType(); workList.add(unit); break; @@ -606,6 +608,9 @@ public void executeChanges() { break; case dropTriggers: query = getDropTriggerSQL(op); + break; + case fillAuditTable: + query = getFillAuditTableSQL(op); break; default: //should not get here if the list is valid, unless a new changetype @@ -625,10 +630,95 @@ public void executeChanges() { } } + String getFillAuditTableSQL(List op) { + + StringBuilder builder = new StringBuilder(); + StringBuilder select = new StringBuilder(); + + boolean firstCol = true; + String schema; + + if (verifiedSchema != null){ + schema = verifiedSchema + "."; + } + else { + schema = ""; + } + + for (DBChangeUnit unit : op) { + switch (unit.changeType) { + case begin: + //nothinig + break; + + case end: + builder.append(")").append(System.lineSeparator()); + select.append(" from ").append(schema).append(unit.tableName).append(System.lineSeparator()); + builder.append(select); + break; + + case fillAuditTable: + builder.append("insert into ").append(schema).append(unit.getAuditTableName()).append(" (").append(System.lineSeparator()); + select.append("select ").append(System.lineSeparator()); + break; + + case addColumn: + case addTriggerAction: + case addTriggerUser: + case addTriggerTimeStamp: + case addTriggerSessionUser: + if (!firstCol){ + builder.append(", "); + select.append(", "); + } + else { + firstCol = false; + } + + builder.append(unit.columnName).append(" "); + builder.append(System.lineSeparator()); + + switch( unit.changeType ) { + case addColumn: + select.append(unit.columnName).append(" "); + select.append(System.lineSeparator()); + break; + case addTriggerAction: + select.append("'L' "); + select.append(System.lineSeparator()); + break; + case addTriggerTimeStamp: + select.append("now() "); + select.append(System.lineSeparator()); + break; + case addTriggerUser: + select.append("user "); + select.append(System.lineSeparator()); + break; + case addTriggerSessionUser: + logger.error("unimplemented DBChangeUnit '{}' for fillAuditTable operation", unit.getChangeType().toString()); + return null; + + } + + break; + + default: + //should not get here if the list is valid, unless a new changetype + //was added that this DMR does not know about. If which case - fail. + logger.error("unimplemented DBChangeUnit '{}' for fillAuditTable operation", unit.getChangeType().toString()); + return null; + } + } + + return builder.toString(); + + } + String getCreateTableSQL(List op) { StringBuilder builder = new StringBuilder(); - StringBuilder constraints = new StringBuilder(); + StringBuilder constraints = new StringBuilder(); DataTypeDef dataTypeDef = null; boolean firstCol = true; String schema; @@ -702,6 +792,9 @@ String getAlterTableSQL(List op) { StringBuilder constraints = new StringBuilder(); DataTypeDef dataTypeDef = null; boolean firstCol = true; + boolean firstUpdateCol = true; + StringBuilder updateSQL = new StringBuilder(); + String schema; if (verifiedSchema != null){ @@ -718,9 +811,22 @@ String getAlterTableSQL(List op) { break; case end: builder.append(constraints); + if ( firstUpdateCol != true ) { + updateSQL.append(System.lineSeparator()).append( "from ").append(schema).append(unit.getTableName()).append( " orig").append(System.lineSeparator()); + + ColumnDef primaryKey = unit.getTableDef().getPrimaryKey(); + + if ( primaryKey != null ) { + updateSQL.append( "where audit.").append( primaryKey.getName() ).append( " = orig." ).append( primaryKey.getName() ); + } else { + logger.warn( "Table " + unit.getTableName() + " has no primary key, can not update audit table data"); + updateSQL = new StringBuilder(); + } + } + builder.append( updateSQL ); break; case alterTable: - builder.append("ALTER TABLE ").append(schema).append(unit.tableName).append(System.lineSeparator()); + builder.append("ALTER TABLE ").append(schema).append(unit.getAuditTableName()).append(System.lineSeparator()); break; case addColumn: if (!firstCol){ @@ -738,8 +844,9 @@ String getAlterTableSQL(List op) { } else { builder.append(unit.columnName).append(" ").append(unit.typeName); -// if (dataTypeDef.create_params != null && unit.size > 0){ - if (dataTypeDef.createWithSize && unit.size > 0){ + +// if (dataTypeDef.create_params != null && unit.size > 0){ + if (dataTypeDef.createWithSize && unit.size > 0){ builder.append(" (").append(unit.size); if (unit.decimalSize > 0){ @@ -747,6 +854,20 @@ String getAlterTableSQL(List op) { } builder.append(") "); } + + // don't genereate update sql for altering the audit table + if ( ! unit.tableName.equals( unit.auditTableName) ) { + if ( firstUpdateCol ) { + firstUpdateCol = false; + + updateSQL.append(";").append( System.lineSeparator() ).append( "update ").append(schema).append(unit.getAuditTableName()).append( " as audit").append(System.lineSeparator()).append( "set "); + updateSQL.append( unit.getColumnName() ).append( " = orig." ).append( unit.getColumnName() ); + } else { + + updateSQL.append(System.lineSeparator()).append( " , " ).append( unit.getColumnName() ).append( " = orig." ).append( unit.getColumnName() ); + } + } + if (!unit.foreignTable.isEmpty()){ builder.append("REFERENCES ").append(unit.foreignTable).append(" (").append(unit.columnName).append(")"); //constraints.append("CONSTRAINT ").append(unit.columnName).append(" REFERENCES ").append(unit.foreignTable); @@ -843,7 +964,7 @@ String getCreateTriggerSQL(List op) { ////////////////////// //generate the when clause for the update trigger - if (columns.size() > whenColumns.size() ){ + if ( true /* columns.size() > whenColumns.size() */ ){ //some columns excluded from update updateConditional.append("AND ("); boolean firstCol = true; @@ -884,9 +1005,9 @@ String getCreateTriggerSQL(List op) { ////////////////////// //generate the insert column valuues for the trigger(s) - insertDetail.append(" SELECT 'insert', user, now()"); - updateDetail.append(" SELECT 'update', user, now()"); - deleteDetail.append(" SELECT 'delete', user, now()"); + insertDetail.append(" SELECT 'I', user, now()"); + updateDetail.append(" SELECT 'U', user, now()"); + deleteDetail.append(" SELECT 'D', user, now()"); if (sessionUserColumn != null){ insertDetail.append(", ").append(this.getSessionUserSQL()); updateDetail.append(", ").append(this.getSessionUserSQL()); @@ -916,6 +1037,7 @@ String getCreateTriggerSQL(List op) { builder.append(" ELSEIF (TG_OP = 'UPDATE' ").append(updateConditional).append(System.lineSeparator()); builder.append(updateDetail); builder.append(" END IF;").append(System.lineSeparator()); + builder.append(" RETURN NEW;"); // This should only happen on an update that does not update anything builder.append("END").append(System.lineSeparator()); builder.append(triggerReference).append(" LANGUAGE plpgsql;").append(System.lineSeparator()); @@ -1061,6 +1183,7 @@ public void executeUpdate (String query){ } catch (SQLException ex) { logger.error("Update failed...", ex); + throw new RuntimeException( "Error applying AuditTable SQL", ex ); } } @@ -1095,7 +1218,7 @@ public int getMaxUserNameLength() { public DataTypeDef getDataType (String typeName){ Map dtds = this.getDataTypes(); - + if (dtds.containsKey(typeName)){ return dtds.get(typeName); } diff --git a/src/main/java/net/certifi/audittablegen/PostgresqlDMR.java b/src/main/java/net/certifi/audittablegen/PostgresqlDMR.java index e13520d..720934d 100644 --- a/src/main/java/net/certifi/audittablegen/PostgresqlDMR.java +++ b/src/main/java/net/certifi/audittablegen/PostgresqlDMR.java @@ -133,6 +133,7 @@ public List getColumns (String tableName){ throw new RuntimeException("No results for DatabaseMetaData.getColumns(" + verifiedSchema + "." + tableName + ")"); } while (rs.next()){ + ColumnDef columnDef = new ColumnDef(); Map columnMetaData = new CaseInsensitiveMap(); for (int i = 1; i <= metaDataColumnCount; i++){ @@ -141,14 +142,20 @@ public List getColumns (String tableName){ columnDef.setName(rs.getString("COLUMN_NAME")); String type_name = rs.getString("TYPE_NAME"); - if ( type_name.equalsIgnoreCase("serial")){ - columnDef.setTypeName("int4"); - } - else { - columnDef.setTypeName(type_name); - } + switch( type_name.toLowerCase() ) { + case "serial": + columnDef.setTypeName("int4"); + break; + case "bigserial": + columnDef.setTypeName("int8"); + break; + default: + columnDef.setTypeName(type_name); + break; + } + columnDef.setSqlType(rs.getInt("DATA_TYPE")); - columnDef.setSize(rs.getInt("COLUMN_SIZE")); + columnDef.setSize( Integer.MAX_VALUE == rs.getInt("COLUMN_SIZE") ? 0 : rs.getInt("COLUMN_SIZE") ); //if a column is maxed don't specify the size columnDef.setDecimalSize(rs.getInt("DECIMAL_DIGITS")); columnDef.setSourceMeta(columnMetaData); @@ -159,8 +166,10 @@ public List getColumns (String tableName){ throw new RuntimeException("Missing DATA_TYPE definition for data type " + columnDef.getTypeName()); } columns.add(columnDef); + } + conn.close(); } catch (SQLException e) { throw Throwables.propagate(e); diff --git a/src/main/java/net/certifi/audittablegen/TableDef.java b/src/main/java/net/certifi/audittablegen/TableDef.java index 61afe8e..17cb0ae 100644 --- a/src/main/java/net/certifi/audittablegen/TableDef.java +++ b/src/main/java/net/certifi/audittablegen/TableDef.java @@ -63,5 +63,18 @@ public List getColumns() { public void setColumns(List columns) { this.columns = columns; } - + + public ColumnDef getPrimaryKey() { + + for ( ColumnDef colDef : columns ) { + + if ( "YES".equals( colDef.sourceMeta.get( "is_autoincrement" ) ) ) { + + return colDef; + } + + } + + return null; + } }