Upgrading libraries, mainly the JSqlParser, to support more queries out of the box.

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres
 to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.15.0] - 2024-01-25
+
+* Upgrading libraries, mainly the JSqlParser, to support more queries out of the box.
+
 ## [2.14.0] - 2023-09-26
 
 ### Added

build.common.gradle

@@ -126,8 +126,8 @@ test {
 }
 
 tasks.findAll { it.name.startsWith("spotbugs") }*.configure {
-    effort = "max"
     excludeFilter = file('../spotbugs-exclude.xml')
+
     reports {
         xml.required = true
         html.required = true

build.gradle

@@ -1,11 +1,13 @@
+import com.github.spotbugs.snom.Confidence
+import com.github.spotbugs.snom.Effort
 import org.eclipse.jgit.api.errors.RefAlreadyExistsException
 
 plugins {
-    id "com.github.spotbugs" version "5.0.13" apply false
+    id "com.github.spotbugs" version "6.0.7"
     id "idea"
-    id 'org.ajoberstar.grgit' version '5.0.0'
-    id 'io.github.gradle-nexus.publish-plugin' version "1.1.0"
-    id 'com.github.johnrengelman.shadow' version '7.0.0' apply false
+    id 'org.ajoberstar.grgit' version '5.2.1'
+    id 'io.github.gradle-nexus.publish-plugin' version "1.3.0"
+    id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
 }
 
 idea.project {
@@ -39,3 +41,8 @@ nexusPublishing {
         }
     }
 }
+
+spotbugs {
+    effort = Effort.valueOf('MAX')
+    reportLevel = Confidence.valueOf('DEFAULT')
+}

build.libraries.gradle

@@ -3,18 +3,18 @@ ext {
 
     libraries = [
             // explicit versions
-            guava                           : "com.google.guava:guava:31.1-jre",
-            jsqlParser                      : "com.github.jsqlparser:jsqlparser:4.6",
+            guava                       : "com.google.guava:guava:33.0.0-jre",
+            jsqlParser                  : "com.github.jsqlparser:jsqlparser:4.8",
             spotbugsAnnotations             : "com.github.spotbugs:spotbugs-annotations:${spotbugs.toolVersion.get()}",
             springBootDependencies          : "org.springframework.boot:spring-boot-dependencies:${springBootVersion}",
-            testContainersMariaDb           : "org.testcontainers:mariadb:1.17.6",
-            twBaseUtils                     : "com.transferwise.common:tw-base-utils:1.10.1",
+            testContainersMariaDb       : "org.testcontainers:mariadb:1.19.3",
+            twBaseUtils                 : "com.transferwise.common:tw-base-utils:1.12.3",
             twContext                       : "com.transferwise.common:tw-context:1.0.0",
             twContextStarter                : "com.transferwise.common:tw-context-starter:1.0.0",
-            twGracefulShutdown              : 'com.transferwise.common:tw-graceful-shutdown:2.11.0',
-            twGracefulShutdownInterfaces    : 'com.transferwise.common:tw-graceful-shutdown-interfaces:2.11.0',
-            twSpyqlCore                     : "com.transferwise.common:tw-spyql-core:1.6.0",
-            twSpyqlStarter                  : "com.transferwise.common:tw-spyql-starter:1.6.0",
+            twGracefulShutdown          : 'com.transferwise.common:tw-graceful-shutdown:2.14.2',
+            twGracefulShutdownInterfaces: 'com.transferwise.common:tw-graceful-shutdown-interfaces:2.14.2',
+            twSpyqlCore                 : "com.transferwise.common:tw-spyql-core:1.6.1",
+            twSpyqlStarter              : "com.transferwise.common:tw-spyql-starter:1.6.1",
 
             // versions managed by spring-boot-dependencies platform
             caffeine                        : "com.github.ben-manes.caffeine:caffeine",
@@ -24,6 +24,7 @@ ext {
 
             hikariCp                        : "com.zaxxer:HikariCP",
             junitJupiter                    : "org.junit.jupiter:junit-jupiter",
+            logbackClassic              : "ch.qos.logback:logback-classic",
             lombok                          : "org.projectlombok:lombok",
             mariadbJavaClient               : "org.mariadb.jdbc:mariadb-java-client",
             micrometerCore                  : "io.micrometer:micrometer-core",

gradle.properties

@@ -1 +1 @@
-version=2.14.0
+version=2.15.0

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -85,9 +85,6 @@ done
 APP_BASE_NAME=${0##*/}
 APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
 
@@ -197,6 +194,10 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
 # Collect all arguments for the java command;
 #   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
 #     shell script including quotes and variable substitutions, so put them in

tw-entrypoints-starter/src/main/java/com/transferwise/common/entrypoints/tableaccessstatistics/TasFlywayConfigurationCustomizer.java

@@ -34,7 +34,7 @@ public boolean canHandleInTransaction(Event event, Context context) {
 
       @Override
       public void handle(Event event, Context context) {
-        if (event == Event.BEFORE_MIGRATE) {
+        if (event == Event.BEFORE_VALIDATE) {
           log.info("Disabling TAS query parsing before Flyway migration.");
 
           var twContext = TwContext.current().createSubContext().asEntryPoint("Flyway", "Flyway");

tw-entrypoints-starter/src/test/java/com/transferwise/common/entrypoints/test/BaseIntTest.java

@@ -39,7 +39,7 @@ public void setup() {
     meterRegistry.getMeters().stream()
         .filter(m -> (m.getId().getName().startsWith("EntryPoints") || m.getId().getName().startsWith("database")) && !(m instanceof Gauge))
         .forEach(m -> {
-          log.info("Removing metric: " + m.getId());
+          log.debug("Removing metric: " + m.getId());
           meterRegistry.remove(m);
         });
     meterCache.clear();

tw-entrypoints/build.gradle

@@ -22,17 +22,8 @@ dependencies {
     implementation libraries.twSpyqlCore
     implementation libraries.twGracefulShutdownInterfaces
 
-    shadow libraries.twContext
-    shadow libraries.commonsLang3
-    shadow libraries.caffeine
-    shadow libraries.guava
-    shadow libraries.micrometerCore
-    shadow libraries.slf4jApi
-    shadow libraries.twBaseUtils
-    shadow libraries.twSpyqlCore
-    shadow libraries.twGracefulShutdownInterfaces
-
     testImplementation libraries.junitJupiter
+    testRuntimeOnly libraries.logbackClassic
 }
 
 apply from: "build-idea-fix.gradle.kts"
@@ -48,10 +39,10 @@ shadowJar {
     manifest {
         attributes 'Implementation-Version': "$project.version"
     }
-    relocate('net.sf.jsqlparser', 'com.transferwise.common.entrypoints.shadow')
+    relocate('net.sf.jsqlparser', 'com.transferwise.common.entrypoints.shadow.net.sf.jsqlparser')
 }
 
-jar.enabled = true
+jar.enabled = false
 jar.dependsOn shadowJar
 
 shadowJar {
@@ -105,8 +96,8 @@ publishing {
                 withXml { xml ->
                     def dependenciesNode = xml.asNode().get('dependencies') ?: xml.asNode().appendNode('dependencies')
 
-                    project.configurations.getByName("shadow").resolvedConfiguration.firstLevelModuleDependencies.forEach {
-                        if (it.configuration != "platform-runtime") {
+                    project.configurations.getByName("runtimeClasspath").resolvedConfiguration.firstLevelModuleDependencies.forEach {
+                        if (it.configuration != "platform-runtime" && it.moduleName != 'jsqlparser') {
                             def dependencyNode = dependenciesNode.appendNode('dependency')
                             dependencyNode.appendNode('groupId', it.moduleGroup)
                             dependencyNode.appendNode('artifactId', it.moduleName)

tw-entrypoints/src/main/java/com/transferwise/common/entrypoints/databaseaccessstatistics/DasUnknownCallsCollector.java

@@ -49,14 +49,14 @@ protected void registerUnknownCalls() {
       final long fetchedRows = das.getAndResetFetchedRowsCount();
 
       TagsSet tagsSet = das.getTagsSet();
-      UnknownCallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "unknownCallMetrics", tagsSet, () -> {
+      UnknownCallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "unknownCallMetrics", tagsSet, (name, tags) -> {
         UnknownCallMeters result = new UnknownCallMeters();
-        result.commits = meterCache.counter(COUNTER_UNREGISTERED_COMMITS, tagsSet);
-        result.rollbacks = meterCache.counter(COUNTER_UNREGISTERED_ROLLBACKS, tagsSet);
-        result.nonTransactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_NT_QUERIES, tagsSet);
-        result.transactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_T_QUERIES, tagsSet);
-        result.timeTakenNs = meterCache.counter(COUNTER_UNREGISTERED_TIME_TAKEN_NS, tagsSet);
-        result.emptyTransactions = meterCache.counter(COUNTER_UNREGISTERED_EMPTY_TRANSACTIONS, tagsSet);
+        result.commits = meterCache.counter(COUNTER_UNREGISTERED_COMMITS, tags);
+        result.rollbacks = meterCache.counter(COUNTER_UNREGISTERED_ROLLBACKS, tags);
+        result.nonTransactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_NT_QUERIES, tags);
+        result.transactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_T_QUERIES, tags);
+        result.timeTakenNs = meterCache.counter(COUNTER_UNREGISTERED_TIME_TAKEN_NS, tags);
+        result.emptyTransactions = meterCache.counter(COUNTER_UNREGISTERED_EMPTY_TRANSACTIONS, tags);
         result.affectedRows = meterCache.counter(COUNTER_UNREGISTERED_AFFECTED_ROWS, tagsSet);
         result.fetchedRows = meterCache.counter(COUNTER_UNREGISTERED_FETCHED_ROWS, tagsSet);
         return result;

tw-entrypoints/src/main/java/com/transferwise/common/entrypoints/databaseaccessstatistics/DatabaseAccessStatisticsEntryPointInterceptor.java

@@ -71,18 +71,18 @@ private void registerCall(TwContext context, Map<String, DatabaseAccessStatistic
       final long timeTakenInDatabaseNs = das.getTimeTakenInDatabaseNs();
       final long fetchedRowsCount = das.getFetchedRowsCount();
 
-      CallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "callMetrics", tagsSet, () -> {
+      CallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "callMetrics", tagsSet, (name, tags) -> {
         CallMeters result = new CallMeters();
-        result.commits = meterCache.summary(SUMMARY_REGISTERED_COMMITS, tagsSet);
-        result.rollbacks = meterCache.summary(SUMMARY_REGISTERED_ROLLBACKS, tagsSet);
-        result.nonTransactionalQueries = meterCache.summary(SUMMARY_REGISTERED_NT_QUERIES, tagsSet);
-        result.transactionalQueries = meterCache.summary(SUMMARY_REGISTERED_T_QUERIES, tagsSet);
-        result.maxConcurrentConnections = meterCache.summary(SUMMARY_REGISTERED_MAX_CONCURRENT_CONNECTIONS, tagsSet);
-        result.remainingOpenConnections = meterCache.summary(SUMMARY_REGISTERED_REMAINING_OPEN_CONNECTIONS, tagsSet);
-        result.emptyTransactions = meterCache.summary(SUMMARY_REGISTERED_EMPTY_TRANSACTIONS, tagsSet);
-        result.affectedRows = meterCache.summary(SUMMARY_REGISTERED_AFFECTED_ROWS, tagsSet);
-        result.fetchedRows = meterCache.summary(SUMMARY_REGISTERED_FETCHED_ROWS, tagsSet);
-        result.timeTakenNs = meterCache.timer(TIMER_REGISTERED_TIME_TAKEN, tagsSet);
+        result.commits = meterCache.summary(SUMMARY_REGISTERED_COMMITS, tags);
+        result.rollbacks = meterCache.summary(SUMMARY_REGISTERED_ROLLBACKS, tags);
+        result.nonTransactionalQueries = meterCache.summary(SUMMARY_REGISTERED_NT_QUERIES, tags);
+        result.transactionalQueries = meterCache.summary(SUMMARY_REGISTERED_T_QUERIES, tags);
+        result.maxConcurrentConnections = meterCache.summary(SUMMARY_REGISTERED_MAX_CONCURRENT_CONNECTIONS, tags);
+        result.remainingOpenConnections = meterCache.summary(SUMMARY_REGISTERED_REMAINING_OPEN_CONNECTIONS, tags);
+        result.emptyTransactions = meterCache.summary(SUMMARY_REGISTERED_EMPTY_TRANSACTIONS, tags);
+        result.affectedRows = meterCache.summary(SUMMARY_REGISTERED_AFFECTED_ROWS, tags);
+        result.fetchedRows = meterCache.summary(SUMMARY_REGISTERED_FETCHED_ROWS, tags);
+        result.timeTakenNs = meterCache.timer(TIMER_REGISTERED_TIME_TAKEN, tags);
         return result;
       });
 

tw-entrypoints/src/main/java/com/transferwise/common/entrypoints/tableaccessstatistics/CustomTablesNamesFinder.java

@@ -1,56 +1,33 @@
 package com.transferwise.common.entrypoints.tableaccessstatistics;
 
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import lombok.Getter;
 import net.sf.jsqlparser.schema.Table;
-import net.sf.jsqlparser.statement.SetStatement;
-import net.sf.jsqlparser.statement.ShowColumnsStatement;
-import net.sf.jsqlparser.statement.Statements;
-import net.sf.jsqlparser.statement.alter.Alter;
-import net.sf.jsqlparser.statement.create.index.CreateIndex;
-import net.sf.jsqlparser.statement.create.view.CreateView;
-import net.sf.jsqlparser.statement.drop.Drop;
-import net.sf.jsqlparser.statement.execute.Execute;
 import net.sf.jsqlparser.util.TablesNamesFinder;
 
 public class CustomTablesNamesFinder extends TablesNamesFinder {
 
-  @Override
-  protected String extractTableName(Table table) {
-    return table.getName();
-  }
-
-  @Override
-  public void visit(Drop drop) {
-    visit(drop.getName());
-  }
-
-  @Override
-  public void visit(CreateIndex createIndex) {
-    visit(createIndex.getTable());
-  }
-
-  @Override
-  public void visit(CreateView createView) {
-    visit(createView.getView());
-  }
+  @Getter
+  private final List<String> tables = new ArrayList<>();
+  private final Set<String> uniqueTables = new HashSet<>();
 
-  @Override
-  public void visit(Alter alter) {
-    visit(alter.getTable());
-  }
-
-  @Override
-  public void visit(Statements stmts) {
-  }
+  /*
+    The super class loses the order of tables visited, as its tables list is based on Set.
 
+    We are providing here the option to get the ordered tables list.
+   */
   @Override
-  public void visit(Execute execute) {
-  }
+  public void visit(Table tableName) {
+    String tableWholeName = extractTableName(tableName);
 
-  @Override
-  public void visit(SetStatement set) {
-  }
+    if (!uniqueTables.contains(tableWholeName)) {
+      uniqueTables.add(tableWholeName);
+      tables.add(tableWholeName);
+    }
 
-  @Override
-  public void visit(ShowColumnsStatement set) {
+    super.visit(tableName);
   }
 }

tw-entrypoints/src/main/java/com/transferwise/common/entrypoints/tableaccessstatistics/SqlParser.java

@@ -4,64 +4,24 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParser;
 import net.sf.jsqlparser.parser.StringProvider;
 import net.sf.jsqlparser.statement.Statements;
 
 public class SqlParser {
 
-  /*
-    DATABASE is reserved keyword in sql parser, so having DATABASE() in sql will just throw error.
-    DATABASE() is however used by some of our internal libraries for MariaDb.
-   */
-  private static final Pattern FUNCTION_REPLACEMENT_PATTERN = Pattern.compile("DATABASE()",
-      Pattern.LITERAL | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
-
-  private static final Pattern ON_CONFLICT_REPLACEMENT_PATTERN = Pattern.compile("on\\s*conflict\\s*\\(.*?\\)",
-      Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.DOTALL);
   private final ExecutorService executorService;
 
   public SqlParser(ExecutorService executorService) {
     this.executorService = executorService;
   }
 
   public Statements parse(String sql, Duration timeout) throws JSQLParserException {
-    sql = replaceFunctions(sql);
-    sql = replaceOnConflicts(sql);
-
     CCJSqlParser parser = newParser(sql).withAllowComplexParsing(true);
     return parseStatementAsync(parser, timeout);
   }
 
-  // Sqlparser 4.6 does not support "on conflict" clause with multiple parameters.
-  // As a workaround, we change the query to a single parameter clause.
-  protected String replaceOnConflicts(String sql) {
-    var matcher = ON_CONFLICT_REPLACEMENT_PATTERN.matcher(sql);
-
-    // 99.99% of sqls don't have it, so let's avoid new string creation for those.
-    if (matcher.find()) {
-      matcher.reset();
-      return matcher.replaceAll(Matcher.quoteReplacement("on conflict (blah)"));
-    }
-
-    return sql;
-  }
-
-  protected String replaceFunctions(String sql) {
-    var matcher = FUNCTION_REPLACEMENT_PATTERN.matcher(sql);
-
-    // 99.99% of sqls don't have it, so let's avoid new string creation for those.
-    if (matcher.find()) {
-      matcher.reset();
-      return matcher.replaceAll(Matcher.quoteReplacement("UNSUPPORTED()"));
-    }
-
-    return sql;
-  }
-
   protected Statements parseStatementAsync(CCJSqlParser parser, Duration timeout) throws JSQLParserException {
     try {
       var future = executorService.submit(parser::Statements);