[NYS2AWS-134] added metrics

xenit-eu · Feb 10, 2025 · 8562088 · 8562088
1 parent 85707a9
commit 8562088
Show file tree

Hide file tree

Showing 11 changed files with 811 additions and 7 deletions.
diff --git a/...form/src/main/amp/config/alfresco/subsystems/HealthProcessor/default/indexing-context.xml b/...form/src/main/amp/config/alfresco/subsystems/HealthProcessor/default/indexing-context.xml
@@ -67,5 +67,6 @@
         <constructor-arg name="searchTrackingComponent" ref="searchTrackingComponent" />
         <constructor-arg name="nodeDAO" ref="nodeDAO" />
         <constructor-arg name="dataSource" ref="dataSource" />
+        <constructor-arg name="meterRegistry" ref="meterRegistry" />
     </bean>
 </beans>
diff --git a/...atform/src/main/amp/config/alfresco/subsystems/HealthProcessor/default/plugin-context.xml b/...atform/src/main/amp/config/alfresco/subsystems/HealthProcessor/default/plugin-context.xml
@@ -26,6 +26,7 @@
         <constructor-arg name="properties" ref="global-properties" />
         <constructor-arg name="transactionHelper" ref="eu.xenit.alfresco.healthprocessor.util.AlfrescoTransactionHelper" />
         <constructor-arg name="nodeDAO" ref="nodeDAO" />
+        <constructor-arg name="meterRegistry" ref="meterRegistry" />
     </bean>
 
     <!-- Solr validation -->

diff --git a/...src/main/java/eu/xenit/alfresco/healthprocessor/indexing/IndexingStrategyFactoryBean.java b/...src/main/java/eu/xenit/alfresco/healthprocessor/indexing/IndexingStrategyFactoryBean.java
@@ -7,6 +7,7 @@
 import eu.xenit.alfresco.healthprocessor.indexing.txnid.TxnIdBasedIndexingStrategy;
 import eu.xenit.alfresco.healthprocessor.indexing.txnid.TxnIdIndexingConfiguration;
 import eu.xenit.alfresco.healthprocessor.util.AttributeStore;
+import io.micrometer.core.instrument.MeterRegistry;
 import lombok.AllArgsConstructor;
 import org.alfresco.repo.domain.node.AbstractNodeDAOImpl;
 import org.alfresco.repo.search.SearchTrackingComponent;
@@ -23,6 +24,7 @@ public final class IndexingStrategyFactoryBean extends AbstractFactoryBean<Index
     private final SearchTrackingComponent searchTrackingComponent;
     private final AbstractNodeDAOImpl nodeDAO;
     private final DataSource dataSource;
+    private final MeterRegistry meterRegistry;
 
     @Override
     public Class<?> getObjectType() {
@@ -41,7 +43,8 @@ private IndexingStrategy createIndexingStrategy(IndexingStrategy.IndexingStrateg
             case LAST_TXNS:
                 return new LastTxnsBasedIndexingStrategy((LastTxnsIndexingConfiguration) configuration, trackingComponent);
             case THRESHOLD:
-                return new ThresholdIndexingStrategy((ThresholdIndexingStrategyConfiguration) configuration, nodeDAO, searchTrackingComponent, dataSource);
+                return new ThresholdIndexingStrategy((ThresholdIndexingStrategyConfiguration) configuration, nodeDAO,
+                        searchTrackingComponent, dataSource, meterRegistry);
             default:
                 throw new IllegalArgumentException("Unknown indexing strategy: "+ indexingStrategy);
         }

diff --git a/.../java/eu/xenit/alfresco/healthprocessor/indexing/threshold/ThresholdIndexingStrategy.java b/.../java/eu/xenit/alfresco/healthprocessor/indexing/threshold/ThresholdIndexingStrategy.java
@@ -4,6 +4,8 @@
 import eu.xenit.alfresco.healthprocessor.indexing.NullCycleProgress;
 import eu.xenit.alfresco.healthprocessor.indexing.SimpleCycleProgress;
 import eu.xenit.alfresco.healthprocessor.reporter.api.CycleProgress;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.binder.MeterBinder;
 import lombok.NonNull;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -21,7 +23,7 @@
 import java.util.function.LongSupplier;
 
 @Slf4j
-public class ThresholdIndexingStrategy implements IndexingStrategy {
+public class ThresholdIndexingStrategy implements IndexingStrategy, MeterBinder {
 
     private final @NonNull ThresholdIndexingStrategyConfiguration configuration;
     private final @NonNull AbstractNodeDAOImpl nodeDAO;
@@ -37,7 +39,8 @@ public class ThresholdIndexingStrategy implements IndexingStrategy {
     public ThresholdIndexingStrategy(@NonNull ThresholdIndexingStrategyConfiguration configuration,
                                      @NonNull AbstractNodeDAOImpl nodeDAO,
                                      @NonNull SearchTrackingComponent searchTrackingComponent,
-                                     @NonNull DataSource dataSource) {
+                                     @NonNull DataSource dataSource,
+                                     @NonNull MeterRegistry meterRegistry) {
         if (configuration.getTransactionsBackgroundWorkers() <= 0)
             throw new IllegalArgumentException(String.format("The amount of background workers must be greater than zero (%d provided).", configuration.getTransactionsBackgroundWorkers()));
 
@@ -52,6 +55,8 @@ public ThresholdIndexingStrategy(@NonNull ThresholdIndexingStrategyConfiguration
         this.transactionIdMergers = new ThresholdIndexingStrategyTransactionIdMerger[configuration.getTransactionsBackgroundWorkers()];
         for (int i = 0; i < configuration.getTransactionsBackgroundWorkers(); i++)
             this.transactionIdMergers[i] = new ThresholdIndexingStrategyTransactionIdMerger(transactionIdFetcher, queuedNodes, configuration, searchTrackingComponent);
+
+        bindTo(meterRegistry);
     }
 
     @Override
@@ -111,4 +116,13 @@ public void onStop() {
     public @NonNull CycleProgress getCycleProgress() {
         return cycleProgress.get();
     }
+
+    @Override
+    public void bindTo(MeterRegistry registry) {
+        state.bindTo(registry);
+        transactionIdFetcher.bindTo(registry);
+
+        registry.gauge("eu.xenit.alfresco.healthprocessor.indexing.threshold.queued-nodes", queuedNodes, BlockingDeque::size);
+        registry.gauge("eu.xenit.alfresco.healthprocessor.indexing.threshold.running-background-threads", runningThreads, value -> value.stream().filter(Thread::isAlive).count());
+    }
 }
diff --git a/.../eu/xenit/alfresco/healthprocessor/indexing/threshold/ThresholdIndexingStrategyState.java b/.../eu/xenit/alfresco/healthprocessor/indexing/threshold/ThresholdIndexingStrategyState.java
@@ -1,5 +1,7 @@
 package eu.xenit.alfresco.healthprocessor.indexing.threshold;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.binder.MeterBinder;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -11,7 +13,7 @@
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
-public class ThresholdIndexingStrategyState {
+public class ThresholdIndexingStrategyState implements MeterBinder {
 
     private long currentTransactionId = -1;
     private long maxTransactionId = -1;
@@ -31,4 +33,11 @@ public void decrementRunningTransactionMergers() {
         runningTransactionMergers--;
     }
 
+    @Override
+    public void bindTo(MeterRegistry registry) {
+        registry.gauge("eu.xenit.alfresco.healthprocessor.indexing.threshold.current-transaction-id", this, ThresholdIndexingStrategyState::getCurrentTransactionId);
+        registry.gauge("eu.xenit.alfresco.healthprocessor.indexing.threshold.max-transaction-id", this, ThresholdIndexingStrategyState::getMaxTransactionId);
+        registry.gauge("eu.xenit.alfresco.healthprocessor.indexing.threshold.running-transaction-mergers", this, ThresholdIndexingStrategyState::getRunningTransactionMergers);
+        registry.gauge("eu.xenit.alfresco.healthprocessor.indexing.threshold.queued-transaction-batches", transactionBatchesQueueSize, AtomicInteger::get);
+    }
 }
diff --git a/...sco/healthprocessor/indexing/threshold/ThresholdIndexingStrategyTransactionIdFetcher.java b/...sco/healthprocessor/indexing/threshold/ThresholdIndexingStrategyTransactionIdFetcher.java
@@ -1,5 +1,7 @@
 package eu.xenit.alfresco.healthprocessor.indexing.threshold;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.binder.MeterBinder;
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.jdbc.core.JdbcTemplate;
@@ -11,7 +13,7 @@
 
 // TODO: add an explanation about why we directly contact the DB.
 @Slf4j
-public class ThresholdIndexingStrategyTransactionIdFetcher implements Runnable {
+public class ThresholdIndexingStrategyTransactionIdFetcher implements Runnable, MeterBinder {
 
     private final static @NonNull String QUERY = "SELECT txn.id as id FROM alf_transaction txn WHERE txn.id BETWEEN %d AND %d ORDER BY txn.id ASC LIMIT %d";
 
@@ -21,6 +23,8 @@ public class ThresholdIndexingStrategyTransactionIdFetcher implements Runnable {
     private final @NonNull ThresholdIndexingStrategyState state;
     private final @NonNull ThresholdIndexingStrategyConfiguration configuration;
 
+    private boolean isRunning = false;
+
     public ThresholdIndexingStrategyTransactionIdFetcher(@NonNull ThresholdIndexingStrategyConfiguration configuration,
                                                          @NonNull DataSource dataSource,
                                                          @NonNull ThresholdIndexingStrategyState state) {
@@ -49,6 +53,7 @@ public ThresholdIndexingStrategyTransactionIdFetcher(@NonNull ThresholdIndexingS
     public void run() {
         log.debug("Starting the ThresholdIndexingStrategyTransactionIdFetcher.");
         try {
+            isRunning = true;
             long currentTransactionId = state.getCurrentTransactionId();
             long maxTransactionId = state.getMaxTransactionId();
             int amountOfTransactionsToFetch = configuration.getTransactionsBackgroundWorkers() * configuration.getTransactionsBatchSize();
@@ -69,6 +74,7 @@ public void run() {
             log.warn("An exception occurred while fetching transactions. Trying to signal the end to the transaction merger(s).", e);
         } finally {
             try {
+                isRunning = false;
                 signalEnd();
             } catch (InterruptedException e) {
                 log.error("The ThresholdIndexingStrategyTransactionIdFetcher has been interrupted while signaling the end to the transaction merger(s). " +
@@ -106,4 +112,8 @@ private void queueTransactions(@NonNull List<@NonNull Long> transactionIDs) thro
         }
     }
 
+    @Override
+    public void bindTo(@NonNull MeterRegistry registry) {
+        registry.gauge("eu.xenit.alfresco.healthprocessor.indexing.threshold.transaction-fetcher.running", this, value -> value.isRunning ? 1 : 0);
+    }
 }
diff --git a/...lfresco/healthprocessor/plugins/solr/SolrUndersizedTransactionsHealthProcessorPlugin.java b/...lfresco/healthprocessor/plugins/solr/SolrUndersizedTransactionsHealthProcessorPlugin.java
@@ -4,6 +4,8 @@
 import eu.xenit.alfresco.healthprocessor.plugins.api.ToggleableHealthProcessorPlugin;
 import eu.xenit.alfresco.healthprocessor.reporter.api.NodeHealthReport;
 import eu.xenit.alfresco.healthprocessor.util.TransactionHelper;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.binder.MeterBinder;
 import lombok.Getter;
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
@@ -24,7 +26,7 @@
 import java.util.stream.Collectors;
 
 @Slf4j
-public class SolrUndersizedTransactionsHealthProcessorPlugin extends ToggleableHealthProcessorPlugin {
+public class SolrUndersizedTransactionsHealthProcessorPlugin extends ToggleableHealthProcessorPlugin implements MeterBinder {
 
     public static final @NonNull String SELECTED_INDEXER_STRATEGY_PROPERTY = "eu.xenit.alfresco.healthprocessor.indexing.strategy";
     private static final @NonNull String MERGER_THREADS_CONFIGURATION_KEY = "merger-threads";
@@ -39,7 +41,8 @@ public class SolrUndersizedTransactionsHealthProcessorPlugin extends ToggleableH
     public SolrUndersizedTransactionsHealthProcessorPlugin(boolean enabled, int mergerThreads,
                                                            @NonNull Properties properties,
                                                            @NonNull TransactionHelper transactionHelper,
-                                                           @NonNull AbstractNodeDAOImpl nodeDAO) {
+                                                           @NonNull AbstractNodeDAOImpl nodeDAO,
+                                                           @NonNull MeterRegistry meterRegistry) {
         super(enabled);
         if (enabled) guaranteeThresholdIndexerIsUsed(properties);
 
@@ -49,6 +52,8 @@ public SolrUndersizedTransactionsHealthProcessorPlugin(boolean enabled, int merg
 
         this.configuration = new HashMap<>(super.getConfiguration());
         this.configuration.put(MERGER_THREADS_CONFIGURATION_KEY, String.valueOf(mergerThreads));
+
+        bindTo(meterRegistry);
     }
 
     @Nonnull
@@ -93,4 +98,8 @@ private static void guaranteeThresholdIndexerIsUsed(@NonNull Properties properti
                     "Please adjust the (%s) property.", expected, property, SELECTED_INDEXER_STRATEGY_PROPERTY));
     }
 
+    @Override
+    public void bindTo(@NonNull MeterRegistry registry) {
+        registry.gauge("eu.xenit.alfresco.healthprocessor.plugin.solr-transaction-merger.merge-queue-size", queuedMergeRequests, AtomicInteger::get);
+    }
 }
diff --git a/prometheus/docker-compose.yaml b/prometheus/docker-compose.yaml
@@ -0,0 +1,121 @@
+# This Docker-compose file contains configuration that is contradictory to the default configuration from the integration tests.
+# To not mess up the integration tests, a version of the Docker-compose file was kept here, that can be used
+# to test the threshold indexing strategy with the solr transaction merger plugin.
+# However, you still need to swap the Docker-compose files in the integration tests to use this one.
+
+version: '3.2'
+services:
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - "../../../../../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana-enterprise
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - '3000:3000'
+
+  alfresco:
+    image: ${DOCKER_IMAGE}
+    ports:
+      - ${COMPOSE_INFLOW_TCP_8080:-8080}
+      - ":8000:8000"
+    environment:
+      - JAVA_OPTS_DEBUG=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:8000
+      - JAVA_XMX=8G
+      - DEBUG=true
+      #      - SOLR_SSL=none
+      - INDEX=solr6
+      - GLOBAL_solr.http.socket.timeout=30000
+      - GLOBAL_solr.http.connection.timeout=30000
+      - GLOBAL_csrf.filter.enabled=false
+      - GLOBAL_messaging.broker.url=failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true
+      - ENABLE_CLUSTERING=true
+
+      - GLOBAL_system.node_table_cleaner.algorithm=V1
+      - GLOBAL_index.tracking.minRecordPurgeAgeDays=0
+
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.indexing.strategy=threshold
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.indexing.threshold.transactions-background-workers=5
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.indexing.threshold.transactions-batch-size=100
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.indexing.threshold.threshold=1000
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.indexing.txn-id.start=-1
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.indexing.txn-id.end=-1
+
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.plugin.solr-transaction-merger.enabled=true
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.plugin.solr-transaction-merger.threads=2
+
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.processing.node-batch-size=1000
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.processing.max-batches-per-second=0
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.plugin.noop.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.reporter.store.max-stored-reports=4000
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.reporter.log.streaming.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.reporter.log.summary.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.reporter.log.progress.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.reporter.alfred-telemetry.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.plugin.content-validation.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.plugin.content-validation.properties=cm:content
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.plugin.solr-index.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.plugin.solr-index.check-transaction=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.fixer.solr-missing-node.enabled=false
+      - GLOBAL_eu.xenit.alfresco.healthprocessor.fixer.solr-duplicate-node.enabled=false
+
+      # Disable this loud logger which sends errors in Alfresco 7.0.
+      - LOG4J_logger.org.alfresco.repo.content.transform.LocalTransformServiceRegistry=OFF
+      # Settings for MQ (is needed!)
+      - GLOBAL_messaging.subsystem.autostart=false
+      - GLOBAL_events.subsystem.autostart=false
+      # Settings for transformation services
+      - GLOBAL_transform.service.enabled=false
+      - GLOBAL_local.transform.service.enabled=false
+      - GLOBAL_legacy.transform.service.enabled=false
+      # Needed to silence transformation routing log spam
+      - GLOBAL_messaging.broker.username=admin
+      - GLOBAL_messaging.broker.password=admin
+      - GLOBAL_messaging.broker.url=vm://localhost?broker.persistent=false
+      # Disable unused services
+      - GLOBAL_cifs.enabled=false
+      - GLOBAL_nfs.enabled=false
+      - GLOBAL_ftp.enabled=false
+      - GLOBAL_system.metadata-query-indexes.ignored=false
+      - GLOBAL_system.workflow.engine.jbpm.enabled=false
+      - GLOBAL_system.workflow.engine.activiti.enabled=false
+      - GLOBAL_system.usages.enabled=false
+      - GLOBAL_replication.enabled=false
+      - GLOBAL_audit.enabled=false
+      - GLOBAL_transferservice.receiver.enabled=false
+      - GLOBAL_home.folder.creation.eager=false
+      - GLOBAL_activities.feed.notifier.enabled=false
+      - GLOBAL_sync.pullJob.enabled=false
+      - GLOBAL_sync.pushJob.enabled=false
+      - GLOBAL_activities.feed.generator.enabled=false
+      - GLOBAL_activities.feed.cleaner.enabled=false
+      - GLOBAL_activities.post.lookup.enabled=false
+      - GLOBAL_activities.post.cleaner.enabled=false
+      - GLOBAL_ooo.enabled=false
+      - GLOBAL_jodconverter.enabled=false
+    depends_on:
+      - postgresql
+  solr:
+    image: docker.io/xenit/alfresco-solr6:2.0
+    environment:
+      - ALFRESCO_HOST=alfresco
+      #      - ALFRESCO_SSL=none
+      - GLOBAL_ALL_alfresco.index.transformContent=false
+      - GLOBAL_solr.suggester.enabled=false
+      - GLOBAL_ALL_alfresco.cron=0/2 * * * * ? *
+
+  postgresql:
+    image: docker.io/xenit/postgres
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_USER=alfresco
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_DB=alfresco
+
+