Initial commit for Zipkin tracing tags.

evcache-zipkin-tracing/build.gradle

@@ -0,0 +1,23 @@
+apply plugin: 'java'
+
+sourceSets.main.java.srcDir 'src/main/java'
+sourceSets.test.java.srcDir 'src/test/java'
+
+dependencies {
+    compile project(':evcache-core')
+    compile group:"io.zipkin.brave", name:"brave", version:"latest.release"
+    testCompile group:"org.testng", name:"testng", version:"7.+"
+    testCompile group:"org.mockito", name:"mockito-all", version:"latest.release"
+}
+
+javadoc {
+    failOnError = false
+}
+
+test {
+    useTestNG()
+    minHeapSize = '1024m'
+    maxHeapSize = '1536m'
+    testLogging.displayGranularity = -1
+    testLogging.showStandardStreams = true
+}

evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingEventListener.java

@@ -0,0 +1,161 @@
+package com.netflix.evcache;
+
+import brave.Span;
+import brave.Tracer;
+import com.netflix.evcache.event.EVCacheEvent;
+import com.netflix.evcache.event.EVCacheEventListener;
+import com.netflix.evcache.pool.EVCacheClient;
+import com.netflix.evcache.pool.EVCacheClientPoolManager;
+import net.spy.memcached.CachedData;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.StringJoiner;
+
+/** Adds tracing tags for EvCache calls. */
+public class EVCacheTracingEventListener implements EVCacheEventListener {
+
+  public static String EVCACHE_SPAN_NAME = "evcache";
+
+  private static Logger logger = LoggerFactory.getLogger(EVCacheTracingEventListener.class);
+
+  private static String CLIENT_SPAN_ATTRIBUTE_KEY = "clientSpanAttributeKey";
+
+  private final Tracer tracer;
+
+  public EVCacheTracingEventListener(EVCacheClientPoolManager poolManager, Tracer tracer) {
+    poolManager.addEVCacheEventListener(this);
+    this.tracer = tracer;
+  }
+
+  @Override
+  public void onStart(EVCacheEvent e) {
+    try {
+      Span clientSpan =
+          this.tracer.nextSpan().kind(Span.Kind.CLIENT).name(EVCACHE_SPAN_NAME).start(e.getStartTime());
+
+      String appName = e.getAppName();
+      this.safeTag(clientSpan, EVCacheTracingTags.APP_NAME, appName);
+
+      String cacheNamePrefix = e.getCacheName();
+      this.safeTag(clientSpan, EVCacheTracingTags.CACHE_NAME_PREFIX, cacheNamePrefix);
+
+      String call = e.getCall().name();
+      this.safeTag(clientSpan, EVCacheTracingTags.CALL, call);
+
+      /**
+       * Note - e.getClients() returns a list of clients associated with the EVCacheEvent.
+       *
+       * <p>Read operation will have only 1 EVCacheClient as reading from just 1 instance of cache
+       * is sufficient. Write operations will have appropriate number of clients as each client will
+       * attempt to write to its cache instance.
+       */
+      String serverGroup;
+      StringJoiner serverGroups = new StringJoiner(",", "[", "]");
+      for (EVCacheClient client : e.getClients()) {
+        serverGroup = client.getServerGroupName();
+        if (StringUtils.isNotBlank(serverGroup)) {
+          serverGroups.add("\"" + serverGroup + "\"");
+        }
+      }
+      clientSpan.tag(EVCacheTracingTags.SERVER_GROUPS, serverGroups.toString());
+
+      /**
+       * Note - EvCache client creates a hash key if the given canonical key size exceeds 255
+       * characters.
+       *
+       * <p>There have been cases where canonical key size exceeded few megabytes. As caching client
+       * creates a hash of such canonical keys and optimizes the storage in the cache servers, it is
+       * safe to annotate hash key instead of canonical key in such cases.
+       */
+      String hashKey;
+      for (EVCacheKey keyObj : e.getEVCacheKeys()) {
+        hashKey = keyObj.getHashKey();
+        if (StringUtils.isNotBlank(hashKey)) {
+          clientSpan.tag(EVCacheTracingTags.HASH_KEY, hashKey);
+        } else {
+          this.safeTag(clientSpan, EVCacheTracingTags.CANONICAL_KEY, keyObj.getCanonicalKey());
+        }
+      }
+
+      /**
+       * Note - tracer.spanInScope(...) method stores Spans in the thread local object.
+       *
+       * <p>As EVCache write operations are asynchronous and quorum based, we are avoiding attaching
+       * clientSpan with tracer.spanInScope(...) method. Instead, we are storing the clientSpan as
+       * an object in the EVCacheEvent's attributes.
+       */
+      e.setAttribute(CLIENT_SPAN_ATTRIBUTE_KEY, clientSpan);
+    } catch (Exception exception) {
+      logger.error("onStart exception", exception);
+    }
+  }
+
+  @Override
+  public void onComplete(EVCacheEvent e) {
+    try {
+      this.onFinishHelper(e, null);
+    } catch (Exception exception) {
+      logger.error("onComplete exception", exception);
+    }
+  }
+
+  @Override
+  public void onError(EVCacheEvent e, Throwable t) {
+    try {
+      this.onFinishHelper(e, t);
+    } catch (Exception exception) {
+      logger.error("onError exception", exception);
+    }
+  }
+
+  /**
+   * On throttle is not a trace event, but it is used to decide whether to throttle. We don't want
+   * to interfere so always return false.
+   */
+  @Override
+  public boolean onThrottle(EVCacheEvent e) throws EVCacheException {
+    return false;
+  }
+
+  private void onFinishHelper(EVCacheEvent e, Throwable t) {
+    Object clientSpanObj = e.getAttribute(CLIENT_SPAN_ATTRIBUTE_KEY);
+
+    // Return if the previously saved Client Span is null
+    if (clientSpanObj == null) {
+      return;
+    }
+
+    Span clientSpan = (Span) clientSpanObj;
+
+    try {
+      if (t != null) {
+        this.safeTag(clientSpan, EVCacheTracingTags.ERROR, t.toString());
+      }
+
+      String status = e.getStatus();
+      this.safeTag(clientSpan, EVCacheTracingTags.STATUS, status);
+
+      long latency = e.getDurationInMillis();
+      clientSpan.tag(EVCacheTracingTags.LATENCY, String.valueOf(latency));
+
+      int ttl = e.getTTL();
+      clientSpan.tag(EVCacheTracingTags.DATA_TTL, String.valueOf(ttl));
+
+      CachedData cachedData = e.getCachedData();
+      if (cachedData != null) {
+        int cachedDataSize = cachedData.getData().length;
+        clientSpan.tag(EVCacheTracingTags.DATA_SIZE, String.valueOf(cachedDataSize));
+      }
+    } finally {
+      clientSpan.finish();
+    }
+  }
+
+  private void safeTag(Span span, String key, String value) {
+    if (StringUtils.isNotBlank(value)) {
+      span.tag(key, value);
+    }
+  }
+}

evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java

@@ -0,0 +1,15 @@
+package com.netflix.evcache;
+
+public class EVCacheTracingTags {
+    public static String CACHE_NAME_PREFIX = "evcache.cache_name_prefix";
+    public static String APP_NAME = "evcache.app_name";
+    public static String STATUS = "evcache.status";
+    public static String LATENCY = "evcache.latency";
+    public static String CALL = "evcache.call";
+    public static String SERVER_GROUPS = "evcache.server_groups";
+    public static String HASH_KEY = "evcache.hash_key";
+    public static String CANONICAL_KEY = "evcache.canonical_key";
+    public static String DATA_TTL = "evcache.data_ttl";
+    public static String DATA_SIZE = "evcache.data_size";
+    public static String ERROR = "evcache.error";
+}

evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java

@@ -0,0 +1,126 @@
+package com.netflix.evcache;
+
+import brave.Tracing;
+import com.netflix.evcache.event.EVCacheEvent;
+import com.netflix.evcache.pool.EVCacheClient;
+import com.netflix.evcache.pool.EVCacheClientPoolManager;
+import net.spy.memcached.CachedData;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import zipkin2.Span;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+public class EVCacheTracingEventListenerUnitTests {
+
+  List<zipkin2.Span> reportedSpans;
+  EVCacheTracingEventListener tracingListener;
+  EVCacheClient mockEVCacheClient;
+  EVCacheEvent mockEVCacheEvent;
+
+  @BeforeMethod
+  public void resetMocks() {
+    mockEVCacheClient = mock(EVCacheClient.class);
+    when(mockEVCacheClient.getServerGroupName()).thenReturn("dummyServerGroupName");
+
+    mockEVCacheEvent = mock(EVCacheEvent.class);
+
+    when(mockEVCacheEvent.getClients()).thenReturn(Arrays.asList(mockEVCacheClient));
+    when(mockEVCacheEvent.getCall()).thenReturn(EVCache.Call.GET);
+
+    when(mockEVCacheEvent.getAppName()).thenReturn("dummyAppName");
+    when(mockEVCacheEvent.getCacheName()).thenReturn("dummyCacheName");
+    when(mockEVCacheEvent.getEVCacheKeys())
+        .thenReturn(Arrays.asList(new EVCacheKey("dummyKey", "dummyCanonicalKey", null)));
+    when(mockEVCacheEvent.getStatus()).thenReturn("success");
+    when(mockEVCacheEvent.getDurationInMillis()).thenReturn(1L);
+    when(mockEVCacheEvent.getTTL()).thenReturn(0);
+    when(mockEVCacheEvent.getCachedData())
+        .thenReturn(new CachedData(1, "dummyData".getBytes(), 255));
+
+    Map<String, Object> eventAttributes = new HashMap<>();
+    doAnswer(
+            new Answer<Void>() {
+              @Override
+              public Void answer(InvocationOnMock invocation) throws Throwable {
+                Object[] arguments = invocation.getArguments();
+                String key = (String) arguments[0];
+                Object value = arguments[1];
+                eventAttributes.put(key, value);
+                return null;
+              }
+            })
+        .when(mockEVCacheEvent)
+        .setAttribute(any(), any());
+
+    doAnswer(
+            new Answer<Object>() {
+              @Override
+              public Object answer(InvocationOnMock invocation) throws Throwable {
+                Object[] arguments = invocation.getArguments();
+                String key = (String) arguments[0];
+                return eventAttributes.get(key);
+              }
+            })
+        .when(mockEVCacheEvent)
+        .getAttribute(any());
+
+    reportedSpans = new ArrayList<>();
+    Tracing tracing = Tracing.newBuilder().spanReporter(reportedSpans::add).build();
+
+    tracingListener =
+        new EVCacheTracingEventListener(mock(EVCacheClientPoolManager.class), tracing.tracer());
+  }
+
+  public void verifyCommonTags(List<zipkin2.Span> spans) {
+    Assert.assertEquals(spans.size(), 1, "Number of expected spans are not matching");
+    zipkin2.Span span = spans.get(0);
+
+    Assert.assertEquals(span.kind(), Span.Kind.CLIENT, "Span Kind are not equal");
+    Assert.assertEquals(
+        span.name(), EVCacheTracingEventListener.EVCACHE_SPAN_NAME, "Cache name are not equal");
+
+    Map<String, String> tags = span.tags();
+    tags.containsKey(EVCacheTracingTags.APP_NAME);
+    tags.containsKey(EVCacheTracingTags.CACHE_NAME_PREFIX);
+    tags.containsKey(EVCacheTracingTags.CALL);
+    tags.containsKey(EVCacheTracingTags.SERVER_GROUPS);
+    tags.containsKey(EVCacheTracingTags.CANONICAL_KEY);
+    tags.containsKey(EVCacheTracingTags.STATUS);
+    tags.containsKey(EVCacheTracingTags.LATENCY);
+    tags.containsKey(EVCacheTracingTags.DATA_TTL);
+    tags.containsKey(EVCacheTracingTags.DATA_SIZE);
+  }
+
+  public void verifyErrorTags(List<zipkin2.Span> spans) {
+    zipkin2.Span span = spans.get(0);
+    Map<String, String> tags = span.tags();
+    tags.containsKey(EVCacheTracingTags.ERROR);
+  }
+
+  @Test
+  public void testEVCacheListenerOnComplete() {
+    tracingListener.onStart(mockEVCacheEvent);
+    tracingListener.onComplete(mockEVCacheEvent);
+
+    verifyCommonTags(reportedSpans);
+  }
+
+  @Test
+  public void testEVCacheListenerOnError() {
+    tracingListener.onStart(mockEVCacheEvent);
+    tracingListener.onError(mockEVCacheEvent, new RuntimeException("Unexpected Error"));
+
+    verifyCommonTags(reportedSpans);
+    verifyErrorTags(reportedSpans);
+  }
+}

settings.gradle

@@ -2,3 +2,4 @@ rootProject.name='EVCache'
 include 'evcache-core'
 include 'evcache-client'
 include 'evcache-client-sample'
+include 'evcache-zipkin-tracing'