Read operation will have only 1 EVCacheClient as reading from just 1 instance of cache
+ * is sufficient. Write operations will have appropriate number of clients as each client will
+ * attempt to write to its cache instance.
+ */
+ String serverGroup;
+ StringJoiner serverGroups = new StringJoiner(",", "[", "]");
+ for (EVCacheClient client : e.getClients()) {
+ serverGroup = client.getServerGroupName();
+ if (StringUtils.isNotBlank(serverGroup)) {
+ serverGroups.add("\"" + serverGroup + "\"");
+ }
+ }
+ clientSpan.tag(EVCacheTracingTags.SERVER_GROUPS, serverGroups.toString());
+
+ /**
+ * Note - EvCache client creates a hash key if the given canonical key size exceeds 255
+ * characters.
+ *
+ *
There have been cases where canonical key size exceeded few megabytes. As caching client
+ * creates a hash of such canonical keys and optimizes the storage in the cache servers, it is
+ * safe to annotate hash key instead of canonical key in such cases.
+ */
+ String hashKey;
+ for (EVCacheKey keyObj : e.getEVCacheKeys()) {
+ hashKey = keyObj.getHashKey();
+ if (StringUtils.isNotBlank(hashKey)) {
+ clientSpan.tag(EVCacheTracingTags.HASH_KEY, hashKey);
+ } else {
+ this.safeTag(clientSpan, EVCacheTracingTags.CANONICAL_KEY, keyObj.getCanonicalKey());
+ }
+ }
+
+ /**
+ * Note - tracer.spanInScope(...) method stores Spans in the thread local object.
+ *
+ *
As EVCache write operations are asynchronous and quorum based, we are avoiding attaching
+ * clientSpan with tracer.spanInScope(...) method. Instead, we are storing the clientSpan as
+ * an object in the EVCacheEvent's attributes.
+ */
+ e.setAttribute(CLIENT_SPAN_ATTRIBUTE_KEY, clientSpan);
+ } catch (Exception exception) {
+ logger.error("onStart exception", exception);
+ }
+ }
+
+ @Override
+ public void onComplete(EVCacheEvent e) {
+ try {
+ this.onFinishHelper(e, null);
+ } catch (Exception exception) {
+ logger.error("onComplete exception", exception);
+ }
+ }
+
+ @Override
+ public void onError(EVCacheEvent e, Throwable t) {
+ try {
+ this.onFinishHelper(e, t);
+ } catch (Exception exception) {
+ logger.error("onError exception", exception);
+ }
+ }
+
+ /**
+ * On throttle is not a trace event, but it is used to decide whether to throttle. We don't want
+ * to interfere so always return false.
+ */
+ @Override
+ public boolean onThrottle(EVCacheEvent e) throws EVCacheException {
+ return false;
+ }
+
+ private void onFinishHelper(EVCacheEvent e, Throwable t) {
+ Object clientSpanObj = e.getAttribute(CLIENT_SPAN_ATTRIBUTE_KEY);
+
+ // Return if the previously saved Client Span is null
+ if (clientSpanObj == null) {
+ return;
+ }
+
+ Span clientSpan = (Span) clientSpanObj;
+
+ try {
+ if (t != null) {
+ this.safeTag(clientSpan, EVCacheTracingTags.ERROR, t.toString());
+ }
+
+ String status = e.getStatus();
+ this.safeTag(clientSpan, EVCacheTracingTags.STATUS, status);
+
+ long latency = e.getDurationInMillis();
+ clientSpan.tag(EVCacheTracingTags.LATENCY, String.valueOf(latency));
+
+ int ttl = e.getTTL();
+ clientSpan.tag(EVCacheTracingTags.DATA_TTL, String.valueOf(ttl));
+
+ CachedData cachedData = e.getCachedData();
+ if (cachedData != null) {
+ int cachedDataSize = cachedData.getData().length;
+ clientSpan.tag(EVCacheTracingTags.DATA_SIZE, String.valueOf(cachedDataSize));
+ }
+ } finally {
+ clientSpan.finish();
+ }
+ }
+
+ private void safeTag(Span span, String key, String value) {
+ if (StringUtils.isNotBlank(value)) {
+ span.tag(key, value);
+ }
+ }
+}
diff --git a/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java b/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java
new file mode 100644
index 00000000..646a0062
--- /dev/null
+++ b/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java
@@ -0,0 +1,15 @@
+package com.netflix.evcache;
+
+public class EVCacheTracingTags {
+ public static String CACHE_NAME_PREFIX = "evcache.cache_name_prefix";
+ public static String APP_NAME = "evcache.app_name";
+ public static String STATUS = "evcache.status";
+ public static String LATENCY = "evcache.latency";
+ public static String CALL = "evcache.call";
+ public static String SERVER_GROUPS = "evcache.server_groups";
+ public static String HASH_KEY = "evcache.hash_key";
+ public static String CANONICAL_KEY = "evcache.canonical_key";
+ public static String DATA_TTL = "evcache.data_ttl";
+ public static String DATA_SIZE = "evcache.data_size";
+ public static String ERROR = "evcache.error";
+}
diff --git a/evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java b/evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java
new file mode 100644
index 00000000..8554c86d
--- /dev/null
+++ b/evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java
@@ -0,0 +1,126 @@
+package com.netflix.evcache;
+
+import brave.Tracing;
+import com.netflix.evcache.event.EVCacheEvent;
+import com.netflix.evcache.pool.EVCacheClient;
+import com.netflix.evcache.pool.EVCacheClientPoolManager;
+import net.spy.memcached.CachedData;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import zipkin2.Span;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+public class EVCacheTracingEventListenerUnitTests {
+
+ List reportedSpans;
+ EVCacheTracingEventListener tracingListener;
+ EVCacheClient mockEVCacheClient;
+ EVCacheEvent mockEVCacheEvent;
+
+ @BeforeMethod
+ public void resetMocks() {
+ mockEVCacheClient = mock(EVCacheClient.class);
+ when(mockEVCacheClient.getServerGroupName()).thenReturn("dummyServerGroupName");
+
+ mockEVCacheEvent = mock(EVCacheEvent.class);
+
+ when(mockEVCacheEvent.getClients()).thenReturn(Arrays.asList(mockEVCacheClient));
+ when(mockEVCacheEvent.getCall()).thenReturn(EVCache.Call.GET);
+
+ when(mockEVCacheEvent.getAppName()).thenReturn("dummyAppName");
+ when(mockEVCacheEvent.getCacheName()).thenReturn("dummyCacheName");
+ when(mockEVCacheEvent.getEVCacheKeys())
+ .thenReturn(Arrays.asList(new EVCacheKey("dummyKey", "dummyCanonicalKey", null)));
+ when(mockEVCacheEvent.getStatus()).thenReturn("success");
+ when(mockEVCacheEvent.getDurationInMillis()).thenReturn(1L);
+ when(mockEVCacheEvent.getTTL()).thenReturn(0);
+ when(mockEVCacheEvent.getCachedData())
+ .thenReturn(new CachedData(1, "dummyData".getBytes(), 255));
+
+ Map eventAttributes = new HashMap<>();
+ doAnswer(
+ new Answer() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] arguments = invocation.getArguments();
+ String key = (String) arguments[0];
+ Object value = arguments[1];
+ eventAttributes.put(key, value);
+ return null;
+ }
+ })
+ .when(mockEVCacheEvent)
+ .setAttribute(any(), any());
+
+ doAnswer(
+ new Answer