From 872aad466bb0d9db96d06a6552a7819e03330826 Mon Sep 17 00:00:00 2001 From: Maulik Pandey Date: Thu, 2 Jan 2020 14:37:07 -0800 Subject: [PATCH] Initial commit for Zipkin tracing tags. --- evcache-zipkin-tracing/build.gradle | 23 +++ .../evcache/EVCacheTracingEventListener.java | 161 ++++++++++++++++++ .../netflix/evcache/EVCacheTracingTags.java | 15 ++ .../EVCacheTracingEventListenerUnitTests.java | 126 ++++++++++++++ settings.gradle | 1 + 5 files changed, 326 insertions(+) create mode 100644 evcache-zipkin-tracing/build.gradle create mode 100644 evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingEventListener.java create mode 100644 evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java create mode 100644 evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java diff --git a/evcache-zipkin-tracing/build.gradle b/evcache-zipkin-tracing/build.gradle new file mode 100644 index 00000000..1c1e49aa --- /dev/null +++ b/evcache-zipkin-tracing/build.gradle @@ -0,0 +1,23 @@ +apply plugin: 'java' + +sourceSets.main.java.srcDir 'src/main/java' +sourceSets.test.java.srcDir 'src/test/java' + +dependencies { + compile project(':evcache-core') + compile group:"io.zipkin.brave", name:"brave", version:"latest.release" + testCompile group:"org.testng", name:"testng", version:"7.+" + testCompile group:"org.mockito", name:"mockito-all", version:"latest.release" +} + +javadoc { + failOnError = false +} + +test { + useTestNG() + minHeapSize = '1024m' + maxHeapSize = '1536m' + testLogging.displayGranularity = -1 + testLogging.showStandardStreams = true +} \ No newline at end of file diff --git a/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingEventListener.java b/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingEventListener.java new file mode 100644 index 00000000..927e85ba --- /dev/null +++ b/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingEventListener.java @@ -0,0 +1,161 @@ +package com.netflix.evcache; + +import brave.Span; +import brave.Tracer; +import com.netflix.evcache.event.EVCacheEvent; +import com.netflix.evcache.event.EVCacheEventListener; +import com.netflix.evcache.pool.EVCacheClient; +import com.netflix.evcache.pool.EVCacheClientPoolManager; +import net.spy.memcached.CachedData; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.StringJoiner; + +/** Adds tracing tags for EvCache calls. */ +public class EVCacheTracingEventListener implements EVCacheEventListener { + + public static String EVCACHE_SPAN_NAME = "evcache"; + + private static Logger logger = LoggerFactory.getLogger(EVCacheTracingEventListener.class); + + private static String CLIENT_SPAN_ATTRIBUTE_KEY = "clientSpanAttributeKey"; + + private final Tracer tracer; + + public EVCacheTracingEventListener(EVCacheClientPoolManager poolManager, Tracer tracer) { + poolManager.addEVCacheEventListener(this); + this.tracer = tracer; + } + + @Override + public void onStart(EVCacheEvent e) { + try { + Span clientSpan = + this.tracer.nextSpan().kind(Span.Kind.CLIENT).name(EVCACHE_SPAN_NAME).start(e.getStartTime()); + + String appName = e.getAppName(); + this.safeTag(clientSpan, EVCacheTracingTags.APP_NAME, appName); + + String cacheNamePrefix = e.getCacheName(); + this.safeTag(clientSpan, EVCacheTracingTags.CACHE_NAME_PREFIX, cacheNamePrefix); + + String call = e.getCall().name(); + this.safeTag(clientSpan, EVCacheTracingTags.CALL, call); + + /** + * Note - e.getClients() returns a list of clients associated with the EVCacheEvent. + * + *

Read operation will have only 1 EVCacheClient as reading from just 1 instance of cache + * is sufficient. Write operations will have appropriate number of clients as each client will + * attempt to write to its cache instance. + */ + String serverGroup; + StringJoiner serverGroups = new StringJoiner(",", "[", "]"); + for (EVCacheClient client : e.getClients()) { + serverGroup = client.getServerGroupName(); + if (StringUtils.isNotBlank(serverGroup)) { + serverGroups.add("\"" + serverGroup + "\""); + } + } + clientSpan.tag(EVCacheTracingTags.SERVER_GROUPS, serverGroups.toString()); + + /** + * Note - EvCache client creates a hash key if the given canonical key size exceeds 255 + * characters. + * + *

There have been cases where canonical key size exceeded few megabytes. As caching client + * creates a hash of such canonical keys and optimizes the storage in the cache servers, it is + * safe to annotate hash key instead of canonical key in such cases. + */ + String hashKey; + for (EVCacheKey keyObj : e.getEVCacheKeys()) { + hashKey = keyObj.getHashKey(); + if (StringUtils.isNotBlank(hashKey)) { + clientSpan.tag(EVCacheTracingTags.HASH_KEY, hashKey); + } else { + this.safeTag(clientSpan, EVCacheTracingTags.CANONICAL_KEY, keyObj.getCanonicalKey()); + } + } + + /** + * Note - tracer.spanInScope(...) method stores Spans in the thread local object. + * + *

As EVCache write operations are asynchronous and quorum based, we are avoiding attaching + * clientSpan with tracer.spanInScope(...) method. Instead, we are storing the clientSpan as + * an object in the EVCacheEvent's attributes. + */ + e.setAttribute(CLIENT_SPAN_ATTRIBUTE_KEY, clientSpan); + } catch (Exception exception) { + logger.error("onStart exception", exception); + } + } + + @Override + public void onComplete(EVCacheEvent e) { + try { + this.onFinishHelper(e, null); + } catch (Exception exception) { + logger.error("onComplete exception", exception); + } + } + + @Override + public void onError(EVCacheEvent e, Throwable t) { + try { + this.onFinishHelper(e, t); + } catch (Exception exception) { + logger.error("onError exception", exception); + } + } + + /** + * On throttle is not a trace event, but it is used to decide whether to throttle. We don't want + * to interfere so always return false. + */ + @Override + public boolean onThrottle(EVCacheEvent e) throws EVCacheException { + return false; + } + + private void onFinishHelper(EVCacheEvent e, Throwable t) { + Object clientSpanObj = e.getAttribute(CLIENT_SPAN_ATTRIBUTE_KEY); + + // Return if the previously saved Client Span is null + if (clientSpanObj == null) { + return; + } + + Span clientSpan = (Span) clientSpanObj; + + try { + if (t != null) { + this.safeTag(clientSpan, EVCacheTracingTags.ERROR, t.toString()); + } + + String status = e.getStatus(); + this.safeTag(clientSpan, EVCacheTracingTags.STATUS, status); + + long latency = e.getDurationInMillis(); + clientSpan.tag(EVCacheTracingTags.LATENCY, String.valueOf(latency)); + + int ttl = e.getTTL(); + clientSpan.tag(EVCacheTracingTags.DATA_TTL, String.valueOf(ttl)); + + CachedData cachedData = e.getCachedData(); + if (cachedData != null) { + int cachedDataSize = cachedData.getData().length; + clientSpan.tag(EVCacheTracingTags.DATA_SIZE, String.valueOf(cachedDataSize)); + } + } finally { + clientSpan.finish(); + } + } + + private void safeTag(Span span, String key, String value) { + if (StringUtils.isNotBlank(value)) { + span.tag(key, value); + } + } +} diff --git a/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java b/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java new file mode 100644 index 00000000..646a0062 --- /dev/null +++ b/evcache-zipkin-tracing/src/main/java/com/netflix/evcache/EVCacheTracingTags.java @@ -0,0 +1,15 @@ +package com.netflix.evcache; + +public class EVCacheTracingTags { + public static String CACHE_NAME_PREFIX = "evcache.cache_name_prefix"; + public static String APP_NAME = "evcache.app_name"; + public static String STATUS = "evcache.status"; + public static String LATENCY = "evcache.latency"; + public static String CALL = "evcache.call"; + public static String SERVER_GROUPS = "evcache.server_groups"; + public static String HASH_KEY = "evcache.hash_key"; + public static String CANONICAL_KEY = "evcache.canonical_key"; + public static String DATA_TTL = "evcache.data_ttl"; + public static String DATA_SIZE = "evcache.data_size"; + public static String ERROR = "evcache.error"; +} diff --git a/evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java b/evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java new file mode 100644 index 00000000..8554c86d --- /dev/null +++ b/evcache-zipkin-tracing/src/test/java/com/netflix/evcache/EVCacheTracingEventListenerUnitTests.java @@ -0,0 +1,126 @@ +package com.netflix.evcache; + +import brave.Tracing; +import com.netflix.evcache.event.EVCacheEvent; +import com.netflix.evcache.pool.EVCacheClient; +import com.netflix.evcache.pool.EVCacheClientPoolManager; +import net.spy.memcached.CachedData; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import zipkin2.Span; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.*; + +public class EVCacheTracingEventListenerUnitTests { + + List reportedSpans; + EVCacheTracingEventListener tracingListener; + EVCacheClient mockEVCacheClient; + EVCacheEvent mockEVCacheEvent; + + @BeforeMethod + public void resetMocks() { + mockEVCacheClient = mock(EVCacheClient.class); + when(mockEVCacheClient.getServerGroupName()).thenReturn("dummyServerGroupName"); + + mockEVCacheEvent = mock(EVCacheEvent.class); + + when(mockEVCacheEvent.getClients()).thenReturn(Arrays.asList(mockEVCacheClient)); + when(mockEVCacheEvent.getCall()).thenReturn(EVCache.Call.GET); + + when(mockEVCacheEvent.getAppName()).thenReturn("dummyAppName"); + when(mockEVCacheEvent.getCacheName()).thenReturn("dummyCacheName"); + when(mockEVCacheEvent.getEVCacheKeys()) + .thenReturn(Arrays.asList(new EVCacheKey("dummyKey", "dummyCanonicalKey", null))); + when(mockEVCacheEvent.getStatus()).thenReturn("success"); + when(mockEVCacheEvent.getDurationInMillis()).thenReturn(1L); + when(mockEVCacheEvent.getTTL()).thenReturn(0); + when(mockEVCacheEvent.getCachedData()) + .thenReturn(new CachedData(1, "dummyData".getBytes(), 255)); + + Map eventAttributes = new HashMap<>(); + doAnswer( + new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + Object[] arguments = invocation.getArguments(); + String key = (String) arguments[0]; + Object value = arguments[1]; + eventAttributes.put(key, value); + return null; + } + }) + .when(mockEVCacheEvent) + .setAttribute(any(), any()); + + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + Object[] arguments = invocation.getArguments(); + String key = (String) arguments[0]; + return eventAttributes.get(key); + } + }) + .when(mockEVCacheEvent) + .getAttribute(any()); + + reportedSpans = new ArrayList<>(); + Tracing tracing = Tracing.newBuilder().spanReporter(reportedSpans::add).build(); + + tracingListener = + new EVCacheTracingEventListener(mock(EVCacheClientPoolManager.class), tracing.tracer()); + } + + public void verifyCommonTags(List spans) { + Assert.assertEquals(spans.size(), 1, "Number of expected spans are not matching"); + zipkin2.Span span = spans.get(0); + + Assert.assertEquals(span.kind(), Span.Kind.CLIENT, "Span Kind are not equal"); + Assert.assertEquals( + span.name(), EVCacheTracingEventListener.EVCACHE_SPAN_NAME, "Cache name are not equal"); + + Map tags = span.tags(); + tags.containsKey(EVCacheTracingTags.APP_NAME); + tags.containsKey(EVCacheTracingTags.CACHE_NAME_PREFIX); + tags.containsKey(EVCacheTracingTags.CALL); + tags.containsKey(EVCacheTracingTags.SERVER_GROUPS); + tags.containsKey(EVCacheTracingTags.CANONICAL_KEY); + tags.containsKey(EVCacheTracingTags.STATUS); + tags.containsKey(EVCacheTracingTags.LATENCY); + tags.containsKey(EVCacheTracingTags.DATA_TTL); + tags.containsKey(EVCacheTracingTags.DATA_SIZE); + } + + public void verifyErrorTags(List spans) { + zipkin2.Span span = spans.get(0); + Map tags = span.tags(); + tags.containsKey(EVCacheTracingTags.ERROR); + } + + @Test + public void testEVCacheListenerOnComplete() { + tracingListener.onStart(mockEVCacheEvent); + tracingListener.onComplete(mockEVCacheEvent); + + verifyCommonTags(reportedSpans); + } + + @Test + public void testEVCacheListenerOnError() { + tracingListener.onStart(mockEVCacheEvent); + tracingListener.onError(mockEVCacheEvent, new RuntimeException("Unexpected Error")); + + verifyCommonTags(reportedSpans); + verifyErrorTags(reportedSpans); + } +} diff --git a/settings.gradle b/settings.gradle index f9ba982f..8d63d7b7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,3 +2,4 @@ rootProject.name='EVCache' include 'evcache-core' include 'evcache-client' include 'evcache-client-sample' +include 'evcache-zipkin-tracing' \ No newline at end of file