From 63844c64f6cd280cbac47be63dea3a2115673116 Mon Sep 17 00:00:00 2001 From: Shashi Madappa Date: Mon, 28 Sep 2020 17:55:01 -0700 Subject: [PATCH 1/3] Passing flag so that we control if the value should be deserialized or not. This will help in displaying admin --- .../java/com/netflix/evcache/EVCacheImpl.java | 51 +++++++++++-------- .../netflix/evcache/EVCacheInternalImpl.java | 3 +- .../netflix/evcache/pool/ChunkTranscoder.java | 2 +- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java index 8c240db7..900baf83 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java @@ -37,6 +37,7 @@ import com.netflix.evcache.operation.EVCacheItemMetaData; import com.netflix.evcache.operation.EVCacheLatchImpl; import com.netflix.evcache.operation.EVCacheOperationFuture; +import com.netflix.evcache.pool.ChunkTranscoder; import com.netflix.evcache.pool.EVCacheClient; import com.netflix.evcache.pool.EVCacheClientPool; import com.netflix.evcache.pool.EVCacheClientPoolManager; @@ -657,7 +658,7 @@ protected EVCacheItem metaGetInternal(String key, Transcoder tc, boole try { final boolean hasZF = hasZoneFallback(); boolean throwEx = hasZF ? false : throwExc; - EVCacheItem data = getEVCacheItem(client, evcKey, tc, throwEx, hasZF, isOriginalKeyHashed); + EVCacheItem data = getEVCacheItem(client, evcKey, tc, throwEx, hasZF, isOriginalKeyHashed, true); if (data == null && hasZF) { final List fbClients = _pool.getEVCacheClientsForReadExcluding(client.getServerGroup()); if (fbClients != null && !fbClients.isEmpty()) { @@ -678,7 +679,7 @@ protected EVCacheItem metaGetInternal(String key, Transcoder tc, boole } } tries++; - data = getEVCacheItem(fbClient, evcKey, tc, throwEx, (i < fbClients.size() - 1) ? true : false, isOriginalKeyHashed); + data = getEVCacheItem(fbClient, evcKey, tc, throwEx, (i < fbClients.size() - 1) ? true : false, isOriginalKeyHashed, true); if (log.isDebugEnabled() && shouldLog()) log.debug("Retry for APP " + _appName + ", key [" + evcKey + (log.isTraceEnabled() ? "], Value [" + data : "") + "], ServerGroup : " + fbClient.getServerGroup()); if (data != null) { client = fbClient; @@ -978,33 +979,39 @@ protected EVCacheItemMetaData getEVCacheItemMetaData(EVCacheClient client, EVCac } } - protected EVCacheItem getEVCacheItem(EVCacheClient client, EVCacheKey evcKey, Transcoder tc, boolean throwException, boolean hasZF, boolean isOriginalKeyHashed) throws Exception { + protected EVCacheItem getEVCacheItem(EVCacheClient client, EVCacheKey evcKey, Transcoder tc, boolean throwException, boolean hasZF, boolean isOriginalKeyHashed, boolean desearilizeEVCacheValue) throws Exception { if (client == null) return null; final Transcoder transcoder = (tc == null) ? ((_transcoder == null) ? (Transcoder) client.getTranscoder() : (Transcoder) _transcoder) : tc; try { String hashKey = isOriginalKeyHashed ? evcKey.getKey() : evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashBytes()); String canonicalKey = evcKey.getCanonicalKey(client.isDuetClient()); if (hashKey != null) { - final EVCacheItem obj = client.metaGet(hashKey, evcacheValueTranscoder, throwException, hasZF); - if (null == obj) return null; - if (obj.getData() instanceof EVCacheValue) { - final EVCacheValue val = (EVCacheValue) obj.getData(); - if (null == val) { - return null; - } - - // compare the key embedded in the value to the original key only if the original key is not passed hashed - if (!isOriginalKeyHashed && !(val.getKey().equals(canonicalKey))) { - incrementFailure(EVCacheMetricsFactory.KEY_HASH_COLLISION, Call.META_GET.name(), EVCacheMetricsFactory.META_GET_OPERATION); + if(desearilizeEVCacheValue) { + final EVCacheItem obj = client.metaGet(hashKey, evcacheValueTranscoder, throwException, hasZF); + if (null == obj) return null; + if (obj.getData() instanceof EVCacheValue) { + final EVCacheValue val = (EVCacheValue) obj.getData(); + if (null == val) { + return null; + } + + // compare the key embedded in the value to the original key only if the original key is not passed hashed + if (!isOriginalKeyHashed && !(val.getKey().equals(canonicalKey))) { + incrementFailure(EVCacheMetricsFactory.KEY_HASH_COLLISION, Call.META_GET.name(), EVCacheMetricsFactory.META_GET_OPERATION); + return null; + } + final CachedData cd = new CachedData(val.getFlags(), val.getValue(), CachedData.MAX_SIZE); + T t = transcoder.decode(cd); + obj.setData(t); + obj.setFlag(val.getFlags()); + return (EVCacheItem) obj; + } else { return null; } - final CachedData cd = new CachedData(val.getFlags(), val.getValue(), CachedData.MAX_SIZE); - T t = transcoder.decode(cd); - obj.setData(t); - obj.setFlag(val.getFlags()); - return (EVCacheItem) obj; } else { - return null; + final EVCacheItem obj = client.metaGet(hashKey, new ChunkTranscoder(), throwException, hasZF); + if (null == obj) return null; + return (EVCacheItem) obj; } } else { return client.metaGet(canonicalKey, transcoder, throwException, hasZF); @@ -1918,8 +1925,8 @@ protected EVCacheLatch set(String key, T value, Transcoder tc, int timeTo CachedData cd = null; CachedData cdHashed = null; for (EVCacheClient client : clients) { - String canonicalKey = evcKey.getCanonicalKey(client.isDuetClient()); - String hashKey = evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashBytes()); + final String canonicalKey = evcKey.getCanonicalKey(client.isDuetClient()); + final String hashKey = evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashBytes()); if(cd == null) { if (tc != null) { cd = tc.encode(value); diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java index 87cf7211..da52b2ab 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java @@ -5,6 +5,7 @@ import com.netflix.evcache.operation.EVCacheItemMetaData; import com.netflix.evcache.pool.EVCacheClient; import com.netflix.evcache.pool.EVCacheClientPoolManager; +import com.netflix.evcache.pool.EVCacheValue; import com.netflix.evcache.pool.ServerGroup; import net.spy.memcached.CachedData; import net.spy.memcached.MemcachedNode; @@ -34,7 +35,7 @@ public Map metaGetPerClient(String key, Transcoder< for (ServerGroup sGroup : instancesByZone.keySet()) { try { for (EVCacheClient client : instancesByZone.get(sGroup)) { - EVCacheItem item = getEVCacheItem(client, evcKey, tc, true, false, isOriginalKeyHashed); + EVCacheItem item = getEVCacheItem(client, evcKey, tc, true, false, isOriginalKeyHashed, false); map.put(client.getNodeLocator().getPrimary(key), null == item ? null : new CachedValues(key, item.getData(), item.getItemMetaData())); } } catch (Exception e) { diff --git a/evcache-core/src/main/java/com/netflix/evcache/pool/ChunkTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/pool/ChunkTranscoder.java index d9d2feb8..d66d44aa 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/pool/ChunkTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/pool/ChunkTranscoder.java @@ -10,7 +10,7 @@ * @author smadappa * */ -class ChunkTranscoder extends BaseSerializingTranscoder implements Transcoder { +public class ChunkTranscoder extends BaseSerializingTranscoder implements Transcoder { public ChunkTranscoder() { super(Integer.MAX_VALUE); From 77a1b6783ab50510290333ce25077c295689ac95 Mon Sep 17 00:00:00 2001 From: Shashi Madappa Date: Mon, 28 Sep 2020 18:45:34 -0700 Subject: [PATCH 2/3] Support for multiple hashes --- .../netflix/evcache/EVCacheInternalImpl.java | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java index da52b2ab..7f54b119 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java @@ -31,18 +31,60 @@ public EVCacheItem metaGet(String key, Transcoder tc, bo public Map metaGetPerClient(String key, Transcoder tc, boolean isOriginalKeyHashed) throws EVCacheException { Map map = new HashMap<>(); final Map> instancesByZone = _pool.getAllInstancesByZone(); + final Map instancesWithNull = new HashMap(); final EVCacheKey evcKey = getEVCacheKey(key); for (ServerGroup sGroup : instancesByZone.keySet()) { try { for (EVCacheClient client : instancesByZone.get(sGroup)) { EVCacheItem item = getEVCacheItem(client, evcKey, tc, true, false, isOriginalKeyHashed, false); - map.put(client.getNodeLocator().getPrimary(key), null == item ? null : new CachedValues(key, item.getData(), item.getItemMetaData())); + if (log.isDebugEnabled()) log.debug("client : " + client + "; item : " + item); + if(item == null) { + instancesWithNull.put(sGroup, client); + } else { + map.put(client.getNodeLocator().getPrimary(key), null == item ? null : new CachedValues(key, item.getData(), item.getItemMetaData())); + } } } catch (Exception e) { log.error("Error getting meta data", e); } } + if (log.isDebugEnabled()) log.debug("map : " + map); + if (log.isDebugEnabled()) log.debug("instancesWithNull : " + instancesWithNull); + if(instancesWithNull.size() > 0 && map.size() > 0) { + final EVCacheTranscoder transcoder = new EVCacheTranscoder(); + String originalKey = null; + for(CachedValues vals : map.values()) { + if (log.isDebugEnabled()) log.debug("vals : " + vals); + try { + Object obj = transcoder.decode(vals.getData()); + if (log.isDebugEnabled()) log.debug("Obj : " + obj); + if(obj instanceof EVCacheValue) { + originalKey = ((EVCacheValue)obj).getKey(); + if (log.isDebugEnabled()) log.debug("original key: " + originalKey); + break; + } + } catch(Exception e) { + log.error("Exception decoding", e); + } + } + if(originalKey != null) { + for(ServerGroup sGroup : instancesWithNull.keySet()) { + if (log.isDebugEnabled()) log.debug("sGroup : " + sGroup); + final EVCacheClient client = instancesWithNull.get(sGroup); + if (log.isDebugEnabled()) log.debug("Client : " + client); + EVCacheItem item; + try { + item = getEVCacheItem(client, getEVCacheKey(originalKey), tc, true, false, false, false); + if (log.isDebugEnabled()) log.debug("item : " + item); + map.put(client.getNodeLocator().getPrimary(originalKey), null == item ? null : new CachedValues(key, item.getData(), item.getItemMetaData())); + } catch (Exception e) { + log.error("Exception getting meta data using original key - " + originalKey, e); + } + } + } + } + if (log.isDebugEnabled()) log.debug("return map : " + map); return map; } From 55f0ce817c615e4e742949b8d1c36203d8cdfc87 Mon Sep 17 00:00:00 2001 From: Shashi Madappa Date: Mon, 28 Sep 2020 20:51:25 -0700 Subject: [PATCH 3/3] Fixed the compile issue --- .../src/main/java/com/netflix/evcache/EVCacheImpl.java | 2 +- .../main/java/com/netflix/evcache/util/KeyHasher.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java index cd21ee04..41e3bba1 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java @@ -1926,7 +1926,7 @@ protected EVCacheLatch set(String key, T value, Transcoder tc, int timeTo CachedData cdHashed = null; for (EVCacheClient client : clients) { final String canonicalKey = evcKey.getCanonicalKey(client.isDuetClient()); - final String hashKey = evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashBytes()); + final String hashKey = evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength()); if(cd == null) { if (tc != null) { cd = tc.encode(value); diff --git a/evcache-core/src/main/java/com/netflix/evcache/util/KeyHasher.java b/evcache-core/src/main/java/com/netflix/evcache/util/KeyHasher.java index 39c9d611..f32e184f 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/util/KeyHasher.java +++ b/evcache-core/src/main/java/com/netflix/evcache/util/KeyHasher.java @@ -132,4 +132,13 @@ private static byte[] getHashedKey(String key, HashingAlgorithm hashingAlgorithm return digest; } + + + public static void main(String args[]) { + String key = "MAP_LT:721af5a5-3452-4b62-86fb-5f31ccde8d99_187978153X28X2787347X1601330156682"; + System.out.println(getHashedKeyEncoded(key, HashingAlgorithm.murmur3, null, null)); + System.out.println(getHashedKeyEncoded(key, HashingAlgorithm.goodfasthash, null, null)); + System.out.println(getHashedKeyEncoded(key, HashingAlgorithm.goodfasthash, 15, null)); + } + }