Skip to content

Commit

Permalink
Merge pull request #92 from Netflix/feature/metaget_evcachevalue_changes
Browse files Browse the repository at this point in the history 
Passing flag so that we control if the value should be deserialized o…
  • Loading branch information
@smadappa
smadappa authored Sep 29, 2020
2 parents 1db854a + 55f0ce8 commit 1c2b137
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 25 deletions.
51 changes: 29 additions & 22 deletions evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.netflix.evcache.operation.EVCacheItemMetaData;
import com.netflix.evcache.operation.EVCacheLatchImpl;
import com.netflix.evcache.operation.EVCacheOperationFuture;
import com.netflix.evcache.pool.ChunkTranscoder;
import com.netflix.evcache.pool.EVCacheClient;
import com.netflix.evcache.pool.EVCacheClientPool;
import com.netflix.evcache.pool.EVCacheClientPoolManager;
Expand Down Expand Up @@ -657,7 +658,7 @@ protected <T> EVCacheItem<T> metaGetInternal(String key, Transcoder<T> tc, boole
try {
final boolean hasZF = hasZoneFallback();
boolean throwEx = hasZF ? false : throwExc;
EVCacheItem<T> data = getEVCacheItem(client, evcKey, tc, throwEx, hasZF, isOriginalKeyHashed);
EVCacheItem<T> data = getEVCacheItem(client, evcKey, tc, throwEx, hasZF, isOriginalKeyHashed, true);
if (data == null && hasZF) {
final List<EVCacheClient> fbClients = _pool.getEVCacheClientsForReadExcluding(client.getServerGroup());
if (fbClients != null && !fbClients.isEmpty()) {
Expand All @@ -678,7 +679,7 @@ protected <T> EVCacheItem<T> metaGetInternal(String key, Transcoder<T> tc, boole
}
}
tries++;
data = getEVCacheItem(fbClient, evcKey, tc, throwEx, (i < fbClients.size() - 1) ? true : false, isOriginalKeyHashed);
data = getEVCacheItem(fbClient, evcKey, tc, throwEx, (i < fbClients.size() - 1) ? true : false, isOriginalKeyHashed, true);
if (log.isDebugEnabled() && shouldLog()) log.debug("Retry for APP " + _appName + ", key [" + evcKey + (log.isTraceEnabled() ? "], Value [" + data : "") + "], ServerGroup : " + fbClient.getServerGroup());
if (data != null) {
client = fbClient;
Expand Down Expand Up @@ -978,33 +979,39 @@ protected EVCacheItemMetaData getEVCacheItemMetaData(EVCacheClient client, EVCac
}
}

protected <T> EVCacheItem<T> getEVCacheItem(EVCacheClient client, EVCacheKey evcKey, Transcoder<T> tc, boolean throwException, boolean hasZF, boolean isOriginalKeyHashed) throws Exception {
protected <T> EVCacheItem<T> getEVCacheItem(EVCacheClient client, EVCacheKey evcKey, Transcoder<T> tc, boolean throwException, boolean hasZF, boolean isOriginalKeyHashed, boolean desearilizeEVCacheValue) throws Exception {
if (client == null) return null;
final Transcoder<T> transcoder = (tc == null) ? ((_transcoder == null) ? (Transcoder<T>) client.getTranscoder() : (Transcoder<T>) _transcoder) : tc;
try {
String hashKey = isOriginalKeyHashed ? evcKey.getKey() : evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength());
String canonicalKey = evcKey.getCanonicalKey(client.isDuetClient());
if (hashKey != null) {
final EVCacheItem<Object> obj = client.metaGet(hashKey, evcacheValueTranscoder, throwException, hasZF);
if (null == obj) return null;
if (obj.getData() instanceof EVCacheValue) {
final EVCacheValue val = (EVCacheValue) obj.getData();
if (null == val) {
return null;
}

// compare the key embedded in the value to the original key only if the original key is not passed hashed
if (!isOriginalKeyHashed && !(val.getKey().equals(canonicalKey))) {
incrementFailure(EVCacheMetricsFactory.KEY_HASH_COLLISION, Call.META_GET.name(), EVCacheMetricsFactory.META_GET_OPERATION);
if(desearilizeEVCacheValue) {
final EVCacheItem<Object> obj = client.metaGet(hashKey, evcacheValueTranscoder, throwException, hasZF);
if (null == obj) return null;
if (obj.getData() instanceof EVCacheValue) {
final EVCacheValue val = (EVCacheValue) obj.getData();
if (null == val) {
return null;
}

// compare the key embedded in the value to the original key only if the original key is not passed hashed
if (!isOriginalKeyHashed && !(val.getKey().equals(canonicalKey))) {
incrementFailure(EVCacheMetricsFactory.KEY_HASH_COLLISION, Call.META_GET.name(), EVCacheMetricsFactory.META_GET_OPERATION);
return null;
}
final CachedData cd = new CachedData(val.getFlags(), val.getValue(), CachedData.MAX_SIZE);
T t = transcoder.decode(cd);
obj.setData(t);
obj.setFlag(val.getFlags());
return (EVCacheItem<T>) obj;
} else {
return null;
}
final CachedData cd = new CachedData(val.getFlags(), val.getValue(), CachedData.MAX_SIZE);
T t = transcoder.decode(cd);
obj.setData(t);
obj.setFlag(val.getFlags());
return (EVCacheItem<T>) obj;
} else {
return null;
final EVCacheItem<CachedData> obj = client.metaGet(hashKey, new ChunkTranscoder(), throwException, hasZF);
if (null == obj) return null;
return (EVCacheItem<T>) obj;
}
} else {
return client.metaGet(canonicalKey, transcoder, throwException, hasZF);
Expand Down Expand Up @@ -1918,8 +1925,8 @@ protected <T> EVCacheLatch set(String key, T value, Transcoder<T> tc, int timeTo
CachedData cd = null;
CachedData cdHashed = null;
for (EVCacheClient client : clients) {
String canonicalKey = evcKey.getCanonicalKey(client.isDuetClient());
String hashKey = evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength());
final String canonicalKey = evcKey.getCanonicalKey(client.isDuetClient());
final String hashKey = evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength());
if(cd == null) {
if (tc != null) {
cd = tc.encode(value);
Expand Down
47 changes: 45 additions & 2 deletions evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.netflix.evcache.operation.EVCacheItemMetaData;
import com.netflix.evcache.pool.EVCacheClient;
import com.netflix.evcache.pool.EVCacheClientPoolManager;
import com.netflix.evcache.pool.EVCacheValue;
import com.netflix.evcache.pool.ServerGroup;
import net.spy.memcached.CachedData;
import net.spy.memcached.MemcachedNode;
Expand All @@ -30,18 +31,60 @@ public EVCacheItem<CachedData> metaGet(String key, Transcoder<CachedData> tc, bo
public Map<MemcachedNode, CachedValues> metaGetPerClient(String key, Transcoder<CachedData> tc, boolean isOriginalKeyHashed) throws EVCacheException {
Map<MemcachedNode, CachedValues> map = new HashMap<>();
final Map<ServerGroup, List<EVCacheClient>> instancesByZone = _pool.getAllInstancesByZone();
final Map<ServerGroup, EVCacheClient> instancesWithNull = new HashMap<ServerGroup, EVCacheClient>();
final EVCacheKey evcKey = getEVCacheKey(key);
for (ServerGroup sGroup : instancesByZone.keySet()) {
try {
for (EVCacheClient client : instancesByZone.get(sGroup)) {
EVCacheItem<CachedData> item = getEVCacheItem(client, evcKey, tc, true, false, isOriginalKeyHashed);
map.put(client.getNodeLocator().getPrimary(key), null == item ? null : new CachedValues(key, item.getData(), item.getItemMetaData()));
EVCacheItem<CachedData> item = getEVCacheItem(client, evcKey, tc, true, false, isOriginalKeyHashed, false);
if (log.isDebugEnabled()) log.debug("client : " + client + "; item : " + item);
if(item == null) {
instancesWithNull.put(sGroup, client);
} else {
map.put(client.getNodeLocator().getPrimary(key), null == item ? null : new CachedValues(key, item.getData(), item.getItemMetaData()));
}
}
} catch (Exception e) {
log.error("Error getting meta data", e);
}
}
if (log.isDebugEnabled()) log.debug("map : " + map);
if (log.isDebugEnabled()) log.debug("instancesWithNull : " + instancesWithNull);
if(instancesWithNull.size() > 0 && map.size() > 0) {
final EVCacheTranscoder transcoder = new EVCacheTranscoder();
String originalKey = null;
for(CachedValues vals : map.values()) {
if (log.isDebugEnabled()) log.debug("vals : " + vals);
try {
Object obj = transcoder.decode(vals.getData());
if (log.isDebugEnabled()) log.debug("Obj : " + obj);
if(obj instanceof EVCacheValue) {
originalKey = ((EVCacheValue)obj).getKey();
if (log.isDebugEnabled()) log.debug("original key: " + originalKey);
break;
}
} catch(Exception e) {
log.error("Exception decoding", e);
}
}
if(originalKey != null) {
for(ServerGroup sGroup : instancesWithNull.keySet()) {
if (log.isDebugEnabled()) log.debug("sGroup : " + sGroup);
final EVCacheClient client = instancesWithNull.get(sGroup);
if (log.isDebugEnabled()) log.debug("Client : " + client);
EVCacheItem<CachedData> item;
try {
item = getEVCacheItem(client, getEVCacheKey(originalKey), tc, true, false, false, false);
if (log.isDebugEnabled()) log.debug("item : " + item);
map.put(client.getNodeLocator().getPrimary(originalKey), null == item ? null : new CachedValues(key, item.getData(), item.getItemMetaData()));
} catch (Exception e) {
log.error("Exception getting meta data using original key - " + originalKey, e);
}
}
}
}

if (log.isDebugEnabled()) log.debug("return map : " + map);
return map;
}

Expand Down
2 changes: 1 addition & 1 deletion evcache-core/src/main/java/com/netflix/evcache/pool/ChunkTranscoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* @author smadappa
*
*/
class ChunkTranscoder extends BaseSerializingTranscoder implements Transcoder<CachedData> {
public class ChunkTranscoder extends BaseSerializingTranscoder implements Transcoder<CachedData> {

public ChunkTranscoder() {
super(Integer.MAX_VALUE);
Expand Down
9 changes: 9 additions & 0 deletions evcache-core/src/main/java/com/netflix/evcache/util/KeyHasher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,4 +132,13 @@ private static byte[] getHashedKey(String key, HashingAlgorithm hashingAlgorithm

return digest;
}


public static void main(String args[]) {
String key = "MAP_LT:721af5a5-3452-4b62-86fb-5f31ccde8d99_187978153X28X2787347X1601330156682";
System.out.println(getHashedKeyEncoded(key, HashingAlgorithm.murmur3, null, null));
System.out.println(getHashedKeyEncoded(key, HashingAlgorithm.goodfasthash, null, null));
System.out.println(getHashedKeyEncoded(key, HashingAlgorithm.goodfasthash, 15, null));
}

}

0 comments on commit 1c2b137

Please sign in to comment.