Skip to content

Commit

Permalink
Merge pull request #33 from CelerityChip/master
Browse files Browse the repository at this point in the history 
Fix issues #25 and #32: Regions ignored when cleaning/shrinking/flushing
  • Loading branch information
@acarteas
acarteas authored Dec 6, 2019
2 parents 6e3776e + 0593aba commit ea39af8
Show file tree
Hide file tree
Showing 5 changed files with 189 additions and 49 deletions.
6 changes: 6 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,12 @@ public string[] GetKeys(string regionName = null)
/// <returns></returns>
public CacheItemPolicy GetPolicy(string key, string regionName = null)

/// <summary>
/// Returns a list of regions, including the root region.
/// </summary>
/// <returns></returns>
public IEnumerable<string> GetRegions()

/// <summary>
/// Used to specify the disk size, in bytes, that can be used by the File Cache.
/// Defaults to long.MaxValue
Expand Down
55 changes: 49 additions & 6 deletions src/FileCache.UnitTests/FileCacheTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ FileCache is distributed under the Apache License 2.0.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Caching;
using System.Threading;

Expand Down Expand Up @@ -165,12 +166,12 @@ public void ShrinkCacheTest()
// Test empty case
_cache.ShrinkCacheToSize(0).Should().Be(0);

// Insert 4 items, and keep track of their size
// Insert 4 items, one of them in a region, and keep track of their size
//sleep to make sure that oldest item gets removed first
_cache["item1"] = "bar1asdfasdfdfskjslkjlkjsdf sdlfkjasdlf asdlfkjskjfkjs d sdkfjksjd";
Thread.Sleep(500);
long size1 = _cache.GetCacheSize();
_cache["item2"] = "bar2sdfjkjk skdfj sdflkj sdlkj lkjkjkjkjkjssss";
_cache.Add("item2", "bar2sdfjkjk skdfj sdflkj sdlkj lkjkjkjkjkjssss", _cache.DefaultPolicy, "region");
Thread.Sleep(500);
long size2 = _cache.GetCacheSize() - size1;
_cache["item3"] = "bar3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff";
Expand All @@ -183,7 +184,7 @@ public void ShrinkCacheTest()
long newSize = _cache.ShrinkCacheToSize(size4 + size3 + size2);
newSize.Should().Be(size4 + size3 + size2);

// Shrink to just smaller than two items (should keep just item4, delete item2 and item3)
// Shrink to just smaller than two items (should keep just item4, delete item2 (in a region) and item3)
newSize = _cache.ShrinkCacheToSize(size3 + size4 - 1);
newSize.Should().Be(size4);

Expand Down Expand Up @@ -229,7 +230,8 @@ public void FlushTest()
{
_cache = new FileCache("FlushTest");

_cache["foo"] = "bar";
_cache.Add("Key1", "Value1", _cache.DefaultPolicy);
_cache.Add("Key2", "Value2", DateTime.Now.AddDays(1), "Region1");
Thread.Sleep(500);

//attempt flush
Expand All @@ -241,6 +243,26 @@ public void FlushTest()
_cache.GetCacheSize().Should().Be(0);
}

[TestMethod]
public void FlushRegionTest()
{
_cache = new FileCache("FlushRegionTest");

_cache.Add("Key1", "Value1", _cache.DefaultPolicy);
_cache.Add("Key2", "Value2", _cache.DefaultPolicy, "Region1");
Thread.Sleep(500);

//attempt flush
_cache.Flush("Region1");

Thread.Sleep(500);

object result = _cache["Key1"];
result.Should().Be("Value1");
object result2 = _cache.Get("Key2", "Region1");
result2.Should().BeNull();
}

[TestMethod]
public void RemoveTest()
{
Expand Down Expand Up @@ -295,8 +317,16 @@ public void ClearCache()
{
_cache = new FileCache();
_cache["MyFirstKey"] = "MyFirstValue";
_cache.Add("MySecondKey", "MySecondValue", _cache.DefaultPolicy, "MyFirstRegion");
object pull = _cache.Get("MyFirstKey");
pull.ToString().Should().Be("MyFirstValue");
object pull2 = _cache.Get("MySecondKey", "MyFirstRegion");
pull2.ToString().Should().Be("MySecondValue");
_cache.Clear();
object result = _cache.Get("MyFirstKey");
result.Should().BeNull();
object result2 = _cache.Get("MySecondKey", "MyFirstRegion");
result2.Should().BeNull();
}

/// <summary>
Expand Down Expand Up @@ -381,11 +411,24 @@ public void CleanCacheTest()

_cache.Add("foo", 1, DateTime.Now); // expires immediately
_cache.Add("bar", 2, DateTime.Now + TimeSpan.FromDays(1)); // set to expire tomorrow
_cache.Add("foo", 1, DateTime.Now, "region"); // expires immediately
_cache.Add("bar", 2, DateTime.Now + TimeSpan.FromDays(1), "region"); // set to expire tomorrow

var keys = _cache.GetKeys().ToList();
keys.Should().Contain("foo");
keys.Should().Contain("bar");
keys = _cache.GetKeys("region").ToList();
keys.Should().Contain("foo");
keys.Should().Contain("bar");

_cache.CleanCache();

_cache["foo"].Should().BeNull();
_cache["bar"].Should().NotBeNull();
keys = _cache.GetKeys().ToList();
keys.Should().NotContain("foo");
keys.Should().Contain("bar");
keys = _cache.GetKeys("region").ToList();
keys.Should().NotContain("foo");
keys.Should().Contain("bar");
}

[TestMethod]
Expand Down
55 changes: 49 additions & 6 deletions src/FileCache.UnitTests/HashedFileCacheTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ FileCache is distributed under the Apache License 2.0.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Caching;
using System.Threading;

Expand Down Expand Up @@ -165,12 +166,12 @@ public void ShrinkCacheTest()
// Test empty case
_cache.ShrinkCacheToSize(0).Should().Be(0);

// Insert 4 items, and keep track of their size
// Insert 4 items, one of them in a region, and keep track of their size
//sleep to make sure that oldest item gets removed first
_cache["item1"] = "bar1asdfasdfdfskjslkjlkjsdf sdlfkjasdlf asdlfkjskjfkjs d sdkfjksjd";
Thread.Sleep(500);
long size1 = _cache.GetCacheSize();
_cache["item2"] = "bar2sdfjkjk skdfj sdflkj sdlkj lkjkjkjkjkjssss";
_cache.Add("item2", "bar2sdfjkjk skdfj sdflkj sdlkj lkjkjkjkjkjssss", _cache.DefaultPolicy, "region");
Thread.Sleep(500);
long size2 = _cache.GetCacheSize() - size1;
_cache["item3"] = "bar3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff";
Expand All @@ -183,7 +184,7 @@ public void ShrinkCacheTest()
long newSize = _cache.ShrinkCacheToSize(size4 + size3 + size2);
newSize.Should().Be(size4 + size3 + size2);

// Shrink to just smaller than two items (should keep just item4, delete item2 and item3)
// Shrink to just smaller than two items (should keep just item4, delete item2 (in a region) and item3)
newSize = _cache.ShrinkCacheToSize(size3 + size4 - 1);
newSize.Should().Be(size4);

Expand Down Expand Up @@ -225,7 +226,8 @@ public void FlushTest()
{
_cache = new FileCache("FlushTest");

_cache["foo"] = "bar";
_cache.Add("Key1", "Value1", _cache.DefaultPolicy);
_cache.Add("Key2", "Value2", DateTime.Now.AddDays(1), "Region1");
Thread.Sleep(500);

//attempt flush
Expand All @@ -237,6 +239,26 @@ public void FlushTest()
_cache.GetCacheSize().Should().Be(0);
}

[TestMethod]
public void FlushRegionTest()
{
_cache = new FileCache("FlushRegionTest");

_cache.Add("Key1", "Value1", _cache.DefaultPolicy);
_cache.Add("Key2", "Value2", _cache.DefaultPolicy, "Region1");
Thread.Sleep(500);

//attempt flush
_cache.Flush("Region1");

Thread.Sleep(500);

object result = _cache["Key1"];
result.Should().Be("Value1");
object result2 = _cache.Get("Key2", "Region1");
result2.Should().BeNull();
}

[TestMethod]
public void RemoveTest()
{
Expand Down Expand Up @@ -291,8 +313,16 @@ public void ClearCache()
{
_cache = new FileCache();
_cache["MyFirstKey"] = "MyFirstValue";
_cache.Add("MySecondKey", "MySecondValue", _cache.DefaultPolicy, "MyFirstRegion");
object pull = _cache.Get("MyFirstKey");
pull.ToString().Should().Be("MyFirstValue");
object pull2 = _cache.Get("MySecondKey", "MyFirstRegion");
pull2.ToString().Should().Be("MySecondValue");
_cache.Clear();
object result = _cache.Get("MyFirstKey");
result.Should().BeNull();
object result2 = _cache.Get("MySecondKey", "MyFirstRegion");
result2.Should().BeNull();
}

/// <summary>
Expand Down Expand Up @@ -377,11 +407,24 @@ public void CleanCacheTest()

_cache.Add("foo", 1, DateTime.Now); // expires immediately
_cache.Add("bar", 2, DateTime.Now + TimeSpan.FromDays(1)); // set to expire tomorrow
_cache.Add("foo", 1, DateTime.Now, "region"); // expires immediately
_cache.Add("bar", 2, DateTime.Now + TimeSpan.FromDays(1), "region"); // set to expire tomorrow

var keys = _cache.GetKeys().ToList();
keys.Should().Contain("foo");
keys.Should().Contain("bar");
keys = _cache.GetKeys("region").ToList();
keys.Should().Contain("foo");
keys.Should().Contain("bar");

_cache.CleanCache();

_cache["foo"].Should().BeNull();
_cache["bar"].Should().NotBeNull();
keys = _cache.GetKeys().ToList();
keys.Should().NotContain("foo");
keys.Should().Contain("bar");
keys = _cache.GetKeys("region").ToList();
keys.Should().NotContain("foo");
keys.Should().Contain("bar");
}
}
}
104 changes: 67 additions & 37 deletions src/FileCache/FileCache.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -461,22 +461,30 @@ public long CleanCache(string regionName = null)
{
if (cLock == null)
return 0;

foreach (string key in GetKeys(regionName))

IEnumerable<string> regions =
string.IsNullOrEmpty(regionName)
? CacheManager.GetRegions()
: new List<string>(1) { regionName };

foreach (var region in regions)
{
CacheItemPolicy policy = GetPolicy(key, regionName);
if (policy.AbsoluteExpiration < DateTime.Now)
foreach (string key in GetKeys(region))
{
try
CacheItemPolicy policy = GetPolicy(key, region);
if (policy.AbsoluteExpiration < DateTime.Now)
{
string cachePath = CacheManager.GetCachePath(key, regionName);
string policyPath = CacheManager.GetPolicyPath(key, regionName);
CacheItemReference ci = new CacheItemReference(key, cachePath, policyPath);
Remove(key, regionName); // CT note: Remove will update CurrentCacheSize
removed += ci.Length;
try
{
string cachePath = CacheManager.GetCachePath(key, region);
string policyPath = CacheManager.GetPolicyPath(key, region);
CacheItemReference ci = new CacheItemReference(key, region, cachePath, policyPath);
Remove(key, region); // CT note: Remove will update CurrentCacheSize
removed += ci.Length;
}
catch (Exception) // skip if the file cannot be accessed
{ }
}
catch (Exception) // skip if the file cannot be accessed
{ }
}
}

Expand Down Expand Up @@ -506,19 +514,27 @@ private long DeleteOldestFiles(long amount, string regionName = null)
//Heap of all CacheReferences
PriortyQueue<CacheItemReference> cacheReferences = new PriortyQueue<CacheItemReference>();

//build a heap of all files in cache region
foreach (string key in GetKeys(regionName))
IEnumerable<string> regions =
string.IsNullOrEmpty(regionName)
? CacheManager.GetRegions()
: new List<string>(1) { regionName };

foreach (var region in regions)
{
try
{
//build item reference
string cachePath = CacheManager.GetCachePath(key, regionName);
string policyPath = CacheManager.GetPolicyPath(key, regionName);
CacheItemReference ci = new CacheItemReference(key, cachePath, policyPath);
cacheReferences.Enqueue(ci);
}
catch(FileNotFoundException)
//build a heap of all files in cache region
foreach (string key in GetKeys(region))
{
try
{
//build item reference
string cachePath = CacheManager.GetCachePath(key, region);
string policyPath = CacheManager.GetPolicyPath(key, region);
CacheItemReference ci = new CacheItemReference(key, region, cachePath, policyPath);
cacheReferences.Enqueue(ci);
}
catch (FileNotFoundException)
{
}
}
}

Expand All @@ -529,7 +545,7 @@ private long DeleteOldestFiles(long amount, string regionName = null)
//remove oldest item
CacheItemReference oldest = cacheReferences.Dequeue();
removedBytes += oldest.Length;
Remove(oldest.Key, regionName);
Remove(oldest.Key, oldest.Region);
}
return removedBytes;
}
Expand Down Expand Up @@ -648,18 +664,26 @@ public void Flush(DateTime minDate, string regionName = null)
return;
}

IEnumerable<string> keys = CacheManager.GetKeys();
foreach (string key in keys)
{
string policyPath = CacheManager.GetPolicyPath(key, regionName);
string cachePath = CacheManager.GetCachePath(key, regionName);
IEnumerable<string> regions =
string.IsNullOrEmpty(regionName)
? CacheManager.GetRegions()
: new List<string>(1) { regionName };

// Update the Cache size
CurrentCacheSize = GetCacheSize();
//if either policy or cache are stale, delete both
if (File.GetLastAccessTime(policyPath) < minDate || File.GetLastAccessTime(cachePath) < minDate)
foreach (var region in regions)
{
IEnumerable<string> keys = CacheManager.GetKeys(region);
foreach (string key in keys)
{
CurrentCacheSize -= CacheManager.DeleteFile(key, regionName);
string policyPath = CacheManager.GetPolicyPath(key, region);
string cachePath = CacheManager.GetCachePath(key, region);

// Update the Cache size
CurrentCacheSize = GetCacheSize();
//if either policy or cache are stale, delete both
if (File.GetLastAccessTime(policyPath) < minDate || File.GetLastAccessTime(cachePath) < minDate)
{
CurrentCacheSize -= CacheManager.DeleteFile(key, region);
}
}
}

Expand Down Expand Up @@ -1003,10 +1027,12 @@ private class CacheItemReference : IComparable<CacheItemReference>
public readonly DateTime LastAccessTime;
public readonly long Length;
public readonly string Key;
public readonly string Region;

public CacheItemReference(string key, string cachePath, string policyPath)
public CacheItemReference(string key, string region, string cachePath, string policyPath)
{
Key = key;
Region = region;
FileInfo cfi = new FileInfo(cachePath);
FileInfo pfi = new FileInfo(policyPath);
cfi.Refresh();
Expand All @@ -1027,8 +1053,12 @@ public int CompareTo(CacheItemReference other)
// that way we delete smaller files first)
i = -1 * Length.CompareTo(other.Length);
if (i == 0)
{
i = Key.CompareTo(other.Key);
{
i = string.Compare(Region, other.Region);
if (i == 0)
{
i = Key.CompareTo(other.Key);
}
}
}

Expand Down
Loading

0 comments on commit ea39af8

Please sign in to comment.