diff --git a/src/Kdyby/Redis/DI/RedisExtension.php b/src/Kdyby/Redis/DI/RedisExtension.php index a7ac5eb..c8d2783 100644 --- a/src/Kdyby/Redis/DI/RedisExtension.php +++ b/src/Kdyby/Redis/DI/RedisExtension.php @@ -78,7 +78,7 @@ public function loadConfiguration() if ($config['journal']) { $builder->addDefinition($this->prefix('cacheJournal')) - ->setClass('Kdyby\Redis\RedisJournal'); + ->setClass('Kdyby\Redis\RedisLuaJournal'); // overwrite $builder->removeDefinition('nette.cacheJournal'); diff --git a/src/Kdyby/Redis/RedisClient.php b/src/Kdyby/Redis/RedisClient.php index b13ee79..a7f2523 100644 --- a/src/Kdyby/Redis/RedisClient.php +++ b/src/Kdyby/Redis/RedisClient.php @@ -46,8 +46,6 @@ * @method discard() Discard all commands issued after MULTI * @method dump(string $key) Return a serialized version of the value stored at the specified key. * @method echo(string $message) Echo the given string - * @method eval(string $script, int $numkeys, string $key1, string $key2 = NULL, string $arg1 = NULL, string $arg2 = NULL) Execute a Lua script server side - * @method evalSha(string $sha1, int $numkeys, string $key1, string $key2 = NULL, string $arg1 = NULL, string $arg2 = NULL) Execute a Lua script server side * @method exists(string $key) Determine if a key exists * @method expire(string $key, int $seconds) Set a key's time to live in seconds * @method expireAt(string $key, int $timestamp) Set the expiration for a key as a UNIX timestamp @@ -208,6 +206,8 @@ class RedisClient extends Nette\Object implements \ArrayAccess */ private $lock; + private static $exceptionCmd = array('evalsha' => 0); + /** @@ -313,6 +313,14 @@ protected function send($cmd, array $args = array()) if ($result instanceof \Redis) { $result = strtolower($cmd) === 'multi' ? 'OK' : 'QUEUED'; + + } elseif ($result === FALSE && ($msg = $this->driver->getLastError())) { + if (!isset(self::$exceptionCmd[strtolower($cmd)])) { + throw new \RedisException($msg); + } + + } else { + $this->driver->clearLastError(); } if ($this->panel) { @@ -326,7 +334,6 @@ protected function send($cmd, array $args = array()) throw new RedisClientException($e->getMessage(), $e->getCode(), $e); } - return $result; } @@ -397,6 +404,25 @@ public function exec() + /** + * Execute a Lua script server side + */ + public function evalScript($script, array $keys = array(), array $args = array()) + { + $script = trim($script); + + $result = $this->send('evalsha', array(sha1($script), array_merge($keys, $args), count($keys))); + if ($result === FALSE && stripos($this->driver->getLastError(), 'NOSCRIPT') !== FALSE) { + $this->driver->clearLastError(); + $sha = $this->driver->script('load', $script); + $result = $this->send('evalsha', array($sha, array_merge($keys, $args), count($keys))); + } + + return $result; + } + + + /** * @return ExclusiveLock */ diff --git a/src/Kdyby/Redis/RedisLuaJournal.php b/src/Kdyby/Redis/RedisLuaJournal.php new file mode 100644 index 0000000..de82dc5 --- /dev/null +++ b/src/Kdyby/Redis/RedisLuaJournal.php @@ -0,0 +1,141 @@ + + */ +class RedisLuaJournal extends Nette\Object implements Nette\Caching\Storages\IJournal +{ + + /** @internal cache structure */ + const NS_NETTE = 'Nette.Journal'; + + /** dependency */ + const PRIORITY = 'priority', + TAGS = 'tags', + KEYS = 'keys'; + + /** + * @var RedisClient + */ + private $client; + + /** + * @var array + */ + private $scriptSha = array(); + + /** + * @var array + */ + private $script = array(); + + + + /** + * @param RedisClient $client + */ + public function __construct(RedisClient $client) + { + $this->client = $client; + } + + + + /** + * Writes entry information into the journal. + * + * @param string $key + * @param array $dp + * + * @return void + */ + public function write($key, array $dp) + { + $args = self::flattenDp($dp); + + $result = $this->client->evalScript($this->getScript('write'), array($key), $args); + if ($result !== TRUE) { + throw new RedisClientException("Failed to successfully execute lua script journal.write($key)"); + } + } + + + + /** + * Cleans entries from journal. + * + * @param array $conds + * + * @return array of removed items or NULL when performing a full cleanup + */ + public function clean(array $conds) + { + $args = self::flattenDp($conds); + + $result = $this->client->evalScript($this->getScript('clean'), array(), $args); + if (!is_array($result) && $result !== TRUE) { + throw new RedisClientException("Failed to successfully execute lua script journal.clean()"); + } + + return is_array($result) ? array_unique($result) : NULL; + } + + + + private static function flattenDp($array) + { + if (isset($array[Cache::TAGS])) { + $array[Cache::TAGS] = (array) $array[Cache::TAGS]; + } + + $res = array(); + foreach (array_intersect_key($array, array_flip(array(Cache::TAGS, Cache::PRIORITY, Cache::ALL))) as $key => $value) { + $res[] = $key; + if (is_array($value)) { + $res[] = count($value); + foreach ($value as $item) { + $res[] = $item; + } + + } else { + $res[] = -1; + $res[] = $value; + } + } + + return $res; + } + + + + private function getScript($name) + { + if (isset($this->script[$name])) { + return $this->script[$name]; + } + + $script = file_get_contents(__DIR__ . '/scripts/common.lua'); + $script .= file_get_contents(__DIR__ . '/scripts/journal.' . $name . '.lua'); + + return $this->script[$name] = $script; + } + +} diff --git a/src/Kdyby/Redis/scripts/common.lua b/src/Kdyby/Redis/scripts/common.lua new file mode 100644 index 0000000..659bb8a --- /dev/null +++ b/src/Kdyby/Redis/scripts/common.lua @@ -0,0 +1,67 @@ + +local formatKey = function (key, suffix) + local res = "Nette.Journal:" .. key:gsub("\x00", ":") + if suffix ~= nil then + res = res .. ":" .. suffix + end + + return res +end + +local priorityEntries = function (priority) + return redis.call('zRangeByScore', formatKey("priority"), 0, priority) +end + +local entryTags = function (key) + return redis.call('lRange', formatKey(key, "tags"), 0, -1) +end + +local tagEntries = function (tag) + return redis.call('lRange', formatKey(tag, "keys"), 0, -1) +end + +local cleanEntry = function (keys) + for i, key in pairs(keys) do + local tags = entryTags(key) + + -- redis.call('multi') + for i, tag in pairs(tags) do + redis.call('lRem', formatKey(tag, "keys"), 0, key) + end + + -- drop tags of entry and priority, in case there are some + redis.call('del', formatKey(key, "tags"), formatKey(key, "priority")) + redis.call('zRem', formatKey("priority"), key) + + -- redis.call('exec') + end +end + +-- builds table from serialized arguments +local readArgs = function (args) + local res = {} + local counter = 0 + local key + local tmp + + for i, item in pairs(args) do + if counter > 0 then + if res[key] == nil then res[key] = {} end + + tmp = res[key] + res[key][#tmp + 1] = item + counter = counter - 1 + + if counter == 0 then key = nil end + + elseif counter < 0 then + res[key] = item + key = nil + + else + if key == nil then key = item else counter = tonumber(item); end + end + end + + return res +end diff --git a/src/Kdyby/Redis/scripts/journal.clean.lua b/src/Kdyby/Redis/scripts/journal.clean.lua new file mode 100644 index 0000000..a3d14a7 --- /dev/null +++ b/src/Kdyby/Redis/scripts/journal.clean.lua @@ -0,0 +1,37 @@ + +local conds = readArgs(ARGV) + +if conds["all"] ~= nil then + -- redis.call('multi') + for i, value in pairs(redis.call('keys', "Nette.Journal:*")) do + redis.call('del', value) + end + -- redis.call('exec') + + return redis.status_reply("Ok") +end + +local entries = {} +if conds["tags"] ~= nil then + for i, tag in pairs(conds["tags"]) do + local found = tagEntries(tag) + if #found > 0 then + cleanEntry(found) + for i, key in pairs(found) do + entries[#entries + 1] = key + end + end + end +end + +if conds["priority"] ~= nil then + local found = priorityEntries(conds["priority"]) + if #found > 0 then + cleanEntry(found) + for i, key in pairs(found) do + entries[#entries + 1] = key + end + end +end + +return entries diff --git a/src/Kdyby/Redis/scripts/journal.write.lua b/src/Kdyby/Redis/scripts/journal.write.lua new file mode 100644 index 0000000..6228792 --- /dev/null +++ b/src/Kdyby/Redis/scripts/journal.write.lua @@ -0,0 +1,24 @@ + +local dp = readArgs(ARGV) + +-- clean the entry key +cleanEntry({KEYS[1]}) + +-- write the entry key +-- redis.call('multi') + +-- add entry to each tag & tag to entry +if dp["tags"] ~= nil then + for i, tag in pairs(dp["tags"]) do + redis.call('rPush', formatKey(tag, "keys") , KEYS[1]) + redis.call('rPush', formatKey(KEYS[1], "tags") , tag) + end +end + +if dp["priority"] ~= nil then + redis.call('zAdd', formatKey("priority"), dp["priority"], KEYS[1]) +end + +-- redis.call('exec') + +return redis.status_reply("Ok") diff --git a/tests/KdybyTests/Redis/AbstractRedisTestCase.php b/tests/KdybyTests/Redis/AbstractRedisTestCase.php index 482709f..eb29209 100644 --- a/tests/KdybyTests/Redis/AbstractRedisTestCase.php +++ b/tests/KdybyTests/Redis/AbstractRedisTestCase.php @@ -33,31 +33,44 @@ abstract class AbstractRedisTestCase extends Tester\TestCase - protected function setUp() + protected function getClient() { + if ($this->client) { + return $this->client; + } + flock(self::$lock = fopen(dirname(TEMP_DIR) . '/lock-redis', 'w'), LOCK_EX); - $this->client = new RedisClient(); + $client = new RedisClient(); try { - $this->client->connect(); + $client->connect(); } catch (RedisClientException $e) { Tester\Helpers::skip($e->getMessage()); } try { - $this->client->assertVersion(); + $client->assertVersion(); } catch (AssertionException $e) { Tester\Helpers::skip($e->getMessage()); } try { - $this->client->flushDb(); + $client->flushDb(); } catch (RedisClientException $e) { Tester\Assert::fail($e->getMessage()); } + + return $this->client = $client; + } + + + + protected function setUp() + { + $this->getClient(); // make sure it's created } @@ -69,6 +82,8 @@ protected function tearDown() @fclose(self::$lock); self::$lock = NULL; } + + $this->client = NULL; } } diff --git a/tests/KdybyTests/Redis/Extension.phpt b/tests/KdybyTests/Redis/Extension.phpt index b8309b3..4055f5c 100644 --- a/tests/KdybyTests/Redis/Extension.phpt +++ b/tests/KdybyTests/Redis/Extension.phpt @@ -44,8 +44,8 @@ class ExtensionTest extends Tester\TestCase { $dic = $this->createContainer(); Assert::true($dic->getService('redis.client') instanceof Kdyby\Redis\RedisClient); - Assert::true($dic->getService('redis.cacheJournal') instanceof Kdyby\Redis\RedisJournal); - Assert::true($dic->getService('nette.cacheJournal') instanceof Kdyby\Redis\RedisJournal); + Assert::true($dic->getService('redis.cacheJournal') instanceof Kdyby\Redis\RedisLuaJournal); + Assert::true($dic->getService('nette.cacheJournal') instanceof Kdyby\Redis\RedisLuaJournal); Assert::true($dic->getService('redis.cacheStorage') instanceof Kdyby\Redis\RedisStorage); Assert::true($dic->getService('cacheStorage') instanceof Kdyby\Redis\RedisStorage); Assert::same(array( diff --git a/tests/KdybyTests/Redis/RedisJournal.phpt b/tests/KdybyTests/Redis/RedisJournal.phpt index 9c38b9b..571c098 100644 --- a/tests/KdybyTests/Redis/RedisJournal.phpt +++ b/tests/KdybyTests/Redis/RedisJournal.phpt @@ -11,8 +11,10 @@ namespace KdybyTests\Redis; use Kdyby\Redis\RedisJournal; +use Kdyby\Redis\RedisLuaJournal; use Nette; use Nette\Caching\Cache; +use Nette\Caching\Storages\IJournal; use Tester; use Tester\Assert; @@ -26,257 +28,300 @@ require_once __DIR__ . '/../bootstrap.php'; class RedisJournalTest extends AbstractRedisTestCase { - - /** - * @var RedisJournal - */ - private $journal; - - - - protected function setUp() + public function dataJournals() { - parent::setUp(); - $this->journal = new RedisJournal($this->client); + return array( + // array(new RedisJournal($this->getClient())), + array(new RedisLuaJournal($this->getClient())), + ); } - public function testRemoveByTag() + /** + * @dataProvider dataJournals + */ + public function testRemoveByTag(IJournal $journal) { - $this->journal->write('ok_test1', array( + Assert::same(0, count($this->getClient()->keys('*'))); + + $journal->write('ok_test1', array( Cache::TAGS => array('test:homepage'), )); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage'))); + Assert::same(2, count($this->getClient()->keys('*'))); + + $result = $journal->clean(array(Cache::TAGS => array('test:homepage'))); Assert::same(1, count($result)); Assert::same('ok_test1', $result[0]); } - public function testRemovingByMultipleTags_OneIsNotDefined() + /** + * @dataProvider dataJournals + */ + public function testRemovingByMultipleTags_OneIsNotDefined(IJournal $journal) { - $this->journal->write('ok_test2', array( + $journal->write('ok_test2', array( Cache::TAGS => array('test:homepage', 'test:homepage2'), )); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage2'))); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage2'))); Assert::same(1, count($result)); Assert::same('ok_test2', $result[0]); } - public function testRemovingByMultipleTags_BothAreOnOneEntry() + /** + * @dataProvider dataJournals + */ + public function testRemovingByMultipleTags_BothAreOnOneEntry(IJournal $journal) { - $this->journal->write('ok_test2b', array( + $journal->write('ok_test2b', array( Cache::TAGS => array('test:homepage', 'test:homepage2'), )); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage', 'test:homepage2'))); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage', 'test:homepage2'))); Assert::same(1, count($result)); Assert::same('ok_test2b', $result[0]); } - public function testRemoveByMultipleTags_TwoSameTags() + /** + * @dataProvider dataJournals + */ + public function testRemoveByMultipleTags_TwoSameTags(IJournal $journal) { - $this->journal->write('ok_test2c', array( + $journal->write('ok_test2c', array( Cache::TAGS => array('test:homepage', 'test:homepage'), )); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage', 'test:homepage'))); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage', 'test:homepage'))); Assert::same(1, count($result)); Assert::same('ok_test2c', $result[0]); } - public function testRemoveByTagAndPriority() + /** + * @dataProvider dataJournals + */ + public function testRemoveByTagAndPriority(IJournal $journal) { - $this->journal->write('ok_test2d', array( + $journal->write('ok_test2d', array( Cache::TAGS => array('test:homepage'), Cache::PRIORITY => 15, )); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage'), Cache::PRIORITY => 20)); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage'), Cache::PRIORITY => 20)); Assert::same(1, count($result)); Assert::same('ok_test2d', $result[0]); } - public function testRemoveByPriority() + /** + * @dataProvider dataJournals + */ + public function testRemoveByPriority(IJournal $journal) { - $this->journal->write('ok_test3', array( + $journal->write('ok_test3', array( Cache::PRIORITY => 10, )); - $result = $this->journal->clean(array(Cache::PRIORITY => 10)); + $result = $journal->clean(array(Cache::PRIORITY => 10)); Assert::same(1, count($result)); Assert::same('ok_test3', $result[0]); } - public function testPriorityAndTag_CleanByTag() + /** + * @dataProvider dataJournals + */ + public function testPriorityAndTag_CleanByTag(IJournal $journal) { - $this->journal->write('ok_test4', array( + $journal->write('ok_test4', array( Cache::TAGS => array('test:homepage'), Cache::PRIORITY => 10, )); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage'))); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage'))); Assert::same(1, count($result)); Assert::same('ok_test4', $result[0]); } - public function testPriorityAndTag_CleanByPriority() + /** + * @dataProvider dataJournals + */ + public function testPriorityAndTag_CleanByPriority(IJournal $journal) { - $this->journal->write('ok_test5', array( + $journal->write('ok_test5', array( Cache::TAGS => array('test:homepage'), Cache::PRIORITY => 10, )); - $result = $this->journal->clean(array(Cache::PRIORITY => 10)); + $result = $journal->clean(array(Cache::PRIORITY => 10)); Assert::same(1, count($result)); Assert::same('ok_test5', $result[0]); } - public function testMultipleWritesAndMultipleClean() + /** + * @dataProvider dataJournals + */ + public function testMultipleWritesAndMultipleClean(IJournal $journal) { for ($i = 1; $i <= 10; $i++) { - $this->journal->write('ok_test6_' . $i, array( + $journal->write('ok_test6_' . $i, array( Cache::TAGS => array('test:homepage', 'test:homepage/' . $i), Cache::PRIORITY => $i, )); } - $result = $this->journal->clean(array(Cache::PRIORITY => 5)); + $result = $journal->clean(array(Cache::PRIORITY => 5)); Assert::same(5, count($result), "clean priority lower then 5"); Assert::same('ok_test6_1', $result[0], "clean priority lower then 5"); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage/7'))); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage/7'))); Assert::same(1, count($result), "clean tag homepage/7"); Assert::same('ok_test6_7', $result[0], "clean tag homepage/7"); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage/4'))); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage/4'))); Assert::same(0, count($result), "clean non exists tag"); - $result = $this->journal->clean(array(Cache::PRIORITY => 4)); + $result = $journal->clean(array(Cache::PRIORITY => 4)); Assert::same(0, count($result), "clean non exists priority"); - $result = $this->journal->clean(array(Cache::TAGS => array('test:homepage'))); + $result = $journal->clean(array(Cache::TAGS => array('test:homepage'))); Assert::same(4, count($result), "clean other"); Assert::same('ok_test6_6', $result[0], "clean other"); } - public function testSpecialChars() + /** + * @dataProvider dataJournals + */ + public function testSpecialChars(IJournal $journal) { - $this->journal->write('ok_test7ščřžýáíé', array( + $journal->write('ok_test7ščřžýáíé', array( Cache::TAGS => array('čšřýýá', 'ýřžčýž/10') )); - $result = $this->journal->clean(array(Cache::TAGS => array('čšřýýá'))); + $result = $journal->clean(array(Cache::TAGS => array('čšřýýá'))); Assert::same(1, count($result)); Assert::same('ok_test7ščřžýáíé', $result[0]); } - public function testDuplicates_SameTag() + /** + * @dataProvider dataJournals + */ + public function testDuplicates_SameTag(IJournal $journal) { - $this->journal->write('ok_test_a', array( + $journal->write('ok_test_a', array( Cache::TAGS => array('homepage') )); - $this->journal->write('ok_test_a', array( + $journal->write('ok_test_a', array( Cache::TAGS => array('homepage') )); - $result = $this->journal->clean(array(Cache::TAGS => array('homepage'))); + $result = $journal->clean(array(Cache::TAGS => array('homepage'))); Assert::same(1, count($result)); Assert::same('ok_test_a', $result[0]); } - public function testDuplicates_SamePriority() + /** + * @dataProvider dataJournals + */ + public function testDuplicates_SamePriority(IJournal $journal) { - $this->journal->write('ok_test_b', array( + $journal->write('ok_test_b', array( Cache::PRIORITY => 12 )); - $this->journal->write('ok_test_b', array( + $journal->write('ok_test_b', array( Cache::PRIORITY => 12 )); - $result = $this->journal->clean(array(Cache::PRIORITY => 12)); + $result = $journal->clean(array(Cache::PRIORITY => 12)); Assert::same(1, count($result)); Assert::same('ok_test_b', $result[0]); } - public function testDuplicates_DifferentTags() + /** + * @dataProvider dataJournals + */ + public function testDuplicates_DifferentTags(IJournal $journal) { - $this->journal->write('ok_test_ba', array( + $journal->write('ok_test_ba', array( Cache::TAGS => array('homepage') )); - $this->journal->write('ok_test_ba', array( + $journal->write('ok_test_ba', array( Cache::TAGS => array('homepage2') )); - $result = $this->journal->clean(array(Cache::TAGS => array('homepage'))); + $result = $journal->clean(array(Cache::TAGS => array('homepage'))); Assert::same(0, count($result)); - $result2 = $this->journal->clean(array(Cache::TAGS => array('homepage2'))); + $result2 = $journal->clean(array(Cache::TAGS => array('homepage2'))); Assert::same(1, count($result2)); Assert::same('ok_test_ba', $result2[0]); } - public function testDuplicates_DifferentPriorities() + /** + * @dataProvider dataJournals + */ + public function testDuplicates_DifferentPriorities(IJournal $journal) { - $this->journal->write('ok_test_bb', array( + $journal->write('ok_test_bb', array( Cache::PRIORITY => 15 )); - $this->journal->write('ok_test_bb', array( + $journal->write('ok_test_bb', array( Cache::PRIORITY => 20 )); - $result = $this->journal->clean(array(Cache::PRIORITY => 30)); + $result = $journal->clean(array(Cache::PRIORITY => 30)); Assert::same(1, count($result)); Assert::same('ok_test_bb', $result[0]); } - public function testCleanAll() + /** + * @dataProvider dataJournals + */ + public function testCleanAll(IJournal $journal) { - $this->journal->write('ok_test_all_tags', array( + $journal->write('ok_test_all_tags', array( Cache::TAGS => array('test:all', 'test:all') )); - $this->journal->write('ok_test_all_priority', array( + $journal->write('ok_test_all_priority', array( Cache::PRIORITY => 5, )); - $result = $this->journal->clean(array(Cache::ALL => TRUE)); + $result = $journal->clean(array(Cache::ALL => TRUE)); Assert::null($result); - $result2 = $this->journal->clean(array(Cache::TAGS => 'test:all')); + $result2 = $journal->clean(array(Cache::TAGS => 'test:all')); Assert::true(empty($result2)); }