From 273d6cda24b153538a4cc6780763418f2456aabd Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Tue, 17 Apr 2012 17:02:57 +0100 Subject: [PATCH 1/7] Initial work to get key expiry working --- lib/redistat/model.rb | 4 ++++ lib/redistat/summary.rb | 25 ++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/redistat/model.rb b/lib/redistat/model.rb index 8357cf7..5b2e0d4 100644 --- a/lib/redistat/model.rb +++ b/lib/redistat/model.rb @@ -46,6 +46,10 @@ module Redistat alias :class_name :scope + def expire(exp) + options[:expire] = exp.is_a?(Hash) ? exp : Hash.new(exp) + end + def connect_to(opts = {}) Connection.create(opts.merge(:ref => name)) options[:connection_ref] = name diff --git a/lib/redistat/summary.rb b/lib/redistat/summary.rb index 2dd1509..def7a84 100644 --- a/lib/redistat/summary.rb +++ b/lib/redistat/summary.rb @@ -5,9 +5,12 @@ module Redistat class << self def default_options - { :enable_grouping => true, - :label_indexing => true, - :connection_ref => nil } + { + :enable_grouping => true, + :label_indexing => true, + :connection_ref => nil, + :expire => {} + } end def buffer @@ -33,26 +36,30 @@ module Redistat if opts[:enable_grouping] stats = inject_group_summaries(stats) key.groups.each do |k| - update_key(k, stats, depth_limit, opts[:connection_ref]) + update_key(k, stats, depth_limit, opts) k.update_index if opts[:label_indexing] end else - update_key(key, stats, depth_limit, opts[:connection_ref]) + update_key(key, stats, depth_limit, opts) end end private - def update_key(key, stats, depth_limit, connection_ref) + def update_key(key, stats, depth_limit, opts) Date::DEPTHS.each do |depth| - update_fields(key, stats, depth, connection_ref) + update_fields(key, stats, depth, opts) break if depth == depth_limit end end - def update_fields(key, stats, depth, connection_ref = nil) + def update_fields(key, stats, depth, opts) stats.each do |field, value| - db(connection_ref).hincrby key.to_s(depth), field, value + db(opts[:connection_ref]).hincrby key.to_s(depth), field, value + end + + if opts[:expire] && !opts[:expire][depth].nil? + db(opts[:connection_ref]).expire key.to_s(depth), opts[:expire][depth] end end From fccf0db68a954c45c25bc5f6bd7fe5a3dafdefc8 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Wed, 18 Apr 2012 12:33:19 +0100 Subject: [PATCH 2/7] Options are optional --- lib/redistat/summary.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/redistat/summary.rb b/lib/redistat/summary.rb index def7a84..b92224a 100644 --- a/lib/redistat/summary.rb +++ b/lib/redistat/summary.rb @@ -32,7 +32,7 @@ module Redistat update(*args) unless buffer.store(*args) end - def update(key, stats, depth_limit, opts) + def update(key, stats, depth_limit, opts = {}) if opts[:enable_grouping] stats = inject_group_summaries(stats) key.groups.each do |k| @@ -46,14 +46,14 @@ module Redistat private - def update_key(key, stats, depth_limit, opts) + def update_key(key, stats, depth_limit, opts = {}) Date::DEPTHS.each do |depth| update_fields(key, stats, depth, opts) break if depth == depth_limit end end - def update_fields(key, stats, depth, opts) + def update_fields(key, stats, depth, opts = {}) stats.each do |field, value| db(opts[:connection_ref]).hincrby key.to_s(depth), field, value end From e15f6376032d9ba327199abce86d16c9ecda812d Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Wed, 18 Apr 2012 15:08:25 +0100 Subject: [PATCH 3/7] Make Model.expire act like the other options, and add tests for it --- lib/redistat/model.rb | 8 ++++++-- spec/model_helper.rb | 1 + spec/model_spec.rb | 2 ++ spec/summary_spec.rb | 15 +++++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/redistat/model.rb b/lib/redistat/model.rb index 5b2e0d4..f494959 100644 --- a/lib/redistat/model.rb +++ b/lib/redistat/model.rb @@ -46,8 +46,12 @@ module Redistat alias :class_name :scope - def expire(exp) - options[:expire] = exp.is_a?(Hash) ? exp : Hash.new(exp) + def expire(exp = nil) + if !exp.nil? + options[:expire] = exp.is_a?(Hash) ? exp : Hash.new(exp) + else + options[:expire] + end end def connect_to(opts = {}) diff --git a/spec/model_helper.rb b/spec/model_helper.rb index f2dbf93..def5046 100644 --- a/spec/model_helper.rb +++ b/spec/model_helper.rb @@ -26,5 +26,6 @@ class ModelHelper4 include Redistat::Model scope "FancyHelper" + expire :hour => 24*3600 end diff --git a/spec/model_spec.rb b/spec/model_spec.rb index cc5fbfb..0c88b27 100644 --- a/spec/model_spec.rb +++ b/spec/model_spec.rb @@ -38,6 +38,7 @@ describe Redistat::Model do ModelHelper2.store_event.should == true ModelHelper2.hashed_label.should == true ModelHelper2.scope.should be_nil + ModelHelper2.expire.should be_nil ModelHelper1.depth.should == nil ModelHelper1.store_event.should == nil @@ -57,6 +58,7 @@ describe Redistat::Model do ModelHelper4.scope.should == "FancyHelper" ModelHelper4.send(:name).should == "FancyHelper" + ModelHelper4.expire.should == {:hour => 24*3600} end it "should store and fetch stats" do diff --git a/spec/summary_spec.rb b/spec/summary_spec.rb index 2f92910..c0e5106 100644 --- a/spec/summary_spec.rb +++ b/spec/summary_spec.rb @@ -10,6 +10,7 @@ describe Redistat::Summary do @date = Time.now @key = Redistat::Key.new(@scope, @label, @date, {:depth => :day}) @stats = {"views" => 3, "visitors" => 2} + @expire = {:hour => 24*3600} end it "should update a single summary properly" do @@ -32,6 +33,20 @@ describe Redistat::Summary do summary["visitors"].should == "1" end + it "should set key expiry properly" do + Redistat::Summary.update_all(@key, @stats, :hour,{:expire => @expire}) + ((24*3600)-1..(24*3600)+1).should include(db.ttl(@key.to_s(:hour))) + [:day, :month, :year].each do |depth| + db.ttl(@key.to_s(depth)).should == -1 + end + + db.flushdb + Redistat::Summary.update_all(@key, @stats, :hour, {:expire => {}}) + [:hour, :day, :month, :year].each do |depth| + db.ttl(@key.to_s(depth)).should == -1 + end + end + it "should update all summaries properly" do Redistat::Summary.update_all(@key, @stats, :sec) [:year, :month, :day, :hour, :min, :sec, :usec].each do |depth| From 6429f07d5bad87b5ff36e008dba4e17cd80c0c9e Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Wed, 18 Apr 2012 15:11:13 +0100 Subject: [PATCH 4/7] Update .travis.yml to cover more Ruby versions --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9592f40..b59edb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ +language: ruby rvm: - 1.8.7 - 1.9.2 - - jruby + - 1.9.3 + - jruby-18mode + - jruby-19mode + - rbx-18mode - ree From a8de80e69e17473d22b66886116f032ad114387a Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Wed, 18 Apr 2012 15:15:14 +0100 Subject: [PATCH 5/7] Remove rbx from .travis.yml Proper support for Rubinus will added in due time. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b59edb5..cdd28fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,5 +5,4 @@ rvm: - 1.9.3 - jruby-18mode - jruby-19mode - - rbx-18mode - ree From c0ecf4bc842ce0d042bbd2b31aa8b5ee02e80b58 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Wed, 18 Apr 2012 16:19:03 +0100 Subject: [PATCH 6/7] Add information about key expiry to readme --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 2d6d829..3aaa151 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,36 @@ This section does need further expanding as there's a lot to cover when it comes to the finder. +## Key Expiry + +Support for expiring keys from Redis is available, allowing you too keep +varying levels of details for X period of time. This allows you easily keep +things nice and tidy by only storing varying levels detailed stats only for as +long as you need. + +In the below example we define how long Redis keys for varying depths are +stored. Second by second stats are available for 10 minutes, minute by minute +stats for 6 hours, hourly stats for 3 months, daily stats for 2 years, and +yearly stats are retained forever. + +```ruby +class ViewStats + include Redistat::Model + + depth :sec + + expire \ + :sec => 10.minutes.to_i, + :min => 6.hours.to_i, + :hour => 3.months.to_i, + :day => 2.years.to_i +end +``` + +Keep in mind that when storing stats for a custom date in the past for +example, the expiry time for the keys will be relative to now. The values you +specify are simply passed to the `Redis#expire` method. + ## Internals From 41f53b9fbacc904497358cf78c35d58a01cfe742 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Wed, 18 Apr 2012 16:42:27 +0100 Subject: [PATCH 7/7] Print a warning to STDERR if Redis version is less than 2.1.3 Prior to v2.1.3, any writes to keys with an expire/TTL set were completely deleted before the write took place. --- lib/redistat/connection.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/redistat/connection.rb b/lib/redistat/connection.rb index f7dcfd9..9b3fff3 100644 --- a/lib/redistat/connection.rb +++ b/lib/redistat/connection.rb @@ -4,6 +4,7 @@ module Redistat module Connection REQUIRED_SERVER_VERSION = "1.3.10" + MIN_EXPIRE_SERVER_VERSION = "2.1.3" # TODO: Create a ConnectionPool instance object using Sychronize mixin to replace Connection class @@ -67,6 +68,10 @@ module Redistat def check_redis_version(conn) raise RedisServerIsTooOld if conn.info["redis_version"] < REQUIRED_SERVER_VERSION + if conn.info["redis_version"] < MIN_EXPIRE_SERVER_VERSION + STDOUT.puts "WARNING: You MUST upgrade Redis to v2.1.3 or later " + + "if you are using key expiry." + end conn end