From 9faa0db7b833f3ad62bd4b22e2b4ce15044fd839 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Wed, 9 Mar 2011 17:05:10 +0000 Subject: [PATCH] drastic change in label indexing --- lib/redistat/finder.rb | 70 +++++++++++++++++++++---------------- lib/redistat/key.rb | 57 +++++++++++++++--------------- lib/redistat/label.rb | 44 ++++++++--------------- lib/redistat/summary.rb | 2 +- spec/event_spec.rb | 8 ++--- spec/finder_spec.rb | 68 ++++++++++++++++++++++-------------- spec/key_spec.rb | 77 +++++++++++++++++++++++++++++++---------- spec/label_spec.rb | 40 +++++---------------- spec/model_spec.rb | 41 ++++++++++++++++++++-- spec/summary_spec.rb | 8 ++--- 10 files changed, 241 insertions(+), 174 deletions(-) diff --git a/lib/redistat/finder.rb b/lib/redistat/finder.rb index 654625c..63c947c 100644 --- a/lib/redistat/finder.rb +++ b/lib/redistat/finder.rb @@ -40,8 +40,12 @@ module Redistat attr_reader :options - def initialize(options = {}) - @options = options + def initialize(opts = {}) + set_options(opts) + end + + def options + @options ||= {} end def all(reload = false) @@ -65,21 +69,27 @@ module Redistat all.each_with_index(&block) end + def children + build_key.children.map { |key| + self.class.new(options.merge(:label => key.label.to_s)) + } + end + def connection_ref(ref) - reset! if @options[:connection_ref] != ref - @options[:connection_ref] = ref + reset! if options[:connection_ref] != ref + options[:connection_ref] = ref self end def scope(scope) - reset! if @options[:scope].to_s != scope - @options[:scope] = Scope.new(scope) + reset! if !options[:scope].nil? && options[:scope].to_s != scope + options[:scope] = Scope.new(scope) self end def label(label) - reset! if @options[:label].raw != label - @options[:label] = Label.new(label) + reset! if !options[:label].nil? && options[:label].to_s != label + options[:label] = Label.new(label) self end @@ -89,34 +99,34 @@ module Redistat alias :date :dates def from(date) - reset! if @options[:from] != date - @options[:from] = date + reset! if options[:from] != date + options[:from] = date self end def till(date) - reset! if @options[:till] != date - @options[:till] = date + reset! if options[:till] != date + options[:till] = date self end alias :until :till def depth(unit) - reset! if @options[:depth] != unit - @options[:depth] = unit + reset! if options[:depth] != unit + options[:depth] = unit self end def interval(unit) - reset! if @options[:interval] != unit - @options[:interval] = unit + reset! if options[:interval] != unit + options[:interval] = unit self end - def find(options = {}) - set_options(options) + def find(opts = {}) + set_options(opts) raise InvalidOptions.new if !valid_options? - if @options[:interval].nil? || !@options[:interval] + if options[:interval].nil? || !options[:interval] find_by_magic else find_by_interval @@ -130,14 +140,14 @@ module Redistat opts.each do |key, value| self.send(key, opts.delete(key)) if self.respond_to?(key) end - @options.merge!(opts) + self.options.merge!(opts) end - def find_by_interval(options = {}) + def find_by_interval raise InvalidOptions.new if !valid_options? key = build_key - col = Collection.new(@options) - col.total = Result.new(@options) + col = Collection.new(options) + col.total = Result.new(options) build_date_sets.each do |set| set[:add].each do |date| result = Result.new @@ -152,11 +162,11 @@ module Redistat col end - def find_by_magic(options = {}) + def find_by_magic raise InvalidOptions.new if !valid_options? key = build_key - col = Collection.new(@options) - col.total = Result.new(@options) + col = Collection.new(options) + col.total = Result.new(options) col << col.total build_date_sets.each do |set| sum = Result.new @@ -174,16 +184,16 @@ module Redistat end def valid_options? - return true if !@options[:scope].blank? && !@options[:label].blank? && !@options[:from].blank? && !@options[:till].blank? + return true if !options[:scope].blank? && !options[:label].blank? && !options[:from].blank? && !options[:till].blank? false end def build_date_sets - Finder::DateSet.new(@options[:from], @options[:till], @options[:depth], @options[:interval]) + Finder::DateSet.new(options[:from], options[:till], options[:depth], options[:interval]) end def build_key - Key.new(@options[:scope], @options[:label]) + Key.new(options[:scope], options[:label]) end def summarize_add_keys(sets, key, sum) @@ -205,7 +215,7 @@ module Redistat end def db - super(@options[:connection_ref]) + super(options[:connection_ref]) end end diff --git a/lib/redistat/key.rb b/lib/redistat/key.rb index 80b7080..5444a49 100644 --- a/lib/redistat/key.rb +++ b/lib/redistat/key.rb @@ -1,7 +1,7 @@ module Redistat class Key + include Redistat::Database - attr_accessor :date attr_accessor :options def initialize(scope, label_name = nil, time_stamp = nil, options = {}) @@ -17,7 +17,7 @@ module Redistat def prefix key = "#{@scope}" - key << "/#{label}" if !label.nil? + key << "/#{label.name}" if !label.nil? key << ":" key end @@ -25,27 +25,12 @@ module Redistat def date=(input) @date = (input.instance_of?(Redistat::Date)) ? input : Date.new(input) # Redistat::Date, not ::Date end + attr_reader :date def depth @options[:depth] end - def label - @label.name - end - - def label=(input) - @label = (input.instance_of?(Redistat::Label)) ? input : Label.create(input, @options) - end - - def label_hash - @label.hash - end - - def label_groups - @label.groups - end - def scope @scope.to_s end @@ -54,18 +39,36 @@ module Redistat @scope = (input.instance_of?(Redistat::Scope)) ? input : Scope.new(input) end - def groups - @groups ||= label_groups.map do |label_name| - self.class.new(@scope, label_name, self.date, @options) + def label=(input) + @label = (input.instance_of?(Redistat::Label)) ? input : Label.create(input, @options) + end + attr_reader :label + + def label_hash + @label.hash + end + + def parent + @parent ||= self.class.new(self.scope, @label.parent, self.date, @options) unless @label.parent.nil? + end + + def children + db.smembers("#{scope}#{LABEL_INDEX}#{@label}").map { |member| + self.class.new(self.scope, "#{@label}#{GROUP_SEPARATOR}#{member}", self.date, @options) + } + end + + def update_index + @label.groups.each do |label| + break if label.parent.nil? + db.sadd("#{scope}#{LABEL_INDEX}#{label.parent}", label.me) end end - def parent_group - @label.parent_group - end - - def update_label_index - @label.update_index + def groups # TODO: Is this useless? + @groups ||= @label.groups.map do |label| + self.class.new(@scope, label, self.date, @options) + end end def to_s(depth = nil) diff --git a/lib/redistat/label.rb b/lib/redistat/label.rb index 748e85d..ef45852 100644 --- a/lib/redistat/label.rb +++ b/lib/redistat/label.rb @@ -2,7 +2,6 @@ module Redistat class Label include Database - attr_reader :raw attr_reader :connection_ref def self.create(name, options = {}) @@ -13,22 +12,25 @@ module Redistat @options = options @raw = str.to_s end + + def to_s + @raw + end def db super(@options[:connection_ref]) end def name - @options[:hashed_label] ? hash : @raw + @options[:hashed_label] ? hash : self.to_s end def hash - @hash ||= Digest::SHA1.hexdigest(@raw) + @hash ||= Digest::SHA1.hexdigest(self.to_s) end def save - @saved = (db.set("#{KEY_LEBELS}#{hash}", @raw) == "OK") if @options[:hashed_label] - update_index if groups.size > 1 # TODO: add a label_indexing option + @saved = (db.set("#{KEY_LEBELS}#{hash}", self.to_s) == "OK") if @options[:hashed_label] self end @@ -37,44 +39,26 @@ module Redistat end def parent - @parent ||= self.class.new(parent_group) + @parent ||= groups[1] if groups.size > 1 end - def parent_group - groups[1] - end - - def group - @raw.split(GROUP_SEPARATOR).last + def me + self.to_s.split(GROUP_SEPARATOR).last end def groups - return @groups if @groups + return @groups unless @groups.nil? @groups = [] parent = "" - @raw.split(GROUP_SEPARATOR).each do |part| + self.to_s.split(GROUP_SEPARATOR).each do |part| if !part.blank? - group = ((parent.blank?) ? "" : "#{parent}/") + part - @groups << group + group = ((parent.blank?) ? "" : "#{parent}#{GROUP_SEPARATOR}") + part + @groups << Label.new(group) parent = group end end @groups.reverse! end - def sub_labels - db.smembers("#{LABEL_INDEX}#{parent_group}").map { |member| - self.class.new("#{parent_group}#{GROUP_SEPARATOR}#{member}") - } - end - - def update_index - groups.each do |group| - label = self.class.new(group) - break if label.parent_group.nil? - db.sadd("#{LABEL_INDEX}#{label.parent_group}", label.group) == "OK" ? true : false - end - end - end end \ No newline at end of file diff --git a/lib/redistat/summary.rb b/lib/redistat/summary.rb index 938baaa..90834ea 100644 --- a/lib/redistat/summary.rb +++ b/lib/redistat/summary.rb @@ -13,7 +13,7 @@ module Redistat stats = inject_group_summaries(stats) key.groups.each { |k| update_key(k, stats, depth_limit, connection_ref) - k.update_label_index # TODO: add a label_indexing option + k.update_index # TODO: add a label_indexing option } else update_key(key, stats, depth_limit, connection_ref) diff --git a/spec/event_spec.rb b/spec/event_spec.rb index 3550c59..e586894 100644 --- a/spec/event_spec.rb +++ b/spec/event_spec.rb @@ -18,7 +18,7 @@ describe Redistat::Event do it "should initialize properly" do @event.id.should be_nil @event.scope.should == @scope - @event.label.should == @label + @event.label.to_s.should == @label @event.label_hash.should == @label_hash @event.date.to_time.to_s.should == @date.to_s @event.stats.should == @stats @@ -33,12 +33,12 @@ describe Redistat::Event do @event.date = @date @event.date.to_time.to_s.should == @date.to_s # label - @event.label.should == @label + @event.label.to_s.should == @label @event.label_hash.should == @label_hash @label = "contact_us" @label_hash = Digest::SHA1.hexdigest(@label) @event.label = @label - @event.label.should == @label + @event.label.to_s.should == @label @event.label_hash.should == @label_hash end @@ -64,7 +64,7 @@ describe Redistat::Event do @event = Redistat::Event.new(@scope, @label, @date, @stats, @options.merge({:store_event => true}), @meta).save fetched = Redistat::Event.find(@scope, @event.id) @event.scope.should == fetched.scope - @event.label.should == fetched.label + @event.label.to_s.should == fetched.label.to_s @event.date.to_s.should == fetched.date.to_s end diff --git a/spec/finder_spec.rb b/spec/finder_spec.rb index f87e8ff..0d8df60 100644 --- a/spec/finder_spec.rb +++ b/spec/finder_spec.rb @@ -9,41 +9,43 @@ describe Redistat::Finder do @label = "about_us" @date = Time.now @key = Redistat::Key.new(@scope, @label, @date, {:depth => :day}) - @stats = {"views" => 3, "visitors" => 2} + @stats = {"views" => 3, "visitors" => 2} + @two_hours_ago = 2.hours.ago + @one_hour_ago = 1.hour.ago end it "should initialize properly" do - two_hours_ago = 2.hours.ago - one_hour_ago = 1.hour.ago - options = {:scope => "PageViews", :label => "Label", :from => two_hours_ago, :till => one_hour_ago, :depth => :hour, :interval => :hour} - - finder = Redistat::Finder.new(options) - finder.options.should == options + options = {:scope => "PageViews", :label => "Label", :from => @two_hours_ago, :till => @one_hour_ago, :depth => :hour, :interval => :hour} finder = Redistat::Finder.new finder.send(:set_options, options) - finder.options.should == options - - finder = Redistat::Finder.dates(two_hours_ago, one_hour_ago).scope("PageViews").label("Label").depth(:hour).interval(:hour) - finder.options.should == options - - finder = Redistat::Finder.scope("PageViews").label("Label").from(two_hours_ago).till(one_hour_ago).depth(:hour).interval(:hour) - finder.options.should == options + finder.options[:scope].should be_a(Redistat::Scope) + finder.options[:scope].to_s.should == options[:scope] + finder.options[:label].should be_a(Redistat::Label) + finder.options[:label].to_s.should == options[:label] + finder.options.should == options.merge(:scope => finder.options[:scope], :label => finder.options[:label]) - finder = Redistat::Finder.label("Label").from(two_hours_ago).till(one_hour_ago).depth(:hour).interval(:hour).scope("PageViews") - finder.options.should == options + finder = Redistat::Finder.dates(@two_hours_ago, @one_hour_ago) + finder.options[:from].should == @two_hours_ago + finder.options[:till].should == @one_hour_ago - finder = Redistat::Finder.from(two_hours_ago).till(one_hour_ago).depth(:hour).interval(:hour).scope("PageViews").label("Label") - finder.options.should == options + finder = Redistat::Finder.scope("hello") + finder.options[:scope].to_s.should == "hello" - finder = Redistat::Finder.till(one_hour_ago).depth(:hour).interval(:hour).scope("PageViews").label("Label").from(two_hours_ago) - finder.options.should == options + finder = Redistat::Finder.label("hello") + finder.options[:label].to_s.should == "hello" - finder = Redistat::Finder.depth(:hour).interval(:hour).scope("PageViews").label("Label").from(two_hours_ago).till(one_hour_ago) - finder.options.should == options + finder = Redistat::Finder.from(@two_hours_ago) + finder.options[:from].should == @two_hours_ago - finder = Redistat::Finder.interval(:hour).scope("PageViews").label("Label").from(two_hours_ago).till(one_hour_ago).depth(:hour) - finder.options.should == options + finder = Redistat::Finder.till(@one_hour_ago) + finder.options[:till].should == @one_hour_ago + + finder = Redistat::Finder.depth(:hour) + finder.options[:depth].should == :hour + + finder = Redistat::Finder.interval(:hour) + finder.options[:interval].should == :hour end @@ -89,6 +91,21 @@ describe Redistat::Finder do lambda { Redistat::Finder.find(:from => 3.hours.ago) }.should raise_error(Redistat::InvalidOptions) end + it "should find children" do + Redistat::Key.new("PageViews", "message/public/die").update_index + Redistat::Key.new("PageViews", "message/public/live").update_index + Redistat::Key.new("PageViews", "message/public/fester").update_index + members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}message/public") # checking 'message/public' + options = {:scope => "PageViews", :label => "message/public", :from => @two_hours_ago, :till => @one_hour_ago, :depth => :hour, :interval => :hour} + finder = Redistat::Finder.new(options) + finder.children.first.should be_a(Redistat::Finder) + subs = finder.children.map { |f| f.options[:label].me } + subs.should have(3).items + subs.should include('die') + subs.should include('live') + subs.should include('fester') + end + describe "Lazy-Loading" do before(:each) do @@ -103,7 +120,6 @@ describe Redistat::Finder do end it "should lazy-load" do - @finder.instance_variable_get("@result").should be_nil stats = @finder.all @finder.instance_variable_get("@result").should_not be_nil @@ -146,7 +162,7 @@ describe Redistat::Finder do res.should == match end - end + end # "Lazy-Loading" # helper methods diff --git a/spec/key_spec.rb b/spec/key_spec.rb index 9623275..dd7ff03 100644 --- a/spec/key_spec.rb +++ b/spec/key_spec.rb @@ -1,8 +1,10 @@ require "spec_helper" describe Redistat::Key do + include Redistat::Database before(:each) do + db.flushdb @scope = "PageViews" @label = "about_us" @label_hash = Digest::SHA1.hexdigest(@label) @@ -12,9 +14,9 @@ describe Redistat::Key do it "should initialize properly" do @key.scope.should == @scope - @key.label.should == @label + @key.label.to_s.should == @label @key.label_hash.should == @label_hash - @key.label_groups.should == @key.instance_variable_get("@label").groups + @key.groups.map { |k| k.instance_variable_get("@label") }.should == @key.instance_variable_get("@label").groups @key.date.should be_instance_of(Redistat::Date) @key.date.to_time.to_s.should == @date.to_s end @@ -52,31 +54,70 @@ describe Redistat::Key do @key.date = @date @key.date.to_time.to_s.should == @date.to_s # label - @key.label.should == @label + @key.label.to_s.should == @label @key.label_hash == @label_hash @label = "contact_us" @label_hash = Digest::SHA1.hexdigest(@label) @key.label = @label - @key.label.should == @label + @key.label.to_s.should == @label @key.label_hash == @label_hash end - it "should create a group of keys from label group" do - label = 'message/public/offensive' - result = [ "message/public/offensive", - "message/public", - "message" ] + describe "Grouping" do + before(:each) do + @label = "message/public/offensive" + @key = Redistat::Key.new(@scope, @label, @date, {:depth => :hour}) + end - key = Redistat::Key.new(@scope, label, @date, {:depth => :hour}) + it "should create a group of keys from label group" do + label = 'message/public/offensive' + result = [ "message/public/offensive", + "message/public", + "message" ] + + key = Redistat::Key.new(@scope, label, @date, {:depth => :hour}) + + key.groups.map { |k| k.label.to_s }.should == result + end - key.label_groups.should == result - key.groups.map { |k| k.label }.should == result - end - - it "should know it's parent label group" do - label = 'message/public/offensive' - key = Redistat::Key.new(@scope, label, @date, {:depth => :hour}) - key.parent_group.should == 'message/public' + it "should know it's parent" do + @key.parent.should be_a(Redistat::Key) + @key.parent.label.to_s.should == 'message/public' + Redistat::Key.new(@scope, 'hello', @date).parent.should be_nil + end + + it "should update label index and return children" do + db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{@key.label.parent}").should == [] + @key.children.should have(0).items + + @key.update_index # indexing 'message/publish/offensive' + Redistat::Key.new("PageViews", "message/public/die").update_index # indexing 'message/publish/die' + Redistat::Key.new("PageViews", "message/public/live").update_index # indexing 'message/publish/live' + + members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{@key.label.parent}") # checking 'message/public' + members.should have(3).item + members.should include('offensive') + members.should include('live') + members.should include('die') + + key = @key.parent + key.children.first.should be_a(Redistat::Key) + key.children.should have(3).item + key.children.map { |k| k.label.me }.should == members + + members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{key.label.parent}") # checking 'message' + members.should have(1).item + members.should include('public') + + key = key.parent + key.children.should have(1).item + key.children.map { |k| k.label.me }.should == members + + members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{key.label.parent}") # checking '' + members.should have(0).item + + key.parent.should be_nil + end end end \ No newline at end of file diff --git a/spec/label_spec.rb b/spec/label_spec.rb index e742b00..8d38b09 100644 --- a/spec/label_spec.rb +++ b/spec/label_spec.rb @@ -32,50 +32,26 @@ describe Redistat::Label do end it "should know it's parent label group" do - @label.parent_group.should == 'message/public' - Redistat::Label.new('hello').parent_group.should be_nil + @label.parent.to_s.should == 'message/public' + Redistat::Label.new('hello').parent.should be_nil end it "should separate label names into groups" do @label.name.should == @name - @label.groups.should == [ "message/public/offensive", - "message/public", - "message" ] + @label.groups.map { |l| l.to_s }.should == [ "message/public/offensive", + "message/public", + "message" ] @name = "/message/public/" @label = Redistat::Label.new(@name) @label.name.should == @name - @label.groups.should == [ "message/public", - "message" ] + @label.groups.map { |l| l.to_s }.should == [ "message/public", + "message" ] @name = "message" @label = Redistat::Label.new(@name) @label.name.should == @name - @label.groups.should == [ "message" ] - end - - it "should update label index" do - db.smembers("#{Redistat::LABEL_INDEX}#{@label.parent_group}").should == [] - @label.update_index - members = db.smembers("#{Redistat::LABEL_INDEX}#{@label.parent_group}") # checking 'message/public' - members.should have(1).item - members.should include('offensive') - members.should == @label.sub_labels.map { |l| l.group } - - name = "message/public/nice" - label = Redistat::Label.new(name) - label.update_index - members = db.smembers("#{Redistat::LABEL_INDEX}#{label.parent_group}") # checking 'message/public' - members.should have(2).items - members.should include('offensive') - members.should include('nice') - members.should == label.sub_labels.map { |l| l.group } - - label = @label.parent - members = db.smembers("#{Redistat::LABEL_INDEX}#{label.parent_group}") # checking 'message' - members.should have(1).item - members.should include('public') - members.should == label.sub_labels.map { |l| l.group } + @label.groups.map { |l| l.to_s }.should == [ "message" ] end end diff --git a/spec/model_spec.rb b/spec/model_spec.rb index 3398bdf..503ba27 100644 --- a/spec/model_spec.rb +++ b/spec/model_spec.rb @@ -22,8 +22,8 @@ describe Redistat::Model do one_hour_ago = 1.hour.ago finder = ModelHelper1.find('label', two_hours_ago, one_hour_ago) finder.should be_a(Redistat::Finder) - finder.options[:scope].should == 'ModelHelper1' - finder.options[:label].should == 'label' + finder.options[:scope].to_s.should == 'ModelHelper1' + finder.options[:label].to_s.should == 'label' finder.options[:from].should == two_hours_ago finder.options[:till].should == one_hour_ago end @@ -82,6 +82,43 @@ describe Redistat::Model do stats.first.should == stats.total end + it "should store and fetch grouping enabled stats" do + ModelHelper1.store("sheep/black", {:count => 6, :weight => 461}, @time.hours_ago(4)) + ModelHelper1.store("sheep/black", {:count => 2, :weight => 156}, @time) + ModelHelper1.store("sheep/white", {:count => 5, :weight => 393}, @time.hours_ago(4)) + ModelHelper1.store("sheep/white", {:count => 4, :weight => 316}, @time) + + stats = ModelHelper1.fetch("sheep/black", @time.hours_ago(2), @time.hours_since(1)) + stats.total["count"].should == 2 + stats.total["weight"].should == 156 + stats.first.should == stats.total + + stats = ModelHelper1.fetch("sheep/black", @time.hours_ago(5), @time.hours_since(1)) + stats.total[:count].should == 8 + stats.total[:weight].should == 617 + stats.first.should == stats.total + + stats = ModelHelper1.fetch("sheep/white", @time.hours_ago(2), @time.hours_since(1)) + stats.total[:count].should == 4 + stats.total[:weight].should == 316 + stats.first.should == stats.total + + stats = ModelHelper1.fetch("sheep/white", @time.hours_ago(5), @time.hours_since(1)) + stats.total[:count].should == 9 + stats.total[:weight].should == 709 + stats.first.should == stats.total + + stats = ModelHelper1.fetch("sheep", @time.hours_ago(2), @time.hours_since(1)) + stats.total[:count].should == 6 + stats.total[:weight].should == 472 + stats.first.should == stats.total + + stats = ModelHelper1.fetch("sheep", @time.hours_ago(5), @time.hours_since(1)) + stats.total[:count].should == 17 + stats.total[:weight].should == 1326 + stats.first.should == stats.total + end + it "should connect to different Redis servers on a per-model basis" do ModelHelper3.redis.client.db.should == 14 diff --git a/spec/summary_spec.rb b/spec/summary_spec.rb index 303c03f..ae1f952 100644 --- a/spec/summary_spec.rb +++ b/spec/summary_spec.rb @@ -83,8 +83,8 @@ describe Redistat::Summary do key = Redistat::Key.new(@scope, label, @date) Redistat::Summary.update_all(key, stats, :hour) - key.groups[0].label.should == "views/about_us" - key.groups[1].label.should == "views" + key.groups[0].label.to_s.should == "views/about_us" + key.groups[1].label.to_s.should == "views" child1 = key.groups[0] parent = key.groups[1] @@ -92,8 +92,8 @@ describe Redistat::Summary do key = Redistat::Key.new(@scope, label, @date) Redistat::Summary.update_all(key, stats, :hour) - key.groups[0].label.should == "views/contact" - key.groups[1].label.should == "views" + key.groups[0].label.to_s.should == "views/contact" + key.groups[1].label.to_s.should == "views" child2 = key.groups[0] summary = db.hgetall(child1.to_s(:hour))