jimeh/redistat
1
0
Fork 0
mirror of https://github.com/jimeh/redistat.git synced 2026-02-19 05:16:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

drastic change in label indexing

Browse Source
This commit is contained in:
Jim Myhrberg
2011-03-09 17:05:10 +00:00
parent 834614ab79
commit 9faa0db7b8
10 changed files with 241 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
lib/redistat/finder.rb
View File

@@ -40,8 +40,12 @@ module Redistat
attr_reader :options
def initialize(options = {})
@options = options
def initialize(opts = {})
set_options(opts)
end
def options
@options ||= {}
end
def all(reload = false)
@@ -65,21 +69,27 @@ module Redistat
all.each_with_index(&block)
end
def children
build_key.children.map { |key|
self.class.new(options.merge(:label => key.label.to_s))
}
end
def connection_ref(ref)
reset! if @options[:connection_ref] != ref
@options[:connection_ref] = ref
reset! if options[:connection_ref] != ref
options[:connection_ref] = ref
self
end
def scope(scope)
reset! if @options[:scope].to_s != scope
@options[:scope] = Scope.new(scope)
reset! if !options[:scope].nil? && options[:scope].to_s != scope
options[:scope] = Scope.new(scope)
self
end
def label(label)
reset! if @options[:label].raw != label
@options[:label] = Label.new(label)
reset! if !options[:label].nil? && options[:label].to_s != label
options[:label] = Label.new(label)
self
end
@@ -89,34 +99,34 @@ module Redistat
alias :date :dates
def from(date)
reset! if @options[:from] != date
@options[:from] = date
reset! if options[:from] != date
options[:from] = date
self
end
def till(date)
reset! if @options[:till] != date
@options[:till] = date
reset! if options[:till] != date
options[:till] = date
self
end
alias :until :till
def depth(unit)
reset! if @options[:depth] != unit
@options[:depth] = unit
reset! if options[:depth] != unit
options[:depth] = unit
self
end
def interval(unit)
reset! if @options[:interval] != unit
@options[:interval] = unit
reset! if options[:interval] != unit
options[:interval] = unit
self
end
def find(options = {})
set_options(options)
def find(opts = {})
set_options(opts)
raise InvalidOptions.new if !valid_options?
if @options[:interval].nil? || !@options[:interval]
if options[:interval].nil? || !options[:interval]
find_by_magic
else
find_by_interval
@@ -130,14 +140,14 @@ module Redistat
opts.each do |key, value|
self.send(key, opts.delete(key)) if self.respond_to?(key)
end
@options.merge!(opts)
self.options.merge!(opts)
end
def find_by_interval(options = {})
def find_by_interval
raise InvalidOptions.new if !valid_options?
key = build_key
col = Collection.new(@options)
col.total = Result.new(@options)
col = Collection.new(options)
col.total = Result.new(options)
build_date_sets.each do |set|
set[:add].each do |date|
result = Result.new
@@ -152,11 +162,11 @@ module Redistat
col
end
def find_by_magic(options = {})
def find_by_magic
raise InvalidOptions.new if !valid_options?
key = build_key
col = Collection.new(@options)
col.total = Result.new(@options)
col = Collection.new(options)
col.total = Result.new(options)
col << col.total
build_date_sets.each do |set|
sum = Result.new
@@ -174,16 +184,16 @@ module Redistat
end
def valid_options?
return true if !@options[:scope].blank? && !@options[:label].blank? && !@options[:from].blank? && !@options[:till].blank?
return true if !options[:scope].blank? && !options[:label].blank? && !options[:from].blank? && !options[:till].blank?
false
end
def build_date_sets
Finder::DateSet.new(@options[:from], @options[:till], @options[:depth], @options[:interval])
Finder::DateSet.new(options[:from], options[:till], options[:depth], options[:interval])
end
def build_key
Key.new(@options[:scope], @options[:label])
Key.new(options[:scope], options[:label])
end
def summarize_add_keys(sets, key, sum)
@@ -205,7 +215,7 @@ module Redistat
end
def db
super(@options[:connection_ref])
super(options[:connection_ref])
end
end

57
lib/redistat/key.rb
View File

@@ -1,7 +1,7 @@
module Redistat
class Key
include Redistat::Database
attr_accessor :date
attr_accessor :options
def initialize(scope, label_name = nil, time_stamp = nil, options = {})
@@ -17,7 +17,7 @@ module Redistat
def prefix
key = "#{@scope}"
key << "/#{label}" if !label.nil?
key << "/#{label.name}" if !label.nil?
key << ":"
key
end
@@ -25,27 +25,12 @@ module Redistat
def date=(input)
@date = (input.instance_of?(Redistat::Date)) ? input : Date.new(input) # Redistat::Date, not ::Date
end
attr_reader :date
def depth
@options[:depth]
end
def label
@label.name
end
def label=(input)
@label = (input.instance_of?(Redistat::Label)) ? input : Label.create(input, @options)
end
def label_hash
@label.hash
end
def label_groups
@label.groups
end
def scope
@scope.to_s
end
@@ -54,18 +39,36 @@ module Redistat
@scope = (input.instance_of?(Redistat::Scope)) ? input : Scope.new(input)
end
def groups
@groups ||= label_groups.map do |label_name|
self.class.new(@scope, label_name, self.date, @options)
def label=(input)
@label = (input.instance_of?(Redistat::Label)) ? input : Label.create(input, @options)
end
attr_reader :label
def label_hash
@label.hash
end
def parent
@parent ||= self.class.new(self.scope, @label.parent, self.date, @options) unless @label.parent.nil?
end
def children
db.smembers("#{scope}#{LABEL_INDEX}#{@label}").map { |member|
self.class.new(self.scope, "#{@label}#{GROUP_SEPARATOR}#{member}", self.date, @options)
}
end
def update_index
@label.groups.each do |label|
break if label.parent.nil?
db.sadd("#{scope}#{LABEL_INDEX}#{label.parent}", label.me)
end
end
def parent_group
@label.parent_group
end
def update_label_index
@label.update_index
def groups # TODO: Is this useless?
@groups ||= @label.groups.map do |label|
self.class.new(@scope, label, self.date, @options)
end
end
def to_s(depth = nil)

44
lib/redistat/label.rb
View File

@@ -2,7 +2,6 @@ module Redistat
class Label
include Database
attr_reader :raw
attr_reader :connection_ref
def self.create(name, options = {})
@@ -13,22 +12,25 @@ module Redistat
@options = options
@raw = str.to_s
end
def to_s
@raw
end
def db
super(@options[:connection_ref])
end
def name
@options[:hashed_label] ? hash : @raw
@options[:hashed_label] ? hash : self.to_s
end
def hash
@hash ||= Digest::SHA1.hexdigest(@raw)
@hash ||= Digest::SHA1.hexdigest(self.to_s)
end
def save
@saved = (db.set("#{KEY_LEBELS}#{hash}", @raw) == "OK") if @options[:hashed_label]
update_index if groups.size > 1 # TODO: add a label_indexing option
@saved = (db.set("#{KEY_LEBELS}#{hash}", self.to_s) == "OK") if @options[:hashed_label]
self
end
@@ -37,44 +39,26 @@ module Redistat
end
def parent
@parent ||= self.class.new(parent_group)
@parent ||= groups[1] if groups.size > 1
end
def parent_group
groups[1]
end
def group
@raw.split(GROUP_SEPARATOR).last
def me
self.to_s.split(GROUP_SEPARATOR).last
end
def groups
return @groups if @groups
return @groups unless @groups.nil?
@groups = []
parent = ""
@raw.split(GROUP_SEPARATOR).each do |part|
self.to_s.split(GROUP_SEPARATOR).each do |part|
if !part.blank?
group = ((parent.blank?) ? "" : "#{parent}/") + part
@groups << group
group = ((parent.blank?) ? "" : "#{parent}#{GROUP_SEPARATOR}") + part
@groups << Label.new(group)
parent = group
end
end
@groups.reverse!
end
def sub_labels
db.smembers("#{LABEL_INDEX}#{parent_group}").map { |member|
self.class.new("#{parent_group}#{GROUP_SEPARATOR}#{member}")
}
end
def update_index
groups.each do |group|
label = self.class.new(group)
break if label.parent_group.nil?
db.sadd("#{LABEL_INDEX}#{label.parent_group}", label.group) == "OK" ? true : false
end
end
end
end

2
lib/redistat/summary.rb
View File

@@ -13,7 +13,7 @@ module Redistat
stats = inject_group_summaries(stats)
key.groups.each { |k|
update_key(k, stats, depth_limit, connection_ref)
k.update_label_index # TODO: add a label_indexing option
k.update_index # TODO: add a label_indexing option
}
else
update_key(key, stats, depth_limit, connection_ref)

8
spec/event_spec.rb
View File

@@ -18,7 +18,7 @@ describe Redistat::Event do
it "should initialize properly" do
@event.id.should be_nil
@event.scope.should == @scope
@event.label.should == @label
@event.label.to_s.should == @label
@event.label_hash.should == @label_hash
@event.date.to_time.to_s.should == @date.to_s
@event.stats.should == @stats
@@ -33,12 +33,12 @@ describe Redistat::Event do
@event.date = @date
@event.date.to_time.to_s.should == @date.to_s
# label
@event.label.should == @label
@event.label.to_s.should == @label
@event.label_hash.should == @label_hash
@label = "contact_us"
@label_hash = Digest::SHA1.hexdigest(@label)
@event.label = @label
@event.label.should == @label
@event.label.to_s.should == @label
@event.label_hash.should == @label_hash
end
@@ -64,7 +64,7 @@ describe Redistat::Event do
@event = Redistat::Event.new(@scope, @label, @date, @stats, @options.merge({:store_event => true}), @meta).save
fetched = Redistat::Event.find(@scope, @event.id)
@event.scope.should == fetched.scope
@event.label.should == fetched.label
@event.label.to_s.should == fetched.label.to_s
@event.date.to_s.should == fetched.date.to_s
end

68
spec/finder_spec.rb
View File

@@ -9,41 +9,43 @@ describe Redistat::Finder do
@label = "about_us"
@date = Time.now
@key = Redistat::Key.new(@scope, @label, @date, {:depth => :day})
@stats = {"views" => 3, "visitors" => 2}
@stats = {"views" => 3, "visitors" => 2}
@two_hours_ago = 2.hours.ago
@one_hour_ago = 1.hour.ago
end
it "should initialize properly" do
two_hours_ago = 2.hours.ago
one_hour_ago = 1.hour.ago
options = {:scope => "PageViews", :label => "Label", :from => two_hours_ago, :till => one_hour_ago, :depth => :hour, :interval => :hour}
finder = Redistat::Finder.new(options)
finder.options.should == options
options = {:scope => "PageViews", :label => "Label", :from => @two_hours_ago, :till => @one_hour_ago, :depth => :hour, :interval => :hour}
finder = Redistat::Finder.new
finder.send(:set_options, options)
finder.options.should == options
finder = Redistat::Finder.dates(two_hours_ago, one_hour_ago).scope("PageViews").label("Label").depth(:hour).interval(:hour)
finder.options.should == options
finder = Redistat::Finder.scope("PageViews").label("Label").from(two_hours_ago).till(one_hour_ago).depth(:hour).interval(:hour)
finder.options.should == options
finder.options[:scope].should be_a(Redistat::Scope)
finder.options[:scope].to_s.should == options[:scope]
finder.options[:label].should be_a(Redistat::Label)
finder.options[:label].to_s.should == options[:label]
finder.options.should == options.merge(:scope => finder.options[:scope], :label => finder.options[:label])
finder = Redistat::Finder.label("Label").from(two_hours_ago).till(one_hour_ago).depth(:hour).interval(:hour).scope("PageViews")
finder.options.should == options
finder = Redistat::Finder.dates(@two_hours_ago, @one_hour_ago)
finder.options[:from].should == @two_hours_ago
finder.options[:till].should == @one_hour_ago
finder = Redistat::Finder.from(two_hours_ago).till(one_hour_ago).depth(:hour).interval(:hour).scope("PageViews").label("Label")
finder.options.should == options
finder = Redistat::Finder.scope("hello")
finder.options[:scope].to_s.should == "hello"
finder = Redistat::Finder.till(one_hour_ago).depth(:hour).interval(:hour).scope("PageViews").label("Label").from(two_hours_ago)
finder.options.should == options
finder = Redistat::Finder.label("hello")
finder.options[:label].to_s.should == "hello"
finder = Redistat::Finder.depth(:hour).interval(:hour).scope("PageViews").label("Label").from(two_hours_ago).till(one_hour_ago)
finder.options.should == options
finder = Redistat::Finder.from(@two_hours_ago)
finder.options[:from].should == @two_hours_ago
finder = Redistat::Finder.interval(:hour).scope("PageViews").label("Label").from(two_hours_ago).till(one_hour_ago).depth(:hour)
finder.options.should == options
finder = Redistat::Finder.till(@one_hour_ago)
finder.options[:till].should == @one_hour_ago
finder = Redistat::Finder.depth(:hour)
finder.options[:depth].should == :hour
finder = Redistat::Finder.interval(:hour)
finder.options[:interval].should == :hour
end
@@ -89,6 +91,21 @@ describe Redistat::Finder do
lambda { Redistat::Finder.find(:from => 3.hours.ago) }.should raise_error(Redistat::InvalidOptions)
end
it "should find children" do
Redistat::Key.new("PageViews", "message/public/die").update_index
Redistat::Key.new("PageViews", "message/public/live").update_index
Redistat::Key.new("PageViews", "message/public/fester").update_index
members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}message/public") # checking 'message/public'
options = {:scope => "PageViews", :label => "message/public", :from => @two_hours_ago, :till => @one_hour_ago, :depth => :hour, :interval => :hour}
finder = Redistat::Finder.new(options)
finder.children.first.should be_a(Redistat::Finder)
subs = finder.children.map { |f| f.options[:label].me }
subs.should have(3).items
subs.should include('die')
subs.should include('live')
subs.should include('fester')
end
describe "Lazy-Loading" do
before(:each) do
@@ -103,7 +120,6 @@ describe Redistat::Finder do
end
it "should lazy-load" do
@finder.instance_variable_get("@result").should be_nil
stats = @finder.all
@finder.instance_variable_get("@result").should_not be_nil
@@ -146,7 +162,7 @@ describe Redistat::Finder do
res.should == match
end
end
end # "Lazy-Loading"
# helper methods

77
spec/key_spec.rb
View File

@@ -1,8 +1,10 @@
require "spec_helper"
describe Redistat::Key do
include Redistat::Database
before(:each) do
db.flushdb
@scope = "PageViews"
@label = "about_us"
@label_hash = Digest::SHA1.hexdigest(@label)
@@ -12,9 +14,9 @@ describe Redistat::Key do
it "should initialize properly" do
@key.scope.should == @scope
@key.label.should == @label
@key.label.to_s.should == @label
@key.label_hash.should == @label_hash
@key.label_groups.should == @key.instance_variable_get("@label").groups
@key.groups.map { |k| k.instance_variable_get("@label") }.should == @key.instance_variable_get("@label").groups
@key.date.should be_instance_of(Redistat::Date)
@key.date.to_time.to_s.should == @date.to_s
end
@@ -52,31 +54,70 @@ describe Redistat::Key do
@key.date = @date
@key.date.to_time.to_s.should == @date.to_s
# label
@key.label.should == @label
@key.label.to_s.should == @label
@key.label_hash == @label_hash
@label = "contact_us"
@label_hash = Digest::SHA1.hexdigest(@label)
@key.label = @label
@key.label.should == @label
@key.label.to_s.should == @label
@key.label_hash == @label_hash
end
it "should create a group of keys from label group" do
label = 'message/public/offensive'
result = [ "message/public/offensive",
"message/public",
"message" ]
describe "Grouping" do
before(:each) do
@label = "message/public/offensive"
@key = Redistat::Key.new(@scope, @label, @date, {:depth => :hour})
end
key = Redistat::Key.new(@scope, label, @date, {:depth => :hour})
it "should create a group of keys from label group" do
label = 'message/public/offensive'
result = [ "message/public/offensive",
"message/public",
"message" ]
key = Redistat::Key.new(@scope, label, @date, {:depth => :hour})
key.groups.map { |k| k.label.to_s }.should == result
end
key.label_groups.should == result
key.groups.map { |k| k.label }.should == result
end
it "should know it's parent label group" do
label = 'message/public/offensive'
key = Redistat::Key.new(@scope, label, @date, {:depth => :hour})
key.parent_group.should == 'message/public'
it "should know it's parent" do
@key.parent.should be_a(Redistat::Key)
@key.parent.label.to_s.should == 'message/public'
Redistat::Key.new(@scope, 'hello', @date).parent.should be_nil
end
it "should update label index and return children" do
db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{@key.label.parent}").should == []
@key.children.should have(0).items
@key.update_index # indexing 'message/publish/offensive'
Redistat::Key.new("PageViews", "message/public/die").update_index # indexing 'message/publish/die'
Redistat::Key.new("PageViews", "message/public/live").update_index # indexing 'message/publish/live'
members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{@key.label.parent}") # checking 'message/public'
members.should have(3).item
members.should include('offensive')
members.should include('live')
members.should include('die')
key = @key.parent
key.children.first.should be_a(Redistat::Key)
key.children.should have(3).item
key.children.map { |k| k.label.me }.should == members
members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{key.label.parent}") # checking 'message'
members.should have(1).item
members.should include('public')
key = key.parent
key.children.should have(1).item
key.children.map { |k| k.label.me }.should == members
members = db.smembers("#{@scope}#{Redistat::LABEL_INDEX}#{key.label.parent}") # checking ''
members.should have(0).item
key.parent.should be_nil
end
end
end

40
spec/label_spec.rb
View File

@@ -32,50 +32,26 @@ describe Redistat::Label do
end
it "should know it's parent label group" do
@label.parent_group.should == 'message/public'
Redistat::Label.new('hello').parent_group.should be_nil
@label.parent.to_s.should == 'message/public'
Redistat::Label.new('hello').parent.should be_nil
end
it "should separate label names into groups" do
@label.name.should == @name
@label.groups.should == [ "message/public/offensive",
"message/public",
"message" ]
@label.groups.map { |l| l.to_s }.should == [ "message/public/offensive",
"message/public",
"message" ]
@name = "/message/public/"
@label = Redistat::Label.new(@name)
@label.name.should == @name
@label.groups.should == [ "message/public",
"message" ]
@label.groups.map { |l| l.to_s }.should == [ "message/public",
"message" ]
@name = "message"
@label = Redistat::Label.new(@name)
@label.name.should == @name
@label.groups.should == [ "message" ]
end
it "should update label index" do
db.smembers("#{Redistat::LABEL_INDEX}#{@label.parent_group}").should == []
@label.update_index
members = db.smembers("#{Redistat::LABEL_INDEX}#{@label.parent_group}") # checking 'message/public'
members.should have(1).item
members.should include('offensive')
members.should == @label.sub_labels.map { |l| l.group }
name = "message/public/nice"
label = Redistat::Label.new(name)
label.update_index
members = db.smembers("#{Redistat::LABEL_INDEX}#{label.parent_group}") # checking 'message/public'
members.should have(2).items
members.should include('offensive')
members.should include('nice')
members.should == label.sub_labels.map { |l| l.group }
label = @label.parent
members = db.smembers("#{Redistat::LABEL_INDEX}#{label.parent_group}") # checking 'message'
members.should have(1).item
members.should include('public')
members.should == label.sub_labels.map { |l| l.group }
@label.groups.map { |l| l.to_s }.should == [ "message" ]
end
end

41
spec/model_spec.rb
View File

@@ -22,8 +22,8 @@ describe Redistat::Model do
one_hour_ago = 1.hour.ago
finder = ModelHelper1.find('label', two_hours_ago, one_hour_ago)
finder.should be_a(Redistat::Finder)
finder.options[:scope].should == 'ModelHelper1'
finder.options[:label].should == 'label'
finder.options[:scope].to_s.should == 'ModelHelper1'
finder.options[:label].to_s.should == 'label'
finder.options[:from].should == two_hours_ago
finder.options[:till].should == one_hour_ago
end
@@ -82,6 +82,43 @@ describe Redistat::Model do
stats.first.should == stats.total
end
it "should store and fetch grouping enabled stats" do
ModelHelper1.store("sheep/black", {:count => 6, :weight => 461}, @time.hours_ago(4))
ModelHelper1.store("sheep/black", {:count => 2, :weight => 156}, @time)
ModelHelper1.store("sheep/white", {:count => 5, :weight => 393}, @time.hours_ago(4))
ModelHelper1.store("sheep/white", {:count => 4, :weight => 316}, @time)
stats = ModelHelper1.fetch("sheep/black", @time.hours_ago(2), @time.hours_since(1))
stats.total["count"].should == 2
stats.total["weight"].should == 156
stats.first.should == stats.total
stats = ModelHelper1.fetch("sheep/black", @time.hours_ago(5), @time.hours_since(1))
stats.total[:count].should == 8
stats.total[:weight].should == 617
stats.first.should == stats.total
stats = ModelHelper1.fetch("sheep/white", @time.hours_ago(2), @time.hours_since(1))
stats.total[:count].should == 4
stats.total[:weight].should == 316
stats.first.should == stats.total
stats = ModelHelper1.fetch("sheep/white", @time.hours_ago(5), @time.hours_since(1))
stats.total[:count].should == 9
stats.total[:weight].should == 709
stats.first.should == stats.total
stats = ModelHelper1.fetch("sheep", @time.hours_ago(2), @time.hours_since(1))
stats.total[:count].should == 6
stats.total[:weight].should == 472
stats.first.should == stats.total
stats = ModelHelper1.fetch("sheep", @time.hours_ago(5), @time.hours_since(1))
stats.total[:count].should == 17
stats.total[:weight].should == 1326
stats.first.should == stats.total
end
it "should connect to different Redis servers on a per-model basis" do
ModelHelper3.redis.client.db.should == 14

8
spec/summary_spec.rb
View File

@@ -83,8 +83,8 @@ describe Redistat::Summary do
key = Redistat::Key.new(@scope, label, @date)
Redistat::Summary.update_all(key, stats, :hour)
key.groups[0].label.should == "views/about_us"
key.groups[1].label.should == "views"
key.groups[0].label.to_s.should == "views/about_us"
key.groups[1].label.to_s.should == "views"
child1 = key.groups[0]
parent = key.groups[1]
@@ -92,8 +92,8 @@ describe Redistat::Summary do
key = Redistat::Key.new(@scope, label, @date)
Redistat::Summary.update_all(key, stats, :hour)
key.groups[0].label.should == "views/contact"
key.groups[1].label.should == "views"
key.groups[0].label.to_s.should == "views/contact"
key.groups[1].label.to_s.should == "views"
child2 = key.groups[0]
summary = db.hgetall(child1.to_s(:hour))