From 1486110d2e020ef2d9b344848e8cf07ec63dd2ce Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Sun, 4 Dec 2016 19:16:57 +0000 Subject: [PATCH] Noramlize Store interface Instead of exposing a key/value get/set style interface, design the interface around the actions that are actually required. This is a first step towards supporting adding a Store to support Google Cloud's Datastore. --- storage/goleveldbstore/store.go | 120 ++++++++++++++--- storage/goleveldbstore/store_test.go | 190 +++++++++++++++++---------- storage/inmemorystore/store.go | 81 ++++++++---- storage/inmemorystore/store_test.go | 164 ++++++++++++++++------- storage/store.go | 21 ++- 5 files changed, 413 insertions(+), 163 deletions(-) diff --git a/storage/goleveldbstore/store.go b/storage/goleveldbstore/store.go index 7725b1b..de4d419 100644 --- a/storage/goleveldbstore/store.go +++ b/storage/goleveldbstore/store.go @@ -1,17 +1,19 @@ package goleveldbstore import ( - "errors" "strconv" + "github.com/jimeh/ozu.io/storage" "github.com/syndtr/goleveldb/leveldb" ) +const errLeveldbNotFound = "leveldb: not found" + // DefaultSequenceKey is used by NextSequence(). var DefaultSequenceKey = []byte("__SEQUENCE_ID__") -// ErrNotFound is returned when Get() tries to fetch a non-existent key. -var ErrNotFound = errors.New("not found") +var uidKeyPrefix = []byte("!") +var urlKeyPrefix = []byte("#") // New creates a new Store using given path to persist data. func New(path string) (*Store, error) { @@ -39,33 +41,117 @@ func (s *Store) Close() error { return s.DB.Close() } -// Get a given key's value. -func (s *Store) Get(key []byte) ([]byte, error) { - value, err := s.DB.Get(key, nil) - if err != nil && err.Error() == "leveldb: not found" { - return nil, ErrNotFound +// Create a given Record. +func (s *Store) Create(uid []byte, url []byte) (*storage.Record, error) { + tx, err := s.DB.OpenTransaction() + if err != nil { + return &storage.Record{}, err } - return value, err + err = tx.Put(s.uidKey(uid), url, nil) + if err != nil { + return &storage.Record{}, err + } + + err = tx.Put(s.urlKey(url), uid, nil) + if err != nil { + return &storage.Record{}, err + } + + err = tx.Commit() + if err != nil { + return &storage.Record{}, err + } + + return &storage.Record{UID: uid, URL: url}, nil } -// Set a given key's to the specified value. -func (s *Store) Set(key []byte, value []byte) error { - return s.DB.Put(key, value, nil) +// FindByUID looks up records based on their UID. +func (s *Store) FindByUID(uid []byte) (*storage.Record, error) { + value, err := s.DB.Get(s.uidKey(uid), nil) + if err != nil { + if err.Error() == errLeveldbNotFound { + return &storage.Record{}, storage.ErrNotFound + } + return &storage.Record{}, err + } + + return &storage.Record{UID: uid, URL: value}, nil } -// Delete a given key. -func (s *Store) Delete(key []byte) error { - return s.DB.Delete(key, nil) +// FindByURL looks up records based on their URL. +func (s *Store) FindByURL(url []byte) (*storage.Record, error) { + value, err := s.DB.Get(s.urlKey(url), nil) + if err != nil { + if err.Error() == errLeveldbNotFound { + return &storage.Record{}, storage.ErrNotFound + } + return &storage.Record{}, err + } + + return &storage.Record{UID: value, URL: url}, nil +} + +// DeleteByUID deletes records based on their UID. +func (s *Store) DeleteByUID(uid []byte) (*storage.Record, error) { + record, err := s.FindByUID(uid) + if err != nil { + return &storage.Record{}, err + } + + s.delete(record) + return record, nil +} + +// DeleteByURL deletes records based on their URL. +func (s *Store) DeleteByURL(url []byte) (*storage.Record, error) { + record, err := s.FindByURL(url) + if err != nil { + return &storage.Record{}, err + } + + err = s.delete(record) + if err != nil { + return &storage.Record{}, err + } + + return record, nil +} + +func (s *Store) delete(r *storage.Record) error { + tx, err := s.DB.OpenTransaction() + if err != nil { + return err + } + + err = tx.Delete(s.uidKey(r.UID), nil) + if err != nil && err.Error() == errLeveldbNotFound { + return err + } + + err = tx.Delete(s.urlKey(r.URL), nil) + if err != nil && err.Error() == errLeveldbNotFound { + return err + } + + return tx.Commit() +} + +func (s *Store) uidKey(uid []byte) []byte { + return append(uidKeyPrefix, uid...) +} + +func (s *Store) urlKey(url []byte) []byte { + return append(urlKeyPrefix, url...) } // NextSequence returns a auto-incrementing int. func (s *Store) NextSequence() (int, error) { - return s.Incr(s.SequenceKey) + return s.incr(s.SequenceKey) } // Incr increments a given key (must be numeric-like value) -func (s *Store) Incr(key []byte) (int, error) { +func (s *Store) incr(key []byte) (int, error) { tx, err := s.DB.OpenTransaction() if err != nil { return -1, err diff --git a/storage/goleveldbstore/store_test.go b/storage/goleveldbstore/store_test.go index d17ad0f..b7e2fa7 100644 --- a/storage/goleveldbstore/store_test.go +++ b/storage/goleveldbstore/store_test.go @@ -13,13 +13,10 @@ import ( var testDbPath = "./goleveldb_test_data" -var examples = []struct { - key []byte - value []byte -}{ - {key: []byte("hello"), value: []byte("world")}, - {key: []byte("foo"), value: []byte("bar")}, - {key: []byte("wtf"), value: []byte("dude")}, +var examples = []storage.Record{ + storage.Record{UID: []byte("Kb8X"), URL: []byte("https://google.com/")}, + storage.Record{UID: []byte("h3mz"), URL: []byte("https://github.com/")}, + storage.Record{UID: []byte("3qxs"), URL: []byte("https://twitter.com/")}, } type StoreSuite struct { @@ -42,7 +39,9 @@ func (s *StoreSuite) TearDownTest() { func (s *StoreSuite) Seed() { for _, e := range examples { - err := s.db.Put(e.key, e.value, nil) + err := s.db.Put(append(uidKeyPrefix, e.UID...), e.URL, nil) + s.Require().NoError(err) + err = s.db.Put(append(urlKeyPrefix, e.URL...), e.UID, nil) s.Require().NoError(err) } } @@ -53,62 +52,112 @@ func (s *StoreSuite) TestStoreInterface() { s.Implements(new(storage.Store), new(Store)) } -func (s *StoreSuite) TestSet() { +func (s *StoreSuite) TestCreate() { for _, e := range examples { - err := s.store.Set(e.key, e.value) + record, err := s.store.Create(e.UID, e.URL) s.NoError(err) + s.Equal(e.UID, record.UID) + s.Equal(e.URL, record.URL) } for _, e := range examples { - result, _ := s.db.Get(e.key, nil) - s.Equal(e.value, result) + recordURL, _ := s.db.Get(append(uidKeyPrefix, e.UID...), nil) + s.Equal(e.URL, recordURL) + recordUID, _ := s.db.Get(append(urlKeyPrefix, e.URL...), nil) + s.Equal(e.UID, recordUID) } } -func (s *StoreSuite) TestGetExisting() { +func (s *StoreSuite) TestFindExistingByUID() { s.Seed() for _, e := range examples { - result, err := s.store.Get(e.key) + record, err := s.store.FindByUID(e.UID) s.NoError(err) - s.Equal(e.value, result) + s.Equal(e.UID, record.UID) + s.Equal(e.URL, record.URL) } } -func (s *StoreSuite) TestGetNonExistant() { - result, err := s.store.Get([]byte("does-not-exist")) - s.Nil(result) +func (s *StoreSuite) TestFindNonExistantByUID() { + record, err := s.store.FindByUID([]byte("nope")) + s.Nil(record.UID) + s.Nil(record.URL) s.EqualError(err, "not found") } -func (s *StoreSuite) TestDeleteExisting() { +func (s *StoreSuite) TestFindExistingByURL() { s.Seed() for _, e := range examples { - value, _ := s.db.Get(e.key, nil) - s.Require().Equal(e.value, value) - - value, err := s.store.Get(e.key) - s.Require().NoError(err) - s.Require().Equal(value, e.value) - - err = s.store.Delete(e.key) + record, err := s.store.FindByURL(e.URL) s.NoError(err) - - value, err = s.store.Get(e.key) - s.Nil(value) - s.EqualError(err, "not found") - - has, _ := s.db.Has(e.key, nil) - s.Equal(false, has) - result, _ := s.db.Get(e.key, nil) - s.Equal([]byte{}, result) + s.Equal(e.UID, record.UID) + s.Equal(e.URL, record.URL) } } -func (s *StoreSuite) TestDeleteNonExistant() { - err := s.store.Delete([]byte("does-not-exist")) - s.NoError(err) +func (s *StoreSuite) TestFindNonExistantByURL() { + record, err := s.store.FindByURL([]byte("http://nope.com/")) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") +} + +func (s *StoreSuite) TestDeleteExistingByUID() { + s.Seed() + + for _, e := range examples { + record, err := s.store.DeleteByUID(e.UID) + s.NoError(err) + s.Equal(record.UID, e.UID) + s.Equal(record.URL, e.URL) + + record, err = s.store.FindByUID(e.UID) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + + record, err = s.store.FindByURL(e.URL) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + } +} + +func (s *StoreSuite) TestDeleteNonExistantByUID() { + record, err := s.store.DeleteByUID([]byte("nope")) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") +} + +func (s *StoreSuite) TestDeleteExistingByURL() { + s.Seed() + + for _, e := range examples { + record, err := s.store.DeleteByURL(e.URL) + s.NoError(err) + s.Equal(record.UID, e.UID) + s.Equal(record.URL, e.URL) + + record, err = s.store.FindByUID(e.UID) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + + record, err = s.store.FindByURL(e.URL) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + } +} + +func (s *StoreSuite) TestDeleteNonExistantByURL() { + record, err := s.store.DeleteByURL([]byte("http://nope/")) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") } func (s *StoreSuite) TestNextSequenceExisting() { @@ -135,14 +184,14 @@ func (s *StoreSuite) TestIncrExisting() { err := s.db.Put(key, []byte("5"), nil) s.Require().NoError(err) - result, err := s.store.Incr(key) + result, err := s.store.incr(key) s.NoError(err) s.Equal(6, result) } func (s *StoreSuite) TestIncrNonExistant() { for i := 1; i < 10; i++ { - result, err := s.store.Incr([]byte("counter")) + result, err := s.store.incr([]byte("counter")) s.NoError(err) s.Equal(i, result) @@ -157,54 +206,57 @@ func TestStoreSuite(t *testing.T) { // Benchmarks -func BenchmarkGet(b *testing.B) { +func BenchmarkCreate(b *testing.B) { store, _ := New(testDbPath) - key := []byte("hello") - value := []byte("world") - _ = store.Set(key, value) + uid := []byte("Kb8X") + url := []byte("https://google.com/") for n := 0; n < b.N; n++ { - _, _ = store.Get(key) + store.Create(append(uid, string(n)...), url) } - _ = store.Close() - _ = os.RemoveAll(testDbPath) + store.Close() + os.RemoveAll(testDbPath) } -func BenchmarkSet(b *testing.B) { +func BenchmarkFindByUID(b *testing.B) { store, _ := New(testDbPath) - key := []byte("hello") - value := []byte("world") + uid := []byte("Kb8X") + url := []byte("https://google.com/") + store.Create(uid, url) for n := 0; n < b.N; n++ { - _ = store.Set(append(key, string(n)...), value) + store.FindByUID(uid) } - _ = store.Close() - _ = os.RemoveAll(testDbPath) + store.Close() + os.RemoveAll(testDbPath) +} + +func BenchmarkFindByURL(b *testing.B) { + store, _ := New(testDbPath) + + uid := []byte("Kb8X") + url := []byte("https://google.com/") + store.Create(uid, url) + + for n := 0; n < b.N; n++ { + store.FindByURL(url) + } + + store.Close() + os.RemoveAll(testDbPath) } func BenchmarkNextSequence(b *testing.B) { store, _ := New(testDbPath) for n := 0; n < b.N; n++ { - _, _ = store.NextSequence() + store.NextSequence() } - _ = store.Close() - _ = os.RemoveAll(testDbPath) -} - -func BenchmarkIncr(b *testing.B) { - store, _ := New(testDbPath) - - key := []byte("incr-benchmark-counter") - for n := 0; n < b.N; n++ { - _, _ = store.Incr(key) - } - - _ = store.Close() - _ = os.RemoveAll(testDbPath) + store.Close() + os.RemoveAll(testDbPath) } diff --git a/storage/inmemorystore/store.go b/storage/inmemorystore/store.go index 9f8f3ad..e44f52e 100644 --- a/storage/inmemorystore/store.go +++ b/storage/inmemorystore/store.go @@ -1,17 +1,16 @@ package inmemorystore import ( - "errors" "sync" -) -// ErrNotFound is returned when Get() tries to fetch a non-existent key. -var ErrNotFound = errors.New("not found") + "github.com/jimeh/ozu.io/storage" +) // New creates a new Store using given path to persist data. func New() (*Store, error) { store := &Store{ - Data: map[string][]byte{}, + UIDMap: map[string][]byte{}, + URLMap: map[string][]byte{}, Sequence: 0, } return store, nil @@ -20,43 +19,79 @@ func New() (*Store, error) { // Store allows storing data into a in-memory map. type Store struct { sync.RWMutex - Data map[string][]byte + UIDMap map[string][]byte + URLMap map[string][]byte Sequence int - Closed bool } // Close database. func (s *Store) Close() error { - s.Data = make(map[string][]byte) + s.Lock() + s.UIDMap = make(map[string][]byte) + s.URLMap = make(map[string][]byte) s.Sequence = 0 + s.Unlock() return nil } -// Get a given key's value. -func (s *Store) Get(key []byte) ([]byte, error) { +// Create a given Record +func (s *Store) Create(uid []byte, url []byte) (*storage.Record, error) { + s.Lock() + s.UIDMap[string(uid)] = url + s.URLMap[string(url)] = uid + s.Unlock() + return &storage.Record{UID: uid, URL: url}, nil +} + +// FindByUID looks up records based on their UID. +func (s *Store) FindByUID(uid []byte) (*storage.Record, error) { s.RLock() - value := s.Data[string(key)] + value := s.UIDMap[string(uid)] s.RUnlock() if value == nil { - return nil, ErrNotFound + return &storage.Record{}, storage.ErrNotFound } - return value, nil + return &storage.Record{UID: uid, URL: value}, nil } -// Set a given key's to the specified value. -func (s *Store) Set(key []byte, value []byte) error { - s.Lock() - s.Data[string(key)] = value - s.Unlock() - return nil +// FindByURL looks up records based on their URL. +func (s *Store) FindByURL(url []byte) (*storage.Record, error) { + s.RLock() + value := s.URLMap[string(url)] + s.RUnlock() + if value == nil { + return &storage.Record{}, storage.ErrNotFound + } + return &storage.Record{UID: value, URL: url}, nil } -// Delete a given key. -func (s *Store) Delete(key []byte) error { +// DeleteByUID deletes records based on their UID. +func (s *Store) DeleteByUID(uid []byte) (*storage.Record, error) { + record, err := s.FindByUID(uid) + if err != nil { + return &storage.Record{}, err + } + + s.delete(record) + return record, nil +} + +// DeleteByURL deletes records based on their URL. +func (s *Store) DeleteByURL(url []byte) (*storage.Record, error) { + record, err := s.FindByURL(url) + if err != nil { + return &storage.Record{}, err + } + + s.delete(record) + return record, nil +} + +func (s *Store) delete(r *storage.Record) { s.Lock() - delete(s.Data, string(key)) + delete(s.UIDMap, string(r.UID)) + delete(s.URLMap, string(r.URL)) s.Unlock() - return nil } // NextSequence returns a auto-incrementing int. diff --git a/storage/inmemorystore/store_test.go b/storage/inmemorystore/store_test.go index 424d18f..0bc116f 100644 --- a/storage/inmemorystore/store_test.go +++ b/storage/inmemorystore/store_test.go @@ -9,13 +9,10 @@ import ( // Setup Suite -var examples = []struct { - key []byte - value []byte -}{ - {key: []byte("hello"), value: []byte("world")}, - {key: []byte("foo"), value: []byte("bar")}, - {key: []byte("wtf"), value: []byte("dude")}, +var examples = []storage.Record{ + storage.Record{UID: []byte("Kb8X"), URL: []byte("https://google.com/")}, + storage.Record{UID: []byte("h3mz"), URL: []byte("https://github.com/")}, + storage.Record{UID: []byte("3qxs"), URL: []byte("https://twitter.com/")}, } type StoreSuite struct { @@ -35,7 +32,8 @@ func (s *StoreSuite) TearDownTest() { func (s *StoreSuite) Seed() { for _, e := range examples { - s.store.Data[string(e.key)] = e.value + s.store.UIDMap[string(e.UID)] = e.URL + s.store.URLMap[string(e.URL)] = e.UID } } @@ -45,60 +43,112 @@ func (s *StoreSuite) TestStoreInterface() { s.Implements(new(storage.Store), new(Store)) } -func (s *StoreSuite) TestSet() { +func (s *StoreSuite) TestCreate() { for _, e := range examples { - err := s.store.Set(e.key, e.value) + record, err := s.store.Create(e.UID, e.URL) s.NoError(err) + s.Equal(e.UID, record.UID) + s.Equal(e.URL, record.URL) } for _, e := range examples { - result, _ := s.store.Data[string(e.key)] - s.Equal(e.value, result) + recordURL, _ := s.store.UIDMap[string(e.UID)] + s.Equal(e.URL, recordURL) + recordUID, _ := s.store.URLMap[string(e.URL)] + s.Equal(e.UID, recordUID) } } -func (s *StoreSuite) TestGetExisting() { +func (s *StoreSuite) TestFindExistingByUID() { s.Seed() for _, e := range examples { - result, err := s.store.Get(e.key) + record, err := s.store.FindByUID(e.UID) s.NoError(err) - s.Equal(e.value, result) + s.Equal(e.UID, record.UID) + s.Equal(e.URL, record.URL) } } -func (s *StoreSuite) TestGetNonExistant() { - result, err := s.store.Get([]byte("does-not-exist")) - s.Nil(result) +func (s *StoreSuite) TestFindNonExistantByUID() { + record, err := s.store.FindByUID([]byte("does-not-exist")) + s.Nil(record.UID) + s.Nil(record.URL) s.EqualError(err, "not found") } -func (s *StoreSuite) TestDeleteExisting() { +func (s *StoreSuite) TestFindExistingByURL() { s.Seed() for _, e := range examples { - value := s.store.Data[string(e.key)] - s.Require().Equal(e.value, value) - - value, err := s.store.Get(e.key) - s.Require().NoError(err) - s.Require().Equal(value, e.value) - - err = s.store.Delete(e.key) + record, err := s.store.FindByURL(e.URL) s.NoError(err) - - value, err = s.store.Get(e.key) - s.Nil(value) - s.EqualError(err, "not found") - - _, has := s.store.Data[string(e.key)] - s.Equal(false, has) + s.Equal(e.UID, record.UID) + s.Equal(e.URL, record.URL) } } -func (s *StoreSuite) TestDeleteNonExistant() { - err := s.store.Delete([]byte("does-not-exist")) - s.NoError(err) +func (s *StoreSuite) TestFindNonExistantByURL() { + record, err := s.store.FindByURL([]byte("http://nope.com/")) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") +} + +func (s *StoreSuite) TestDeleteExistingByUID() { + s.Seed() + + for _, e := range examples { + record, err := s.store.DeleteByUID(e.UID) + s.NoError(err) + s.Equal(record.UID, e.UID) + s.Equal(record.URL, e.URL) + + record, err = s.store.FindByUID(e.UID) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + + record, err = s.store.FindByURL(e.URL) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + } +} + +func (s *StoreSuite) TestDeleteNonExistantByUID() { + record, err := s.store.DeleteByUID([]byte("nope")) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") +} + +func (s *StoreSuite) TestDeleteExistingByURL() { + s.Seed() + + for _, e := range examples { + record, err := s.store.DeleteByURL(e.URL) + s.NoError(err) + s.Equal(record.UID, e.UID) + s.Equal(record.URL, e.URL) + + record, err = s.store.FindByUID(e.UID) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + + record, err = s.store.FindByURL(e.URL) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") + } +} + +func (s *StoreSuite) TestDeleteNonExistantByURL() { + record, err := s.store.DeleteByURL([]byte("http://nope/")) + s.Nil(record.UID) + s.Nil(record.URL) + s.EqualError(err, "not found") } func (s *StoreSuite) TestNextSequenceExisting() { @@ -126,39 +176,53 @@ func TestStoreSuite(t *testing.T) { // Benchmarks -func BenchmarkGet(b *testing.B) { +func BenchmarkCreate(b *testing.B) { store, _ := New() - key := []byte("hello") - value := []byte("world") - _ = store.Set(key, value) + uid := []byte("Kb8X") + url := []byte("https://google.com/") for n := 0; n < b.N; n++ { - _, _ = store.Get(key) + store.Create(append(uid, string(n)...), url) } - _ = store.Close() + store.Close() } -func BenchmarkSet(b *testing.B) { +func BenchmarkFindByUID(b *testing.B) { store, _ := New() - key := []byte("hello") - value := []byte("world") + uid := []byte("Kb8X") + url := []byte("https://google.com/") + store.Create(uid, url) for n := 0; n < b.N; n++ { - _ = store.Set(append(key, string(n)...), value) + store.FindByUID(uid) } - _ = store.Close() + store.Close() +} + +func BenchmarkFindByURL(b *testing.B) { + store, _ := New() + + uid := []byte("Kb8X") + url := []byte("https://google.com/") + store.Create(uid, url) + + for n := 0; n < b.N; n++ { + store.FindByURL(url) + } + + store.Close() } func BenchmarkNextSequence(b *testing.B) { store, _ := New() for n := 0; n < b.N; n++ { - _, _ = store.NextSequence() + store.NextSequence() } - _ = store.Close() + store.Close() } diff --git a/storage/store.go b/storage/store.go index c79431d..6514e1b 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1,10 +1,23 @@ package storage -// Store defines a standard interface for storage. +import "errors" + +// ErrNotFound is the default error message when data is not found. +var ErrNotFound = errors.New("not found") + +// Store defines a standard interface for storage type Store interface { Close() error - Get([]byte) ([]byte, error) - Set([]byte, []byte) error - Delete([]byte) error + Create(UID []byte, URL []byte) (*Record, error) + FindByUID(UID []byte) (*Record, error) + FindByURL(URL []byte) (*Record, error) + DeleteByUID(UID []byte) (*Record, error) + DeleteByURL(URL []byte) (*Record, error) NextSequence() (int, error) } + +// Record provides a standard way to refer to a shortened URL. +type Record struct { + UID []byte + URL []byte +}