Noramlize Store interface

Instead of exposing a key/value get/set style interface, design the
interface around the actions that are actually required.

This is a first step towards supporting adding a Store to support
Google Cloud's Datastore.
This commit is contained in:
2016-12-04 19:16:57 +00:00
parent d544df0b6d
commit 1486110d2e
5 changed files with 413 additions and 163 deletions

View File

@@ -1,17 +1,19 @@
package goleveldbstore
import (
"errors"
"strconv"
"github.com/jimeh/ozu.io/storage"
"github.com/syndtr/goleveldb/leveldb"
)
const errLeveldbNotFound = "leveldb: not found"
// DefaultSequenceKey is used by NextSequence().
var DefaultSequenceKey = []byte("__SEQUENCE_ID__")
// ErrNotFound is returned when Get() tries to fetch a non-existent key.
var ErrNotFound = errors.New("not found")
var uidKeyPrefix = []byte("!")
var urlKeyPrefix = []byte("#")
// New creates a new Store using given path to persist data.
func New(path string) (*Store, error) {
@@ -39,33 +41,117 @@ func (s *Store) Close() error {
return s.DB.Close()
}
// Get a given key's value.
func (s *Store) Get(key []byte) ([]byte, error) {
value, err := s.DB.Get(key, nil)
if err != nil && err.Error() == "leveldb: not found" {
return nil, ErrNotFound
// Create a given Record.
func (s *Store) Create(uid []byte, url []byte) (*storage.Record, error) {
tx, err := s.DB.OpenTransaction()
if err != nil {
return &storage.Record{}, err
}
return value, err
err = tx.Put(s.uidKey(uid), url, nil)
if err != nil {
return &storage.Record{}, err
}
err = tx.Put(s.urlKey(url), uid, nil)
if err != nil {
return &storage.Record{}, err
}
err = tx.Commit()
if err != nil {
return &storage.Record{}, err
}
return &storage.Record{UID: uid, URL: url}, nil
}
// Set a given key's to the specified value.
func (s *Store) Set(key []byte, value []byte) error {
return s.DB.Put(key, value, nil)
// FindByUID looks up records based on their UID.
func (s *Store) FindByUID(uid []byte) (*storage.Record, error) {
value, err := s.DB.Get(s.uidKey(uid), nil)
if err != nil {
if err.Error() == errLeveldbNotFound {
return &storage.Record{}, storage.ErrNotFound
}
return &storage.Record{}, err
}
return &storage.Record{UID: uid, URL: value}, nil
}
// Delete a given key.
func (s *Store) Delete(key []byte) error {
return s.DB.Delete(key, nil)
// FindByURL looks up records based on their URL.
func (s *Store) FindByURL(url []byte) (*storage.Record, error) {
value, err := s.DB.Get(s.urlKey(url), nil)
if err != nil {
if err.Error() == errLeveldbNotFound {
return &storage.Record{}, storage.ErrNotFound
}
return &storage.Record{}, err
}
return &storage.Record{UID: value, URL: url}, nil
}
// DeleteByUID deletes records based on their UID.
func (s *Store) DeleteByUID(uid []byte) (*storage.Record, error) {
record, err := s.FindByUID(uid)
if err != nil {
return &storage.Record{}, err
}
s.delete(record)
return record, nil
}
// DeleteByURL deletes records based on their URL.
func (s *Store) DeleteByURL(url []byte) (*storage.Record, error) {
record, err := s.FindByURL(url)
if err != nil {
return &storage.Record{}, err
}
err = s.delete(record)
if err != nil {
return &storage.Record{}, err
}
return record, nil
}
func (s *Store) delete(r *storage.Record) error {
tx, err := s.DB.OpenTransaction()
if err != nil {
return err
}
err = tx.Delete(s.uidKey(r.UID), nil)
if err != nil && err.Error() == errLeveldbNotFound {
return err
}
err = tx.Delete(s.urlKey(r.URL), nil)
if err != nil && err.Error() == errLeveldbNotFound {
return err
}
return tx.Commit()
}
func (s *Store) uidKey(uid []byte) []byte {
return append(uidKeyPrefix, uid...)
}
func (s *Store) urlKey(url []byte) []byte {
return append(urlKeyPrefix, url...)
}
// NextSequence returns a auto-incrementing int.
func (s *Store) NextSequence() (int, error) {
return s.Incr(s.SequenceKey)
return s.incr(s.SequenceKey)
}
// Incr increments a given key (must be numeric-like value)
func (s *Store) Incr(key []byte) (int, error) {
func (s *Store) incr(key []byte) (int, error) {
tx, err := s.DB.OpenTransaction()
if err != nil {
return -1, err

View File

@@ -13,13 +13,10 @@ import (
var testDbPath = "./goleveldb_test_data"
var examples = []struct {
key []byte
value []byte
}{
{key: []byte("hello"), value: []byte("world")},
{key: []byte("foo"), value: []byte("bar")},
{key: []byte("wtf"), value: []byte("dude")},
var examples = []storage.Record{
storage.Record{UID: []byte("Kb8X"), URL: []byte("https://google.com/")},
storage.Record{UID: []byte("h3mz"), URL: []byte("https://github.com/")},
storage.Record{UID: []byte("3qxs"), URL: []byte("https://twitter.com/")},
}
type StoreSuite struct {
@@ -42,7 +39,9 @@ func (s *StoreSuite) TearDownTest() {
func (s *StoreSuite) Seed() {
for _, e := range examples {
err := s.db.Put(e.key, e.value, nil)
err := s.db.Put(append(uidKeyPrefix, e.UID...), e.URL, nil)
s.Require().NoError(err)
err = s.db.Put(append(urlKeyPrefix, e.URL...), e.UID, nil)
s.Require().NoError(err)
}
}
@@ -53,62 +52,112 @@ func (s *StoreSuite) TestStoreInterface() {
s.Implements(new(storage.Store), new(Store))
}
func (s *StoreSuite) TestSet() {
func (s *StoreSuite) TestCreate() {
for _, e := range examples {
err := s.store.Set(e.key, e.value)
record, err := s.store.Create(e.UID, e.URL)
s.NoError(err)
s.Equal(e.UID, record.UID)
s.Equal(e.URL, record.URL)
}
for _, e := range examples {
result, _ := s.db.Get(e.key, nil)
s.Equal(e.value, result)
recordURL, _ := s.db.Get(append(uidKeyPrefix, e.UID...), nil)
s.Equal(e.URL, recordURL)
recordUID, _ := s.db.Get(append(urlKeyPrefix, e.URL...), nil)
s.Equal(e.UID, recordUID)
}
}
func (s *StoreSuite) TestGetExisting() {
func (s *StoreSuite) TestFindExistingByUID() {
s.Seed()
for _, e := range examples {
result, err := s.store.Get(e.key)
record, err := s.store.FindByUID(e.UID)
s.NoError(err)
s.Equal(e.value, result)
s.Equal(e.UID, record.UID)
s.Equal(e.URL, record.URL)
}
}
func (s *StoreSuite) TestGetNonExistant() {
result, err := s.store.Get([]byte("does-not-exist"))
s.Nil(result)
func (s *StoreSuite) TestFindNonExistantByUID() {
record, err := s.store.FindByUID([]byte("nope"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestDeleteExisting() {
func (s *StoreSuite) TestFindExistingByURL() {
s.Seed()
for _, e := range examples {
value, _ := s.db.Get(e.key, nil)
s.Require().Equal(e.value, value)
value, err := s.store.Get(e.key)
s.Require().NoError(err)
s.Require().Equal(value, e.value)
err = s.store.Delete(e.key)
record, err := s.store.FindByURL(e.URL)
s.NoError(err)
value, err = s.store.Get(e.key)
s.Nil(value)
s.EqualError(err, "not found")
has, _ := s.db.Has(e.key, nil)
s.Equal(false, has)
result, _ := s.db.Get(e.key, nil)
s.Equal([]byte{}, result)
s.Equal(e.UID, record.UID)
s.Equal(e.URL, record.URL)
}
}
func (s *StoreSuite) TestDeleteNonExistant() {
err := s.store.Delete([]byte("does-not-exist"))
s.NoError(err)
func (s *StoreSuite) TestFindNonExistantByURL() {
record, err := s.store.FindByURL([]byte("http://nope.com/"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestDeleteExistingByUID() {
s.Seed()
for _, e := range examples {
record, err := s.store.DeleteByUID(e.UID)
s.NoError(err)
s.Equal(record.UID, e.UID)
s.Equal(record.URL, e.URL)
record, err = s.store.FindByUID(e.UID)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
record, err = s.store.FindByURL(e.URL)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
}
func (s *StoreSuite) TestDeleteNonExistantByUID() {
record, err := s.store.DeleteByUID([]byte("nope"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestDeleteExistingByURL() {
s.Seed()
for _, e := range examples {
record, err := s.store.DeleteByURL(e.URL)
s.NoError(err)
s.Equal(record.UID, e.UID)
s.Equal(record.URL, e.URL)
record, err = s.store.FindByUID(e.UID)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
record, err = s.store.FindByURL(e.URL)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
}
func (s *StoreSuite) TestDeleteNonExistantByURL() {
record, err := s.store.DeleteByURL([]byte("http://nope/"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestNextSequenceExisting() {
@@ -135,14 +184,14 @@ func (s *StoreSuite) TestIncrExisting() {
err := s.db.Put(key, []byte("5"), nil)
s.Require().NoError(err)
result, err := s.store.Incr(key)
result, err := s.store.incr(key)
s.NoError(err)
s.Equal(6, result)
}
func (s *StoreSuite) TestIncrNonExistant() {
for i := 1; i < 10; i++ {
result, err := s.store.Incr([]byte("counter"))
result, err := s.store.incr([]byte("counter"))
s.NoError(err)
s.Equal(i, result)
@@ -157,54 +206,57 @@ func TestStoreSuite(t *testing.T) {
// Benchmarks
func BenchmarkGet(b *testing.B) {
func BenchmarkCreate(b *testing.B) {
store, _ := New(testDbPath)
key := []byte("hello")
value := []byte("world")
_ = store.Set(key, value)
uid := []byte("Kb8X")
url := []byte("https://google.com/")
for n := 0; n < b.N; n++ {
_, _ = store.Get(key)
store.Create(append(uid, string(n)...), url)
}
_ = store.Close()
_ = os.RemoveAll(testDbPath)
store.Close()
os.RemoveAll(testDbPath)
}
func BenchmarkSet(b *testing.B) {
func BenchmarkFindByUID(b *testing.B) {
store, _ := New(testDbPath)
key := []byte("hello")
value := []byte("world")
uid := []byte("Kb8X")
url := []byte("https://google.com/")
store.Create(uid, url)
for n := 0; n < b.N; n++ {
_ = store.Set(append(key, string(n)...), value)
store.FindByUID(uid)
}
_ = store.Close()
_ = os.RemoveAll(testDbPath)
store.Close()
os.RemoveAll(testDbPath)
}
func BenchmarkFindByURL(b *testing.B) {
store, _ := New(testDbPath)
uid := []byte("Kb8X")
url := []byte("https://google.com/")
store.Create(uid, url)
for n := 0; n < b.N; n++ {
store.FindByURL(url)
}
store.Close()
os.RemoveAll(testDbPath)
}
func BenchmarkNextSequence(b *testing.B) {
store, _ := New(testDbPath)
for n := 0; n < b.N; n++ {
_, _ = store.NextSequence()
store.NextSequence()
}
_ = store.Close()
_ = os.RemoveAll(testDbPath)
}
func BenchmarkIncr(b *testing.B) {
store, _ := New(testDbPath)
key := []byte("incr-benchmark-counter")
for n := 0; n < b.N; n++ {
_, _ = store.Incr(key)
}
_ = store.Close()
_ = os.RemoveAll(testDbPath)
store.Close()
os.RemoveAll(testDbPath)
}

View File

@@ -1,17 +1,16 @@
package inmemorystore
import (
"errors"
"sync"
)
// ErrNotFound is returned when Get() tries to fetch a non-existent key.
var ErrNotFound = errors.New("not found")
"github.com/jimeh/ozu.io/storage"
)
// New creates a new Store using given path to persist data.
func New() (*Store, error) {
store := &Store{
Data: map[string][]byte{},
UIDMap: map[string][]byte{},
URLMap: map[string][]byte{},
Sequence: 0,
}
return store, nil
@@ -20,43 +19,79 @@ func New() (*Store, error) {
// Store allows storing data into a in-memory map.
type Store struct {
sync.RWMutex
Data map[string][]byte
UIDMap map[string][]byte
URLMap map[string][]byte
Sequence int
Closed bool
}
// Close database.
func (s *Store) Close() error {
s.Data = make(map[string][]byte)
s.Lock()
s.UIDMap = make(map[string][]byte)
s.URLMap = make(map[string][]byte)
s.Sequence = 0
s.Unlock()
return nil
}
// Get a given key's value.
func (s *Store) Get(key []byte) ([]byte, error) {
// Create a given Record
func (s *Store) Create(uid []byte, url []byte) (*storage.Record, error) {
s.Lock()
s.UIDMap[string(uid)] = url
s.URLMap[string(url)] = uid
s.Unlock()
return &storage.Record{UID: uid, URL: url}, nil
}
// FindByUID looks up records based on their UID.
func (s *Store) FindByUID(uid []byte) (*storage.Record, error) {
s.RLock()
value := s.Data[string(key)]
value := s.UIDMap[string(uid)]
s.RUnlock()
if value == nil {
return nil, ErrNotFound
return &storage.Record{}, storage.ErrNotFound
}
return value, nil
return &storage.Record{UID: uid, URL: value}, nil
}
// Set a given key's to the specified value.
func (s *Store) Set(key []byte, value []byte) error {
s.Lock()
s.Data[string(key)] = value
s.Unlock()
return nil
// FindByURL looks up records based on their URL.
func (s *Store) FindByURL(url []byte) (*storage.Record, error) {
s.RLock()
value := s.URLMap[string(url)]
s.RUnlock()
if value == nil {
return &storage.Record{}, storage.ErrNotFound
}
return &storage.Record{UID: value, URL: url}, nil
}
// Delete a given key.
func (s *Store) Delete(key []byte) error {
// DeleteByUID deletes records based on their UID.
func (s *Store) DeleteByUID(uid []byte) (*storage.Record, error) {
record, err := s.FindByUID(uid)
if err != nil {
return &storage.Record{}, err
}
s.delete(record)
return record, nil
}
// DeleteByURL deletes records based on their URL.
func (s *Store) DeleteByURL(url []byte) (*storage.Record, error) {
record, err := s.FindByURL(url)
if err != nil {
return &storage.Record{}, err
}
s.delete(record)
return record, nil
}
func (s *Store) delete(r *storage.Record) {
s.Lock()
delete(s.Data, string(key))
delete(s.UIDMap, string(r.UID))
delete(s.URLMap, string(r.URL))
s.Unlock()
return nil
}
// NextSequence returns a auto-incrementing int.

View File

@@ -9,13 +9,10 @@ import (
// Setup Suite
var examples = []struct {
key []byte
value []byte
}{
{key: []byte("hello"), value: []byte("world")},
{key: []byte("foo"), value: []byte("bar")},
{key: []byte("wtf"), value: []byte("dude")},
var examples = []storage.Record{
storage.Record{UID: []byte("Kb8X"), URL: []byte("https://google.com/")},
storage.Record{UID: []byte("h3mz"), URL: []byte("https://github.com/")},
storage.Record{UID: []byte("3qxs"), URL: []byte("https://twitter.com/")},
}
type StoreSuite struct {
@@ -35,7 +32,8 @@ func (s *StoreSuite) TearDownTest() {
func (s *StoreSuite) Seed() {
for _, e := range examples {
s.store.Data[string(e.key)] = e.value
s.store.UIDMap[string(e.UID)] = e.URL
s.store.URLMap[string(e.URL)] = e.UID
}
}
@@ -45,60 +43,112 @@ func (s *StoreSuite) TestStoreInterface() {
s.Implements(new(storage.Store), new(Store))
}
func (s *StoreSuite) TestSet() {
func (s *StoreSuite) TestCreate() {
for _, e := range examples {
err := s.store.Set(e.key, e.value)
record, err := s.store.Create(e.UID, e.URL)
s.NoError(err)
s.Equal(e.UID, record.UID)
s.Equal(e.URL, record.URL)
}
for _, e := range examples {
result, _ := s.store.Data[string(e.key)]
s.Equal(e.value, result)
recordURL, _ := s.store.UIDMap[string(e.UID)]
s.Equal(e.URL, recordURL)
recordUID, _ := s.store.URLMap[string(e.URL)]
s.Equal(e.UID, recordUID)
}
}
func (s *StoreSuite) TestGetExisting() {
func (s *StoreSuite) TestFindExistingByUID() {
s.Seed()
for _, e := range examples {
result, err := s.store.Get(e.key)
record, err := s.store.FindByUID(e.UID)
s.NoError(err)
s.Equal(e.value, result)
s.Equal(e.UID, record.UID)
s.Equal(e.URL, record.URL)
}
}
func (s *StoreSuite) TestGetNonExistant() {
result, err := s.store.Get([]byte("does-not-exist"))
s.Nil(result)
func (s *StoreSuite) TestFindNonExistantByUID() {
record, err := s.store.FindByUID([]byte("does-not-exist"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestDeleteExisting() {
func (s *StoreSuite) TestFindExistingByURL() {
s.Seed()
for _, e := range examples {
value := s.store.Data[string(e.key)]
s.Require().Equal(e.value, value)
value, err := s.store.Get(e.key)
s.Require().NoError(err)
s.Require().Equal(value, e.value)
err = s.store.Delete(e.key)
record, err := s.store.FindByURL(e.URL)
s.NoError(err)
value, err = s.store.Get(e.key)
s.Nil(value)
s.EqualError(err, "not found")
_, has := s.store.Data[string(e.key)]
s.Equal(false, has)
s.Equal(e.UID, record.UID)
s.Equal(e.URL, record.URL)
}
}
func (s *StoreSuite) TestDeleteNonExistant() {
err := s.store.Delete([]byte("does-not-exist"))
s.NoError(err)
func (s *StoreSuite) TestFindNonExistantByURL() {
record, err := s.store.FindByURL([]byte("http://nope.com/"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestDeleteExistingByUID() {
s.Seed()
for _, e := range examples {
record, err := s.store.DeleteByUID(e.UID)
s.NoError(err)
s.Equal(record.UID, e.UID)
s.Equal(record.URL, e.URL)
record, err = s.store.FindByUID(e.UID)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
record, err = s.store.FindByURL(e.URL)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
}
func (s *StoreSuite) TestDeleteNonExistantByUID() {
record, err := s.store.DeleteByUID([]byte("nope"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestDeleteExistingByURL() {
s.Seed()
for _, e := range examples {
record, err := s.store.DeleteByURL(e.URL)
s.NoError(err)
s.Equal(record.UID, e.UID)
s.Equal(record.URL, e.URL)
record, err = s.store.FindByUID(e.UID)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
record, err = s.store.FindByURL(e.URL)
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
}
func (s *StoreSuite) TestDeleteNonExistantByURL() {
record, err := s.store.DeleteByURL([]byte("http://nope/"))
s.Nil(record.UID)
s.Nil(record.URL)
s.EqualError(err, "not found")
}
func (s *StoreSuite) TestNextSequenceExisting() {
@@ -126,39 +176,53 @@ func TestStoreSuite(t *testing.T) {
// Benchmarks
func BenchmarkGet(b *testing.B) {
func BenchmarkCreate(b *testing.B) {
store, _ := New()
key := []byte("hello")
value := []byte("world")
_ = store.Set(key, value)
uid := []byte("Kb8X")
url := []byte("https://google.com/")
for n := 0; n < b.N; n++ {
_, _ = store.Get(key)
store.Create(append(uid, string(n)...), url)
}
_ = store.Close()
store.Close()
}
func BenchmarkSet(b *testing.B) {
func BenchmarkFindByUID(b *testing.B) {
store, _ := New()
key := []byte("hello")
value := []byte("world")
uid := []byte("Kb8X")
url := []byte("https://google.com/")
store.Create(uid, url)
for n := 0; n < b.N; n++ {
_ = store.Set(append(key, string(n)...), value)
store.FindByUID(uid)
}
_ = store.Close()
store.Close()
}
func BenchmarkFindByURL(b *testing.B) {
store, _ := New()
uid := []byte("Kb8X")
url := []byte("https://google.com/")
store.Create(uid, url)
for n := 0; n < b.N; n++ {
store.FindByURL(url)
}
store.Close()
}
func BenchmarkNextSequence(b *testing.B) {
store, _ := New()
for n := 0; n < b.N; n++ {
_, _ = store.NextSequence()
store.NextSequence()
}
_ = store.Close()
store.Close()
}

View File

@@ -1,10 +1,23 @@
package storage
// Store defines a standard interface for storage.
import "errors"
// ErrNotFound is the default error message when data is not found.
var ErrNotFound = errors.New("not found")
// Store defines a standard interface for storage
type Store interface {
Close() error
Get([]byte) ([]byte, error)
Set([]byte, []byte) error
Delete([]byte) error
Create(UID []byte, URL []byte) (*Record, error)
FindByUID(UID []byte) (*Record, error)
FindByURL(URL []byte) (*Record, error)
DeleteByUID(UID []byte) (*Record, error)
DeleteByURL(URL []byte) (*Record, error)
NextSequence() (int, error)
}
// Record provides a standard way to refer to a shortened URL.
type Record struct {
UID []byte
URL []byte
}