From 791ed82aee4b8b6f8aeea42d231b11212c22c482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Wed, 28 Aug 2019 19:04:43 +0200 Subject: [PATCH 1/5] add a Size field to Query's Result Rational is that some datastore can easilty get the size of a value when listing only keys, so we might as well have a way to pass that. One way this could be useful is when implementing a cache for Has/GetSize. --- basic_ds.go | 2 +- namespace/namespace_test.go | 10 +++++----- query/query.go | 1 + query/query_impl.go | 2 +- test/basic_tests.go | 1 + 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/basic_ds.go b/basic_ds.go index 08ae2af..9847676 100644 --- a/basic_ds.go +++ b/basic_ds.go @@ -61,7 +61,7 @@ func (d *MapDatastore) Delete(key Key) (err error) { func (d *MapDatastore) Query(q dsq.Query) (dsq.Results, error) { re := make([]dsq.Entry, 0, len(d.values)) for k, v := range d.values { - e := dsq.Entry{Key: k.String()} + e := dsq.Entry{Key: k.String(), Size: len(v)} if !q.KeysOnly { e.Value = v } diff --git a/namespace/namespace_test.go b/namespace/namespace_test.go index e368286..3d9bd83 100644 --- a/namespace/namespace_test.go +++ b/namespace/namespace_test.go @@ -101,9 +101,9 @@ func (ks *DSSuite) TestQuery(c *C) { c.Check(err, Equals, nil) expect := []dsq.Entry{ - {Key: "/bar", Value: []byte("/foo/bar")}, - {Key: "/bar/baz", Value: []byte("/foo/bar/baz")}, - {Key: "/baz/abc", Value: []byte("/foo/baz/abc")}, + {Key: "/bar", Size: len([]byte("/foo/bar")), Value: []byte("/foo/bar")}, + {Key: "/bar/baz", Size: len([]byte("/foo/bar/baz")), Value: []byte("/foo/bar/baz")}, + {Key: "/baz/abc", Size: len([]byte("/foo/baz/abc")), Value: []byte("/foo/baz/abc")}, } results, err := qres.Rest() @@ -122,8 +122,8 @@ func (ks *DSSuite) TestQuery(c *C) { c.Check(err, Equals, nil) expect = []dsq.Entry{ - {Key: "/bar", Value: []byte("/foo/bar")}, - {Key: "/bar/baz", Value: []byte("/foo/bar/baz")}, + {Key: "/bar", Size: len([]byte("/foo/bar")), Value: []byte("/foo/bar")}, + {Key: "/bar/baz", Size: len([]byte("/foo/bar/baz")), Value: []byte("/foo/bar/baz")}, } results, err = qres.Rest() diff --git a/query/query.go b/query/query.go index 42e69c4..09b9d98 100644 --- a/query/query.go +++ b/query/query.go @@ -115,6 +115,7 @@ func (q Query) String() string { // Entry is a query result entry. type Entry struct { Key string // cant be ds.Key because circular imports ...!!! + Size int // Might be zero if the datastore doesn't support listing the size with KeysOnly Value []byte // Will be nil if KeysOnly has been passed. Expiration time.Time // Entry expiration timestamp if requested and supported (see TTLDatastore). } diff --git a/query/query_impl.go b/query/query_impl.go index 6c2e422..175a4c2 100644 --- a/query/query_impl.go +++ b/query/query_impl.go @@ -136,7 +136,7 @@ func NaiveQueryApply(q Query, qr Results) Results { func ResultEntriesFrom(keys []string, vals [][]byte) []Entry { re := make([]Entry, len(keys)) for i, k := range keys { - re[i] = Entry{Key: k, Value: vals[i]} + re[i] = Entry{Key: k, Size: len(vals[i]), Value: vals[i]} } return re } diff --git a/test/basic_tests.go b/test/basic_tests.go index 862a634..519ccba 100644 --- a/test/basic_tests.go +++ b/test/basic_tests.go @@ -315,6 +315,7 @@ func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, count int) { value := randValue() input = append(input, dsq.Entry{ Key: key, + Size: len(value), Value: value, }) } From d4ed581a8d0fb520f2d55505117293fb317714d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Fri, 4 Oct 2019 15:02:17 +0900 Subject: [PATCH 2/5] query: allow to always request the size alongside KeysOnly, allow impl to always return the size if no perf cost --- autobatch/autobatch.go | 2 +- query/query.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/autobatch/autobatch.go b/autobatch/autobatch.go index 66d701e..4eb089b 100644 --- a/autobatch/autobatch.go +++ b/autobatch/autobatch.go @@ -8,7 +8,7 @@ import ( dsq "github.com/ipfs/go-datastore/query" ) -// Datastore implements a go-datatsore. +// Datastore implements a go-datastore. type Datastore struct { child ds.Batching diff --git a/query/query.go b/query/query.go index 09b9d98..39b08b2 100644 --- a/query/query.go +++ b/query/query.go @@ -66,6 +66,9 @@ type Query struct { Offset int // skip given number of results KeysOnly bool // return only keys. ReturnExpirations bool // return expirations (see TTLDatastore) + ReturnsSize bool // always return sizes. If not set, datastore impl can return + // // it anyway if it doesn't involve a performance cost. If KeysOnly + // // is not set, Size should always be set. } // String returns a string represenation of the Query for debugging/validation @@ -115,9 +118,10 @@ func (q Query) String() string { // Entry is a query result entry. type Entry struct { Key string // cant be ds.Key because circular imports ...!!! - Size int // Might be zero if the datastore doesn't support listing the size with KeysOnly Value []byte // Will be nil if KeysOnly has been passed. Expiration time.Time // Entry expiration timestamp if requested and supported (see TTLDatastore). + Size int // Might be -1 if the datastore doesn't support listing the size with KeysOnly + // // or if ReturnsSizes is not set } // Result is a special entry that includes an error, so that the client From 3c4e26d382cdff0d391d720e9f8eb2782c0f5037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Fri, 22 Nov 2019 13:33:33 +0100 Subject: [PATCH 3/5] query: don't forget to pass ReturnSize in mount's sub Query --- mount/mount.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mount/mount.go b/mount/mount.go index 7c32a82..b3113a9 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -227,6 +227,7 @@ func (d *Datastore) Query(master query.Query) (query.Results, error) { Orders: master.Orders, KeysOnly: master.KeysOnly, ReturnExpirations: master.ReturnExpirations, + ReturnsSize: master.ReturnsSize, } prefix := ds.NewKey(childQuery.Prefix) From b089db1e398bd65f1cc4f316ab3519c71e57bba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Fri, 22 Nov 2019 13:37:18 +0100 Subject: [PATCH 4/5] query: rename ReturnsSize to ReturnsSizes --- mount/mount.go | 2 +- query/query.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mount/mount.go b/mount/mount.go index b3113a9..c092c26 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -227,7 +227,7 @@ func (d *Datastore) Query(master query.Query) (query.Results, error) { Orders: master.Orders, KeysOnly: master.KeysOnly, ReturnExpirations: master.ReturnExpirations, - ReturnsSize: master.ReturnsSize, + ReturnsSizes: master.ReturnsSizes, } prefix := ds.NewKey(childQuery.Prefix) diff --git a/query/query.go b/query/query.go index 39b08b2..b6bfa45 100644 --- a/query/query.go +++ b/query/query.go @@ -66,7 +66,7 @@ type Query struct { Offset int // skip given number of results KeysOnly bool // return only keys. ReturnExpirations bool // return expirations (see TTLDatastore) - ReturnsSize bool // always return sizes. If not set, datastore impl can return + ReturnsSizes bool // always return sizes. If not set, datastore impl can return // // it anyway if it doesn't involve a performance cost. If KeysOnly // // is not set, Size should always be set. } From 68a77964d1eb0a20bf478b1a44fa274d22ebea53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Fri, 22 Nov 2019 14:49:37 +0100 Subject: [PATCH 5/5] query: add tests for ReturnSizes --- test/basic_tests.go | 8 +++++++- test/suite.go | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/test/basic_tests.go b/test/basic_tests.go index 519ccba..b1c73de 100644 --- a/test/basic_tests.go +++ b/test/basic_tests.go @@ -301,6 +301,10 @@ func SubtestFilter(t *testing.T, ds dstore.Datastore) { test(new(testFilter)) } +func SubtestReturnSizes(t *testing.T, ds dstore.Datastore) { + subtestQuery(t, ds, dsq.Query{ReturnsSizes: true}, 100) +} + func randValue() []byte { value := make([]byte, 64) rand.Read(value) @@ -376,7 +380,9 @@ func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, count int) { if !q.KeysOnly && !bytes.Equal(actual[i].Value, expected[i].Value) { t.Errorf("value mismatch for result %d (key=%q)", i, expected[i].Key) } - + if q.ReturnsSizes && actual[i].Size <= 0 { + t.Errorf("for result %d, expected size > 0 with ReturnsSizes", i) + } } t.Log("deleting all keys") diff --git a/test/suite.go b/test/suite.go index 20d0f69..3d17401 100644 --- a/test/suite.go +++ b/test/suite.go @@ -18,6 +18,7 @@ var BasicSubtests = []func(t *testing.T, ds dstore.Datastore){ SubtestLimit, SubtestFilter, SubtestManyKeysAndQuery, + SubtestReturnSizes, } // BatchSubtests is a list of all basic batching datastore tests.