From f811b49c1bd8e53ae5fda47a2a3429ec78e7c44f Mon Sep 17 00:00:00 2001 From: Jens Alfke Date: Wed, 22 Feb 2023 12:05:03 -0800 Subject: [PATCH] Moved document, revision, attachment stuff into new docmodel package --- db/attachment.go | 170 +------- db/attachment_compaction.go | 9 +- db/attachment_test.go | 117 +----- db/blip_connected_client.go | 3 +- db/blip_handler.go | 19 +- db/blip_sync_context.go | 9 +- db/change_cache.go | 5 +- db/change_cache_test.go | 5 +- db/changes.go | 2 +- db/crud.go | 155 +++---- db/crud_test.go | 13 +- db/database.go | 13 +- db/database_collection.go | 6 + db/database_test.go | 3 +- db/doc_model.go | 58 +++ db/import.go | 23 +- db/import_listener.go | 3 +- db/repair_bucket.go | 3 +- db/revision.go | 364 +---------------- db/revision_cache_interface.go | 13 +- db/revision_cache_test.go | 7 +- db/revision_test.go | 173 -------- db/sg_replicate_conflict_resolver.go | 3 +- db/special_docs.go | 3 +- docmodel/attachment.go | 155 +++++++ docmodel/attachment_test.go | 105 +++++ {db => docmodel}/data_for_test.go | 2 +- {db => docmodel}/document.go | 107 ++--- {db => docmodel}/document_test.go | 10 +- docmodel/revision.go | 377 ++++++++++++++++++ docmodel/revision_test.go | 191 +++++++++ {db => docmodel}/revtree.go | 76 ++-- {db => docmodel}/revtree_data_test.go | 2 +- {db => docmodel}/revtree_test.go | 144 +++---- rest/admin_api.go | 3 +- rest/api_benchmark_test.go | 6 +- rest/api_test.go | 3 +- rest/attachment_test.go | 4 +- rest/blip_client_test.go | 3 +- rest/bulk_api.go | 9 +- rest/config_database.go | 3 +- rest/doc_api.go | 13 +- rest/multipart.go | 9 +- rest/replicatortest/replicator_test.go | 15 +- rest/replicatortest/replicator_test_helper.go | 3 +- rest/utilities_testing.go | 22 +- rest/utilities_testing_test.go | 2 +- 47 files changed, 1311 insertions(+), 1132 deletions(-) create mode 100644 db/doc_model.go create mode 100644 docmodel/attachment.go create mode 100644 docmodel/attachment_test.go rename {db => docmodel}/data_for_test.go (99%) rename {db => docmodel}/document.go (92%) rename {db => docmodel}/document_test.go (96%) create mode 100644 docmodel/revision.go create mode 100644 docmodel/revision_test.go rename {db => docmodel}/revtree.go (92%) rename {db => docmodel}/revtree_data_test.go (99%) rename {db => docmodel}/revtree_test.go (92%) diff --git a/db/attachment.go b/db/attachment.go index fdcd66e62e..8dbfc744d4 100644 --- a/db/attachment.go +++ b/db/attachment.go @@ -10,24 +10,16 @@ package db import ( "context" - "crypto/rand" "crypto/sha1" "crypto/sha256" "encoding/base64" - "errors" - "fmt" "net/http" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) -const ( - // AttVersion1 attachments are persisted to the bucket based on attachment body digest. - AttVersion1 int = 1 - - // AttVersion2 attachments are persisted to the bucket based on docID and body digest. - AttVersion2 int = 2 -) +const maxAttachmentSizeBytes = 20 * 1024 * 1024 var ( // ErrAttachmentVersion is thrown in case of any error in parsing version from the attachment meta. @@ -37,29 +29,6 @@ var ( ErrAttachmentMeta = base.HTTPErrorf(http.StatusBadRequest, "Invalid _attachments") ) -// AttachmentData holds the attachment key and value bytes. -type AttachmentData map[string][]byte - -// A map of keys -> DocAttachments. -type AttachmentMap map[string]*DocAttachment - -// A struct which models an attachment. Currently only used by test code, however -// new code or refactoring in the main codebase should try to use where appropriate. -type DocAttachment struct { - ContentType string `json:"content_type,omitempty"` - Digest string `json:"digest,omitempty"` - Length int `json:"length,omitempty"` - Revpos int `json:"revpos,omitempty"` - Stub bool `json:"stub,omitempty"` - Version int `json:"ver,omitempty"` - Data []byte `json:"-"` // tell json marshal/unmarshal to ignore this field -} - -// ErrAttachmentTooLarge is returned when an attempt to attach an oversize attachment is made. -var ErrAttachmentTooLarge = errors.New("attachment too large") - -const maxAttachmentSizeBytes = 20 * 1024 * 1024 - // Given Attachments Meta to be stored in the database, storeAttachments goes through the map, finds attachments with // inline bodies, copies the bodies into the Couchbase db, and replaces the bodies with the 'digest' attributes which // are the keys to retrieving them. @@ -79,19 +48,19 @@ func (db *DatabaseCollectionWithUser) storeAttachments(ctx context.Context, doc data := meta["data"] if data != nil { // Attachment contains data, so store it in the db: - attachment, err := DecodeAttachment(data) + attachment, err := docmodel.DecodeAttachment(data) if err != nil { return nil, err } digest := Sha1DigestKey(attachment) - key := MakeAttachmentKey(AttVersion2, doc.ID, digest) + key := MakeAttachmentKey(docmodel.AttVersion2, doc.ID, digest) newAttachmentData[key] = attachment newMeta := map[string]interface{}{ "stub": true, "digest": digest, "revpos": generation, - "ver": AttVersion2, + "ver": docmodel.AttVersion2, } if contentType, ok := meta["content_type"].(string); ok { newMeta["content_type"] = contentType @@ -146,7 +115,7 @@ func retrieveV2AttachmentKeys(docID string, docAttachments AttachmentsMeta) (att if !ok { return nil, ErrAttachmentMeta } - version, _ := GetAttachmentVersion(meta) + version, _ := docmodel.GetAttachmentVersion(meta) if version != AttVersion2 { continue } @@ -167,10 +136,10 @@ func (db *DatabaseCollectionWithUser) retrieveAncestorAttachments(ctx context.Co } // No non-pruned ancestor is available - if commonAncestor := doc.History.findAncestorFromSet(doc.CurrentRev, docHistory); commonAncestor != "" { + if commonAncestor := doc.History.FindAncestorFromSet(doc.CurrentRev, docHistory); commonAncestor != "" { parentAttachments := make(map[string]interface{}) - commonAncestorGen := int64(genOfRevID(commonAncestor)) - for name, activeAttachment := range GetBodyAttachments(doc.Body()) { + commonAncestorGen := int64(docmodel.GenOfRevID(commonAncestor)) + for name, activeAttachment := range docmodel.GetBodyAttachments(doc.Body()) { if attachmentMeta, ok := activeAttachment.(map[string]interface{}); ok { activeRevpos, ok := base.ToInt64(attachmentMeta["revpos"]) if ok && activeRevpos <= commonAncestorGen { @@ -204,7 +173,7 @@ func (c *DatabaseCollection) loadAttachmentsData(attachments AttachmentsMeta, mi if !ok { return nil, base.RedactErrorf("Unable to load attachment for doc: %v with name: %v and revpos: %v due to unexpected digest field: %v", base.UD(docid), base.UD(attachmentName), revpos, digest) } - version, ok := GetAttachmentVersion(meta) + version, ok := docmodel.GetAttachmentVersion(meta) if !ok { return nil, base.RedactErrorf("Unable to load attachment for doc: %v with name: %v, revpos: %v and digest: %v due to unexpected version value: %v", base.UD(docid), base.UD(attachmentName), revpos, digest, version) } @@ -221,14 +190,6 @@ func (c *DatabaseCollection) loadAttachmentsData(attachments AttachmentsMeta, mi return newAttachments, nil } -// DeleteAttachmentVersion removes attachment versions from the AttachmentsMeta map specified. -func DeleteAttachmentVersion(attachments AttachmentsMeta) { - for _, value := range attachments { - meta := value.(map[string]interface{}) - delete(meta, "ver") - } -} - // GetAttachment retrieves an attachment given its key. func (c *DatabaseCollection) GetAttachment(key string) ([]byte, error) { v, _, err := c.dataStore.GetRaw(key) @@ -248,7 +209,7 @@ func (db *DatabaseCollectionWithUser) setAttachments(ctx context.Context, attach for key, data := range attachments { attachmentSize := int64(len(data)) if attachmentSize > int64(maxAttachmentSizeBytes) { - return ErrAttachmentTooLarge + return docmodel.ErrAttachmentTooLarge } _, err := db.dataStore.AddRaw(key, 0, data) if err == nil { @@ -270,7 +231,7 @@ type AttachmentCallback func(name string, digest string, knownData []byte, meta // to its digest. If the attachment isn't known, the callback can return data for it, which will // be added to the metadata as a "data" property. func (c *DatabaseCollection) ForEachStubAttachment(body Body, minRevpos int, docID string, existingDigests map[string]string, callback AttachmentCallback) error { - atts := GetBodyAttachments(body) + atts := docmodel.GetBodyAttachments(body) if atts == nil && body[BodyAttachments] != nil { return base.HTTPErrorf(http.StatusBadRequest, "Invalid _attachments") } @@ -319,103 +280,14 @@ func (c *DatabaseCollection) ForEachStubAttachment(body Body, minRevpos int, doc return nil } -func GetAttachmentVersion(meta map[string]interface{}) (int, bool) { - ver, ok := meta["ver"] - if !ok { - return AttVersion1, true - } - val, ok := base.ToInt64(ver) - return int(val), ok -} - -// GenerateProofOfAttachment returns a nonce and proof for an attachment body. -func GenerateProofOfAttachment(attachmentData []byte) (nonce []byte, proof string, err error) { - nonce = make([]byte, 20) - if _, err := rand.Read(nonce); err != nil { - return nil, "", base.HTTPErrorf(http.StatusInternalServerError, fmt.Sprintf("Failed to generate random data: %s", err)) - } - proof = ProveAttachment(attachmentData, nonce) - base.TracefCtx(context.Background(), base.KeyCRUD, "Generated nonce %v and proof %q for attachment: %v", nonce, proof, attachmentData) - return nonce, proof, nil -} - -// ProveAttachment returns the proof for an attachment body and nonce pair. -func ProveAttachment(attachmentData, nonce []byte) (proof string) { - d := sha1.New() - d.Write([]byte{byte(len(nonce))}) - d.Write(nonce) - d.Write(attachmentData) - proof = "sha1-" + base64.StdEncoding.EncodeToString(d.Sum(nil)) - base.TracefCtx(context.Background(), base.KeyCRUD, "Generated proof %q using nonce %v for attachment: %v", proof, nonce, attachmentData) - return proof -} - // ////// HELPERS: -// Returns _attachments property from body, when found. Checks for either map[string]interface{} (unmarshalled with body), -// or AttachmentsMeta (written by body by SG) -func GetBodyAttachments(body Body) AttachmentsMeta { - switch atts := body[BodyAttachments].(type) { - case AttachmentsMeta: - return atts - case map[string]interface{}: - return AttachmentsMeta(atts) - default: - return nil - } -} -// AttachmentDigests returns a list of attachment digests contained in the given AttachmentsMeta -func AttachmentDigests(attachments AttachmentsMeta) []string { - var digests = make([]string, 0, len(attachments)) - for _, att := range attachments { - if attMap, ok := att.(map[string]interface{}); ok { - if digest, ok := attMap["digest"]; ok { - if digestString, ok := digest.(string); ok { - digests = append(digests, digestString) - } - } - } - } - return digests -} - -// AttachmentStorageMeta holds the metadata for building -// the key for attachment storage and retrieval. -type AttachmentStorageMeta struct { - digest string - version int -} - -// ToAttachmentStorageMeta returns a slice of AttachmentStorageMeta, which is contains the -// necessary metadata properties to build the key for attachment storage and retrieval. -func ToAttachmentStorageMeta(attachments AttachmentsMeta) []AttachmentStorageMeta { - meta := make([]AttachmentStorageMeta, 0, len(attachments)) - for _, att := range attachments { - if attMap, ok := att.(map[string]interface{}); ok { - if digest, ok := attMap["digest"]; ok { - if digestString, ok := digest.(string); ok { - version, _ := GetAttachmentVersion(attMap) - m := AttachmentStorageMeta{ - digest: digestString, - version: version, - } - meta = append(meta, m) - } - } - } - } - return meta -} - -func DecodeAttachment(att interface{}) ([]byte, error) { - switch att := att.(type) { - case []byte: - return att, nil - case string: - return base64.StdEncoding.DecodeString(att) - default: - return nil, base.HTTPErrorf(400, "invalid attachment data (type %T)", att) +// MakeAttachmentKey returns the unique for attachment storage and retrieval. +func MakeAttachmentKey(version int, docID, digest string) string { + if version == AttVersion2 { + return base.Att2Prefix + sha256Digest([]byte(docID)) + ":" + digest } + return base.AttPrefix + digest } func Sha1DigestKey(data []byte) string { @@ -424,14 +296,6 @@ func Sha1DigestKey(data []byte) string { return "sha1-" + base64.StdEncoding.EncodeToString(digester.Sum(nil)) } -// MakeAttachmentKey returns the unique for attachment storage and retrieval. -func MakeAttachmentKey(version int, docID, digest string) string { - if version == AttVersion2 { - return base.Att2Prefix + sha256Digest([]byte(docID)) + ":" + digest - } - return base.AttPrefix + digest -} - // sha256Digest returns sha256 digest of the input bytes encoded // by using the standard base64 encoding, as defined in RFC 4648. func sha256Digest(key []byte) string { diff --git a/db/attachment_compaction.go b/db/attachment_compaction.go index aa39f38d7d..24c93169c6 100644 --- a/db/attachment_compaction.go +++ b/db/attachment_compaction.go @@ -20,6 +20,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" ) const ( @@ -211,7 +212,7 @@ func getAttachmentSyncData(dataType uint8, data []byte) (*AttachmentCompactionDa var documentBody []byte if dataType&base.MemcachedDataTypeXattr != 0 { - body, xattr, _, err := parseXattrStreamData(base.SyncXattrName, "", data) + body, xattr, _, err := docmodel.ParseXattrStreamData(base.SyncXattrName, "", data) if err != nil { if errors.Is(err, base.ErrXattrNotFound) { return nil, nil @@ -275,7 +276,7 @@ func handleAttachments(attachmentKeyMap map[string]string, docKey string, attach for attName, attachmentMeta := range attachmentsMap { attMetaMap := attachmentMeta - attVer, ok := GetAttachmentVersion(attMetaMap) + attVer, ok := docmodel.GetAttachmentVersion(attMetaMap) if !ok { continue } @@ -312,7 +313,7 @@ func attachmentCompactSweepPhase(ctx context.Context, dataStore base.DataStore, // If the data contains an xattr then the attachment likely has a compaction ID, need to check this value if event.DataType&base.MemcachedDataTypeXattr != 0 { - _, xattr, _, err := parseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value) + _, xattr, _, err := docmodel.ParseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value) if err != nil && !errors.Is(err, base.ErrXattrNotFound) { base.WarnfCtx(ctx, "[%s] Unexpected error occurred attempting to parse attachment xattr: %v", compactionLoggingID, err) return true @@ -424,7 +425,7 @@ func attachmentCompactCleanupPhase(ctx context.Context, dataStore base.DataStore return true } - _, xattr, _, err := parseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value) + _, xattr, _, err := docmodel.ParseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value) if err != nil && !errors.Is(err, base.ErrXattrNotFound) { base.WarnfCtx(ctx, "[%s] Unexpected error occurred attempting to parse attachment xattr: %v", compactionLoggingID, err) return true diff --git a/db/attachment_test.go b/db/attachment_test.go index 408abc1411..66be4406de 100644 --- a/db/attachment_test.go +++ b/db/attachment_test.go @@ -11,18 +11,17 @@ package db import ( "context" "encoding/base64" - "encoding/json" "errors" "fmt" "log" "math/rand" "net/http" "strconv" - "strings" "testing" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -340,7 +339,7 @@ func TestAttachmentCASRetryAfterNewAttachment(t *testing.T) { // 4. Get the document, check attachments finalDoc, err := collection.Get1xBody(ctx, "doc1") - attachments := GetBodyAttachments(finalDoc) + attachments := docmodel.GetBodyAttachments(finalDoc) assert.True(t, attachments != nil, "_attachments should be present in GET response") attachment, attachmentOk := attachments["hello.txt"].(map[string]interface{}) assert.True(t, attachmentOk, "hello.txt attachment should be present in GET response") @@ -404,7 +403,7 @@ func TestAttachmentCASRetryDuringNewAttachment(t *testing.T) { finalDoc, err := collection.Get1xBody(ctx, "doc1") log.Printf("get doc attachments: %v", finalDoc) - attachments := GetBodyAttachments(finalDoc) + attachments := docmodel.GetBodyAttachments(finalDoc) assert.True(t, attachments != nil, "_attachments should be present in GET response") attachment, attachmentOk := attachments["hello.txt"].(map[string]interface{}) assert.True(t, attachmentOk, "hello.txt attachment should be present in GET response") @@ -497,66 +496,6 @@ func TestForEachStubAttachmentErrors(t *testing.T) { assert.Contains(t, err.Error(), "Can't work with this digest value!") } -func TestGenerateProofOfAttachment(t *testing.T) { - base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) - - attData := []byte(`hello world`) - - nonce, proof1, err := GenerateProofOfAttachment(attData) - require.NoError(t, err) - assert.True(t, len(nonce) >= 20, "nonce should be at least 20 bytes") - assert.NotEmpty(t, proof1) - assert.True(t, strings.HasPrefix(proof1, "sha1-")) - - proof2 := ProveAttachment(attData, nonce) - assert.NotEmpty(t, proof1, "") - assert.True(t, strings.HasPrefix(proof1, "sha1-")) - - assert.Equal(t, proof1, proof2, "GenerateProofOfAttachment and ProveAttachment produced different proofs.") -} - -func TestDecodeAttachmentError(t *testing.T) { - attr, err := DecodeAttachment(make([]int, 1)) - assert.Nil(t, attr, "Attachment of data (type []int) should not get decoded.") - assert.Error(t, err, "It should throw 400 invalid attachment data (type []int)") - assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) - - attr, err = DecodeAttachment(make([]float64, 1)) - assert.Nil(t, attr, "Attachment of data (type []float64) should not get decoded.") - assert.Error(t, err, "It should throw 400 invalid attachment data (type []float64)") - assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) - - attr, err = DecodeAttachment(make([]string, 1)) - assert.Nil(t, attr, "Attachment of data (type []string) should not get decoded.") - assert.Error(t, err, "It should throw 400 invalid attachment data (type []string)") - assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) - - attr, err = DecodeAttachment(make(map[string]int, 1)) - assert.Nil(t, attr, "Attachment of data (type map[string]int) should not get decoded.") - assert.Error(t, err, "It should throw 400 invalid attachment data (type map[string]int)") - assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) - - attr, err = DecodeAttachment(make(map[string]float64, 1)) - assert.Nil(t, attr, "Attachment of data (type map[string]float64) should not get decoded.") - assert.Error(t, err, "It should throw 400 invalid attachment data (type map[string]float64)") - assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) - - attr, err = DecodeAttachment(make(map[string]string, 1)) - assert.Error(t, err, "should throw 400 invalid attachment data (type map[string]float64)") - assert.Error(t, err, "It 400 invalid attachment data (type map[string]string)") - assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) - - book := struct { - author string - title string - price float64 - }{author: "William Shakespeare", title: "Hamlet", price: 7.99} - attr, err = DecodeAttachment(book) - assert.Nil(t, attr) - assert.Error(t, err, "It should throw 400 invalid attachment data (type struct { author string; title string; price float64 })") - assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) -} - func TestSetAttachment(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close(ctx) @@ -1373,7 +1312,7 @@ func TestAllowedAttachments(t *testing.T) { docIDForAllowedAttKey = "" } for _, att := range meta { - key := allowedAttachmentKey(docIDForAllowedAttKey, att.digest, activeSubprotocol) + key := allowedAttachmentKey(docIDForAllowedAttKey, att.Digest, activeSubprotocol) require.True(t, isAllowedAttachment(ctx, key)) } } @@ -1384,7 +1323,7 @@ func TestAllowedAttachments(t *testing.T) { docIDForAllowedAttKey = "" } for _, att := range meta { - key := allowedAttachmentKey(docIDForAllowedAttKey, att.digest, activeSubprotocol) + key := allowedAttachmentKey(docIDForAllowedAttKey, att.Digest, activeSubprotocol) require.False(t, isAllowedAttachment(ctx, key)) } } @@ -1394,8 +1333,8 @@ func TestAllowedAttachments(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := &BlipSyncContext{} meta := []AttachmentStorageMeta{ - {digest: "digest1", version: tt.inputAttVersion}, - {digest: "digest2", version: tt.inputAttVersion}, + {Digest: "digest1", Version: tt.inputAttVersion}, + {Digest: "digest2", Version: tt.inputAttVersion}, } docID := "doc1" @@ -1412,13 +1351,13 @@ func TestAllowedAttachments(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := &BlipSyncContext{} meta := []AttachmentStorageMeta{ - {digest: "digest1", version: tt.inputAttVersion}, - {digest: "digest1", version: tt.inputAttVersion}, + {Digest: "digest1", Version: tt.inputAttVersion}, + {Digest: "digest1", Version: tt.inputAttVersion}, } docID := "doc1" ctx.addAllowedAttachments(docID, meta, tt.inputBlipProtocol) - key := allowedAttachmentKey(docID, meta[0].digest, tt.inputBlipProtocol) + key := allowedAttachmentKey(docID, meta[0].Digest, tt.inputBlipProtocol) require.True(t, isAllowedAttachment(ctx, key)) ctx.removeAllowedAttachments(docID, meta, tt.inputBlipProtocol) @@ -1431,8 +1370,8 @@ func TestAllowedAttachments(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := &BlipSyncContext{} meta := []AttachmentStorageMeta{ - {digest: "digest1", version: tt.inputAttVersion}, - {digest: "digest2", version: tt.inputAttVersion}, + {Digest: "digest1", Version: tt.inputAttVersion}, + {Digest: "digest2", Version: tt.inputAttVersion}, } docID1 := "doc1" @@ -1461,8 +1400,8 @@ func TestAllowedAttachments(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := &BlipSyncContext{} meta := []AttachmentStorageMeta{ - {digest: "digest1", version: tt.inputAttVersion}, - {digest: "digest1", version: tt.inputAttVersion}, + {Digest: "digest1", Version: tt.inputAttVersion}, + {Digest: "digest1", Version: tt.inputAttVersion}, } docID1 := "doc1" @@ -1492,12 +1431,12 @@ func TestAllowedAttachments(t *testing.T) { ctx := &BlipSyncContext{} docID1 := "doc1" - att1Meta := []AttachmentStorageMeta{{digest: "att1", version: tt.inputAttVersion}} + att1Meta := []AttachmentStorageMeta{{Digest: "att1", Version: tt.inputAttVersion}} ctx.addAllowedAttachments(docID1, att1Meta, tt.inputBlipProtocol) requireIsAttachmentAllowedTrue(t, ctx, docID1, att1Meta, tt.inputBlipProtocol) docID2 := "doc2" - att2Meta := []AttachmentStorageMeta{{digest: "att2", version: tt.inputAttVersion}} + att2Meta := []AttachmentStorageMeta{{Digest: "att2", Version: tt.inputAttVersion}} ctx.addAllowedAttachments(docID2, att2Meta, tt.inputBlipProtocol) requireIsAttachmentAllowedTrue(t, ctx, docID2, att2Meta, tt.inputBlipProtocol) @@ -1511,30 +1450,6 @@ func TestAllowedAttachments(t *testing.T) { } } -func TestGetAttVersion(t *testing.T) { - var tests = []struct { - name string - inputAttVersion interface{} - expectedValidAttVersion bool - expectedAttVersion int - }{ - {"int attachment version", AttVersion2, true, AttVersion2}, - {"float64 attachment version", float64(AttVersion2), true, AttVersion2}, - {"invalid json.Number attachment version", json.Number("foo"), false, 0}, - {"valid json.Number attachment version", json.Number(strconv.Itoa(AttVersion2)), true, AttVersion2}, - {"invaid string attachment version", strconv.Itoa(AttVersion2), false, 0}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - meta := map[string]interface{}{"ver": tt.inputAttVersion} - version, ok := GetAttachmentVersion(meta) - assert.Equal(t, tt.expectedValidAttVersion, ok) - assert.Equal(t, tt.expectedAttVersion, version) - }) - } -} - func TestLargeAttachments(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close(ctx) diff --git a/db/blip_connected_client.go b/db/blip_connected_client.go index 960add23ee..7aeef2b9e9 100644 --- a/db/blip_connected_client.go +++ b/db/blip_connected_client.go @@ -19,6 +19,7 @@ import ( "github.com/couchbase/go-blip" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) //////// GETREV: @@ -49,7 +50,7 @@ func (bh *blipHandler) handleGetRev(rq *blip.Message) error { // Still need to stamp _attachments into BLIP messages if len(rev.Attachments) > 0 { - DeleteAttachmentVersion(rev.Attachments) + docmodel.DeleteAttachmentVersion(rev.Attachments) body[BodyAttachments] = rev.Attachments } diff --git a/db/blip_handler.go b/db/blip_handler.go index 0baa5f789a..b1fca6d217 100644 --- a/db/blip_handler.go +++ b/db/blip_handler.go @@ -26,6 +26,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" ) // handlersByProfile defines the routes for each message profile (verb) of an incoming request to the function that handles it. @@ -1044,12 +1045,12 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // Look at attachments with revpos > the last common ancestor's minRevpos := 1 if len(history) > 0 { - currentDoc, rawDoc, err := bh.collection.GetDocumentWithRaw(bh.loggingCtx, docID, DocUnmarshalSync) + currentDoc, rawDoc, err := bh.collection.GetDocumentWithRaw(bh.loggingCtx, docID, docmodel.DocUnmarshalSync) // If we're able to obtain current doc data then we should use the common ancestor generation++ for min revpos // as we will already have any attachments on the common ancestor so don't need to ask for them. // Otherwise we'll have to go as far back as we can in the doc history and choose the last entry in there. if err == nil { - commonAncestor := currentDoc.History.findAncestorFromSet(currentDoc.CurrentRev, history) + commonAncestor := currentDoc.History.FindAncestorFromSet(currentDoc.CurrentRev, history) minRevpos, _ = ParseRevID(commonAncestor) minRevpos++ rawBucketDoc = rawDoc @@ -1065,7 +1066,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // Do we have a previous doc? If not don't need to do this check if currentBucketDoc != nil { - bodyAtts := GetBodyAttachments(body) + bodyAtts := docmodel.GetBodyAttachments(body) currentDigests = make(map[string]string, len(bodyAtts)) for name, value := range bodyAtts { // Check if we have this attachment name already, if we do, continue check @@ -1125,7 +1126,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err return err } - newDoc.DocAttachments = GetBodyAttachments(body) + newDoc.DocAttachments = docmodel.GetBodyAttachments(body) delete(body, BodyAttachments) newDoc.UpdateBody(body) } @@ -1199,7 +1200,7 @@ func (bh *blipHandler) handleProveAttachment(rq *blip.Message) error { return base.HTTPErrorf(http.StatusInternalServerError, fmt.Sprintf("Error getting client attachment: %v", err)) } - proof := ProveAttachment(attData, nonce) + proof := docmodel.ProveAttachment(attData, nonce) resp := rq.Response() resp.SetBody([]byte(proof)) @@ -1309,7 +1310,7 @@ func (bh *blipHandler) sendGetAttachment(sender *blip.Sender, docID string, name // This is to prevent clients from creating a doc with a digest for an attachment they otherwise can't access, in order to download it. func (bh *blipHandler) sendProveAttachment(sender *blip.Sender, docID, name, digest string, knownData []byte) error { base.DebugfCtx(bh.loggingCtx, base.KeySync, " Verifying attachment %q for doc %s (digest %s)", base.UD(name), base.UD(docID), digest) - nonce, proof, err := GenerateProofOfAttachment(knownData) + nonce, proof, err := docmodel.GenerateProofOfAttachment(knownData) if err != nil { return err } @@ -1402,14 +1403,14 @@ func (bsc *BlipSyncContext) addAllowedAttachments(docID string, attMeta []Attach bsc.allowedAttachments = make(map[string]AllowedAttachment, 100) } for _, attachment := range attMeta { - key := allowedAttachmentKey(docID, attachment.digest, activeSubprotocol) + key := allowedAttachmentKey(docID, attachment.Digest, activeSubprotocol) att, found := bsc.allowedAttachments[key] if found { att.counter++ bsc.allowedAttachments[key] = att } else { bsc.allowedAttachments[key] = AllowedAttachment{ - version: attachment.version, + version: attachment.Version, counter: 1, docID: docID, } @@ -1428,7 +1429,7 @@ func (bsc *BlipSyncContext) removeAllowedAttachments(docID string, attMeta []Att defer bsc.allowedAttachmentsLock.Unlock() for _, attachment := range attMeta { - key := allowedAttachmentKey(docID, attachment.digest, activeSubprotocol) + key := allowedAttachmentKey(docID, attachment.Digest, activeSubprotocol) att, found := bsc.allowedAttachments[key] if found { if n := att.counter; n > 1 { diff --git a/db/blip_sync_context.go b/db/blip_sync_context.go index 0877eb59ee..c796afcd1e 100644 --- a/db/blip_sync_context.go +++ b/db/blip_sync_context.go @@ -23,6 +23,7 @@ import ( "github.com/couchbase/go-blip" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) const ( @@ -585,12 +586,12 @@ func (bsc *BlipSyncContext) sendRevision(sender *blip.Sender, docID, revID strin } base.TracefCtx(bsc.loggingCtx, base.KeySync, "sendRevision, rev attachments for %s/%s are %v", base.UD(docID), revID, base.UD(rev.Attachments)) - attachmentStorageMeta := ToAttachmentStorageMeta(rev.Attachments) + attachmentStorageMeta := docmodel.ToAttachmentStorageMeta(rev.Attachments) var bodyBytes []byte if base.IsEnterpriseEdition() { // Still need to stamp _attachments into BLIP messages if len(rev.Attachments) > 0 { - DeleteAttachmentVersion(rev.Attachments) + docmodel.DeleteAttachmentVersion(rev.Attachments) bodyBytes, err = base.InjectJSONProperties(rev.BodyBytes, base.KVPair{Key: BodyAttachments, Val: rev.Attachments}) if err != nil { return err @@ -606,7 +607,7 @@ func (bsc *BlipSyncContext) sendRevision(sender *blip.Sender, docID, revID strin // Still need to stamp _attachments into BLIP messages if len(rev.Attachments) > 0 { - DeleteAttachmentVersion(rev.Attachments) + docmodel.DeleteAttachmentVersion(rev.Attachments) body[BodyAttachments] = rev.Attachments } @@ -628,7 +629,7 @@ func (bsc *BlipSyncContext) sendRevision(sender *blip.Sender, docID, revID strin func digests(meta []AttachmentStorageMeta) []string { digests := make([]string, len(meta)) for _, m := range meta { - digests = append(digests, m.digest) + digests = append(digests, m.Digest) } return digests } diff --git a/db/change_cache.go b/db/change_cache.go index 5c3192e28b..88d7636f75 100644 --- a/db/change_cache.go +++ b/db/change_cache.go @@ -26,6 +26,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" ) const ( @@ -375,7 +376,7 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) { } // First unmarshal the doc (just its metadata, to save time/memory): - syncData, rawBody, _, rawUserXattr, err := UnmarshalDocumentSyncDataFromFeed(docJSON, event.DataType, collection.userXattrKey(), false) + syncData, rawBody, _, rawUserXattr, err := docmodel.UnmarshalDocumentSyncDataFromFeed(docJSON, event.DataType, collection.userXattrKey(), false) if err != nil { // Avoid log noise related to failed unmarshaling of binary documents. if event.DataType != base.MemcachedDataTypeRaw { @@ -401,7 +402,7 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) { // If not using xattrs and no sync metadata found, check whether we're mid-upgrade and attempting to read a doc w/ metadata stored in xattr // before ignoring the mutation. if !collection.UseXattrs() && !syncData.HasValidSyncData() { - migratedDoc, _ := collection.checkForUpgrade(docID, DocUnmarshalNoHistory) + migratedDoc, _ := collection.checkForUpgrade(docID, docmodel.DocUnmarshalNoHistory) if migratedDoc != nil && migratedDoc.Cas == event.Cas { base.InfofCtx(c.logCtx, base.KeyCache, "Found mobile xattr on doc %q without %s property - caching, assuming upgrade in progress.", base.UD(docID), base.SyncPropertyName) syncData = &migratedDoc.SyncData diff --git a/db/change_cache_test.go b/db/change_cache_test.go index c2922d8068..9cd4eb5096 100644 --- a/db/change_cache_test.go +++ b/db/change_cache_test.go @@ -22,6 +22,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -2041,7 +2042,7 @@ func (f *testDocChangedFeed) reset() { } } -// makeFeedBytes creates a DCP mutation message w/ xattr (reverse of parseXattrStreamData) +// makeFeedBytes creates a DCP mutation message w/ xattr (reverse of docmodel.ParseXattrStreamData) func makeFeedBytes(xattrKey, xattrValue, docValue string) []byte { xattrKeyBytes := []byte(xattrKey) xattrValueBytes := []byte(xattrValue) @@ -2070,7 +2071,7 @@ func TestMakeFeedBytes(t *testing.T) { rawBytes := makeFeedBytes(base.SyncPropertyName, `{"rev":"foo"}`, `{"k":"val"}`) - body, xattr, _, err := parseXattrStreamData(base.SyncXattrName, "", rawBytes) + body, xattr, _, err := docmodel.ParseXattrStreamData(base.SyncXattrName, "", rawBytes) assert.NoError(t, err) require.Len(t, body, 11) require.Len(t, xattr, 13) diff --git a/db/changes.go b/db/changes.go index 964dba7bc7..3b9d5e0093 100644 --- a/db/changes.go +++ b/db/changes.go @@ -152,7 +152,7 @@ func (db *DatabaseCollectionWithUser) AddDocInstanceToChangeEntry(ctx context.Co revID := entry.Changes[0]["rev"] if includeConflicts { - doc.History.forEachLeaf(func(leaf *RevInfo) { + doc.History.ForEachLeaf(func(leaf *RevInfo) { if leaf.ID != revID { if !leaf.Deleted { entry.Deleted = false diff --git a/db/crud.go b/db/crud.go index 76b9740cc6..9b94b39f66 100644 --- a/db/crud.go +++ b/db/crud.go @@ -22,6 +22,7 @@ import ( "github.com/couchbase/sync_gateway/auth" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" "github.com/pkg/errors" ) @@ -87,7 +88,7 @@ func (c *DatabaseCollection) GetDocumentWithRaw(ctx context.Context, docid strin return nil, nil, getErr } - doc, err = unmarshalDocument(key, rawDoc) + doc, err = docmodel.UnmarshalDocument(key, rawDoc) if err != nil { return nil, nil, err } @@ -119,7 +120,7 @@ func (c *DatabaseCollection) GetDocWithXattr(key string, unmarshalLevel Document } var unmarshalErr error - doc, unmarshalErr = unmarshalDocumentWithXattr(key, rawBucketDoc.Body, rawBucketDoc.Xattr, rawBucketDoc.UserXattr, rawBucketDoc.Cas, unmarshalLevel) + doc, unmarshalErr = docmodel.UnmarshalDocumentWithXattr(key, rawBucketDoc.Body, rawBucketDoc.Xattr, rawBucketDoc.UserXattr, rawBucketDoc.Cas, unmarshalLevel) if unmarshalErr != nil { return nil, nil, unmarshalErr } @@ -146,7 +147,7 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( } // Unmarshal xattr only - doc, unmarshalErr := unmarshalDocumentWithXattr(docid, nil, rawXattr, rawUserXattr, cas, DocUnmarshalSync) + doc, unmarshalErr := docmodel.UnmarshalDocumentWithXattr(docid, nil, rawXattr, rawUserXattr, cas, DocUnmarshalSync) if unmarshalErr != nil { return emptySyncData, unmarshalErr } @@ -175,7 +176,7 @@ func (c *DatabaseCollection) GetDocSyncData(ctx context.Context, docid string) ( return emptySyncData, err } - docRoot := documentRoot{ + docRoot := docmodel.DocumentRoot{ SyncData: &SyncData{History: make(RevTree)}, } if err := base.JSONUnmarshal(rawDocBytes, &docRoot); err != nil { @@ -242,7 +243,7 @@ func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Contex requestedHistory = nil } if requestedHistory != nil { - _, requestedHistory = trimEncodedRevisionsToAncestor(requestedHistory, historyFrom, maxHistory) + _, requestedHistory = docmodel.TrimEncodedRevisionsToAncestor(requestedHistory, historyFrom, maxHistory) } return rev.Mutable1xBody(db, requestedHistory, attachmentsSince, showExp) @@ -289,7 +290,7 @@ func (db *DatabaseCollectionWithUser) getRev(ctx context.Context, docid, revid s requestedHistory = nil } if requestedHistory != nil { - _, requestedHistory = trimEncodedRevisionsToAncestor(requestedHistory, historyFrom, maxHistory) + _, requestedHistory = docmodel.TrimEncodedRevisionsToAncestor(requestedHistory, historyFrom, maxHistory) } isAuthorized, redactedRev := db.authorizeUserForChannels(docid, revision.RevID, revision.Channels, revision.Deleted, requestedHistory) @@ -348,7 +349,7 @@ func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromR if fromRevision.Delta != nil { if fromRevision.Delta.ToRevID == toRevID { - isAuthorized, redactedBody := db.authorizeUserForChannels(docID, toRevID, fromRevision.Delta.ToChannels, fromRevision.Delta.ToDeleted, encodeRevisions(docID, fromRevision.Delta.RevisionHistory)) + isAuthorized, redactedBody := db.authorizeUserForChannels(docID, toRevID, fromRevision.Delta.ToChannels, fromRevision.Delta.ToDeleted, docmodel.EncodeRevisions(docID, fromRevision.Delta.RevisionHistory)) if !isAuthorized { return nil, &redactedBody, nil } @@ -407,15 +408,15 @@ func (db *DatabaseCollectionWithUser) GetDelta(ctx context.Context, docID, fromR if fromRevision.Attachments != nil { // the delta library does not handle deltas in non builtin types, // so we need the map[string]interface{} type conversion here - DeleteAttachmentVersion(fromRevision.Attachments) + docmodel.DeleteAttachmentVersion(fromRevision.Attachments) fromBodyCopy[BodyAttachments] = map[string]interface{}(fromRevision.Attachments) } var toRevAttStorageMeta []AttachmentStorageMeta if toRevision.Attachments != nil { // Flatten the AttachmentsMeta into a list of digest version pairs. - toRevAttStorageMeta = ToAttachmentStorageMeta(toRevision.Attachments) - DeleteAttachmentVersion(toRevision.Attachments) + toRevAttStorageMeta = docmodel.ToAttachmentStorageMeta(toRevision.Attachments) + docmodel.DeleteAttachmentVersion(toRevision.Attachments) toBodyCopy[BodyAttachments] = map[string]interface{}(toRevision.Attachments) } @@ -451,7 +452,7 @@ func (col *DatabaseCollectionWithUser) authorizeUserForChannels(docID, revID str redactedRev.BodyBytes = []byte(base.EmptyDocument) } else { // ... but removals are still denoted by the _removed property in the body, even for 2.x replication - redactedRev.BodyBytes = []byte(RemovedRedactedDocument) + redactedRev.BodyBytes = []byte(docmodel.RemovedRedactedDocument) } return false, redactedRev } @@ -505,11 +506,11 @@ func (col *DatabaseCollectionWithUser) authorizeDoc(doc *Document, revid string) // Gets a revision of a document. If it's obsolete it will be loaded from the database if possible. // inline "_attachments" properties in the body will be extracted and returned separately if present (pre-2.5 metadata, or backup revisions) func (c *DatabaseCollection) getRevision(ctx context.Context, doc *Document, revid string) (bodyBytes []byte, body Body, attachments AttachmentsMeta, err error) { - bodyBytes = doc.getRevisionBodyJSON(ctx, revid, c.RevisionBodyLoader) + bodyBytes = doc.GetRevisionBodyJSON(ctx, revid, c.RevisionBodyLoader) // No inline body, so look for separate doc: if bodyBytes == nil { - if !doc.History.contains(revid) { + if !doc.History.Contains(revid) { return nil, nil, nil, ErrMissing } @@ -521,7 +522,7 @@ func (c *DatabaseCollection) getRevision(ctx context.Context, doc *Document, rev // optimistically grab the doc body and to store as a pre-unmarshalled version, as well as anticipating no inline attachments. if doc.CurrentRev == revid { - body = doc._body + body = doc.PeekBody() attachments = doc.Attachments } @@ -544,9 +545,9 @@ func mergeAttachments(pre25Attachments, docAttachments AttachmentsMeta) Attachme if len(pre25Attachments)+len(docAttachments) == 0 { return nil // noop } else if len(pre25Attachments) == 0 { - return copyMap(docAttachments) + return docmodel.CopyMap(docAttachments) } else if len(docAttachments) == 0 { - return copyMap(pre25Attachments) + return docmodel.CopyMap(pre25Attachments) } merged := make(AttachmentsMeta, len(docAttachments)) @@ -624,9 +625,9 @@ func extractInlineAttachments(bodyBytes []byte) (attachments AttachmentsMeta, cl // If no ancestor has any JSON, returns nil but no error. func (db *DatabaseCollectionWithUser) getAncestorJSON(ctx context.Context, doc *Document, revid string) ([]byte, error) { for { - if revid = doc.History.getParent(revid); revid == "" { + if revid = doc.History.GetParent(revid); revid == "" { return nil, nil - } else if body := doc.getRevisionBodyJSON(ctx, revid, db.RevisionBodyLoader); body != nil { + } else if body := doc.GetRevisionBodyJSON(ctx, revid, db.RevisionBodyLoader); body != nil { return body, nil } } @@ -650,7 +651,7 @@ func (db *DatabaseCollectionWithUser) get1xRevFromDoc(ctx context.Context, doc * if doc.History[revid].Deleted { bodyBytes = []byte(base.EmptyDocument) } else { - bodyBytes = []byte(RemovedRedactedDocument) + bodyBytes = []byte(docmodel.RemovedRedactedDocument) removed = true } } else { @@ -679,11 +680,11 @@ func (db *DatabaseCollectionWithUser) get1xRevFromDoc(ctx context.Context, doc * } if listRevisions { - validatedHistory, getHistoryErr := doc.History.getHistory(revid) + validatedHistory, getHistoryErr := doc.History.GetHistory(revid) if getHistoryErr != nil { return nil, removed, getHistoryErr } - kvPairs = append(kvPairs, base.KVPair{Key: BodyRevisions, Val: encodeRevisions(doc.ID, validatedHistory)}) + kvPairs = append(kvPairs, base.KVPair{Key: BodyRevisions, Val: docmodel.EncodeRevisions(doc.ID, validatedHistory)}) } bodyBytes, err = base.InjectJSONProperties(bodyBytes, kvPairs...) @@ -755,11 +756,11 @@ func (db *DatabaseCollectionWithUser) backupAncestorRevs(ctx context.Context, do var json []byte ancestorRevId := newDoc.RevID for { - if ancestorRevId = doc.History.getParent(ancestorRevId); ancestorRevId == "" { + if ancestorRevId = doc.History.GetParent(ancestorRevId); ancestorRevId == "" { // No ancestors with JSON found. Check if we need to back up current rev for delta sync, then return db.backupRevisionJSON(ctx, doc.ID, newDoc.RevID, "", newBodyBytes, nil, doc.Attachments) return - } else if json = doc.getRevisionBodyJSON(ctx, ancestorRevId, db.RevisionBodyLoader); json != nil { + } else if json = doc.GetRevisionBodyJSON(ctx, ancestorRevId, db.RevisionBodyLoader); json != nil { break } } @@ -771,7 +772,7 @@ func (db *DatabaseCollectionWithUser) backupAncestorRevs(ctx context.Context, do if ancestorRevId == doc.CurrentRev { doc.RemoveBody() } else { - doc.removeRevisionBody(ancestorRevId) + doc.RemoveRevisionBody(ancestorRevId) } } @@ -831,7 +832,7 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod } // Pull out attachments - newDoc.DocAttachments = GetBodyAttachments(body) + newDoc.DocAttachments = docmodel.GetBodyAttachments(body) delete(body, BodyAttachments) delete(body, BodyRevisions) @@ -881,12 +882,12 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod generation++ } } - } else if !doc.History.isLeaf(matchRev) || db.IsIllegalConflict(ctx, doc, matchRev, deleted, false, nil) { + } else if !doc.History.IsLeaf(matchRev) || db.IsIllegalConflict(ctx, doc, matchRev, deleted, false, nil) { conflictErr = base.HTTPErrorf(http.StatusConflict, "Document revision conflict") } // Make up a new _rev, and add it to the history: - bodyWithoutInternalProps, wasStripped := stripInternalProperties(body) + bodyWithoutInternalProps, wasStripped := docmodel.StripInternalProperties(body) canonicalBytesForRevID, err := base.JSONMarshalCanonical(bodyWithoutInternalProps) if err != nil { return nil, nil, false, nil, err @@ -904,7 +905,7 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod // If no special properties were stripped and document wasn't deleted, the canonical bytes represent the current // body. In this scenario, store canonical bytes as newDoc._rawBody if !wasStripped && !isDeleted { - newDoc._rawBody = canonicalBytesForRevID + newDoc.PokeRawBody(canonicalBytesForRevID) } // Handle telling the user if there is a conflict @@ -933,9 +934,9 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod return nil, nil, false, nil, err } - newRev := CreateRevIDWithBytes(generation, matchRev, canonicalBytesForRevID) + newRev := docmodel.CreateRevIDWithBytes(generation, matchRev, canonicalBytesForRevID) - if err := doc.History.addRevision(newDoc.ID, RevInfo{ID: newRev, Parent: matchRev, Deleted: deleted}); err != nil { + if err := doc.History.AddRevision(newDoc.ID, RevInfo{ID: newRev, Parent: matchRev, Deleted: deleted}); err != nil { base.InfofCtx(ctx, base.KeyCRUD, "Failed to add revision ID: %s, for doc: %s, error: %v", newRev, base.UD(docid), err) return nil, nil, false, nil, base.ErrRevTreeAddRevFailure } @@ -993,7 +994,7 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx c currentRevIndex := len(docHistory) parent := "" for i, revid := range docHistory { - if doc.History.contains(revid) { + if doc.History.Contains(revid) { currentRevIndex = i parent = revid break @@ -1028,7 +1029,7 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx c currentRevIndex = len(docHistory) parent = "" for i, revid := range docHistory { - if doc.History.contains(revid) { + if doc.History.Contains(revid) { currentRevIndex = i parent = revid break @@ -1039,7 +1040,7 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithConflictResolution(ctx c // Add all the new-to-me revisions to the rev tree: for i := currentRevIndex - 1; i >= 0; i-- { - err := doc.History.addRevision(newDoc.ID, + err := doc.History.AddRevision(newDoc.ID, RevInfo{ ID: docHistory[i], Parent: parent, @@ -1086,7 +1087,7 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithBody(ctx context.Context delete(body, BodyId) delete(body, BodyRevisions) - newDoc.DocAttachments = GetBodyAttachments(body) + newDoc.DocAttachments = docmodel.GetBodyAttachments(body) delete(body, BodyAttachments) newDoc.UpdateBody(body) @@ -1209,7 +1210,7 @@ func (db *DatabaseCollectionWithUser) resolveDocLocalWins(ctx context.Context, l if !localDoc.Deleted { // If the local doc is not a tombstone, we're just rewriting it as a child of the remote - newRevID = CreateRevIDWithBytes(remoteGeneration+1, remoteRevID, docBodyBytes) + newRevID = docmodel.CreateRevIDWithBytes(remoteGeneration+1, remoteRevID, docBodyBytes) } else { // If the local doc is a tombstone, we're going to end up with both the local and remote branches tombstoned, // and need to ensure the remote branch is the winning branch. To do that, we inject entries into the remote @@ -1223,10 +1224,10 @@ func (db *DatabaseCollectionWithUser) resolveDocLocalWins(ctx context.Context, l for i := 0; i < requiredAdditionalRevs; i++ { injectedGeneration++ remoteLeafRevID := docHistory[0] - injectedRevID := CreateRevIDWithBytes(injectedGeneration, remoteLeafRevID, injectedRevBody) + injectedRevID := docmodel.CreateRevIDWithBytes(injectedGeneration, remoteLeafRevID, injectedRevBody) docHistory = append([]string{injectedRevID}, docHistory...) } - newRevID = CreateRevIDWithBytes(injectedGeneration+1, docHistory[0], docBodyBytes) + newRevID = docmodel.CreateRevIDWithBytes(injectedGeneration+1, docHistory[0], docBodyBytes) } // Update the history for the incoming doc to prepend the cloned revID @@ -1243,7 +1244,7 @@ func (db *DatabaseCollectionWithUser) resolveDocLocalWins(ctx context.Context, l // to have their revpos updated when we rewrite the rev as a child of the remote branch. if remoteDoc.DocAttachments != nil { // Identify generation of common ancestor and new rev - commonAncestorRevID := localDoc.SyncData.History.findAncestorFromSet(localDoc.CurrentRev, docHistory) + commonAncestorRevID := localDoc.SyncData.History.FindAncestorFromSet(localDoc.CurrentRev, docHistory) commonAncestorGen := 0 if commonAncestorRevID != "" { commonAncestorGen, _ = ParseRevID(commonAncestorRevID) @@ -1265,7 +1266,7 @@ func (db *DatabaseCollectionWithUser) resolveDocLocalWins(ctx context.Context, l } } - remoteDoc._rawBody = docBodyBytes + remoteDoc.PokeRawBody(docBodyBytes) // Tombstone the local revision localRevID := localDoc.CurrentRev @@ -1303,7 +1304,7 @@ func (db *DatabaseCollectionWithUser) resolveDocMerge(ctx context.Context, local remoteRevID := remoteDoc.RevID remoteGeneration, _ := ParseRevID(remoteRevID) - mergedRevID, err := CreateRevID(remoteGeneration+1, remoteRevID, mergedBody) + mergedRevID, err := docmodel.CreateRevID(remoteGeneration+1, remoteRevID, mergedBody) if err != nil { return "", nil, err } @@ -1311,7 +1312,7 @@ func (db *DatabaseCollectionWithUser) resolveDocMerge(ctx context.Context, local // Update the remote document's body to the merge result remoteDoc.RevID = mergedRevID remoteDoc.RemoveBody() - remoteDoc._body = mergedBody + remoteDoc.PokeBody(mergedBody) // Update the history for the remote doc to prepend the merged revID docHistory = append([]string{mergedRevID}, docHistory...) @@ -1334,9 +1335,9 @@ func (db *DatabaseCollectionWithUser) tombstoneActiveRevision(ctx context.Contex } // Create tombstone - newGeneration := genOfRevID(revID) + 1 - newRevID := CreateRevIDWithBytes(newGeneration, revID, []byte(DeletedDocument)) - err = doc.History.addRevision(doc.ID, + newGeneration := docmodel.GenOfRevID(revID) + 1 + newRevID := docmodel.CreateRevIDWithBytes(newGeneration, revID, []byte(docmodel.DeletedDocument)) + err = doc.History.AddRevision(doc.ID, RevInfo{ ID: newRevID, Parent: revID, @@ -1356,13 +1357,13 @@ func (db *DatabaseCollectionWithUser) tombstoneActiveRevision(ctx context.Contex return newRevID, nil } -func (doc *Document) updateWinningRevAndSetDocFlags() { +func updateWinningRevAndSetDocFlags(doc *Document) { var branched, inConflict bool - doc.CurrentRev, branched, inConflict = doc.History.winningRevision() - doc.setFlag(channels.Deleted, doc.History[doc.CurrentRev].Deleted) - doc.setFlag(channels.Conflict, inConflict) - doc.setFlag(channels.Branched, branched) - if doc.hasFlag(channels.Deleted) { + doc.CurrentRev, branched, inConflict = doc.History.WinningRevision() + doc.SetFlag(channels.Deleted, doc.History[doc.CurrentRev].Deleted) + doc.SetFlag(channels.Conflict, inConflict) + doc.SetFlag(channels.Branched, branched) + if doc.HasFlag(channels.Deleted) { doc.SyncData.TombstonedAt = time.Now().Unix() } else { doc.SyncData.TombstonedAt = 0 @@ -1397,7 +1398,7 @@ func (db *DatabaseCollectionWithUser) storeOldBodyInRevTreeAndUpdateCurrent(ctx bodyAtts[attName] = attMeta } - version, _ := GetAttachmentVersion(attMetaMap) + version, _ := docmodel.GetAttachmentVersion(attMetaMap) if version == AttVersion2 { oldDocHasAttachments = true } @@ -1417,20 +1418,20 @@ func (db *DatabaseCollectionWithUser) storeOldBodyInRevTreeAndUpdateCurrent(ctx if marshalErr != nil { base.WarnfCtx(ctx, "Unable to marshal document body properties for storage in rev tree: %v", marshalErr) } - doc.setNonWinningRevisionBody(prevCurrentRev, oldBodyJson, db.AllowExternalRevBodyStorage(), oldDocHasAttachments) + doc.SetNonWinningRevisionBody(prevCurrentRev, oldBodyJson, db.AllowExternalRevBodyStorage(), oldDocHasAttachments) } // Store the new revision body into the doc: - doc.setRevisionBody(newRevID, newDoc, db.AllowExternalRevBodyStorage(), newDocHasAttachments) + doc.SetRevisionBody(newRevID, newDoc, db.AllowExternalRevBodyStorage(), newDocHasAttachments) doc.SyncData.Attachments = newDoc.DocAttachments if doc.CurrentRev == newRevID { doc.NewestRev = "" - doc.setFlag(channels.Hidden, false) + doc.SetFlag(channels.Hidden, false) } else { doc.NewestRev = newRevID - doc.setFlag(channels.Hidden, true) + doc.SetFlag(channels.Hidden, true) if doc.CurrentRev != prevCurrentRev { - doc.promoteNonWinningRevisionBody(doc.CurrentRev, db.RevisionBodyLoader) + doc.PromoteNonWinningRevisionBody(doc.CurrentRev, db.RevisionBodyLoader) } } } @@ -1510,7 +1511,7 @@ func (db *DatabaseCollectionWithUser) addAttachments(ctx context.Context, newAtt // Need to check and add attachments here to ensure the attachment is within size constraints err := db.setAttachments(ctx, newAttachments) if err != nil { - if errors.Is(err, ErrAttachmentTooLarge) || err.Error() == "document value was too large" { + if errors.Is(err, docmodel.ErrAttachmentTooLarge) || err.Error() == "document value was too large" { err = base.HTTPErrorf(http.StatusRequestEntityTooLarge, "Attachment too large") } else { err = errors.Wrap(err, "Error adding attachment") @@ -1606,7 +1607,7 @@ func (db *DatabaseContext) assignSequence(ctx context.Context, docSequence uint6 return unusedSequences, nil } -func (doc *Document) updateExpiry(syncExpiry, updatedExpiry *uint32, expiry uint32) (finalExp *uint32) { +func updateExpiry(doc *Document, syncExpiry, updatedExpiry *uint32, expiry uint32) (finalExp *uint32) { if syncExpiry != nil { finalExp = syncExpiry } else if updatedExpiry != nil { @@ -1692,7 +1693,7 @@ func (col *DatabaseCollectionWithUser) documentUpdateFunc(ctx context.Context, d } prevCurrentRev := doc.CurrentRev - doc.updateWinningRevAndSetDocFlags() + updateWinningRevAndSetDocFlags(doc) newDocHasAttachments := len(newAttachments) > 0 col.storeOldBodyInRevTreeAndUpdateCurrent(ctx, doc, prevCurrentRev, newRevID, newDoc, newDocHasAttachments) @@ -1730,12 +1731,12 @@ func (col *DatabaseCollectionWithUser) documentUpdateFunc(ctx context.Context, d if newRevID != doc.CurrentRev { channelSet, access, roles, syncExpiry, oldBodyJSON, err = col.recalculateSyncFnForActiveRev(ctx, doc, metaMap, newRevID) } - _, err = doc.updateChannels(ctx, channelSet) + _, err = doc.UpdateChannels(ctx, channelSet) if err != nil { return } - changedAccessPrincipals = doc.Access.updateAccess(doc, access) - changedRoleAccessUsers = doc.RoleAccess.updateAccess(doc, roles) + changedAccessPrincipals = doc.Access.UpdateAccess(doc, access) + changedRoleAccessUsers = doc.RoleAccess.UpdateAccess(doc, roles) } else { base.DebugfCtx(ctx, base.KeyCRUD, "updateDoc(%q): Rev %q leaves %q still current", @@ -1743,12 +1744,12 @@ func (col *DatabaseCollectionWithUser) documentUpdateFunc(ctx context.Context, d } // Prune old revision history to limit the number of revisions: - if pruned := doc.pruneRevisions(col.revsLimit(), doc.CurrentRev); pruned > 0 { + if pruned := doc.PruneRevisions(col.revsLimit(), doc.CurrentRev); pruned > 0 { base.DebugfCtx(ctx, base.KeyCRUD, "updateDoc(%q): Pruned %d old revisions", base.UD(doc.ID), pruned) } - updatedExpiry = doc.updateExpiry(syncExpiry, updatedExpiry, expiry) - err = doc.persistModifiedRevisionBodies(col.dataStore) + updatedExpiry = updateExpiry(doc, syncExpiry, updatedExpiry, expiry) + err = doc.PersistModifiedRevisionBodies(col.dataStore) if err != nil { return } @@ -1792,7 +1793,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do // Update the document, storing metadata in _sync property _, err = db.dataStore.Update(key, expiry, func(currentValue []byte) (raw []byte, syncFuncExpiry *uint32, isDelete bool, err error) { // Be careful: this block can be invoked multiple times if there are races! - if doc, err = unmarshalDocument(docid, currentValue); err != nil { + if doc, err = docmodel.UnmarshalDocument(docid, currentValue); err != nil { return } previousAttachments, err = getAttachmentIDsForLeafRevisions(ctx, db, doc, newRevID) @@ -1808,7 +1809,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do } docSequence = doc.Sequence - inConflict = doc.hasFlag(channels.Conflict) + inConflict = doc.HasFlag(channels.Conflict) // Return the new raw document value for the bucket to store. raw, err = doc.MarshalBodyAndSync() base.DebugfCtx(ctx, base.KeyCRUD, "Saving doc (seq: #%d, id: %v rev: %v)", doc.Sequence, base.UD(doc.ID), doc.CurrentRev) @@ -1831,14 +1832,14 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do // Update the document, storing metadata in extended attribute casOut, err = db.dataStore.WriteUpdateWithXattr(key, base.SyncXattrName, db.userXattrKey(), expiry, opts, existingDoc, func(currentValue []byte, currentXattr []byte, currentUserXattr []byte, cas uint64) (raw []byte, rawXattr []byte, deleteDoc bool, syncFuncExpiry *uint32, err error) { // Be careful: this block can be invoked multiple times if there are races! - if doc, err = unmarshalDocumentWithXattr(docid, currentValue, currentXattr, currentUserXattr, cas, DocUnmarshalAll); err != nil { + if doc, err = docmodel.UnmarshalDocumentWithXattr(docid, currentValue, currentXattr, currentUserXattr, cas, DocUnmarshalAll); err != nil { return } prevCurrentRev = doc.CurrentRev // Check whether Sync Data originated in body if currentXattr == nil && doc.Sequence > 0 { - doc.inlineSyncData = true + doc.InlineSyncData = true } previousAttachments, err = getAttachmentIDsForLeafRevisions(ctx, db, doc, newRevID) @@ -1853,7 +1854,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do return } docSequence = doc.Sequence - inConflict = doc.hasFlag(channels.Conflict) + inConflict = doc.HasFlag(channels.Conflict) currentRevFromHistory, ok := doc.History[doc.CurrentRev] if !ok { @@ -1934,7 +1935,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do if doc.History[newRevID] != nil { // Store the new revision in the cache - history, getHistoryErr := doc.History.getHistory(newRevID) + history, getHistoryErr := doc.History.GetHistory(newRevID) if getHistoryErr != nil { return nil, "", getHistoryErr } @@ -1950,7 +1951,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do DocID: docid, RevID: newRevID, BodyBytes: storedDocBytes, - History: encodeRevisions(docid, history), + History: docmodel.EncodeRevisions(docid, history), Channels: revChannels, Attachments: doc.Attachments, Expiry: doc.Expiry, @@ -2011,7 +2012,7 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do } // Remove any obsolete non-winning revision bodies - doc.deleteRemovedRevisionBodies(db.dataStore) + doc.DeleteRemovedRevisionBodies(db.dataStore) // Mark affected users/roles as needing to recompute their channel access: db.MarkPrincipalsChanged(ctx, docid, newRevID, changedAccessPrincipals, changedRoleAccessUsers, doc.Sequence) @@ -2035,7 +2036,7 @@ func getAttachmentIDsForLeafRevisions(ctx context.Context, db *DatabaseCollectio // Can safely ignore the getInfo error as the only event this should happen in is if there is no entry for the given // rev, however, given we have just got that rev from GetLeavesFiltered we can be sure that rev exists in history documentLeafRevisions := doc.History.GetLeavesFiltered(func(revId string) bool { - revInfo, _ := doc.History.getInfo(revId) + revInfo, _ := doc.History.GetInfo(revId) return revInfo.HasAttachments && revId != newRevID }) @@ -2441,7 +2442,7 @@ func (db *DatabaseCollectionWithUser) RevDiff(ctx context.Context, docid string, missing = revids return } - doc, err := unmarshalDocumentWithXattr(docid, nil, xattrValue, nil, cas, DocUnmarshalSync) + doc, err := docmodel.UnmarshalDocumentWithXattr(docid, nil, xattrValue, nil, cas, DocUnmarshalSync) if err != nil { base.ErrorfCtx(ctx, "RevDiff(%q) Doc Unmarshal Failed: %T %v", base.UD(docid), err, err) } @@ -2464,11 +2465,11 @@ func (db *DatabaseCollectionWithUser) RevDiff(ctx context.Context, docid string, revidsSet := base.SetFromArray(revids) possibleSet := make(map[string]bool) for _, revid := range revids { - if !revtree.contains(revid) { + if !revtree.Contains(revid) { missing = append(missing, revid) // Look at the doc's leaves for a known possible ancestor: if gen, _ := ParseRevID(revid); gen > 1 { - revtree.forEachLeaf(func(possible *RevInfo) { + revtree.ForEachLeaf(func(possible *RevInfo) { if !revidsSet.Contains(possible.ID) { possibleGen, _ := ParseRevID(possible.ID) if possibleGen < gen && possibleGen >= gen-100 { diff --git a/db/crud_test.go b/db/crud_test.go index ea799b60c6..666c06d95f 100644 --- a/db/crud_test.go +++ b/db/crud_test.go @@ -17,6 +17,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,30 +27,30 @@ type treeDoc struct { } type treeMeta struct { - RevTree revTreeList `json:"history"` + RevTree docmodel.RevTreeList `json:"history"` } -// Retrieve the raw doc from the bucket, and unmarshal sync history as revTreeList, to validate low-level storage -func getRevTreeList(dataStore sgbucket.DataStore, key string, useXattrs bool) (revTreeList, error) { +// Retrieve the raw doc from the bucket, and unmarshal sync history as docmodel.RevTreeList, to validate low-level storage +func getRevTreeList(dataStore sgbucket.DataStore, key string, useXattrs bool) (docmodel.RevTreeList, error) { switch useXattrs { case true: var rawDoc, rawXattr []byte _, getErr := dataStore.GetWithXattr(key, base.SyncXattrName, "", &rawDoc, &rawXattr, nil) if getErr != nil { - return revTreeList{}, getErr + return docmodel.RevTreeList{}, getErr } var treeMeta treeMeta err := base.JSONUnmarshal(rawXattr, &treeMeta) if err != nil { - return revTreeList{}, err + return docmodel.RevTreeList{}, err } return treeMeta.RevTree, nil default: rawDoc, _, err := dataStore.GetRaw(key) if err != nil { - return revTreeList{}, err + return docmodel.RevTreeList{}, err } var doc treeDoc err = base.JSONUnmarshal(rawDoc, &doc) diff --git a/db/database.go b/db/database.go index a88fbbc577..646846f7d1 100644 --- a/db/database.go +++ b/db/database.go @@ -23,6 +23,7 @@ import ( "github.com/couchbase/sync_gateway/auth" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" pkgerrors "github.com/pkg/errors" ) @@ -1836,7 +1837,7 @@ func (db *DatabaseCollectionWithUser) getResyncedDocument(ctx context.Context, d // Run the sync fn over each current/leaf revision, in case there are conflicts: changed := 0 - doc.History.forEachLeaf(func(rev *RevInfo) { + doc.History.ForEachLeaf(func(rev *RevInfo) { bodyBytes, _, err := db.get1xRevFromDoc(ctx, doc, rev.ID, false) if err != nil { base.WarnfCtx(ctx, "Error getting rev from doc %s/%s %s", base.UD(docid), rev.ID, err) @@ -1868,9 +1869,9 @@ func (db *DatabaseCollectionWithUser) getResyncedDocument(ctx context.Context, d forceUpdate = true } - changedChannels, err := doc.updateChannels(ctx, channels) - changed = len(doc.Access.updateAccess(doc, access)) + - len(doc.RoleAccess.updateAccess(doc, roles)) + + changedChannels, err := doc.UpdateChannels(ctx, channels) + changed = len(doc.Access.UpdateAccess(doc, access)) + + len(doc.RoleAccess.UpdateAccess(doc, roles)) + len(changedChannels) if err != nil { return @@ -1898,7 +1899,7 @@ func (db *DatabaseCollectionWithUser) resyncDocument(ctx context.Context, docid, if currentValue == nil || len(currentValue) == 0 { return nil, nil, deleteDoc, nil, base.ErrUpdateCancel } - doc, err := unmarshalDocumentWithXattr(docid, currentValue, currentXattr, currentUserXattr, cas, DocUnmarshalAll) + doc, err := docmodel.UnmarshalDocumentWithXattr(docid, currentValue, currentXattr, currentUserXattr, cas, DocUnmarshalAll) if err != nil { return nil, nil, deleteDoc, nil, err } @@ -1926,7 +1927,7 @@ func (db *DatabaseCollectionWithUser) resyncDocument(ctx context.Context, docid, if currentValue == nil { return nil, nil, false, base.ErrUpdateCancel // someone deleted it?! } - doc, err := unmarshalDocument(docid, currentValue) + doc, err := docmodel.UnmarshalDocument(docid, currentValue) if err != nil { return nil, nil, false, err } diff --git a/db/database_collection.go b/db/database_collection.go index 68bac60050..030432f18c 100644 --- a/db/database_collection.go +++ b/db/database_collection.go @@ -326,3 +326,9 @@ func (c *DatabaseCollection) UpdateSyncFun(ctx context.Context, syncFun string) } return } + +// RevisionBodyLoader retrieves a non-winning revision body stored outside the document metadata +func (c *DatabaseCollection) RevisionBodyLoader(key string) ([]byte, error) { + body, _, err := c.dataStore.GetRaw(key) + return body, err +} diff --git a/db/database_test.go b/db/database_test.go index a5831dcdb7..4cd996a081 100644 --- a/db/database_test.go +++ b/db/database_test.go @@ -28,6 +28,7 @@ import ( "github.com/couchbase/sync_gateway/auth" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" "github.com/robertkrimen/otto/underscore" "github.com/stretchr/testify/assert" ) @@ -2858,7 +2859,7 @@ func Test_getUpdatedDocument(t *testing.T) { raw, _, err := db.Bucket.DefaultDataStore().GetRaw(docID) require.NoError(t, err) - doc, err := unmarshalDocument(docID, raw) + doc, err := docmodel.UnmarshalDocument(docID, raw) require.NoError(t, err) collection := GetSingleDatabaseCollectionWithUser(t, db) diff --git a/db/doc_model.go b/db/doc_model.go new file mode 100644 index 0000000000..e3804484ec --- /dev/null +++ b/db/doc_model.go @@ -0,0 +1,58 @@ +/* +Copyright 2023-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package db + +import "github.com/couchbase/sync_gateway/docmodel" + +// Declare aliases for various types & constants that were moved to docmodel, +// to reduce the amount of code touched throughout the project: + +type AttachmentsMeta = docmodel.AttachmentsMeta +type AttachmentData = docmodel.AttachmentData +type AttachmentStorageMeta = docmodel.AttachmentStorageMeta +type Body = docmodel.Body +type ChannelSetEntry = docmodel.ChannelSetEntry +type DocAttachment = docmodel.DocAttachment +type Document = docmodel.Document +type DocumentUnmarshalLevel = docmodel.DocumentUnmarshalLevel +type Revisions = docmodel.Revisions +type RevInfo = docmodel.RevInfo +type RevKey = docmodel.RevKey +type RevTree = docmodel.RevTree +type SyncData = docmodel.SyncData +type UserAccessMap = docmodel.UserAccessMap + +const ( + AttVersion1 = docmodel.AttVersion1 + AttVersion2 = docmodel.AttVersion2 + + BodyDeleted = docmodel.BodyDeleted + BodyRev = docmodel.BodyRev + BodyId = docmodel.BodyId + BodyRevisions = docmodel.BodyRevisions + BodyAttachments = docmodel.BodyAttachments + BodyPurged = docmodel.BodyPurged + BodyExpiry = docmodel.BodyExpiry + BodyRemoved = docmodel.BodyRemoved + BodyInternalPrefix = docmodel.BodyInternalPrefix + + DocUnmarshalAll = docmodel.DocUnmarshalAll + DocUnmarshalSync = docmodel.DocUnmarshalSync + DocUnmarshalNoHistory = docmodel.DocUnmarshalNoHistory + DocUnmarshalRev = docmodel.DocUnmarshalRev + DocUnmarshalCAS = docmodel.DocUnmarshalCAS + DocUnmarshalNone = docmodel.DocUnmarshalNone + + RevisionsStart = docmodel.RevisionsStart + RevisionsIds = docmodel.RevisionsIds +) + +func ParseRevID(revid string) (int, string) { return docmodel.ParseRevID(revid) } diff --git a/db/import.go b/db/import.go index f3061f100b..4d6430ee25 100644 --- a/db/import.go +++ b/db/import.go @@ -20,6 +20,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" + "github.com/couchbase/sync_gateway/docmodel" "github.com/robertkrimen/otto" ) @@ -71,12 +72,12 @@ func (db *DatabaseCollectionWithUser) ImportDoc(ctx context.Context, docid strin // but should refactor import processing to support using the already-unmarshalled doc. existingBucketDoc := &sgbucket.BucketDocument{ Cas: existingDoc.Cas, - UserXattr: existingDoc.rawUserXattr, + UserXattr: existingDoc.RawUserXattr(), } // If we marked this as having inline Sync Data ensure that the existingBucketDoc we pass to importDoc has syncData // in the body so we can detect this and perform the migrate - if existingDoc.inlineSyncData { + if existingDoc.InlineSyncData { existingBucketDoc.Body, err = existingDoc.MarshalJSON() existingBucketDoc.Xattr = nil } else { @@ -169,7 +170,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin updatedExpiry = &expiry } - if doc.inlineSyncData { + if doc.InlineSyncData { existingDoc.Body, err = doc.MarshalBodyAndSync() } else { existingDoc.Body, err = doc.BodyBytes() @@ -183,7 +184,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin // If the existing doc is a legacy SG write (_sync in body), check for migrate instead of import. _, ok := body[base.SyncPropertyName] - if ok || doc.inlineSyncData { + if ok || doc.InlineSyncData { migratedDoc, requiresImport, migrateErr := db.migrateMetadata(ctx, newDoc.ID, body, existingDoc, &mutationOptions) if migrateErr != nil { return nil, nil, false, updatedExpiry, migrateErr @@ -262,7 +263,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin rawBodyForRevID = existingDoc.Body } else { var bodyWithoutInternalProps Body - bodyWithoutInternalProps, wasStripped = stripInternalProperties(body) + bodyWithoutInternalProps, wasStripped = docmodel.StripInternalProperties(body) rawBodyForRevID, err = base.JSONMarshalCanonical(bodyWithoutInternalProps) if err != nil { return nil, nil, false, nil, err @@ -278,14 +279,14 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin parentRev := doc.CurrentRev generation, _ := ParseRevID(parentRev) generation++ - newRev = CreateRevIDWithBytes(generation, parentRev, rawBodyForRevID) + newRev = docmodel.CreateRevIDWithBytes(generation, parentRev, rawBodyForRevID) if err != nil { return nil, nil, false, updatedExpiry, err } base.DebugfCtx(ctx, base.KeyImport, "Created new rev ID for doc %q / %q", base.UD(newDoc.ID), newRev) // body[BodyRev] = newRev newDoc.RevID = newRev - err := doc.History.addRevision(newDoc.ID, RevInfo{ID: newRev, Parent: parentRev, Deleted: isDelete}) + err := doc.History.AddRevision(newDoc.ID, RevInfo{ID: newRev, Parent: parentRev, Deleted: isDelete}) if err != nil { base.InfofCtx(ctx, base.KeyImport, "Error adding new rev ID for doc %q / %q, Error: %v", base.UD(newDoc.ID), newRev, err) } @@ -307,7 +308,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin newDoc.UpdateBody(body) if !wasStripped && !isDelete { - newDoc._rawBody = rawBodyForRevID + newDoc.PokeRawBody(rawBodyForRevID) } // Existing attachments are preserved while importing an updated body - we don't (currently) support changing @@ -359,7 +360,7 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin func (db *DatabaseCollectionWithUser) migrateMetadata(ctx context.Context, docid string, body Body, existingDoc *sgbucket.BucketDocument, opts *sgbucket.MutateInOptions) (docOut *Document, requiresImport bool, err error) { // Unmarshal the existing doc in legacy SG format - doc, unmarshalErr := unmarshalDocument(docid, existingDoc.Body) + doc, unmarshalErr := docmodel.UnmarshalDocument(docid, existingDoc.Body) if unmarshalErr != nil { return nil, false, unmarshalErr } @@ -372,7 +373,7 @@ func (db *DatabaseCollectionWithUser) migrateMetadata(ctx context.Context, docid } // Move any large revision bodies to external storage - err = doc.migrateRevisionBodies(db.dataStore) + err = doc.MigrateRevisionBodies(db.dataStore) if err != nil { base.InfofCtx(ctx, base.KeyMigrate, "Error migrating revision bodies to external storage, doc %q, (cas=%d), Error: %v", base.UD(docid), doc.Cas, err) } @@ -384,7 +385,7 @@ func (db *DatabaseCollectionWithUser) migrateMetadata(ctx context.Context, docid } // Use WriteWithXattr to handle both normal migration and tombstone migration (xattr creation, body delete) - isDelete := doc.hasFlag(channels.Deleted) + isDelete := doc.HasFlag(channels.Deleted) deleteBody := isDelete && len(existingDoc.Body) > 0 casOut, writeErr := db.dataStore.WriteWithXattr(docid, base.SyncXattrName, existingDoc.Expiry, existingDoc.Cas, opts, value, xattrValue, isDelete, deleteBody) if writeErr == nil { diff --git a/db/import_listener.go b/db/import_listener.go index bd2b92015b..222e40ade7 100644 --- a/db/import_listener.go +++ b/db/import_listener.go @@ -17,6 +17,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) // ImportListener manages the import DCP feed. ProcessFeedEvent is triggered for each feed events, @@ -184,7 +185,7 @@ func (il *importListener) ImportFeedEvent(event sgbucket.FeedEvent) { return } - syncData, rawBody, rawXattr, rawUserXattr, err := UnmarshalDocumentSyncDataFromFeed(event.Value, event.DataType, collectionCtx.userXattrKey(), false) + syncData, rawBody, rawXattr, rawUserXattr, err := docmodel.UnmarshalDocumentSyncDataFromFeed(event.Value, event.DataType, collectionCtx.userXattrKey(), false) if err != nil { base.DebugfCtx(il.loggingCtx, base.KeyImport, "Found sync metadata, but unable to unmarshal for feed document %q. Will not be imported. Error: %v", base.UD(event.Key), err) if err == base.ErrEmptyMetadata { diff --git a/db/repair_bucket.go b/db/repair_bucket.go index 2ca3bf0c0a..b066e9b34f 100644 --- a/db/repair_bucket.go +++ b/db/repair_bucket.go @@ -15,6 +15,7 @@ import ( "time" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) // Enum for the different repair jobs (eg, repairing rev tree cycles) @@ -337,7 +338,7 @@ func RepairJobRevTreeCycles(docId string, originalCBDoc []byte) (transformedCBDo base.DebugfCtx(context.TODO(), base.KeyCRUD, "RepairJobRevTreeCycles() called with doc id: %v", base.UD(docId)) defer base.DebugfCtx(context.TODO(), base.KeyCRUD, "RepairJobRevTreeCycles() finished. Doc id: %v. transformed: %v. err: %v", base.UD(docId), base.UD(transformed), err) - doc, errUnmarshal := unmarshalDocument(docId, originalCBDoc) + doc, errUnmarshal := docmodel.UnmarshalDocument(docId, originalCBDoc) if errUnmarshal != nil { return nil, false, errUnmarshal } diff --git a/db/revision.go b/db/revision.go index 12b6dc6bea..1976d46a13 100644 --- a/db/revision.go +++ b/db/revision.go @@ -9,224 +9,13 @@ package db import ( - "bytes" "context" - "crypto/md5" - "errors" "fmt" - "net/http" - "strconv" - "strings" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) -// The body of a CouchDB document/revision as decoded from JSON. -type Body map[string]interface{} - -const ( - BodyDeleted = "_deleted" - BodyRev = "_rev" - BodyId = "_id" - BodyRevisions = "_revisions" - BodyAttachments = "_attachments" - BodyPurged = "_purged" - BodyExpiry = "_exp" - BodyRemoved = "_removed" - BodyInternalPrefix = "_sync_" // New internal properties prefix (CBG-1995) -) - -// A revisions property found within a Body. Expected to be of the form: -// -// Revisions["start"]: int64, starting generation number -// Revisions["ids"]: []string, list of digests -// -// Used as map[string]interface{} instead of Revisions struct because it's unmarshalled -// along with Body, and we don't need the overhead of allocating a new object -type Revisions map[string]interface{} - -const ( - RevisionsStart = "start" - RevisionsIds = "ids" -) - -type BodyCopyType int - -const ( - BodyDeepCopy BodyCopyType = iota // Performs a deep copy (json marshal/unmarshal) - BodyShallowCopy // Performs a shallow copy (copies top level properties, doesn't iterate into nested properties) - BodyNoCopy // Doesn't copy - callers must not mutate the response -) - -func (b *Body) Unmarshal(data []byte) error { - - if len(data) == 0 { - return errors.New("Unexpected empty JSON input to body.Unmarshal") - } - - // Use decoder for unmarshalling to preserve large numbers - decoder := base.JSONDecoder(bytes.NewReader(data)) - decoder.UseNumber() - if err := decoder.Decode(b); err != nil { - return err - } - return nil -} - -func (body Body) Copy(copyType BodyCopyType) Body { - switch copyType { - case BodyShallowCopy: - return body.ShallowCopy() - case BodyDeepCopy: - return body.DeepCopy() - case BodyNoCopy: - return body - default: - base.InfofCtx(context.Background(), base.KeyCRUD, "Unexpected copy type specified in body.Copy - defaulting to shallow copy. copyType: %d", copyType) - return body.ShallowCopy() - } -} - -func (body Body) ShallowCopy() Body { - if body == nil { - return body - } - copied := make(Body, len(body)) - for key, value := range body { - copied[key] = value - } - return copied -} - -func (body Body) DeepCopy() Body { - var copiedBody Body - err := base.DeepCopyInefficient(&copiedBody, body) - if err != nil { - base.InfofCtx(context.Background(), base.KeyCRUD, "Error copying body: %v", err) - } - return copiedBody -} - -func (revisions Revisions) ShallowCopy() Revisions { - copied := make(Revisions, len(revisions)) - for key, value := range revisions { - copied[key] = value - } - return copied -} - -// Version of doc.History.findAncestorFromSet that works against formatted Revisions. -// Returns the most recent ancestor found in revisions -func (revisions Revisions) findAncestor(ancestors []string) (revId string) { - - start, ids := splitRevisionList(revisions) - for _, id := range ids { - revid := fmt.Sprintf("%d-%s", start, id) - for _, a := range ancestors { - if a == revid { - return a - } - } - start-- - } - return "" -} - -// ParseRevisions returns revisions as a slice of revids. -func (revisions Revisions) ParseRevisions() []string { - start, ids := splitRevisionList(revisions) - if ids == nil { - return nil - } - result := make([]string, 0, len(ids)) - for _, id := range ids { - result = append(result, fmt.Sprintf("%d-%s", start, id)) - start-- - } - return result -} - -// Returns revisions as a slice of ancestor revids, from the parent to the target ancestor. -func (revisions Revisions) parseAncestorRevisions(toAncestorRevID string) []string { - start, ids := splitRevisionList(revisions) - if ids == nil || len(ids) < 2 { - return nil - } - result := make([]string, 0) - - // Start at the parent, end at toAncestorRevID - start = start - 1 - for i := 1; i < len(ids); i++ { - revID := fmt.Sprintf("%d-%s", start, ids[i]) - result = append(result, revID) - if revID == toAncestorRevID { - break - } - start-- - } - return result -} - -func (attachments AttachmentsMeta) ShallowCopy() AttachmentsMeta { - if attachments == nil { - return attachments - } - return copyMap(attachments) -} - -func copyMap(sourceMap map[string]interface{}) map[string]interface{} { - copy := make(map[string]interface{}, len(sourceMap)) - for k, v := range sourceMap { - if valueMap, ok := v.(map[string]interface{}); ok { - copiedValue := copyMap(valueMap) - copy[k] = copiedValue - } else { - copy[k] = v - } - } - return copy -} - -// Returns the expiry as uint32 (using getExpiry), and removes the _exp property from the body -func (body Body) ExtractExpiry() (uint32, error) { - - exp, present, err := body.getExpiry() - if !present || err != nil { - return exp, err - } - delete(body, "_exp") - - return exp, nil -} - -func (body Body) ExtractDeleted() bool { - deleted, _ := body[BodyDeleted].(bool) - delete(body, BodyDeleted) - return deleted -} - -func (body Body) ExtractRev() string { - revid, _ := body[BodyRev].(string) - delete(body, BodyRev) - return revid -} - -// Looks up the _exp property in the document, and turns it into a Couchbase Server expiry value, as: -func (body Body) getExpiry() (uint32, bool, error) { - rawExpiry, ok := body["_exp"] - if !ok { - return 0, false, nil // _exp not present - } - expiry, err := base.ReflectExpiry(rawExpiry) - if err != nil || expiry == nil { - return 0, false, err - } - return *expiry, true, err -} - -// nonJSONPrefix is used to ensure old revision bodies aren't hidden from N1QL/Views. -const nonJSONPrefix = byte(1) - // Looks up the raw JSON data of a revision that's been archived to a separate doc. // If the revision isn't found (e.g. has been deleted by compaction) returns 404 error. func (c *DatabaseCollection) getOldRevisionJSON(ctx context.Context, docid string, revid string) ([]byte, error) { @@ -237,7 +26,7 @@ func (c *DatabaseCollection) getOldRevisionJSON(ctx context.Context, docid strin } if data != nil { // Strip out the non-JSON prefix - if len(data) > 0 && data[0] == nonJSONPrefix { + if len(data) > 0 && data[0] == docmodel.NonJSONPrefix { data = data[1:] } base.DebugfCtx(ctx, base.KeyCRUD, "Got old revision %q / %q --> %d bytes", base.UD(docid), revid, len(data)) @@ -300,7 +89,7 @@ func (db *DatabaseCollectionWithUser) setOldRevisionJSON(ctx context.Context, do // To ensure it's not available via N1QL, need to prefix the raw bytes with non-JSON data. // Copying byte slice to make sure we don't modify the version stored in the revcache. nonJSONBytes := make([]byte, 1, len(body)+1) - nonJSONBytes[0] = nonJSONPrefix + nonJSONBytes[0] = docmodel.NonJSONPrefix nonJSONBytes = append(nonJSONBytes, body...) err := db.dataStore.SetRaw(oldRevisionKey(docid, revid), expiry, nil, nonJSONBytes) if err == nil { @@ -333,150 +122,3 @@ func (c *DatabaseCollection) PurgeOldRevisionJSON(ctx context.Context, docid str func oldRevisionKey(docid string, revid string) string { return fmt.Sprintf("%s%s:%d:%s", base.RevPrefix, docid, len(revid), revid) } - -// Version of FixJSONNumbers (see base/util.go) that operates on a Body -func (body Body) FixJSONNumbers() { - for k, v := range body { - body[k] = base.FixJSONNumbers(v) - } -} - -func CreateRevID(generation int, parentRevID string, body Body) (string, error) { - // This should produce the same results as TouchDB. - strippedBody, _ := stripInternalProperties(body) - encoding, err := base.JSONMarshalCanonical(strippedBody) - if err != nil { - return "", err - } - return CreateRevIDWithBytes(generation, parentRevID, encoding), nil -} - -func CreateRevIDWithBytes(generation int, parentRevID string, bodyBytes []byte) string { - digester := md5.New() - digester.Write([]byte{byte(len(parentRevID))}) - digester.Write([]byte(parentRevID)) - digester.Write(bodyBytes) - return fmt.Sprintf("%d-%x", generation, digester.Sum(nil)) -} - -// Returns the generation number (numeric prefix) of a revision ID. -func genOfRevID(revid string) int { - if revid == "" { - return 0 - } - var generation int - n, _ := fmt.Sscanf(revid, "%d-", &generation) - if n < 1 || generation < 1 { - base.WarnfCtx(context.Background(), "genOfRevID unsuccessful for %q", revid) - return -1 - } - return generation -} - -// Splits a revision ID into generation number and hex digest. -func ParseRevID(revid string) (int, string) { - if revid == "" { - return 0, "" - } - - idx := strings.Index(revid, "-") - if idx == -1 { - base.WarnfCtx(context.Background(), "parseRevID found no separator in rev %q", revid) - return -1, "" - } - - gen, err := strconv.Atoi(revid[:idx]) - if err != nil { - base.WarnfCtx(context.Background(), "parseRevID unexpected generation in rev %q: %s", revid, err) - return -1, "" - } else if gen < 1 { - base.WarnfCtx(context.Background(), "parseRevID unexpected generation in rev %q", revid) - return -1, "" - } - - return gen, revid[idx+1:] -} - -// compareRevIDs compares the two rev IDs and returns: -// 1 if id1 is 'greater' than id2 -// -1 if id1 is 'less' than id2 -// 0 if the two are equal. -func compareRevIDs(id1, id2 string) int { - gen1, sha1 := ParseRevID(id1) - gen2, sha2 := ParseRevID(id2) - switch { - case gen1 > gen2: - return 1 - case gen1 < gen2: - return -1 - case sha1 > sha2: - return 1 - case sha1 < sha2: - return -1 - } - return 0 -} - -// stripInternalProperties returns a copy of the given body with all internal underscore-prefixed keys removed, except _attachments and _deleted. -func stripInternalProperties(b Body) (Body, bool) { - return stripSpecialProperties(b, true) -} - -// stripAllInternalProperties returns a copy of the given body with all underscore-prefixed keys removed. -func stripAllSpecialProperties(b Body) (Body, bool) { - return stripSpecialProperties(b, false) -} - -// stripSpecialPropertiesExcept returns a copy of the given body with underscore-prefixed keys removed. -// Set internalOnly to only strip internal properties except _deleted and _attachments -func stripSpecialProperties(b Body, internalOnly bool) (sb Body, foundSpecialProps bool) { - // Assume no properties removed for the initial capacity to reduce allocs on large docs. - stripped := make(Body, len(b)) - for k, v := range b { - // Property is not stripped if: - // - It is blank - // - Does not start with an underscore ('_') - // - Is not an internal special property (this check is excluded when internalOnly = false) - if k == "" || k[0] != '_' || (internalOnly && (!strings.HasPrefix(k, BodyInternalPrefix) && - !base.StringSliceContains([]string{ - base.SyncPropertyName, - BodyId, - BodyRev, - BodyRevisions, - BodyExpiry, - BodyPurged, - BodyRemoved, - }, k))) { - // property is allowed - stripped[k] = v - } else { - foundSpecialProps = true - } - } - - if foundSpecialProps { - return stripped, true - } else { - // Return original body if nothing was removed - return b, false - } -} - -func GetStringArrayProperty(body map[string]interface{}, property string) ([]string, error) { - if raw, exists := body[property]; !exists { - return nil, nil - } else if strings, ok := raw.([]string); ok { - return strings, nil - } else if items, ok := raw.([]interface{}); ok { - strings := make([]string, len(items)) - for i := 0; i < len(items); i++ { - strings[i], ok = items[i].(string) - if !ok { - return nil, base.HTTPErrorf(http.StatusBadRequest, property+" must be a string array") - } - } - return strings, nil - } else { - return nil, base.HTTPErrorf(http.StatusBadRequest, property+" must be a string array") - } -} diff --git a/db/revision_cache_interface.go b/db/revision_cache_interface.go index c3fc20aca4..c3e2887985 100644 --- a/db/revision_cache_interface.go +++ b/db/revision_cache_interface.go @@ -16,6 +16,7 @@ import ( "time" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) const ( @@ -184,7 +185,7 @@ func (rev *DocumentRevision) Mutable1xBody(db *DatabaseCollectionWithUser, reque if len(rev.Attachments) > 0 { minRevpos := 1 if len(attachmentsSince) > 0 { - ancestor := rev.History.findAncestor(attachmentsSince) + ancestor := rev.History.FindAncestor(attachmentsSince) if ancestor != "" { minRevpos, _ = ParseRevID(ancestor) minRevpos++ @@ -194,12 +195,12 @@ func (rev *DocumentRevision) Mutable1xBody(db *DatabaseCollectionWithUser, reque if err != nil { return nil, err } - DeleteAttachmentVersion(bodyAtts) + docmodel.DeleteAttachmentVersion(bodyAtts) b[BodyAttachments] = bodyAtts } } else if rev.Attachments != nil { // Stamp attachment metadata back into the body - DeleteAttachmentVersion(rev.Attachments) + docmodel.DeleteAttachmentVersion(rev.Attachments) b[BodyAttachments] = rev.Attachments } @@ -239,7 +240,7 @@ func newRevCacheDelta(deltaBytes []byte, fromRevID string, toRevision DocumentRe DeltaBytes: deltaBytes, AttachmentStorageMeta: toRevAttStorageMeta, ToChannels: toRevision.Channels, - RevisionHistory: toRevision.History.parseAncestorRevisions(fromRevID), + RevisionHistory: toRevision.History.ParseAncestorRevisions(fromRevID), ToDeleted: deleted, } } @@ -278,11 +279,11 @@ func revCacheLoaderForDocument(ctx context.Context, backingStore RevisionCacheBa } deleted = doc.History[revid].Deleted - validatedHistory, getHistoryErr := doc.History.getHistory(revid) + validatedHistory, getHistoryErr := doc.History.GetHistory(revid) if getHistoryErr != nil { return bodyBytes, body, history, channels, removed, nil, deleted, nil, getHistoryErr } - history = encodeRevisions(doc.ID, validatedHistory) + history = docmodel.EncodeRevisions(doc.ID, validatedHistory) channels = doc.History[revid].Channels return bodyBytes, body, history, channels, removed, attachments, deleted, doc.Expiry, err diff --git a/db/revision_cache_test.go b/db/revision_cache_test.go index eb87a55573..21dd408661 100644 --- a/db/revision_cache_test.go +++ b/db/revision_cache_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -40,10 +41,10 @@ func (t *testBackingStore) GetDocument(ctx context.Context, docid string, unmars } } - doc = NewDocument(docid) - doc._body = Body{ + doc = docmodel.NewDocument(docid) + doc.PokeBody(Body{ "testing": true, - } + }) doc.CurrentRev = "1-abc" doc.History = RevTree{ doc.CurrentRev: { diff --git a/db/revision_test.go b/db/revision_test.go index fc9faaaff4..2e290bc960 100644 --- a/db/revision_test.go +++ b/db/revision_test.go @@ -11,8 +11,6 @@ licenses/APL2.txt. package db import ( - "fmt" - "log" "testing" "github.com/couchbase/sync_gateway/base" @@ -20,81 +18,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParseRevID(t *testing.T) { - - var generation int - var digest string - - generation, _ = ParseRevID("ljlkjl") - log.Printf("generation: %v", generation) - assert.True(t, generation == -1, "Expected -1 generation for invalid rev id") - - generation, digest = ParseRevID("1-ljlkjl") - log.Printf("generation: %v, digest: %v", generation, digest) - assert.True(t, generation == 1, "Expected 1 generation") - assert.True(t, digest == "ljlkjl", "Unexpected digest") - - generation, digest = ParseRevID("2222-") - log.Printf("generation: %v, digest: %v", generation, digest) - assert.True(t, generation == 2222, "Expected invalid generation") - assert.True(t, digest == "", "Unexpected digest") - - generation, digest = ParseRevID("333-a") - log.Printf("generation: %v, digest: %v", generation, digest) - assert.True(t, generation == 333, "Expected generation") - assert.True(t, digest == "a", "Unexpected digest") - -} - -func TestBodyUnmarshal(t *testing.T) { - - tests := []struct { - name string - inputBytes []byte - expectedBody Body - }{ - {"empty bytes", []byte(""), nil}, - {"null", []byte("null"), Body(nil)}, - {"{}", []byte("{}"), Body{}}, - {"example body", []byte(`{"test":true}`), Body{"test": true}}, - } - - for _, test := range tests { - t.Run(test.name, func(ts *testing.T) { - var b Body - err := b.Unmarshal(test.inputBytes) - - // Unmarshal using json.Unmarshal for comparison below - var jsonUnmarshalBody Body - unmarshalErr := base.JSONUnmarshal(test.inputBytes, &jsonUnmarshalBody) - - if unmarshalErr != nil { - // If json.Unmarshal returns error for input, body.Unmarshal should do the same - assert.True(t, err != nil, fmt.Sprintf("Expected error when unmarshalling %s", test.name)) - } else { - assert.NoError(t, err, fmt.Sprintf("Expected no error when unmarshalling %s", test.name)) - assert.Equal(t, test.expectedBody, b) // Check against expected body - assert.Equal(t, jsonUnmarshalBody, b) // Check against json.Unmarshal results - } - - }) - } -} - -func TestParseRevisionsToAncestor(t *testing.T) { - revisions := Revisions{RevisionsStart: 5, RevisionsIds: []string{"five", "four", "three", "two", "one"}} - - assert.Equal(t, []string{"4-four", "3-three"}, revisions.parseAncestorRevisions("3-three")) - assert.Equal(t, []string{"4-four"}, revisions.parseAncestorRevisions("4-four")) - assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.parseAncestorRevisions("1-one")) - assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.parseAncestorRevisions("5-five")) - assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.parseAncestorRevisions("0-zero")) - assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.parseAncestorRevisions("3-threeve")) - - shortRevisions := Revisions{RevisionsStart: 3, RevisionsIds: []string{"three"}} - assert.Equal(t, []string(nil), shortRevisions.parseAncestorRevisions("2-two")) -} - // TestBackupOldRevision ensures that old revisions are kept around temporarily for in-flight requests and delta sync purposes. func TestBackupOldRevision(t *testing.T) { base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) @@ -146,99 +69,3 @@ func TestBackupOldRevision(t *testing.T) { assert.Equal(t, "404 missing", err.Error()) } } - -func TestStripSpecialProperties(t *testing.T) { - testCases := []struct { - name string - input Body - stripInternalOnly bool - stripped bool // Should at least 1 property have been stripped - newBodyIfStripped *Body - }{ - { - name: "No special", - input: Body{"test": 0, "bob": 0, "alice": 0}, - stripInternalOnly: false, - stripped: false, - newBodyIfStripped: nil, - }, - { - name: "No special internal only", - input: Body{"test": 0, "bob": 0, "alice": 0}, - stripInternalOnly: true, - stripped: false, - newBodyIfStripped: nil, - }, - { - name: "Strip special props", - input: Body{"_test": 0, "test": 0, "_attachments": 0, "_id": 0}, - stripInternalOnly: false, - stripped: true, - newBodyIfStripped: &Body{"test": 0}, - }, - { - name: "Strip internal special props", - input: Body{"_test": 0, "test": 0, "_rev": 0, "_exp": 0}, - stripInternalOnly: true, - stripped: true, - newBodyIfStripped: &Body{"_test": 0, "test": 0}, - }, - { - name: "Confirm internal attachments and deleted skipped", - input: Body{"_test": 0, "test": 0, "_attachments": 0, "_rev": 0, "_deleted": 0}, - stripInternalOnly: true, - stripped: true, - newBodyIfStripped: &Body{"_test": 0, "test": 0, "_attachments": 0, "_deleted": 0}, - }, - } - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - stripped, specialProps := stripSpecialProperties(test.input, test.stripInternalOnly) - assert.Equal(t, test.stripped, specialProps) - if test.stripped { - assert.Equal(t, *test.newBodyIfStripped, stripped) - return - } - assert.Equal(t, test.input, stripped) - }) - } -} - -func BenchmarkSpecialProperties(b *testing.B) { - noSpecialBody := Body{ - "asdf": "qwerty", "a": true, "b": true, "c": true, - "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, - "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, - } - - specialBody := noSpecialBody.Copy(BodyShallowCopy) - specialBody[BodyId] = "abc123" - specialBody[BodyRev] = "1-abc" - - tests := []struct { - name string - body Body - }{ - { - "no special", - noSpecialBody, - }, - { - "special", - specialBody, - }, - } - - for _, t := range tests { - b.Run(t.name+"-stripInternalProperties", func(bb *testing.B) { - for i := 0; i < bb.N; i++ { - stripInternalProperties(t.body) - } - }) - b.Run(t.name+"-stripAllInternalProperties", func(bb *testing.B) { - for i := 0; i < bb.N; i++ { - stripAllSpecialProperties(t.body) - } - }) - } -} diff --git a/db/sg_replicate_conflict_resolver.go b/db/sg_replicate_conflict_resolver.go index 8305590eb9..195749324d 100644 --- a/db/sg_replicate_conflict_resolver.go +++ b/db/sg_replicate_conflict_resolver.go @@ -18,6 +18,7 @@ import ( sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" "github.com/robertkrimen/otto" ) @@ -153,7 +154,7 @@ func DefaultConflictResolver(conflict Conflict) (result Body, err error) { localRevID, _ := conflict.LocalDocument[BodyRev].(string) remoteRevID, _ := conflict.RemoteDocument[BodyRev].(string) - if compareRevIDs(localRevID, remoteRevID) >= 0 { + if docmodel.CompareRevIDs(localRevID, remoteRevID) >= 0 { return conflict.LocalDocument, nil } else { return conflict.RemoteDocument, nil diff --git a/db/special_docs.go b/db/special_docs.go index 5cca59318d..6c251a5759 100644 --- a/db/special_docs.go +++ b/db/special_docs.go @@ -13,6 +13,7 @@ import ( "net/http" "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/docmodel" ) const DocTypeLocal = "local" @@ -107,7 +108,7 @@ func putSpecial(dataStore base.DataStore, doctype string, docid string, matchRev func (c *DatabaseCollection) PutSpecial(doctype string, docid string, body Body) (string, error) { matchRev, _ := body[BodyRev].(string) - body, _ = stripAllSpecialProperties(body) + body, _ = docmodel.StripAllSpecialProperties(body) return c.putSpecial(doctype, docid, matchRev, body) } diff --git a/docmodel/attachment.go b/docmodel/attachment.go new file mode 100644 index 0000000000..db1b8d5002 --- /dev/null +++ b/docmodel/attachment.go @@ -0,0 +1,155 @@ +// Copyright 2012-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package docmodel + +import ( + "context" + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "errors" + "fmt" + "net/http" + + "github.com/couchbase/sync_gateway/base" +) + +const ( + // AttVersion1 attachments are persisted to the bucket based on attachment body digest. + AttVersion1 int = 1 + + // AttVersion2 attachments are persisted to the bucket based on docID and body digest. + AttVersion2 int = 2 +) + +// AttachmentData holds the attachment key and value bytes. +type AttachmentData map[string][]byte + +// A struct which models an attachment. Currently only used by test code, however +// new code or refactoring in the main codebase should try to use where appropriate. +type DocAttachment struct { + ContentType string `json:"content_type,omitempty"` + Digest string `json:"digest,omitempty"` + Length int `json:"length,omitempty"` + Revpos int `json:"revpos,omitempty"` + Stub bool `json:"stub,omitempty"` + Version int `json:"ver,omitempty"` + Data []byte `json:"-"` // tell json marshal/unmarshal to ignore this field +} + +// ErrAttachmentTooLarge is returned when an attempt to attach an oversize attachment is made. +var ErrAttachmentTooLarge = errors.New("attachment too large") + +// DeleteAttachmentVersion removes attachment versions from the AttachmentsMeta map specified. +func DeleteAttachmentVersion(attachments AttachmentsMeta) { + for _, value := range attachments { + meta := value.(map[string]interface{}) + delete(meta, "ver") + } +} + +func GetAttachmentVersion(meta map[string]interface{}) (int, bool) { + ver, ok := meta["ver"] + if !ok { + return AttVersion1, true + } + val, ok := base.ToInt64(ver) + return int(val), ok +} + +// GenerateProofOfAttachment returns a nonce and proof for an attachment body. +func GenerateProofOfAttachment(attachmentData []byte) (nonce []byte, proof string, err error) { + nonce = make([]byte, 20) + if _, err := rand.Read(nonce); err != nil { + return nil, "", base.HTTPErrorf(http.StatusInternalServerError, fmt.Sprintf("Failed to generate random data: %s", err)) + } + proof = ProveAttachment(attachmentData, nonce) + base.TracefCtx(context.Background(), base.KeyCRUD, "Generated nonce %v and proof %q for attachment: %v", nonce, proof, attachmentData) + return nonce, proof, nil +} + +// ProveAttachment returns the proof for an attachment body and nonce pair. +func ProveAttachment(attachmentData, nonce []byte) (proof string) { + d := sha1.New() + d.Write([]byte{byte(len(nonce))}) + d.Write(nonce) + d.Write(attachmentData) + proof = "sha1-" + base64.StdEncoding.EncodeToString(d.Sum(nil)) + base.TracefCtx(context.Background(), base.KeyCRUD, "Generated proof %q using nonce %v for attachment: %v", proof, nonce, attachmentData) + return proof +} + +//////// HELPERS: + +// Returns _attachments property from body, when found. Checks for either map[string]interface{} (unmarshalled with body), +// or AttachmentsMeta (written by body by SG) +func GetBodyAttachments(body Body) AttachmentsMeta { + switch atts := body[BodyAttachments].(type) { + case AttachmentsMeta: + return atts + case map[string]interface{}: + return AttachmentsMeta(atts) + default: + return nil + } +} + +// AttachmentDigests returns a list of attachment digests contained in the given AttachmentsMeta +func AttachmentDigests(attachments AttachmentsMeta) []string { + var digests = make([]string, 0, len(attachments)) + for _, att := range attachments { + if attMap, ok := att.(map[string]interface{}); ok { + if digest, ok := attMap["digest"]; ok { + if digestString, ok := digest.(string); ok { + digests = append(digests, digestString) + } + } + } + } + return digests +} + +// AttachmentStorageMeta holds the metadata for building +// the key for attachment storage and retrieval. +type AttachmentStorageMeta struct { + Digest string + Version int +} + +// ToAttachmentStorageMeta returns a slice of AttachmentStorageMeta, which is contains the +// necessary metadata properties to build the key for attachment storage and retrieval. +func ToAttachmentStorageMeta(attachments AttachmentsMeta) []AttachmentStorageMeta { + meta := make([]AttachmentStorageMeta, 0, len(attachments)) + for _, att := range attachments { + if attMap, ok := att.(map[string]interface{}); ok { + if digest, ok := attMap["digest"]; ok { + if digestString, ok := digest.(string); ok { + version, _ := GetAttachmentVersion(attMap) + m := AttachmentStorageMeta{ + Digest: digestString, + Version: version, + } + meta = append(meta, m) + } + } + } + } + return meta +} + +func DecodeAttachment(att interface{}) ([]byte, error) { + switch att := att.(type) { + case []byte: + return att, nil + case string: + return base64.StdEncoding.DecodeString(att) + default: + return nil, base.HTTPErrorf(400, "invalid attachment data (type %T)", att) + } +} diff --git a/docmodel/attachment_test.go b/docmodel/attachment_test.go new file mode 100644 index 0000000000..bf8e23e2c0 --- /dev/null +++ b/docmodel/attachment_test.go @@ -0,0 +1,105 @@ +// Copyright 2012-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package docmodel + +import ( + "encoding/json" + "net/http" + "strconv" + "strings" + "testing" + + "github.com/couchbase/sync_gateway/base" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateProofOfAttachment(t *testing.T) { + base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) + + attData := []byte(`hello world`) + + nonce, proof1, err := GenerateProofOfAttachment(attData) + require.NoError(t, err) + assert.True(t, len(nonce) >= 20, "nonce should be at least 20 bytes") + assert.NotEmpty(t, proof1) + assert.True(t, strings.HasPrefix(proof1, "sha1-")) + + proof2 := ProveAttachment(attData, nonce) + assert.NotEmpty(t, proof1, "") + assert.True(t, strings.HasPrefix(proof1, "sha1-")) + + assert.Equal(t, proof1, proof2, "GenerateProofOfAttachment and ProveAttachment produced different proofs.") +} + +func TestDecodeAttachmentError(t *testing.T) { + attr, err := DecodeAttachment(make([]int, 1)) + assert.Nil(t, attr, "Attachment of data (type []int) should not get decoded.") + assert.Error(t, err, "It should throw 400 invalid attachment data (type []int)") + assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) + + attr, err = DecodeAttachment(make([]float64, 1)) + assert.Nil(t, attr, "Attachment of data (type []float64) should not get decoded.") + assert.Error(t, err, "It should throw 400 invalid attachment data (type []float64)") + assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) + + attr, err = DecodeAttachment(make([]string, 1)) + assert.Nil(t, attr, "Attachment of data (type []string) should not get decoded.") + assert.Error(t, err, "It should throw 400 invalid attachment data (type []string)") + assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) + + attr, err = DecodeAttachment(make(map[string]int, 1)) + assert.Nil(t, attr, "Attachment of data (type map[string]int) should not get decoded.") + assert.Error(t, err, "It should throw 400 invalid attachment data (type map[string]int)") + assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) + + attr, err = DecodeAttachment(make(map[string]float64, 1)) + assert.Nil(t, attr, "Attachment of data (type map[string]float64) should not get decoded.") + assert.Error(t, err, "It should throw 400 invalid attachment data (type map[string]float64)") + assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) + + attr, err = DecodeAttachment(make(map[string]string, 1)) + assert.Error(t, err, "should throw 400 invalid attachment data (type map[string]float64)") + assert.Error(t, err, "It 400 invalid attachment data (type map[string]string)") + assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) + + book := struct { + author string + title string + price float64 + }{author: "William Shakespeare", title: "Hamlet", price: 7.99} + attr, err = DecodeAttachment(book) + assert.Nil(t, attr) + assert.Error(t, err, "It should throw 400 invalid attachment data (type struct { author string; title string; price float64 })") + assert.Contains(t, err.Error(), strconv.Itoa(http.StatusBadRequest)) +} + +func TestGetAttVersion(t *testing.T) { + var tests = []struct { + name string + inputAttVersion interface{} + expectedValidAttVersion bool + expectedAttVersion int + }{ + {"int attachment version", AttVersion2, true, AttVersion2}, + {"float64 attachment version", float64(AttVersion2), true, AttVersion2}, + {"invalid json.Number attachment version", json.Number("foo"), false, 0}, + {"valid json.Number attachment version", json.Number(strconv.Itoa(AttVersion2)), true, AttVersion2}, + {"invaid string attachment version", strconv.Itoa(AttVersion2), false, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + meta := map[string]interface{}{"ver": tt.inputAttVersion} + version, ok := GetAttachmentVersion(meta) + assert.Equal(t, tt.expectedValidAttVersion, ok) + assert.Equal(t, tt.expectedAttVersion, version) + }) + } +} diff --git a/db/data_for_test.go b/docmodel/data_for_test.go similarity index 99% rename from db/data_for_test.go rename to docmodel/data_for_test.go index eb9fb45036..7d297f0fc3 100644 --- a/db/data_for_test.go +++ b/docmodel/data_for_test.go @@ -8,7 +8,7 @@ be governed by the Apache License, Version 2.0, included in the file licenses/APL2.txt. */ -package db +package docmodel var testdocProblematicRevTrees = []string{ testdocProblematicRevTree1, diff --git a/db/document.go b/docmodel/document.go similarity index 92% rename from db/document.go rename to docmodel/document.go index 3f1c1ba8d1..a99c1b5870 100644 --- a/db/document.go +++ b/docmodel/document.go @@ -6,7 +6,7 @@ // software will be governed by the Apache License, Version 2.0, included in // the file licenses/APL2.txt. -package db +package docmodel import ( "bytes" @@ -178,7 +178,7 @@ type Document struct { DocExpiry uint32 RevID string DocAttachments AttachmentsMeta - inlineSyncData bool + InlineSyncData bool } type revOnlySyncData struct { @@ -212,7 +212,7 @@ func (doc *Document) MarshalBodyAndSync() (retBytes []byte, err error) { } func (doc *Document) IsDeleted() bool { - return doc.hasFlag(channels.Deleted) + return doc.HasFlag(channels.Deleted) } func (doc *Document) BodyWithSpecialProperties() ([]byte, error) { @@ -366,7 +366,7 @@ func userXattrCrc32cHash(userXattr []byte) string { // Unmarshals a document from JSON data. The doc ID isn't in the data and must be given. Uses decode to ensure // UseNumber handling is applied to numbers in the body. -func unmarshalDocument(docid string, data []byte) (*Document, error) { +func UnmarshalDocument(docid string, data []byte) (*Document, error) { doc := NewDocument(docid) if len(data) > 0 { decoder := base.JSONDecoder(bytes.NewReader(data)) @@ -383,11 +383,11 @@ func unmarshalDocument(docid string, data []byte) (*Document, error) { return doc, nil } -func unmarshalDocumentWithXattr(docid string, data []byte, xattrData []byte, userXattrData []byte, cas uint64, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) { +func UnmarshalDocumentWithXattr(docid string, data []byte, xattrData []byte, userXattrData []byte, cas uint64, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) { if xattrData == nil || len(xattrData) == 0 { // If no xattr data, unmarshal as standard doc - doc, err = unmarshalDocument(docid, data) + doc, err = UnmarshalDocument(docid, data) } else { doc = NewDocument(docid) err = doc.UnmarshalWithXattr(data, xattrData, unmarshalLevel) @@ -407,7 +407,7 @@ func unmarshalDocumentWithXattr(docid string, data []byte, xattrData []byte, use // Unmarshals just a document's sync metadata from JSON data. // (This is somewhat faster, if all you need is the sync data without the doc body.) func UnmarshalDocumentSyncData(data []byte, needHistory bool) (*SyncData, error) { - var root documentRoot + var root DocumentRoot if needHistory { root.SyncData = &SyncData{History: make(RevTree)} } @@ -435,7 +435,7 @@ func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey if dataType&base.MemcachedDataTypeXattr != 0 { var syncXattr []byte var userXattr []byte - body, syncXattr, userXattr, err = parseXattrStreamData(base.SyncXattrName, userXattrKey, data) + body, syncXattr, userXattr, err = ParseXattrStreamData(base.SyncXattrName, userXattrKey, data) if err != nil { return nil, nil, nil, nil, err } @@ -468,17 +468,17 @@ func UnmarshalDocumentFromFeed(docid string, cas uint64, data []byte, dataType u if dataType&base.MemcachedDataTypeXattr != 0 { var syncXattr []byte var userXattr []byte - body, syncXattr, userXattr, err = parseXattrStreamData(base.SyncXattrName, userXattrKey, data) + body, syncXattr, userXattr, err = ParseXattrStreamData(base.SyncXattrName, userXattrKey, data) if err != nil { return nil, err } - return unmarshalDocumentWithXattr(docid, body, syncXattr, userXattr, cas, DocUnmarshalAll) + return UnmarshalDocumentWithXattr(docid, body, syncXattr, userXattr, cas, DocUnmarshalAll) } - return unmarshalDocument(docid, data) + return UnmarshalDocument(docid, data) } -// parseXattrStreamData returns the raw bytes of the body and the requested xattr (when present) from the raw DCP data bytes. +// docmodel.ParseXattrStreamData returns the raw bytes of the body and the requested xattr (when present) from the raw DCP data bytes. // Details on format (taken from https://docs.google.com/document/d/18UVa5j8KyufnLLy29VObbWRtoBn9vs8pcxttuMt6rz8/edit#heading=h.caqiui1pmmmb.): /* When the XATTR bit is set the first uint32_t in the body contains the size of the entire XATTR section. @@ -502,7 +502,7 @@ func UnmarshalDocumentFromFeed(docid string, cas uint64, data []byte, dataType u The 0x00 byte after the key saves us from storing a key length, and the trailing 0x00 is just for convenience to allow us to use string functions to search in them. */ -func parseXattrStreamData(xattrName string, userXattrName string, data []byte) (body []byte, xattr []byte, userXattr []byte, err error) { +func ParseXattrStreamData(xattrName string, userXattrName string, data []byte) (body []byte, xattr []byte, userXattr []byte, err error) { if len(data) < 4 { return nil, nil, nil, base.ErrEmptyMetadata @@ -643,11 +643,11 @@ func (doc *Document) IsSGWrite(ctx context.Context, rawBody []byte) (isSGWrite b return true, true, false } -func (doc *Document) hasFlag(flag uint8) bool { +func (doc *Document) HasFlag(flag uint8) bool { return doc.Flags&flag != 0 } -func (doc *Document) setFlag(flag uint8, state bool) { +func (doc *Document) SetFlag(flag uint8, state bool) { if state { doc.Flags |= flag } else { @@ -665,14 +665,8 @@ func (doc *Document) newestRevID() string { // RevLoaderFunc and RevWriterFunc manage persistence of non-winning revision bodies that are stored outside the document. type RevLoaderFunc func(key string) ([]byte, error) -// RevisionBodyLoader retrieves a non-winning revision body stored outside the document metadata -func (c *DatabaseCollection) RevisionBodyLoader(key string) ([]byte, error) { - body, _, err := c.dataStore.GetRaw(key) - return body, err -} - // Fetches the body of a revision as a map, or nil if it's not available. -func (doc *Document) getRevisionBody(revid string, loader RevLoaderFunc) Body { +func (doc *Document) GetRevisionBody(revid string, loader RevLoaderFunc) Body { var body Body if revid == doc.CurrentRev { body = doc.Body() @@ -686,7 +680,7 @@ func (doc *Document) getRevisionBody(revid string, loader RevLoaderFunc) Body { // or was previously requested), loader function is used to retrieve from the bucket. func (doc *Document) getNonWinningRevisionBody(revid string, loader RevLoaderFunc) Body { var body Body - bodyBytes, found := doc.History.getRevisionBody(revid, loader) + bodyBytes, found := doc.History.GetRevisionBody(revid, loader) if !found || len(bodyBytes) == 0 { return nil } @@ -699,7 +693,7 @@ func (doc *Document) getNonWinningRevisionBody(revid string, loader RevLoaderFun } // Fetches the body of a revision as JSON, or nil if it's not available. -func (doc *Document) getRevisionBodyJSON(ctx context.Context, revid string, loader RevLoaderFunc) []byte { +func (doc *Document) GetRevisionBodyJSON(ctx context.Context, revid string, loader RevLoaderFunc) []byte { var bodyJSON []byte if revid == doc.CurrentRev { var marshalErr error @@ -708,13 +702,13 @@ func (doc *Document) getRevisionBodyJSON(ctx context.Context, revid string, load base.WarnfCtx(ctx, "Marshal error when retrieving active current revision body: %v", marshalErr) } } else { - bodyJSON, _ = doc.History.getRevisionBody(revid, loader) + bodyJSON, _ = doc.History.GetRevisionBody(revid, loader) } return bodyJSON } -func (doc *Document) removeRevisionBody(revID string) { - removedBodyKey := doc.History.removeRevisionBody(revID) +func (doc *Document) RemoveRevisionBody(revID string) { + removedBodyKey := doc.History.RemoveRevisionBody(revID) if removedBodyKey != "" { if doc.removedRevisionBodyKeys == nil { doc.removedRevisionBodyKeys = make(map[string]string) @@ -724,15 +718,15 @@ func (doc *Document) removeRevisionBody(revID string) { } // makeBodyActive moves a previously non-winning revision body from the rev tree to the document body -func (doc *Document) promoteNonWinningRevisionBody(revid string, loader RevLoaderFunc) { +func (doc *Document) PromoteNonWinningRevisionBody(revid string, loader RevLoaderFunc) { // If the new revision is not current, transfer the current revision's // body to the top level doc._body: doc.UpdateBody(doc.getNonWinningRevisionBody(revid, loader)) - doc.removeRevisionBody(revid) + doc.RemoveRevisionBody(revid) } -func (doc *Document) pruneRevisions(maxDepth uint32, keepRev string) int { - numPruned, prunedTombstoneBodyKeys := doc.History.pruneRevisions(maxDepth, keepRev) +func (doc *Document) PruneRevisions(maxDepth uint32, keepRev string) int { + numPruned, prunedTombstoneBodyKeys := doc.History.PruneRevisions(maxDepth, keepRev) for revID, bodyKey := range prunedTombstoneBodyKeys { if doc.removedRevisionBodyKeys == nil { doc.removedRevisionBodyKeys = make(map[string]string) @@ -743,29 +737,29 @@ func (doc *Document) pruneRevisions(maxDepth uint32, keepRev string) int { } // Adds a revision body (as Body) to a document. Removes special properties first. -func (doc *Document) setRevisionBody(revid string, newDoc *Document, storeInline, hasAttachments bool) { +func (doc *Document) SetRevisionBody(revid string, newDoc *Document, storeInline, hasAttachments bool) { if revid == doc.CurrentRev { doc._body = newDoc._body doc._rawBody = newDoc._rawBody } else { bodyBytes, _ := newDoc.BodyBytes() - doc.setNonWinningRevisionBody(revid, bodyBytes, storeInline, hasAttachments) + doc.SetNonWinningRevisionBody(revid, bodyBytes, storeInline, hasAttachments) } } // Adds a revision body (as []byte) to a document. Flags for external storage when appropriate -func (doc *Document) setNonWinningRevisionBody(revid string, body []byte, storeInline bool, hasAttachments bool) { +func (doc *Document) SetNonWinningRevisionBody(revid string, body []byte, storeInline bool, hasAttachments bool) { revBodyKey := "" if !storeInline && len(body) > MaximumInlineBodySize { revBodyKey = generateRevBodyKey(doc.ID, revid) doc.addedRevisionBodies = append(doc.addedRevisionBodies, revid) } - doc.History.setRevisionBody(revid, body, revBodyKey, hasAttachments) + doc.History.SetRevisionBody(revid, body, revBodyKey, hasAttachments) } -// persistModifiedRevisionBodies writes new non-inline revisions to the bucket. +// PersistModifiedRevisionBodies writes new non-inline revisions to the bucket. // Should be invoked BEFORE the document is successfully committed. -func (doc *Document) persistModifiedRevisionBodies(datastore sgbucket.DataStore) error { +func (doc *Document) PersistModifiedRevisionBodies(datastore sgbucket.DataStore) error { for _, revID := range doc.addedRevisionBodies { // if this rev is also in the delete set, skip add/delete @@ -775,7 +769,7 @@ func (doc *Document) persistModifiedRevisionBodies(datastore sgbucket.DataStore) continue } - revInfo, err := doc.History.getInfo(revID) + revInfo, err := doc.History.GetInfo(revID) if revInfo == nil || err != nil { return err } @@ -794,9 +788,9 @@ func (doc *Document) persistModifiedRevisionBodies(datastore sgbucket.DataStore) return nil } -// deleteRemovedRevisionBodies deletes obsolete non-inline revisions from the bucket. +// DeleteRemovedRevisionBodies deletes obsolete non-inline revisions from the bucket. // Should be invoked AFTER the document is successfully committed. -func (doc *Document) deleteRemovedRevisionBodies(dataStore base.DataStore) { +func (doc *Document) DeleteRemovedRevisionBodies(dataStore base.DataStore) { for _, revBodyKey := range doc.removedRevisionBodyKeys { deleteErr := dataStore.Delete(revBodyKey) @@ -813,10 +807,10 @@ func (doc *Document) persistRevisionBody(datastore sgbucket.DataStore, key strin } // Move any large revision bodies to external document storage -func (doc *Document) migrateRevisionBodies(dataStore base.DataStore) error { +func (doc *Document) MigrateRevisionBodies(dataStore base.DataStore) error { for _, revID := range doc.History.GetLeaves() { - revInfo, err := doc.History.getInfo(revID) + revInfo, err := doc.History.GetInfo(revID) if err != nil { continue } @@ -947,7 +941,7 @@ func (doc *Document) addToChannelSetHistory(channelName string, historyEntry Cha // Updates the Channels property of a document object with current & past channels. // Returns the set of channels that have changed (document joined or left in this revision) -func (doc *Document) updateChannels(ctx context.Context, newChannels base.Set) (changedChannels base.Set, err error) { +func (doc *Document) UpdateChannels(ctx context.Context, newChannels base.Set) (changedChannels base.Set, err error) { var changed []string oldChannels := doc.Channels if oldChannels == nil { @@ -961,7 +955,7 @@ func (doc *Document) updateChannels(ctx context.Context, newChannels base.Set) ( oldChannels[channel] = &channels.ChannelRemoval{ Seq: curSequence, RevID: doc.CurrentRev, - Deleted: doc.hasFlag(channels.Deleted)} + Deleted: doc.HasFlag(channels.Deleted)} doc.updateChannelHistory(channel, curSequence, false) changed = append(changed, channel) } @@ -1017,7 +1011,7 @@ func (doc *Document) IsChannelRemoval(revID string) (bodyBytes []byte, history R } // Build revision history for revID - revHistory, err := doc.History.getHistory(revID) + revHistory, err := doc.History.GetHistory(revID) if err != nil { return nil, nil, nil, false, false, err } @@ -1026,14 +1020,14 @@ func (doc *Document) IsChannelRemoval(revID string) (bodyBytes []byte, history R if len(revHistory) == 0 { revHistory = []string{revID} } - history = encodeRevisions(doc.ID, revHistory) + history = EncodeRevisions(doc.ID, revHistory) return bodyBytes, history, activeChannels, true, isDelete, nil } // Updates a document's channel/role UserAccessMap with new access settings from an AccessMap. // Returns an array of the user/role names whose access has changed as a result. -func (accessMap *UserAccessMap) updateAccess(doc *Document, newAccess channels.AccessMap) (changedUsers []string) { +func (accessMap *UserAccessMap) UpdateAccess(doc *Document, newAccess channels.AccessMap) (changedUsers []string) { // Update users already appearing in doc.Access: for name, access := range *accessMap { if access.UpdateAtSequence(newAccess[name], doc.Sequence) { @@ -1065,7 +1059,7 @@ func (accessMap *UserAccessMap) updateAccess(doc *Document, newAccess channels.A // ////// MARSHALING //////// -type documentRoot struct { +type DocumentRoot struct { SyncData *SyncData `json:"_sync"` } @@ -1076,7 +1070,7 @@ func (doc *Document) UnmarshalJSON(data []byte) error { } // Unmarshal only sync data (into a typed struct) - syncData := documentRoot{SyncData: &SyncData{History: make(RevTree)}} + syncData := DocumentRoot{SyncData: &SyncData{History: make(RevTree)}} err := base.JSONUnmarshal(data, &syncData) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to UnmarshalJSON() doc with id: %s. Error: %v", base.UD(doc.ID), err)) @@ -1212,3 +1206,18 @@ func (doc *Document) MarshalWithXattr() (data []byte, xdata []byte, err error) { return data, xdata, nil } + +// (This method only exists for db.DatabaseCollectionWithUser.ImportDoc().) +func (doc *Document) RawUserXattr() []byte { + return doc.rawUserXattr +} + +// The methods below should be removed -- they exist only because when Document was in the db +// package, some code directly read/wrote the internal `_body` and `_rawBody` fields. +// These methods keep that code working, but the real fix is to change it to not grope these +// fields (but also without losing performance...) --Jens Feb 2023 + +func (doc *Document) PeekBody() Body { return doc._body } +func (doc *Document) PeekRawBody() []byte { return doc._rawBody } +func (doc *Document) PokeBody(b Body) { doc._body = b } +func (doc *Document) PokeRawBody(b []byte) { doc._rawBody = b } diff --git a/db/document_test.go b/docmodel/document_test.go similarity index 96% rename from db/document_test.go rename to docmodel/document_test.go index e15d49ca0a..44bf76eb0f 100644 --- a/db/document_test.go +++ b/docmodel/document_test.go @@ -8,7 +8,7 @@ be governed by the Apache License, Version 2.0, included in the file licenses/APL2.txt. */ -package db +package docmodel import ( "bytes" @@ -133,7 +133,7 @@ func BenchmarkDocUnmarshal(b *testing.B) { for _, bm := range unmarshalBenchmarks { b.Run(bm.name, func(b *testing.B) { for i := 0; i < b.N; i++ { - _, _ = unmarshalDocumentWithXattr("doc_1k", doc1k_body, doc1k_meta, nil, 1, bm.unmarshalLevel) + _, _ = UnmarshalDocumentWithXattr("doc_1k", doc1k_body, doc1k_meta, nil, 1, bm.unmarshalLevel) } }) } @@ -205,19 +205,19 @@ func TestParseXattr(t *testing.T) { dcpBody = append(dcpBody, zeroByte) dcpBody = append(dcpBody, body...) - resultBody, resultXattr, _, err := parseXattrStreamData(base.SyncXattrName, "", dcpBody) + resultBody, resultXattr, _, err := ParseXattrStreamData(base.SyncXattrName, "", dcpBody) assert.NoError(t, err, "Unexpected error parsing dcp body") assert.Equal(t, string(body), string(resultBody)) assert.Equal(t, string(xattrValue), string(resultXattr)) // Attempt to retrieve non-existent xattr - resultBody, resultXattr, _, err = parseXattrStreamData("nonexistent", "", dcpBody) + resultBody, resultXattr, _, err = ParseXattrStreamData("nonexistent", "", dcpBody) assert.NoError(t, err, "Unexpected error parsing dcp body") assert.Equal(t, string(body), string(resultBody)) assert.Equal(t, "", string(resultXattr)) // Attempt to retrieve xattr from empty dcp body - emptyBody, emptyXattr, _, emptyErr := parseXattrStreamData(base.SyncXattrName, "", []byte{}) + emptyBody, emptyXattr, _, emptyErr := ParseXattrStreamData(base.SyncXattrName, "", []byte{}) assert.Equal(t, base.ErrEmptyMetadata, emptyErr) assert.True(t, emptyBody == nil, "Nil body expected") assert.True(t, emptyXattr == nil, "Nil xattr expected") diff --git a/docmodel/revision.go b/docmodel/revision.go new file mode 100644 index 0000000000..23ab87e0de --- /dev/null +++ b/docmodel/revision.go @@ -0,0 +1,377 @@ +// Copyright 2012-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. + +package docmodel + +import ( + "bytes" + "context" + "crypto/md5" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/couchbase/sync_gateway/base" +) + +// The body of a CouchDB document/revision as decoded from JSON. +type Body map[string]interface{} + +const ( + BodyDeleted = "_deleted" + BodyRev = "_rev" + BodyId = "_id" + BodyRevisions = "_revisions" + BodyAttachments = "_attachments" + BodyPurged = "_purged" + BodyExpiry = "_exp" + BodyRemoved = "_removed" + BodyInternalPrefix = "_sync_" // New internal properties prefix (CBG-1995) +) + +// A revisions property found within a Body. Expected to be of the form: +// +// Revisions["start"]: int64, starting generation number +// Revisions["ids"]: []string, list of digests +// +// Used as map[string]interface{} instead of Revisions struct because it's unmarshalled +// along with Body, and we don't need the overhead of allocating a new object +type Revisions map[string]interface{} + +const ( + RevisionsStart = "start" + RevisionsIds = "ids" +) + +type BodyCopyType int + +const ( + BodyDeepCopy BodyCopyType = iota // Performs a deep copy (json marshal/unmarshal) + BodyShallowCopy // Performs a shallow copy (copies top level properties, doesn't iterate into nested properties) + BodyNoCopy // Doesn't copy - callers must not mutate the response +) + +func (b *Body) Unmarshal(data []byte) error { + + if len(data) == 0 { + return errors.New("Unexpected empty JSON input to body.Unmarshal") + } + + // Use decoder for unmarshalling to preserve large numbers + decoder := base.JSONDecoder(bytes.NewReader(data)) + decoder.UseNumber() + if err := decoder.Decode(b); err != nil { + return err + } + return nil +} + +func (body Body) Copy(copyType BodyCopyType) Body { + switch copyType { + case BodyShallowCopy: + return body.ShallowCopy() + case BodyDeepCopy: + return body.DeepCopy() + case BodyNoCopy: + return body + default: + base.InfofCtx(context.Background(), base.KeyCRUD, "Unexpected copy type specified in body.Copy - defaulting to shallow copy. copyType: %d", copyType) + return body.ShallowCopy() + } +} + +func (body Body) ShallowCopy() Body { + if body == nil { + return body + } + copied := make(Body, len(body)) + for key, value := range body { + copied[key] = value + } + return copied +} + +func (body Body) DeepCopy() Body { + var copiedBody Body + err := base.DeepCopyInefficient(&copiedBody, body) + if err != nil { + base.InfofCtx(context.Background(), base.KeyCRUD, "Error copying body: %v", err) + } + return copiedBody +} + +func (revisions Revisions) ShallowCopy() Revisions { + copied := make(Revisions, len(revisions)) + for key, value := range revisions { + copied[key] = value + } + return copied +} + +// Version of doc.History.findAncestorFromSet that works against formatted Revisions. +// Returns the most recent ancestor found in revisions +func (revisions Revisions) FindAncestor(ancestors []string) (revId string) { + + start, ids := splitRevisionList(revisions) + for _, id := range ids { + revid := fmt.Sprintf("%d-%s", start, id) + for _, a := range ancestors { + if a == revid { + return a + } + } + start-- + } + return "" +} + +// ParseRevisions returns revisions as a slice of revids. +func (revisions Revisions) ParseRevisions() []string { + start, ids := splitRevisionList(revisions) + if ids == nil { + return nil + } + result := make([]string, 0, len(ids)) + for _, id := range ids { + result = append(result, fmt.Sprintf("%d-%s", start, id)) + start-- + } + return result +} + +// Returns revisions as a slice of ancestor revids, from the parent to the target ancestor. +func (revisions Revisions) ParseAncestorRevisions(toAncestorRevID string) []string { + start, ids := splitRevisionList(revisions) + if ids == nil || len(ids) < 2 { + return nil + } + result := make([]string, 0) + + // Start at the parent, end at toAncestorRevID + start = start - 1 + for i := 1; i < len(ids); i++ { + revID := fmt.Sprintf("%d-%s", start, ids[i]) + result = append(result, revID) + if revID == toAncestorRevID { + break + } + start-- + } + return result +} + +func (attachments AttachmentsMeta) ShallowCopy() AttachmentsMeta { + if attachments == nil { + return attachments + } + return CopyMap(attachments) +} + +func CopyMap(sourceMap map[string]interface{}) map[string]interface{} { + copy := make(map[string]interface{}, len(sourceMap)) + for k, v := range sourceMap { + if valueMap, ok := v.(map[string]interface{}); ok { + copiedValue := CopyMap(valueMap) + copy[k] = copiedValue + } else { + copy[k] = v + } + } + return copy +} + +// Returns the expiry as uint32 (using getExpiry), and removes the _exp property from the body +func (body Body) ExtractExpiry() (uint32, error) { + + exp, present, err := body.getExpiry() + if !present || err != nil { + return exp, err + } + delete(body, "_exp") + + return exp, nil +} + +func (body Body) ExtractDeleted() bool { + deleted, _ := body[BodyDeleted].(bool) + delete(body, BodyDeleted) + return deleted +} + +func (body Body) ExtractRev() string { + revid, _ := body[BodyRev].(string) + delete(body, BodyRev) + return revid +} + +// Looks up the _exp property in the document, and turns it into a Couchbase Server expiry value, as: +func (body Body) getExpiry() (uint32, bool, error) { + rawExpiry, ok := body["_exp"] + if !ok { + return 0, false, nil // _exp not present + } + expiry, err := base.ReflectExpiry(rawExpiry) + if err != nil || expiry == nil { + return 0, false, err + } + return *expiry, true, err +} + +// NonJSONPrefix is used to ensure old revision bodies aren't hidden from N1QL/Views. +const NonJSONPrefix = byte(1) + +// ////// UTILITY FUNCTIONS: + +// Version of FixJSONNumbers (see base/util.go) that operates on a Body +func (body Body) FixJSONNumbers() { + for k, v := range body { + body[k] = base.FixJSONNumbers(v) + } +} + +func CreateRevID(generation int, parentRevID string, body Body) (string, error) { + // This should produce the same results as TouchDB. + strippedBody, _ := StripInternalProperties(body) + encoding, err := base.JSONMarshalCanonical(strippedBody) + if err != nil { + return "", err + } + return CreateRevIDWithBytes(generation, parentRevID, encoding), nil +} + +func CreateRevIDWithBytes(generation int, parentRevID string, bodyBytes []byte) string { + digester := md5.New() + digester.Write([]byte{byte(len(parentRevID))}) + digester.Write([]byte(parentRevID)) + digester.Write(bodyBytes) + return fmt.Sprintf("%d-%x", generation, digester.Sum(nil)) +} + +// Returns the generation number (numeric prefix) of a revision ID. +func GenOfRevID(revid string) int { + if revid == "" { + return 0 + } + var generation int + n, _ := fmt.Sscanf(revid, "%d-", &generation) + if n < 1 || generation < 1 { + base.WarnfCtx(context.Background(), "GenOfRevID unsuccessful for %q", revid) + return -1 + } + return generation +} + +// Splits a revision ID into generation number and hex digest. +func ParseRevID(revid string) (int, string) { + if revid == "" { + return 0, "" + } + + idx := strings.Index(revid, "-") + if idx == -1 { + base.WarnfCtx(context.Background(), "parseRevID found no separator in rev %q", revid) + return -1, "" + } + + gen, err := strconv.Atoi(revid[:idx]) + if err != nil { + base.WarnfCtx(context.Background(), "parseRevID unexpected generation in rev %q: %s", revid, err) + return -1, "" + } else if gen < 1 { + base.WarnfCtx(context.Background(), "parseRevID unexpected generation in rev %q", revid) + return -1, "" + } + + return gen, revid[idx+1:] +} + +// CompareRevIDs compares the two rev IDs and returns: +// 1 if id1 is 'greater' than id2 +// -1 if id1 is 'less' than id2 +// 0 if the two are equal. +func CompareRevIDs(id1, id2 string) int { + gen1, sha1 := ParseRevID(id1) + gen2, sha2 := ParseRevID(id2) + switch { + case gen1 > gen2: + return 1 + case gen1 < gen2: + return -1 + case sha1 > sha2: + return 1 + case sha1 < sha2: + return -1 + } + return 0 +} + +// StripInternalProperties returns a copy of the given body with all internal underscore-prefixed keys removed, except _attachments and _deleted. +func StripInternalProperties(b Body) (Body, bool) { + return stripSpecialProperties(b, true) +} + +// stripAllInternalProperties returns a copy of the given body with all underscore-prefixed keys removed. +func StripAllSpecialProperties(b Body) (Body, bool) { + return stripSpecialProperties(b, false) +} + +// stripSpecialPropertiesExcept returns a copy of the given body with underscore-prefixed keys removed. +// Set internalOnly to only strip internal properties except _deleted and _attachments +func stripSpecialProperties(b Body, internalOnly bool) (sb Body, foundSpecialProps bool) { + // Assume no properties removed for the initial capacity to reduce allocs on large docs. + stripped := make(Body, len(b)) + for k, v := range b { + // Property is not stripped if: + // - It is blank + // - Does not start with an underscore ('_') + // - Is not an internal special property (this check is excluded when internalOnly = false) + if k == "" || k[0] != '_' || (internalOnly && (!strings.HasPrefix(k, BodyInternalPrefix) && + !base.StringSliceContains([]string{ + base.SyncPropertyName, + BodyId, + BodyRev, + BodyRevisions, + BodyExpiry, + BodyPurged, + BodyRemoved, + }, k))) { + // property is allowed + stripped[k] = v + } else { + foundSpecialProps = true + } + } + + if foundSpecialProps { + return stripped, true + } else { + // Return original body if nothing was removed + return b, false + } +} + +func GetStringArrayProperty(body map[string]interface{}, property string) ([]string, error) { + if raw, exists := body[property]; !exists { + return nil, nil + } else if strings, ok := raw.([]string); ok { + return strings, nil + } else if items, ok := raw.([]interface{}); ok { + strings := make([]string, len(items)) + for i := 0; i < len(items); i++ { + strings[i], ok = items[i].(string) + if !ok { + return nil, base.HTTPErrorf(http.StatusBadRequest, property+" must be a string array") + } + } + return strings, nil + } else { + return nil, base.HTTPErrorf(http.StatusBadRequest, property+" must be a string array") + } +} diff --git a/docmodel/revision_test.go b/docmodel/revision_test.go new file mode 100644 index 0000000000..8ae79ce7df --- /dev/null +++ b/docmodel/revision_test.go @@ -0,0 +1,191 @@ +/* +Copyright 2017-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package docmodel + +import ( + "fmt" + "log" + "testing" + + "github.com/couchbase/sync_gateway/base" + "github.com/stretchr/testify/assert" +) + +func TestParseRevID(t *testing.T) { + + var generation int + var digest string + + generation, _ = ParseRevID("ljlkjl") + log.Printf("generation: %v", generation) + assert.True(t, generation == -1, "Expected -1 generation for invalid rev id") + + generation, digest = ParseRevID("1-ljlkjl") + log.Printf("generation: %v, digest: %v", generation, digest) + assert.True(t, generation == 1, "Expected 1 generation") + assert.True(t, digest == "ljlkjl", "Unexpected digest") + + generation, digest = ParseRevID("2222-") + log.Printf("generation: %v, digest: %v", generation, digest) + assert.True(t, generation == 2222, "Expected invalid generation") + assert.True(t, digest == "", "Unexpected digest") + + generation, digest = ParseRevID("333-a") + log.Printf("generation: %v, digest: %v", generation, digest) + assert.True(t, generation == 333, "Expected generation") + assert.True(t, digest == "a", "Unexpected digest") + +} + +func TestBodyUnmarshal(t *testing.T) { + + tests := []struct { + name string + inputBytes []byte + expectedBody Body + }{ + {"empty bytes", []byte(""), nil}, + {"null", []byte("null"), Body(nil)}, + {"{}", []byte("{}"), Body{}}, + {"example body", []byte(`{"test":true}`), Body{"test": true}}, + } + + for _, test := range tests { + t.Run(test.name, func(ts *testing.T) { + var b Body + err := b.Unmarshal(test.inputBytes) + + // Unmarshal using json.Unmarshal for comparison below + var jsonUnmarshalBody Body + unmarshalErr := base.JSONUnmarshal(test.inputBytes, &jsonUnmarshalBody) + + if unmarshalErr != nil { + // If json.Unmarshal returns error for input, body.Unmarshal should do the same + assert.True(t, err != nil, fmt.Sprintf("Expected error when unmarshalling %s", test.name)) + } else { + assert.NoError(t, err, fmt.Sprintf("Expected no error when unmarshalling %s", test.name)) + assert.Equal(t, test.expectedBody, b) // Check against expected body + assert.Equal(t, jsonUnmarshalBody, b) // Check against json.Unmarshal results + } + + }) + } +} + +func TestParseRevisionsToAncestor(t *testing.T) { + revisions := Revisions{RevisionsStart: 5, RevisionsIds: []string{"five", "four", "three", "two", "one"}} + + assert.Equal(t, []string{"4-four", "3-three"}, revisions.ParseAncestorRevisions("3-three")) + assert.Equal(t, []string{"4-four"}, revisions.ParseAncestorRevisions("4-four")) + assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.ParseAncestorRevisions("1-one")) + assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.ParseAncestorRevisions("5-five")) + assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.ParseAncestorRevisions("0-zero")) + assert.Equal(t, []string{"4-four", "3-three", "2-two", "1-one"}, revisions.ParseAncestorRevisions("3-threeve")) + + shortRevisions := Revisions{RevisionsStart: 3, RevisionsIds: []string{"three"}} + assert.Equal(t, []string(nil), shortRevisions.ParseAncestorRevisions("2-two")) +} + +func TestStripSpecialProperties(t *testing.T) { + testCases := []struct { + name string + input Body + stripInternalOnly bool + stripped bool // Should at least 1 property have been stripped + newBodyIfStripped *Body + }{ + { + name: "No special", + input: Body{"test": 0, "bob": 0, "alice": 0}, + stripInternalOnly: false, + stripped: false, + newBodyIfStripped: nil, + }, + { + name: "No special internal only", + input: Body{"test": 0, "bob": 0, "alice": 0}, + stripInternalOnly: true, + stripped: false, + newBodyIfStripped: nil, + }, + { + name: "Strip special props", + input: Body{"_test": 0, "test": 0, "_attachments": 0, "_id": 0}, + stripInternalOnly: false, + stripped: true, + newBodyIfStripped: &Body{"test": 0}, + }, + { + name: "Strip internal special props", + input: Body{"_test": 0, "test": 0, "_rev": 0, "_exp": 0}, + stripInternalOnly: true, + stripped: true, + newBodyIfStripped: &Body{"_test": 0, "test": 0}, + }, + { + name: "Confirm internal attachments and deleted skipped", + input: Body{"_test": 0, "test": 0, "_attachments": 0, "_rev": 0, "_deleted": 0}, + stripInternalOnly: true, + stripped: true, + newBodyIfStripped: &Body{"_test": 0, "test": 0, "_attachments": 0, "_deleted": 0}, + }, + } + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + stripped, specialProps := stripSpecialProperties(test.input, test.stripInternalOnly) + assert.Equal(t, test.stripped, specialProps) + if test.stripped { + assert.Equal(t, *test.newBodyIfStripped, stripped) + return + } + assert.Equal(t, test.input, stripped) + }) + } +} + +func BenchmarkSpecialProperties(b *testing.B) { + noSpecialBody := Body{ + "asdf": "qwerty", "a": true, "b": true, "c": true, + "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, + "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, + } + + specialBody := noSpecialBody.Copy(BodyShallowCopy) + specialBody[BodyId] = "abc123" + specialBody[BodyRev] = "1-abc" + + tests := []struct { + name string + body Body + }{ + { + "no special", + noSpecialBody, + }, + { + "special", + specialBody, + }, + } + + for _, t := range tests { + b.Run(t.name+"-stripInternalProperties", func(bb *testing.B) { + for i := 0; i < bb.N; i++ { + StripInternalProperties(t.body) + } + }) + b.Run(t.name+"-stripAllInternalProperties", func(bb *testing.B) { + for i := 0; i < bb.N; i++ { + StripAllSpecialProperties(t.body) + } + }) + } +} diff --git a/db/revtree.go b/docmodel/revtree.go similarity index 92% rename from db/revtree.go rename to docmodel/revtree.go index 3320af37ad..cbc26b2780 100644 --- a/db/revtree.go +++ b/docmodel/revtree.go @@ -6,7 +6,7 @@ // software will be governed by the Apache License, Version 2.0, included in // the file licenses/APL2.txt. -package db +package docmodel import ( "bytes" @@ -43,7 +43,7 @@ type RevTree map[string]*RevInfo // The form in which a RevTree is stored in JSON. For space-efficiency it's stored as an array of // rev IDs, with a parallel array of parent indexes. Ordering in the arrays doesn't matter. // So the parent of Revs[i] is Revs[Parents[i]] (unless Parents[i] == -1, which denotes a root.) -type revTreeList struct { +type RevTreeList struct { Revs []string `json:"revs"` // The revision IDs Parents []int `json:"parents"` // Index of parent of each revision (-1 if root) Deleted []int `json:"deleted,omitempty"` // Indexes of revisions that are deletions @@ -56,7 +56,7 @@ type revTreeList struct { func (tree RevTree) MarshalJSON() ([]byte, error) { n := len(tree) - rep := revTreeList{ + rep := RevTreeList{ Revs: make([]string, n), Parents: make([]int, n), Channels: make([]base.Set, n), @@ -120,7 +120,7 @@ func (tree *RevTree) UnmarshalJSON(inputjson []byte) (err error) { // base.Warnf(base.KeyAll, "No RevTree for input %q", inputjson) return nil } - var rep revTreeList + var rep RevTreeList err = base.JSONUnmarshal(inputjson, &rep) if err != nil { return @@ -178,7 +178,7 @@ func (tree *RevTree) UnmarshalJSON(inputjson []byte) (err error) { func (tree RevTree) ContainsCycles() bool { containsCycles := false for _, leafRevision := range tree.GetLeaves() { - _, revHistoryErr := tree.getHistory(leafRevision) + _, revHistoryErr := tree.GetHistory(leafRevision) if revHistoryErr != nil { containsCycles = true } @@ -220,7 +220,7 @@ func (tree RevTree) RepairCycles() (err error) { } // Iterate over leaves - tree.forEachLeaf(leafProcessor) + tree.ForEachLeaf(leafProcessor) return nil } @@ -232,17 +232,17 @@ func (tree RevTree) RepairCycles() (err error) { // where the parent generation is *higher* than the node generation, which is never a valid scenario. // Likewise, detect situations where the parent generation is equal to the node generation, which is also invalid. func (node RevInfo) ParentGenGTENodeGen() bool { - return genOfRevID(node.Parent) >= genOfRevID(node.ID) + return GenOfRevID(node.Parent) >= GenOfRevID(node.ID) } // Returns true if the RevTree has an entry for this revid. -func (tree RevTree) contains(revid string) bool { +func (tree RevTree) Contains(revid string) bool { _, exists := tree[revid] return exists } // Returns the RevInfo for a revision ID, or panics if it's not found -func (tree RevTree) getInfo(revid string) (*RevInfo, error) { +func (tree RevTree) GetInfo(revid string) (*RevInfo, error) { info, exists := tree[revid] if !exists { return nil, errors.New("getInfo can't find rev: " + revid) @@ -252,8 +252,8 @@ func (tree RevTree) getInfo(revid string) (*RevInfo, error) { // Returns the parent ID of a revid. The parent is "" if the revid is a root. // Panics if the revid is not in the map at all. -func (tree RevTree) getParent(revid string) string { - info, err := tree.getInfo(revid) +func (tree RevTree) GetParent(revid string) string { + info, err := tree.GetInfo(revid) if err != nil { return "" } @@ -286,7 +286,7 @@ func (tree RevTree) GetLeavesFiltered(filter func(revId string) bool) []string { } -func (tree RevTree) forEachLeaf(callback func(*RevInfo)) { +func (tree RevTree) ForEachLeaf(callback func(*RevInfo)) { isParent := map[string]bool{} for _, info := range tree { isParent[info.Parent] = true @@ -299,8 +299,8 @@ func (tree RevTree) forEachLeaf(callback func(*RevInfo)) { } } -func (tree RevTree) isLeaf(revid string) bool { - if !tree.contains(revid) { +func (tree RevTree) IsLeaf(revid string) bool { + if !tree.Contains(revid) { return false } for _, info := range tree { @@ -313,18 +313,18 @@ func (tree RevTree) isLeaf(revid string) bool { // Finds the "winning" revision, the one that should be treated as the default. // This is the leaf revision whose (!deleted, generation, hash) tuple compares the highest. -func (tree RevTree) winningRevision() (winner string, branched bool, inConflict bool) { +func (tree RevTree) WinningRevision() (winner string, branched bool, inConflict bool) { winnerExists := false leafCount := 0 activeLeafCount := 0 - tree.forEachLeaf(func(info *RevInfo) { + tree.ForEachLeaf(func(info *RevInfo) { exists := !info.Deleted leafCount++ if exists { activeLeafCount++ } if (exists && !winnerExists) || - ((exists == winnerExists) && compareRevIDs(info.ID, winner) > 0) { + ((exists == winnerExists) && CompareRevIDs(info.ID, winner) > 0) { winner = info.ID winnerExists = exists } @@ -336,7 +336,7 @@ func (tree RevTree) winningRevision() (winner string, branched bool, inConflict // Given a revision and a set of possible ancestors, finds the one that is the most recent // ancestor of the revision; if none are ancestors, returns "". -func (tree RevTree) findAncestorFromSet(revid string, ancestors []string) string { +func (tree RevTree) FindAncestorFromSet(revid string, ancestors []string) string { // OPT: This is slow... for revid != "" { for _, a := range ancestors { @@ -344,7 +344,7 @@ func (tree RevTree) findAncestorFromSet(revid string, ancestors []string) string return a } } - info, err := tree.getInfo(revid) + info, err := tree.GetInfo(revid) if err != nil { break } @@ -354,26 +354,26 @@ func (tree RevTree) findAncestorFromSet(revid string, ancestors []string) string } // Records a revision in a RevTree. -func (tree RevTree) addRevision(docid string, info RevInfo) (err error) { +func (tree RevTree) AddRevision(docid string, info RevInfo) (err error) { revid := info.ID if revid == "" { - err = errors.New(fmt.Sprintf("doc: %v, RevTree addRevision, empty revid is illegal", docid)) + err = errors.New(fmt.Sprintf("doc: %v, RevTree AddRevision, empty revid is illegal", docid)) return } - if tree.contains(revid) { - err = errors.New(fmt.Sprintf("doc: %v, RevTree addRevision, already contains rev %q", docid, revid)) + if tree.Contains(revid) { + err = errors.New(fmt.Sprintf("doc: %v, RevTree AddRevision, already contains rev %q", docid, revid)) return } parent := info.Parent - if parent != "" && !tree.contains(parent) { - err = errors.New(fmt.Sprintf("doc: %v, RevTree addRevision, parent id %q is missing", docid, parent)) + if parent != "" && !tree.Contains(parent) { + err = errors.New(fmt.Sprintf("doc: %v, RevTree AddRevision, parent id %q is missing", docid, parent)) return } tree[revid] = &info return nil } -func (tree RevTree) getRevisionBody(revid string, loader RevLoaderFunc) ([]byte, bool) { +func (tree RevTree) GetRevisionBody(revid string, loader RevLoaderFunc) ([]byte, bool) { if revid == "" { // TODO: CBG-1948 panic("Illegal empty revision ID") @@ -401,13 +401,13 @@ func (tree RevTree) getRevisionBody(revid string, loader RevLoaderFunc) ([]byte, // as if the transient backup had expired. We are not attempting to repair the rev tree, as reclaiming storage // is a much lower priority than avoiding write errors, and want to avoid introducing additional conflict scenarios. // The invalid rev tree bodies will eventually be pruned through normal revision tree pruning. - if len(info.Body) > 0 && info.Body[0] == nonJSONPrefix { + if len(info.Body) > 0 && info.Body[0] == NonJSONPrefix { return nil, false } return info.Body, true } -func (tree RevTree) setRevisionBody(revid string, body []byte, bodyKey string, hasAttachments bool) { +func (tree RevTree) SetRevisionBody(revid string, body []byte, bodyKey string, hasAttachments bool) { if revid == "" { // TODO: CBG-1948 panic("Illegal empty revision ID") @@ -423,7 +423,7 @@ func (tree RevTree) setRevisionBody(revid string, body []byte, bodyKey string, h info.HasAttachments = hasAttachments } -func (tree RevTree) removeRevisionBody(revid string) (deletedBodyKey string) { +func (tree RevTree) RemoveRevisionBody(revid string) (deletedBodyKey string) { info, found := tree[revid] if !found { base.ErrorfCtx(context.Background(), "RemoveRevisionBody called for revid not in tree: %v", revid) @@ -461,7 +461,7 @@ func (tree RevTree) copy() RevTree { // // pruned: number of revisions pruned // prunedTombstoneBodyKeys: set of tombstones with external body storage that were pruned, as map[revid]bodyKey -func (tree RevTree) pruneRevisions(maxDepth uint32, keepRev string) (pruned int, prunedTombstoneBodyKeys map[string]string) { +func (tree RevTree) PruneRevisions(maxDepth uint32, keepRev string) (pruned int, prunedTombstoneBodyKeys map[string]string) { if len(tree) <= int(maxDepth) { return @@ -583,7 +583,7 @@ func (tree RevTree) FindShortestNonTombstonedBranchFromLeaves(leaves []string) ( // This is a tombstoned branch, skip it continue } - gen := genOfRevID(revid) + gen := GenOfRevID(revid) if gen > 0 && gen < genShortestNonTSBranch { genShortestNonTSBranch = gen found = true @@ -604,7 +604,7 @@ func (tree RevTree) FindLongestTombstonedBranch() (generation int) { func (tree RevTree) FindLongestTombstonedBranchFromLeaves(leaves []string) (generation int) { genLongestTSBranch := 0 for _, revid := range leaves { - gen := genOfRevID(revid) + gen := GenOfRevID(revid) if tree[revid].Deleted { if gen > genLongestTSBranch { genLongestTSBranch = gen @@ -710,7 +710,7 @@ func (tree RevTree) RenderGraphvizDot() string { } // Iterate over leaves - tree.forEachLeaf(leafProcessor) + tree.ForEachLeaf(leafProcessor) // Finish graphviz dot file resultBuffer.WriteString("}") @@ -721,12 +721,12 @@ func (tree RevTree) RenderGraphvizDot() string { // Returns the history of a revid as an array of revids in reverse chronological order. // Returns error if detects cycle(s) in rev tree -func (tree RevTree) getHistory(revid string) ([]string, error) { +func (tree RevTree) GetHistory(revid string) ([]string, error) { maxHistory := len(tree) history := make([]string, 0, 5) for revid != "" { - info, err := tree.getInfo(revid) + info, err := tree.GetInfo(revid) if err != nil { break } @@ -751,7 +751,7 @@ func ParseRevisions(body Body) []string { if !ok { return nil } - if genOfRevID(revid) < 1 { + if GenOfRevID(revid) < 1 { return nil } oneRev := make([]string, 0, 1) @@ -787,7 +787,7 @@ func splitRevisionList(revisions Revisions) (int, []string) { // Standard CouchDB encoding of a revision list: digests without numeric generation prefixes go in // the "ids" property, and the first (largest) generation number in the "start" property. // The docID parameter is informational only - and used when logging edge cases. -func encodeRevisions(docID string, revs []string) Revisions { +func EncodeRevisions(docID string, revs []string) Revisions { ids := make([]string, len(revs)) var start int for i, revid := range revs { @@ -806,7 +806,7 @@ func encodeRevisions(docID string, revs []string) Revisions { // trim the history to stop at the first ancestor revID. If no ancestors are found, trim to // length maxUnmatchedLen. // TODO: Document/rename what the boolean result return value represents -func trimEncodedRevisionsToAncestor(revs Revisions, ancestors []string, maxUnmatchedLen int) (result bool, trimmedRevs Revisions) { +func TrimEncodedRevisionsToAncestor(revs Revisions, ancestors []string, maxUnmatchedLen int) (result bool, trimmedRevs Revisions) { trimmedRevs = revs diff --git a/db/revtree_data_test.go b/docmodel/revtree_data_test.go similarity index 99% rename from db/revtree_data_test.go rename to docmodel/revtree_data_test.go index 327b37169a..83bbcdb9f0 100644 --- a/db/revtree_data_test.go +++ b/docmodel/revtree_data_test.go @@ -8,7 +8,7 @@ be governed by the Apache License, Version 2.0, included in the file licenses/APL2.txt. */ -package db +package docmodel const largeRevTree = ` { diff --git a/db/revtree_test.go b/docmodel/revtree_test.go similarity index 92% rename from db/revtree_test.go rename to docmodel/revtree_test.go index 77b291f5d4..2fa3912291 100644 --- a/db/revtree_test.go +++ b/docmodel/revtree_test.go @@ -6,7 +6,7 @@ // software will be governed by the Apache License, Version 2.0, included in // the file licenses/APL2.txt. -package db +package docmodel import ( "fmt" @@ -158,7 +158,7 @@ func getMultiBranchTestRevtree1(unconflictedBranchNumRevs, winningBranchNumRevs Parent: parentRevId, Deleted: true, } - err := revTree.addRevision("testdoc", revInfo) + err := revTree.AddRevision("testdoc", revInfo) if err != nil { panic(fmt.Sprintf("Error: %v", err)) } @@ -230,20 +230,20 @@ func TestRevTreeMarshal(t *testing.T) { } func TestRevTreeAccess(t *testing.T) { - assert.True(t, testmap.contains("3-three"), "contains 3 failed") - assert.True(t, testmap.contains("1-one"), "contains 1 failed") - assert.False(t, testmap.contains("foo"), "contains false positive") + assert.True(t, testmap.Contains("3-three"), "contains 3 failed") + assert.True(t, testmap.Contains("1-one"), "contains 1 failed") + assert.False(t, testmap.Contains("foo"), "contains false positive") } func TestRevTreeParentAccess(t *testing.T) { - parent := testmap.getParent("3-three") + parent := testmap.GetParent("3-three") assert.Equal(t, "2-two", parent) - parent = testmap.getParent("1-one") + parent = testmap.GetParent("1-one") assert.Equal(t, "", parent) } func TestRevTreeGetHistory(t *testing.T) { - history, err := testmap.getHistory("3-three") + history, err := testmap.GetHistory("3-three") assert.True(t, err == nil) assert.Equal(t, []string{"3-three", "2-two", "1-one"}, history) } @@ -258,7 +258,7 @@ func TestRevTreeGetLeaves(t *testing.T) { func TestRevTreeForEachLeaf(t *testing.T) { var leaves []string - branchymap.forEachLeaf(func(rev *RevInfo) { + branchymap.ForEachLeaf(func(rev *RevInfo) { leaves = append(leaves, rev.ID) }) sort.Strings(leaves) @@ -269,66 +269,66 @@ func TestRevTreeAddRevision(t *testing.T) { tempmap := testmap.copy() assert.Equal(t, testmap, tempmap) - err := tempmap.addRevision("testdoc", RevInfo{ID: "4-four", Parent: "3-three"}) + err := tempmap.AddRevision("testdoc", RevInfo{ID: "4-four", Parent: "3-three"}) require.NoError(t, err) - assert.Equal(t, "3-three", tempmap.getParent("4-four")) + assert.Equal(t, "3-three", tempmap.GetParent("4-four")) } func TestRevTreeAddRevisionWithEmptyID(t *testing.T) { tempmap := testmap.copy() assert.Equal(t, testmap, tempmap) - err := tempmap.addRevision("testdoc", RevInfo{Parent: "3-three"}) - assert.Equal(t, fmt.Sprintf("doc: %v, RevTree addRevision, empty revid is illegal", "testdoc"), err.Error()) + err := tempmap.AddRevision("testdoc", RevInfo{Parent: "3-three"}) + assert.Equal(t, fmt.Sprintf("doc: %v, RevTree AddRevision, empty revid is illegal", "testdoc"), err.Error()) } func TestRevTreeAddDuplicateRevID(t *testing.T) { tempmap := testmap.copy() assert.Equal(t, testmap, tempmap) - err := tempmap.addRevision("testdoc", RevInfo{ID: "2-two", Parent: "1-one"}) - assert.Equal(t, fmt.Sprintf("doc: %v, RevTree addRevision, already contains rev %q", "testdoc", "2-two"), err.Error()) + err := tempmap.AddRevision("testdoc", RevInfo{ID: "2-two", Parent: "1-one"}) + assert.Equal(t, fmt.Sprintf("doc: %v, RevTree AddRevision, already contains rev %q", "testdoc", "2-two"), err.Error()) } func TestRevTreeAddRevisionWithMissingParent(t *testing.T) { tempmap := testmap.copy() assert.Equal(t, testmap, tempmap) - err := tempmap.addRevision("testdoc", RevInfo{ID: "5-five", Parent: "4-four"}) - assert.Equal(t, fmt.Sprintf("doc: %v, RevTree addRevision, parent id %q is missing", "testdoc", "4-four"), err.Error()) + err := tempmap.AddRevision("testdoc", RevInfo{ID: "5-five", Parent: "4-four"}) + assert.Equal(t, fmt.Sprintf("doc: %v, RevTree AddRevision, parent id %q is missing", "testdoc", "4-four"), err.Error()) } func TestRevTreeCompareRevIDs(t *testing.T) { - assert.Equal(t, 0, compareRevIDs("1-aaa", "1-aaa")) - assert.Equal(t, -1, compareRevIDs("1-aaa", "5-aaa")) - assert.Equal(t, 1, compareRevIDs("10-aaa", "5-aaa")) - assert.Equal(t, 1, compareRevIDs("1-bbb", "1-aaa")) - assert.Equal(t, 1, compareRevIDs("5-bbb", "1-zzz")) + assert.Equal(t, 0, CompareRevIDs("1-aaa", "1-aaa")) + assert.Equal(t, -1, CompareRevIDs("1-aaa", "5-aaa")) + assert.Equal(t, 1, CompareRevIDs("10-aaa", "5-aaa")) + assert.Equal(t, 1, CompareRevIDs("1-bbb", "1-aaa")) + assert.Equal(t, 1, CompareRevIDs("5-bbb", "1-zzz")) } func TestRevTreeIsLeaf(t *testing.T) { - assert.True(t, branchymap.isLeaf("3-three"), "isLeaf failed on 3-three") - assert.True(t, branchymap.isLeaf("3-drei"), "isLeaf failed on 3-drei") - assert.False(t, branchymap.isLeaf("2-two"), "isLeaf failed on 2-two") - assert.False(t, branchymap.isLeaf("bogus"), "isLeaf failed on 'bogus") - assert.False(t, branchymap.isLeaf(""), "isLeaf failed on ''") + assert.True(t, branchymap.IsLeaf("3-three"), "isLeaf failed on 3-three") + assert.True(t, branchymap.IsLeaf("3-drei"), "isLeaf failed on 3-drei") + assert.False(t, branchymap.IsLeaf("2-two"), "isLeaf failed on 2-two") + assert.False(t, branchymap.IsLeaf("bogus"), "isLeaf failed on 'bogus") + assert.False(t, branchymap.IsLeaf(""), "isLeaf failed on ''") } func TestRevTreeWinningRev(t *testing.T) { tempmap := branchymap.copy() - winner, branched, conflict := tempmap.winningRevision() + winner, branched, conflict := tempmap.WinningRevision() assert.Equal(t, "3-three", winner) assert.True(t, branched) assert.True(t, conflict) - err := tempmap.addRevision("testdoc", RevInfo{ID: "4-four", Parent: "3-three"}) + err := tempmap.AddRevision("testdoc", RevInfo{ID: "4-four", Parent: "3-three"}) require.NoError(t, err) - winner, branched, conflict = tempmap.winningRevision() + winner, branched, conflict = tempmap.WinningRevision() assert.Equal(t, "4-four", winner) assert.True(t, branched) assert.True(t, conflict) - err = tempmap.addRevision("testdoc", RevInfo{ID: "5-five", Parent: "4-four", Deleted: true}) + err = tempmap.AddRevision("testdoc", RevInfo{ID: "5-five", Parent: "4-four", Deleted: true}) require.NoError(t, err) - winner, branched, conflict = tempmap.winningRevision() + winner, branched, conflict = tempmap.WinningRevision() assert.Equal(t, "3-drei", winner) assert.True(t, branched) assert.False(t, conflict) @@ -358,18 +358,18 @@ func TestPruneRevisions(t *testing.T) { assert.Equal(t, uint32(3), tempmap["1-one"].depth) // Prune: - pruned, _ := tempmap.pruneRevisions(1000, "") + pruned, _ := tempmap.PruneRevisions(1000, "") assert.Equal(t, 0, pruned) - pruned, _ = tempmap.pruneRevisions(3, "") + pruned, _ = tempmap.PruneRevisions(3, "") assert.Equal(t, 0, pruned) - pruned, _ = tempmap.pruneRevisions(2, "") + pruned, _ = tempmap.PruneRevisions(2, "") assert.Equal(t, 1, pruned) assert.Equal(t, 4, len(tempmap)) assert.Equal(t, (*RevInfo)(nil), tempmap["1-one"]) assert.Equal(t, "", tempmap["2-two"].Parent) // Make sure leaves are never pruned: - pruned, _ = tempmap.pruneRevisions(1, "") + pruned, _ = tempmap.PruneRevisions(1, "") assert.Equal(t, 2, pruned) assert.Equal(t, 2, len(tempmap)) assert.True(t, tempmap["3-three"] != nil) @@ -388,7 +388,7 @@ func TestPruneRevsSingleBranch(t *testing.T) { maxDepth := uint32(20) expectedNumPruned := numRevs - int(maxDepth) - numPruned, _ := revTree.pruneRevisions(maxDepth, "") + numPruned, _ := revTree.PruneRevisions(maxDepth, "") assert.Equal(t, expectedNumPruned, numPruned) } @@ -410,7 +410,7 @@ func TestPruneRevsOneWinningOneNonwinningBranch(t *testing.T) { maxDepth := uint32(2) - revTree.pruneRevisions(maxDepth, "") + revTree.PruneRevisions(maxDepth, "") assert.Equal(t, int(maxDepth), revTree.LongestBranch()) @@ -433,7 +433,7 @@ func TestPruneRevsOneWinningOneOldTombstonedBranch(t *testing.T) { maxDepth := uint32(2) - revTree.pruneRevisions(maxDepth, "") + revTree.PruneRevisions(maxDepth, "") assert.True(t, revTree.LongestBranch() == int(maxDepth)) @@ -465,7 +465,7 @@ func TestPruneRevsOneWinningOneOldAndOneRecentTombstonedBranch(t *testing.T) { maxDepth := uint32(2) - revTree.pruneRevisions(maxDepth, "") + revTree.PruneRevisions(maxDepth, "") assert.True(t, revTree.LongestBranch() == int(maxDepth)) @@ -474,7 +474,7 @@ func TestPruneRevsOneWinningOneOldAndOneRecentTombstonedBranch(t *testing.T) { assert.Equal(t, 1, len(tombstonedLeaves)) tombstonedLeaf := tombstonedLeaves[0] - tombstonedBranch, err := revTree.getHistory(tombstonedLeaf) + tombstonedBranch, err := revTree.GetHistory(tombstonedLeaf) assert.True(t, err == nil) assert.Equal(t, int(maxDepth), len(tombstonedBranch)) @@ -574,7 +574,7 @@ func TestPruneRevisionsPostIssue2651ThreeBranches(t *testing.T) { revTree := getMultiBranchTestRevtree1(50, 100, branchSpecs) maxDepth := uint32(50) - numPruned, _ := revTree.pruneRevisions(maxDepth, "") + numPruned, _ := revTree.PruneRevisions(maxDepth, "") t.Logf("numPruned: %v", numPruned) t.Logf("LongestBranch: %v", revTree.LongestBranch()) @@ -603,7 +603,7 @@ func TestPruneRevsSingleTombstonedBranch(t *testing.T) { expectedNumPruned += 1 // To account for the tombstone revision in the branchspec, which is spearate from NumRevs - numPruned, _ := revTree.pruneRevisions(maxDepth, "") + numPruned, _ := revTree.PruneRevisions(maxDepth, "") log.Printf("RevTreeAfter pruning: %v", revTree.RenderGraphvizDot()) @@ -661,7 +661,7 @@ func TestPruneDisconnectedRevTreeWithLongWinningBranch(t *testing.T) { maxDepth := uint32(7) - revTree.pruneRevisions(maxDepth, "") + revTree.PruneRevisions(maxDepth, "") if dumpRevTreeDotFiles { err := os.WriteFile("/tmp/TestPruneDisconnectedRevTreeWithLongWinningBranch_pruned1.dot", []byte(revTree.RenderGraphvizDot()), 0666) @@ -683,7 +683,7 @@ func TestPruneDisconnectedRevTreeWithLongWinningBranch(t *testing.T) { require.NoError(t, err) } - revTree.pruneRevisions(maxDepth, "") + revTree.PruneRevisions(maxDepth, "") if dumpRevTreeDotFiles { err := os.WriteFile("/tmp/TestPruneDisconnectedRevTreeWithLongWinningBranch_pruned_final.dot", []byte(revTree.RenderGraphvizDot()), 0666) @@ -759,55 +759,55 @@ func BenchmarkEncodeRevisions(b *testing.B) { docID := b.Name() + "-" + test.name b.Run(test.name, func(b *testing.B) { for i := 0; i < b.N; i++ { - _ = encodeRevisions(docID, test.input) + _ = EncodeRevisions(docID, test.input) } }) } } func TestEncodeRevisions(t *testing.T) { - encoded := encodeRevisions(t.Name(), []string{"5-huey", "4-dewey", "3-louie"}) + encoded := EncodeRevisions(t.Name(), []string{"5-huey", "4-dewey", "3-louie"}) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey", "dewey", "louie"}}, encoded) } func TestEncodeRevisionsGap(t *testing.T) { - encoded := encodeRevisions(t.Name(), []string{"5-huey", "3-louie"}) + encoded := EncodeRevisions(t.Name(), []string{"5-huey", "3-louie"}) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey", "louie"}}, encoded) } func TestEncodeRevisionsZero(t *testing.T) { - encoded := encodeRevisions(t.Name(), []string{"1-foo", "0-bar"}) + encoded := EncodeRevisions(t.Name(), []string{"1-foo", "0-bar"}) assert.Equal(t, Revisions{RevisionsStart: 1, RevisionsIds: []string{"foo", ""}}, encoded) } func TestTrimEncodedRevisionsToAncestor(t *testing.T) { - encoded := encodeRevisions(t.Name(), []string{"5-huey", "4-dewey", "3-louie", "2-screwy"}) + encoded := EncodeRevisions(t.Name(), []string{"5-huey", "4-dewey", "3-louie", "2-screwy"}) - result, trimmedRevs := trimEncodedRevisionsToAncestor(encoded, []string{"3-walter", "17-gretchen", "1-fooey"}, 1000) + result, trimmedRevs := TrimEncodedRevisionsToAncestor(encoded, []string{"3-walter", "17-gretchen", "1-fooey"}, 1000) assert.True(t, result) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey", "dewey", "louie", "screwy"}}, trimmedRevs) - result, trimmedRevs = trimEncodedRevisionsToAncestor(trimmedRevs, []string{"3-walter", "3-louie", "1-fooey"}, 2) + result, trimmedRevs = TrimEncodedRevisionsToAncestor(trimmedRevs, []string{"3-walter", "3-louie", "1-fooey"}, 2) assert.True(t, result) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey", "dewey", "louie"}}, trimmedRevs) - result, trimmedRevs = trimEncodedRevisionsToAncestor(trimmedRevs, []string{"3-walter", "3-louie", "1-fooey"}, 3) + result, trimmedRevs = TrimEncodedRevisionsToAncestor(trimmedRevs, []string{"3-walter", "3-louie", "1-fooey"}, 3) assert.True(t, result) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey", "dewey", "louie"}}, trimmedRevs) - result, trimmedRevs = trimEncodedRevisionsToAncestor(trimmedRevs, []string{"3-walter", "3-louie", "5-huey"}, 3) + result, trimmedRevs = TrimEncodedRevisionsToAncestor(trimmedRevs, []string{"3-walter", "3-louie", "5-huey"}, 3) assert.True(t, result) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey"}}, trimmedRevs) // Check maxLength with no ancestors: - encoded = encodeRevisions(t.Name(), []string{"5-huey", "4-dewey", "3-louie", "2-screwy"}) + encoded = EncodeRevisions(t.Name(), []string{"5-huey", "4-dewey", "3-louie", "2-screwy"}) - result, trimmedRevs = trimEncodedRevisionsToAncestor(encoded, nil, 6) + result, trimmedRevs = TrimEncodedRevisionsToAncestor(encoded, nil, 6) assert.True(t, result) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey", "dewey", "louie", "screwy"}}, trimmedRevs) - result, trimmedRevs = trimEncodedRevisionsToAncestor(trimmedRevs, nil, 2) + result, trimmedRevs = TrimEncodedRevisionsToAncestor(trimmedRevs, nil, 2) assert.True(t, result) assert.Equal(t, Revisions{RevisionsStart: 5, RevisionsIds: []string{"huey", "dewey"}}, trimmedRevs) } @@ -817,7 +817,7 @@ func TestRevsHistoryInfiniteLoop(t *testing.T) { docId := "testdocProblematicRevTree" - rawDoc, err := unmarshalDocument(docId, []byte(testdocProblematicRevTree1)) + rawDoc, err := UnmarshalDocument(docId, []byte(testdocProblematicRevTree1)) if err != nil { t.Fatalf("Error unmarshalling doc: %v", err) } @@ -846,7 +846,7 @@ func TestRepairRevsHistoryWithCycles(t *testing.T) { docId := "testdocProblematicRevTree" - rawDoc, err := unmarshalDocument(docId, []byte(testdocProblematicRevTree)) + rawDoc, err := UnmarshalDocument(docId, []byte(testdocProblematicRevTree)) if err != nil { t.Fatalf("Error unmarshalling doc %d: %v", i, err) } @@ -858,7 +858,7 @@ func TestRepairRevsHistoryWithCycles(t *testing.T) { // This function will be called back for every leaf node in tree leafProcessor := func(leaf *RevInfo) { - _, err := rawDoc.History.getHistory(leaf.ID) + _, err := rawDoc.History.GetHistory(leaf.ID) if err != nil { t.Fatalf("GetHistory() returned error: %v", err) } @@ -866,7 +866,7 @@ func TestRepairRevsHistoryWithCycles(t *testing.T) { } // Iterate over leaves and make sure none of them have a history with cycles - rawDoc.History.forEachLeaf(leafProcessor) + rawDoc.History.ForEachLeaf(leafProcessor) } @@ -947,14 +947,14 @@ func TestRevisionPruningLoop(t *testing.T) { func addAndGet(t *testing.T, revTree RevTree, revID string, parentRevID string, isTombstone bool) error { revBody := []byte(`{"foo":"bar"}`) - err := revTree.addRevision("foobar", RevInfo{ + err := revTree.AddRevision("foobar", RevInfo{ ID: revID, Parent: parentRevID, Body: revBody, Deleted: isTombstone, }) require.NoError(t, err) - history, err := revTree.getHistory(revID) + history, err := revTree.GetHistory(revID) log.Printf("addAndGet. Tree length: %d. History for new rev: %v", len(revTree), history) return err } @@ -985,7 +985,7 @@ func TestPruneRevisionsWithDisconnected(t *testing.T) { "73-abc": {ID: "73-abc", Parent: "72-abc", Deleted: true}, } - prunedCount, _ := revTree.pruneRevisions(4, "") + prunedCount, _ := revTree.PruneRevisions(4, "") assert.Equal(t, 10, prunedCount) remainingKeys := make([]string, 0, len(revTree)) @@ -998,16 +998,16 @@ func TestPruneRevisionsWithDisconnected(t *testing.T) { } func addPruneAndGet(revTree RevTree, revID string, parentRevID string, revBody []byte, revsLimit uint32, tombstone bool) (numPruned int, err error) { - _ = revTree.addRevision("doc", RevInfo{ + _ = revTree.AddRevision("doc", RevInfo{ ID: revID, Parent: parentRevID, Body: revBody, Deleted: tombstone, }) - numPruned, _ = revTree.pruneRevisions(revsLimit, revID) + numPruned, _ = revTree.PruneRevisions(revsLimit, revID) // Get history for new rev (checks for loops) - history, err := revTree.getHistory(revID) + history, err := revTree.GetHistory(revID) log.Printf("addPruneAndGet. Tree length: %d. Num pruned: %d. History for new rev: %v", len(revTree), numPruned, history) return numPruned, err @@ -1019,7 +1019,7 @@ func getHistoryWithTimeout(rawDoc *Document, revId string, timeout time.Duration errChannel := make(chan error) go func() { - history, err := rawDoc.History.getHistory(revId) + history, err := rawDoc.History.GetHistory(revId) if err != nil { errChannel <- err } else { @@ -1064,7 +1064,7 @@ func BenchmarkRevTreePruning(b *testing.B) { revTree := getMultiBranchTestRevtree1(50, 100, branchSpecs) b.StartTimer() - revTree.pruneRevisions(50, "") + revTree.PruneRevisions(50, "") } } @@ -1087,7 +1087,7 @@ func BenchmarkRevtreeUnmarshal(b *testing.B) { }) b.Run("Marshal into revTreeList", func(b *testing.B) { - revTree := revTreeList{} + revTree := RevTreeList{} for i := 0; i < b.N; i++ { _ = base.JSONUnmarshal(treeJson, &revTree) } @@ -1124,7 +1124,7 @@ func addRevs(revTree RevTree, startingParentRevId string, numRevs int, revDigest Deleted: false, Channels: channels, } - _ = revTree.addRevision("testdoc", revInfo) + _ = revTree.AddRevision("testdoc", revInfo) generation += 1 @@ -1172,7 +1172,7 @@ func (tree RevTree) LongestBranch() int { } } - tree.forEachLeaf(leafProcessor) + tree.ForEachLeaf(leafProcessor) return longestBranch diff --git a/rest/admin_api.go b/rest/admin_api.go index dbe8e3c9a8..ac12994002 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -21,6 +21,7 @@ import ( "github.com/couchbase/sync_gateway/auth" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" "github.com/google/uuid" "github.com/gorilla/mux" pkgerrors "github.com/pkg/errors" @@ -1013,7 +1014,7 @@ func (h *handler) handleGetRawDoc() error { rawBytes := []byte(base.EmptyDocument) if includeDoc { if doc.IsDeleted() { - rawBytes = []byte(db.DeletedDocument) + rawBytes = []byte(docmodel.DeletedDocument) } else { docRawBodyBytes, err := doc.BodyBytes() if err != nil { diff --git a/rest/api_benchmark_test.go b/rest/api_benchmark_test.go index 8ddc3b36a2..36a5a51b11 100644 --- a/rest/api_benchmark_test.go +++ b/rest/api_benchmark_test.go @@ -268,10 +268,10 @@ func BenchmarkReadOps_RevsDiff(b *testing.B) { defer PurgeDoc(rt, "doc1k") // Create target doc for revs_diff: - doc1k_bulkDocs_meta := `"_id":"doc1k", - "_rev":"12-abc", + doc1k_bulkDocs_meta := `"_id":"doc1k", + "_rev":"12-abc", "_revisions":{ - "start": 12, + "start": 12, "ids": ["abc", "eleven", "ten", "nine"] },` diff --git a/rest/api_test.go b/rest/api_test.go index e7c4a9da81..99e6d358a9 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -34,6 +34,7 @@ import ( "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" "github.com/couchbaselabs/walrus" "github.com/robertkrimen/otto/underscore" "github.com/stretchr/testify/assert" @@ -2606,7 +2607,7 @@ func TestDocChannelSetPruning(t *testing.T) { syncData, err := rt.GetDatabase().GetSingleDatabaseCollection().GetDocSyncData(base.TestCtx(t), "doc") assert.NoError(t, err) - require.Len(t, syncData.ChannelSetHistory, db.DocumentHistoryMaxEntriesPerChannel) + require.Len(t, syncData.ChannelSetHistory, docmodel.DocumentHistoryMaxEntriesPerChannel) assert.Equal(t, "a", syncData.ChannelSetHistory[0].Name) assert.Equal(t, uint64(1), syncData.ChannelSetHistory[0].Start) assert.Equal(t, uint64(12), syncData.ChannelSetHistory[0].End) diff --git a/rest/attachment_test.go b/rest/attachment_test.go index 0bc04d3005..a1d01a3334 100644 --- a/rest/attachment_test.go +++ b/rest/attachment_test.go @@ -958,8 +958,8 @@ func TestAttachmentRevposPre25Metadata(t *testing.T) { response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc1", "") RequireStatus(t, response, 200) var body struct { - Test bool `json:"test"` - Attachments db.AttachmentMap `json:"_attachments"` + Test bool `json:"test"` + Attachments AttachmentMap `json:"_attachments"` } require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) assert.False(t, body.Test) diff --git a/rest/blip_client_test.go b/rest/blip_client_test.go index 13558aee7e..30920ec02d 100644 --- a/rest/blip_client_test.go +++ b/rest/blip_client_test.go @@ -25,6 +25,7 @@ import ( "github.com/couchbase/go-blip" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -121,7 +122,7 @@ func (btr *BlipTesterReplicator) initHandlers(btc *BlipTesterClient) { panic(fmt.Sprintf("error getting client attachment: %v", err)) } - proof := db.ProveAttachment(attData, nonce) + proof := docmodel.ProveAttachment(attData, nonce) resp := msg.Response() resp.SetBody([]byte(proof)) diff --git a/rest/bulk_api.go b/rest/bulk_api.go index a3150280bd..24b1fc16b5 100644 --- a/rest/bulk_api.go +++ b/rest/bulk_api.go @@ -23,6 +23,7 @@ import ( "github.com/couchbase/sync_gateway/base" ch "github.com/couchbase/sync_gateway/channels" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" ) // HTTP handler for _all_docs @@ -39,7 +40,7 @@ func (h *handler) handleAllDocs() error { if h.rq.Method == "POST" { input, err := h.readJSON() if err == nil { - if explicitDocIDs, _ = db.GetStringArrayProperty(input, "keys"); explicitDocIDs == nil { + if explicitDocIDs, _ = docmodel.GetStringArrayProperty(input, "keys"); explicitDocIDs == nil { err = base.HTTPErrorf(http.StatusBadRequest, "Bad/missing keys") } } @@ -399,7 +400,7 @@ func (h *handler) handleBulkGet() error { if docid == "" || !revok { err = base.HTTPErrorf(http.StatusBadRequest, "Invalid doc/rev ID in _bulk_get") } else { - attsSince, err = db.GetStringArrayProperty(doc, "atts_since") + attsSince, err = docmodel.GetStringArrayProperty(doc, "atts_since") if showRevs { docRevsLimit = globalRevsLimit @@ -414,7 +415,7 @@ func (h *handler) handleBulkGet() error { } if docRevsLimit > 0 { - revsFrom, err = db.GetStringArrayProperty(doc, "revs_from") + revsFrom, err = docmodel.GetStringArrayProperty(doc, "revs_from") if revsFrom == nil { revsFrom = attsSince // revs_from defaults to same value as atts_since } @@ -510,7 +511,7 @@ func (h *handler) handleBulkDocs() error { docid, revid, _, err = h.collection.Post(h.ctx(), doc) } } else { - revisions := db.ParseRevisions(doc) + revisions := docmodel.ParseRevisions(doc) if revisions == nil { err = base.HTTPErrorf(http.StatusBadRequest, "Bad _revisions") } else { diff --git a/rest/config_database.go b/rest/config_database.go index 5d55c99aca..f67c966297 100644 --- a/rest/config_database.go +++ b/rest/config_database.go @@ -12,6 +12,7 @@ import ( "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" ) // RuntimeDatabaseConfig is the non-persisted database config that has the persisted DatabaseConfig embedded @@ -66,7 +67,7 @@ func GenerateDatabaseConfigVersionID(previousRevID string, dbConfig *DbConfig) ( previousGen, previousRev := db.ParseRevID(previousRevID) generation := previousGen + 1 - hash := db.CreateRevIDWithBytes(generation, previousRev, encodedBody) + hash := docmodel.CreateRevIDWithBytes(generation, previousRev, encodedBody) return hash, nil } diff --git a/rest/doc_api.go b/rest/doc_api.go index 129c0e6b95..64e09687c5 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -21,6 +21,7 @@ import ( "github.com/couchbase/sync_gateway/auth" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" ) // HTTP handler for a GET of a document @@ -204,7 +205,7 @@ func (h *handler) handleGetAttachment() error { return base.HTTPErrorf(http.StatusNotFound, "missing attachment %s", attachmentName) } digest := meta["digest"].(string) - version, ok := db.GetAttachmentVersion(meta) + version, ok := docmodel.GetAttachmentVersion(meta) if !ok { return db.ErrAttachmentVersion } @@ -332,7 +333,7 @@ func (h *handler) handlePutAttachment() error { } // find attachment (if it existed) - attachments := db.GetBodyAttachments(body) + attachments := docmodel.GetBodyAttachments(body) if attachments == nil { attachments = make(map[string]interface{}) } @@ -393,7 +394,7 @@ func (h *handler) handleDeleteAttachment() error { } // get document attachments and check if attachment exists - attachments := db.GetBodyAttachments(body) + attachments := docmodel.GetBodyAttachments(body) if _, ok := attachments[attachmentName]; !ok { return base.HTTPErrorf(http.StatusNotFound, "Attachment %s is not found", attachmentName) } @@ -467,7 +468,7 @@ func (h *handler) handlePutDoc() error { h.setEtag(newRev) } else { // Replicator-style PUT with new_edits=false: - revisions := db.ParseRevisions(body) + revisions := docmodel.ParseRevisions(body) if revisions == nil { return base.HTTPErrorf(http.StatusBadRequest, "Bad _revisions") } @@ -521,7 +522,7 @@ func (h *handler) handlePutDocReplicator2(docid string, roundTrip bool) (err err deleted, _ := h.getOptBoolQuery("deleted", false) newDoc.Deleted = deleted - newDoc.RevID = db.CreateRevIDWithBytes(generation, parentRev, bodyBytes) + newDoc.RevID = docmodel.CreateRevIDWithBytes(generation, parentRev, bodyBytes) history := []string{newDoc.RevID} if parentRev != "" { @@ -543,7 +544,7 @@ func (h *handler) handlePutDocReplicator2(docid string, roundTrip bool) (err err if bytes.Contains(bodyBytes, []byte(db.BodyAttachments)) { body := newDoc.Body() - newDoc.DocAttachments = db.GetBodyAttachments(body) + newDoc.DocAttachments = docmodel.GetBodyAttachments(body) delete(body, db.BodyAttachments) newDoc.UpdateBody(body) } diff --git a/rest/multipart.go b/rest/multipart.go index b850762400..ea672490f0 100644 --- a/rest/multipart.go +++ b/rest/multipart.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" "github.com/pkg/errors" "github.com/couchbase/sync_gateway/base" @@ -126,13 +127,13 @@ func writeJSONPart(writer *multipart.Writer, contentType string, body db.Body, c func WriteMultipartDocument(ctx context.Context, cblReplicationPullStats *base.CBLReplicationPullStats, body db.Body, writer *multipart.Writer, compress bool) { // First extract the attachments that should follow: following := []attInfo{} - for name, value := range db.GetBodyAttachments(body) { + for name, value := range docmodel.GetBodyAttachments(body) { meta := value.(map[string]interface{}) if meta["stub"] != true { var err error var info attInfo info.contentType, _ = meta["content_type"].(string) - info.data, err = db.DecodeAttachment(meta["data"]) + info.data, err = docmodel.DecodeAttachment(meta["data"]) if info.data == nil { base.WarnfCtx(ctx, "Couldn't decode attachment %q of doc %q: %v", base.UD(name), base.UD(body[db.BodyId]), err) meta["stub"] = true @@ -167,7 +168,7 @@ func WriteMultipartDocument(ctx context.Context, cblReplicationPullStats *base.C } func hasInlineAttachments(body db.Body) bool { - for _, value := range db.GetBodyAttachments(body) { + for _, value := range docmodel.GetBodyAttachments(body) { if meta, ok := value.(map[string]interface{}); ok && meta["data"] != nil { return true } @@ -227,7 +228,7 @@ func ReadMultipartDocument(reader *multipart.Reader) (db.Body, error) { // Collect the attachments with a "follows" property, which will appear as MIME parts: followingAttachments := map[string]map[string]interface{}{} - for name, value := range db.GetBodyAttachments(body) { + for name, value := range docmodel.GetBodyAttachments(body) { if meta := value.(map[string]interface{}); meta["follows"] == true { followingAttachments[name] = meta } diff --git a/rest/replicatortest/replicator_test.go b/rest/replicatortest/replicator_test.go index 79a9efd52a..d469c7a938 100644 --- a/rest/replicatortest/replicator_test.go +++ b/rest/replicatortest/replicator_test.go @@ -29,6 +29,7 @@ import ( "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" "github.com/couchbase/sync_gateway/rest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1370,7 +1371,7 @@ func TestReplicationConfigChange(t *testing.T) { bulkDocs := ` { "docs": - [ + [ {"channels": ["ChannelOne"], "_id": "doc_1"}, {"channels": ["ChannelOne"], "_id": "doc_2"}, {"channels": ["ChannelOne"], "_id": "doc_3"}, @@ -3751,7 +3752,7 @@ func TestActiveReplicatorPullConflict(t *testing.T) { return mergedDoc; }`, expectedLocalBody: db.Body{"source": "merged"}, - expectedLocalRevID: db.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"merged"}`)), // rev for merged body, with parent 1-b + expectedLocalRevID: docmodel.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"merged"}`)), // rev for merged body, with parent 1-b expectedResolutionType: db.ConflictResolutionMerge, }, { @@ -3762,7 +3763,7 @@ func TestActiveReplicatorPullConflict(t *testing.T) { remoteRevID: "1-b", conflictResolver: `function(conflict) {return conflict.LocalDocument;}`, expectedLocalBody: db.Body{"source": "local"}, - expectedLocalRevID: db.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"local"}`)), // rev for local body, transposed under parent 1-b + expectedLocalRevID: docmodel.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"local"}`)), // rev for local body, transposed under parent 1-b expectedResolutionType: db.ConflictResolutionLocal, }, { @@ -3981,7 +3982,7 @@ func TestActiveReplicatorPushAndPullConflict(t *testing.T) { return mergedDoc; }`, expectedBody: []byte(`{"source": "merged"}`), - expectedRevID: db.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"merged"}`)), // rev for merged body, with parent 1-b + expectedRevID: docmodel.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"merged"}`)), // rev for merged body, with parent 1-b }, { name: "localWins", @@ -3991,7 +3992,7 @@ func TestActiveReplicatorPushAndPullConflict(t *testing.T) { remoteRevID: "1-b", conflictResolver: `function(conflict) {return conflict.LocalDocument;}`, expectedBody: []byte(`{"source": "local"}`), - expectedRevID: db.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"local"}`)), // rev for local body, transposed under parent 1-b + expectedRevID: docmodel.CreateRevIDWithBytes(2, "1-b", []byte(`{"source":"local"}`)), // rev for local body, transposed under parent 1-b }, { name: "localWinsRemoteTombstone", @@ -4002,7 +4003,7 @@ func TestActiveReplicatorPushAndPullConflict(t *testing.T) { commonAncestorRevID: "1-a", conflictResolver: `function(conflict) {return conflict.LocalDocument;}`, expectedBody: []byte(`{"source": "local"}`), - expectedRevID: db.CreateRevIDWithBytes(3, "2-b", []byte(`{"source":"local"}`)), // rev for local body, transposed under parent 2-b + expectedRevID: docmodel.CreateRevIDWithBytes(3, "2-b", []byte(`{"source":"local"}`)), // rev for local body, transposed under parent 2-b }, } @@ -5641,7 +5642,7 @@ func TestActiveReplicatorPullConflictReadWriteIntlProps(t *testing.T) { base.LongRunningTest(t) createRevID := func(generation int, parentRevID string, body db.Body) string { - rev, err := db.CreateRevID(generation, parentRevID, body) + rev, err := docmodel.CreateRevID(generation, parentRevID, body) require.NoError(t, err, "Error creating revision") return rev } diff --git a/rest/replicatortest/replicator_test_helper.go b/rest/replicatortest/replicator_test_helper.go index aae2d4d08e..623c934bd5 100644 --- a/rest/replicatortest/replicator_test_helper.go +++ b/rest/replicatortest/replicator_test_helper.go @@ -15,6 +15,7 @@ import ( "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" "github.com/couchbase/sync_gateway/rest" "github.com/couchbaselabs/walrus" "github.com/stretchr/testify/assert" @@ -102,7 +103,7 @@ func createOrUpdateDoc(t *testing.T, rt *rest.RestTester, docID, revID, bodyValu return rest.RespRevID(t, resp) } func getTestRevpos(t *testing.T, doc db.Body, attachmentKey string) (revpos int) { - attachments := db.GetBodyAttachments(doc) + attachments := docmodel.GetBodyAttachments(doc) if attachments == nil { return 0 } diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index b2ab7fd6f5..cec03447fa 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -37,6 +37,7 @@ import ( "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/channels" "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/docmodel" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1677,6 +1678,9 @@ func (bt *BlipTester) GetDocAtRev(requestedDocID, requestedDocRev string) (resul } +// A map of keys -> DocAttachments. +type AttachmentMap map[string]*docmodel.DocAttachment + type SendRevWithAttachmentInput struct { docId string revId string @@ -1713,7 +1717,7 @@ func (bt *BlipTester) SendRevWithAttachment(input SendRevWithAttachmentInput) (s } } - doc.SetAttachments(db.AttachmentMap{ + doc.SetAttachments(AttachmentMap{ input.attachmentName: &myAttachment, }) @@ -2051,7 +2055,7 @@ func (e ExpectedChange) Equals(change []interface{}) error { // - _attachments // // This struct wraps a map and provides convenience methods for getting at the special -// fields with the appropriate types (string in the id/rev case, db.AttachmentMap in the attachments case). +// fields with the appropriate types (string in the id/rev case, AttachmentMap in the attachments case). // Currently only used in tests, but if similar functionality needed in primary codebase, could be moved. type RestDocument map[string]interface{} @@ -2086,23 +2090,23 @@ func (d RestDocument) SetRevID(revId string) { d[db.BodyRev] = revId } -func (d RestDocument) SetAttachments(attachments db.AttachmentMap) { +func (d RestDocument) SetAttachments(attachments AttachmentMap) { d[db.BodyAttachments] = attachments } -func (d RestDocument) GetAttachments() (db.AttachmentMap, error) { +func (d RestDocument) GetAttachments() (AttachmentMap, error) { rawAttachments, hasAttachments := d[db.BodyAttachments] // If the map doesn't even have the _attachments key, return an empty attachments map if !hasAttachments { - return db.AttachmentMap{}, nil + return AttachmentMap{}, nil } // Otherwise, create an AttachmentMap from the value in the raw map - attachmentMap := db.AttachmentMap{} + attachmentMap := AttachmentMap{} switch v := rawAttachments.(type) { - case db.AttachmentMap: + case AttachmentMap: // If it's already an AttachmentMap (maybe due to previous call to SetAttachments), then return as-is return v, nil default: @@ -2112,11 +2116,11 @@ func (d RestDocument) GetAttachments() (db.AttachmentMap, error) { // marshal attachmentVal into a byte array, then unmarshal into a DocAttachment attachmentValMarshalled, err := base.JSONMarshal(attachmentVal) if err != nil { - return db.AttachmentMap{}, err + return AttachmentMap{}, err } docAttachment := db.DocAttachment{} if err := base.JSONUnmarshal(attachmentValMarshalled, &docAttachment); err != nil { - return db.AttachmentMap{}, err + return AttachmentMap{}, err } attachmentMap[attachmentName] = &docAttachment diff --git a/rest/utilities_testing_test.go b/rest/utilities_testing_test.go index 660b91fcee..773adc75e3 100644 --- a/rest/utilities_testing_test.go +++ b/rest/utilities_testing_test.go @@ -69,7 +69,7 @@ func TestDocumentUnmarshal(t *testing.T) { func TestAttachmentRoundTrip(t *testing.T) { doc := RestDocument{} - attachmentMap := db.AttachmentMap{ + attachmentMap := AttachmentMap{ "foo": &db.DocAttachment{ ContentType: "application/octet-stream", Digest: "WHATEVER",