From 1171e9db0da7abf265e6efb7e41e3f0f26238404 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 7 Jun 2019 23:40:38 -0500 Subject: [PATCH 01/24] wip: engine2 passing test with updates service --- backend/app/cmd/server.go | 26 +- backend/app/rest/api/rest_test.go | 6 +- backend/app/store/engine/bolt_accessor.go | 28 +- backend/app/store/engine/bolt_admin.go | 4 +- backend/app/store/engine/engine.go | 8 - backend/app/store/engine2/bolt.go | 863 ++++++++++++++++++++++ backend/app/store/engine2/bolt_test.go | 774 +++++++++++++++++++ backend/app/store/engine2/engine.go | 128 ++++ backend/app/store/engine2/engine_test.go | 55 ++ backend/app/store/remote/remote.go | 134 ++++ backend/app/store/remote/remote_test.go | 133 ++++ backend/app/store/service/service.go | 187 ++++- 12 files changed, 2282 insertions(+), 64 deletions(-) create mode 100644 backend/app/store/engine2/bolt.go create mode 100644 backend/app/store/engine2/bolt_test.go create mode 100644 backend/app/store/engine2/engine.go create mode 100644 backend/app/store/engine2/engine_test.go create mode 100644 backend/app/store/remote/remote.go create mode 100644 backend/app/store/remote/remote_test.go diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index a13d29e1f0..0ac0e87323 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -31,7 +31,7 @@ import ( "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/service" ) @@ -249,7 +249,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) { log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL) dataService := &service.DataStore{ - Interface: storeEngine, + Engine: storeEngine, EditDuration: s.EditDuration, AdminStore: adminStore, MaxCommentSize: s.MaxCommentSize, @@ -401,7 +401,7 @@ func (a *serverApp) activateBackup(ctx context.Context) { } // makeDataStore creates store for all sites -func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { +func (s *ServerCommand) makeDataStore() (result engine2.Interface, err error) { log.Printf("[INFO] make data store, type=%s", s.Store.Type) switch s.Store.Type { @@ -409,18 +409,18 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { if err = makeDirs(s.Store.Bolt.Path); err != nil { return nil, errors.Wrap(err, "failed to create bolt store") } - sites := []engine.BoltSite{} + sites := []engine2.BoltSite{} for _, site := range s.Sites { - sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)}) + sites = append(sites, engine2.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)}) } - result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...) - case "mongo": - mgServer, e := s.makeMongo() - if e != nil { - return result, errors.Wrap(e, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") - result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) + result, err = engine2.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...) + // case "mongo": + // mgServer, e := s.makeMongo() + // if e != nil { + // return result, errors.Wrap(e, "failed to create mongo server") + // } + // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") + // result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) default: return nil, errors.Errorf("unsupported store type %s", s.Store.Type) } diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index a0e8566a6b..d4fedc3144 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -31,7 +31,7 @@ import ( "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/service" ) @@ -291,7 +291,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { os.RemoveAll("/tmp/ava-remark42") os.RemoveAll("/tmp/pics-remark42") - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) + b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) memCache, err := cache.NewMemoryCache() @@ -301,7 +301,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}}) dataStore := &service.DataStore{ - Interface: b, + Engine: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, AdminStore: adminStore, diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go index 1a555937ed..c900d7747c 100644 --- a/backend/app/store/engine/bolt_accessor.go +++ b/backend/app/store/engine/bolt_accessor.go @@ -105,7 +105,7 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { } // serialize comment to json []byte for bolt and save - if e = b.save(postBkt, []byte(comment.ID), comment); e != nil { + if e = b.save(postBkt, comment.ID, comment); e != nil { return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL) } @@ -206,7 +206,7 @@ func (b *BoltDB) Last(siteID string, max int, since time.Time) (comments []store } comment := store.Comment{} - if e = b.load(postBkt, []byte(commentID), &comment); e != nil { + if e = b.load(postBkt, commentID, &comment); e != nil { log.Printf("[WARN] can't load comment for %s from store %s", commentID, url) continue } @@ -263,7 +263,7 @@ func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err postURL := string(k) infoBkt := tx.Bucket([]byte(infoBucketName)) info := store.PostInfo{} - if e := b.load(infoBkt, []byte(postURL), &info); e != nil { + if e := b.load(infoBkt, postURL, &info); e != nil { return errors.Wrapf(e, "can't load info for %s", postURL) } list = append(list, info) @@ -287,7 +287,7 @@ func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, e info := store.PostInfo{} err = bdb.View(func(tx *bolt.Tx) error { infoBkt := tx.Bucket([]byte(infoBucketName)) - if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil { + if e := b.load(infoBkt, locator.URL, &info); e != nil { return errors.Wrapf(e, "can't load info for %s", locator.URL) } return nil @@ -391,7 +391,7 @@ func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Com if e != nil { return e } - return b.load(bucket, []byte(commentID), &comment) + return b.load(bucket, commentID, &comment) }) return comment, err } @@ -417,7 +417,7 @@ func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error { if e != nil { return e } - return b.save(bucket, []byte(comment.ID), comment) + return b.save(bucket, comment.ID, comment) }) } @@ -467,7 +467,7 @@ func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) } // save marshaled value to key for bucket. Should run in update tx -func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) { +func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) { if value == nil { return errors.Errorf("can't save nil value for %s", key) } @@ -475,15 +475,15 @@ func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err erro if jerr != nil { return errors.Wrap(jerr, "can't marshal comment") } - if err = bkt.Put(key, jdata); err != nil { + if err = bkt.Put([]byte(key), jdata); err != nil { return errors.Wrapf(err, "failed to save key %s", key) } return nil } // load and unmarshal json value by key from bucket. Should run in view tx -func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error { - value := bkt.Get(key) +func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error { + value := bkt.Get([]byte(key)) if value == nil { return errors.Errorf("no value for %s", key) } @@ -501,7 +501,7 @@ func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) { infoBkt := tx.Bucket([]byte(infoBucketName)) info := store.PostInfo{} - if err := b.load(infoBkt, []byte(postURL), &info); err != nil { + if err := b.load(infoBkt, postURL, &info); err != nil { info = store.PostInfo{} } if val == 0 { // get current count, don't update @@ -509,13 +509,13 @@ func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) { } info.Count += val - return info.Count, b.save(infoBkt, []byte(postURL), &info) + return info.Count, b.save(infoBkt, postURL, &info) } func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) { infoBkt := tx.Bucket([]byte(infoBucketName)) info := store.PostInfo{} - if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil { + if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil { info = store.PostInfo{ Count: 0, URL: comment.Locator.URL, @@ -525,7 +525,7 @@ func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, er } info.Count++ info.LastTS = comment.Timestamp - return info, b.save(infoBkt, []byte(comment.Locator.URL), &info) + return info, b.save(infoBkt, comment.Locator.URL, &info) } func (b *BoltDB) db(siteID string) (*bolt.DB, error) { diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine/bolt_admin.go index 429ac991bb..8924f5609f 100644 --- a/backend/app/store/engine/bolt_admin.go +++ b/backend/app/store/engine/bolt_admin.go @@ -29,13 +29,13 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele } comment := store.Comment{} - if err = b.load(postBkt, []byte(commentID), &comment); err != nil { + if err = b.load(postBkt, commentID, &comment); err != nil { return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) } // set deleted status and clear fields comment.SetDeleted(mode) - if err = b.save(postBkt, []byte(commentID), comment); err != nil { + if err = b.save(postBkt, commentID, comment); err != nil { return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) } diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go index fb6b25139e..5c6a68271c 100644 --- a/backend/app/store/engine/engine.go +++ b/backend/app/store/engine/engine.go @@ -13,14 +13,6 @@ import ( // NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate //go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go" -// UserRequest is the request send to get comments by user -type UserRequest struct { - SiteID string - UserID string - Limit int - Skip int -} - // Interface defines methods provided by low-level storage engine type Interface interface { Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id diff --git a/backend/app/store/engine2/bolt.go b/backend/app/store/engine2/bolt.go new file mode 100644 index 0000000000..2a5c6937d8 --- /dev/null +++ b/backend/app/store/engine2/bolt.go @@ -0,0 +1,863 @@ +package engine2 + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "strings" + "time" + + bolt "github.com/coreos/bbolt" + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store" +) + +// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe. +// there are 5 types of top-level buckets: +// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment +// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID +// value is not full comment but a reference combined from post-url+commentID +// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value +// is a nested bucket named userID with kv as ts:reference +// - blocking info sits in "block" bucket. Key is userID, value - ts +// - counts per post to keep number of comments. Key is post url, value - count +// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts +type BoltDB struct { + dbs map[string]*bolt.DB +} + +const ( + // top level buckets + postsBucketName = "posts" + lastBucketName = "last" + userBucketName = "users" + blocksBucketName = "block" + infoBucketName = "info" + readonlyBucketName = "readonly" + verifiedBucketName = "verified" + + tsNano = "2006-01-02T15:04:05.000000000Z07:00" +) + +// BoltSite defines single site param +type BoltSite struct { + FileName string // full path to boltdb + SiteID string // ID of given site +} + +// NewBoltDB makes persistent boltdb-based store. For each site new boltdb file created +func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) { + log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options) + result := BoltDB{dbs: make(map[string]*bolt.DB)} + for _, site := range sites { + db, err := bolt.Open(site.FileName, 0600, &options) + if err != nil { + return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName) + } + + // make top-level buckets + topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, infoBucketName, + readonlyBucketName, verifiedBucketName} + err = db.Update(func(tx *bolt.Tx) error { + for _, bktName := range topBuckets { + if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { + return errors.Wrapf(e, "failed to create top level bucket %s", bktName) + } + } + return nil + }) + + if err != nil { + return nil, errors.Wrap(err, "failed to create top level bucket)") + } + + result.dbs[site.SiteID] = db + log.Printf("[DEBUG] bolt store created for %s", site.SiteID) + } + return &result, nil +} + +// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket +func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { + + bdb, err := b.db(comment.Locator.SiteID) + if err != nil { + return "", err + } + + if b.checkFlag(FlagRequest{Locator: comment.Locator, Flag: ReadOnly}) { + return "", errors.Errorf("post %s is read-only", comment.Locator.URL) + } + + err = bdb.Update(func(tx *bolt.Tx) (err error) { + var postBkt, lastBkt, userBkt *bolt.Bucket + + if postBkt, err = b.makePostBucket(tx, comment.Locator.URL); err != nil { + return err + } + // check if key already in store, reject doubles + if postBkt.Get([]byte(comment.ID)) != nil { + return errors.Errorf("key %s already in store", comment.ID) + } + + // serialize comment to json []byte for bolt and save + if err = b.save(postBkt, comment.ID, comment); err != nil { + return errors.Wrapf(err, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL) + } + + ref := b.makeRef(comment) // reference combines url and comment id + + // add reference to comment to "last" bucket + lastBkt = tx.Bucket([]byte(lastBucketName)) + commentTs := []byte(comment.Timestamp.Format(tsNano)) + if err = lastBkt.Put(commentTs, ref); err != nil { + return errors.Wrapf(err, "can't put reference %s to %s", ref, lastBucketName) + } + + // add reference to commentID to "users" bucket + if userBkt, err = b.getUserBucket(tx, comment.User.ID); err != nil { + return errors.Wrapf(err, "can't get bucket %s", comment.User.ID) + } + // put into individual user's bucket with ts as a key + if err = userBkt.Put(commentTs, ref); err != nil { + return errors.Wrapf(err, "failed to put user comment %s for %s", comment.ID, comment.User.ID) + } + + // set info with the count for post url + if _, err = b.setInfo(tx, comment); err != nil { + return errors.Wrapf(err, "failed to set info for %s", comment.Locator) + } + return nil + }) + + return comment.ID, err +} + +// Get returns comment for locator.URL and commentID string +func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { + + bdb, err := b.db(locator.SiteID) + if err != nil { + return comment, err + } + + err = bdb.View(func(tx *bolt.Tx) error { + bucket, e := b.getPostBucket(tx, locator.URL) + if e != nil { + return e + } + return b.load(bucket, commentID, &comment) + }) + return comment, err +} + +// Find returns all comments for given request and sorts results +func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) { + comments = []store.Comment{} + + bdb, err := b.db(req.Locator.SiteID) + if err != nil { + return nil, err + } + + switch { + case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url + err = bdb.View(func(tx *bolt.Tx) error { + + bucket, e := b.getPostBucket(tx, req.Locator.URL) + if e != nil { + return e + } + + return bucket.ForEach(func(k, v []byte) error { + comment := store.Comment{} + if e = json.Unmarshal(v, &comment); e != nil { + return errors.Wrap(e, "failed to unmarshal") + } + comments = append(comments, comment) + return nil + }) + }) + case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site + comments, err = b.lastComments(req.Locator.SiteID, req.Limit, req.Since) + case req.Locator.SiteID != "" && req.UserID != "": // find comments for user + comments, err = b.userComments(req.Locator.SiteID, req.UserID, req.Limit, req.Skip) + } + + if err != nil { + return nil, err + } + return SortComments(comments, req.Sort), nil +} + +// Flag sets and gets flag values +func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) { + if req.Update == FlagNonSet { // read flag value, no update requested + return b.checkFlag(req), nil + } + + // write flag value + return b.setFlag(req) +} + +// Update for locator.URL with mutable part of comment +func (b *BoltDB) Update(locator store.Locator, comment store.Comment) error { + + if curComment, err := b.Get(locator, comment.ID); err == nil { + // preserve immutable fields + comment.ParentID = curComment.ParentID + comment.Locator = curComment.Locator + comment.Timestamp = curComment.Timestamp + comment.User = curComment.User + } + + bdb, err := b.db(locator.SiteID) + if err != nil { + return err + } + + return bdb.Update(func(tx *bolt.Tx) error { + bucket, e := b.getPostBucket(tx, locator.URL) + if e != nil { + return e + } + return b.save(bucket, comment.ID, comment) + }) +} + +// Count returns number of comments for post or user +func (b *BoltDB) Count(req FindRequest) (count int, err error) { + + bdb, err := b.db(req.Locator.SiteID) + if err != nil { + return 0, err + } + + if req.Locator.URL != "" { // comment's count for post + err = bdb.View(func(tx *bolt.Tx) error { + var e error + count, e = b.count(tx, req.Locator.URL, 0) + return e + }) + return count, err + } + + if req.UserID != "" { // comment's count for user + err = bdb.View(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(userBucketName)) + userIDBkt := usersBkt.Bucket([]byte(req.UserID)) + if userIDBkt == nil { + return errors.Errorf("no comments for user %s in store for %s site", req.UserID, req.Locator.SiteID) + } + stats := userIDBkt.Stats() + count = stats.KeyN + return nil + }) + return count, err + } + + return 0, errors.Errorf("invalid count request %+v", req) +} + +// Info get post(s) meta info +func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) { + + bdb, err := b.db(req.Locator.SiteID) + if err != nil { + return []store.PostInfo{}, err + } + + if req.Locator.URL != "" { // post info + info := store.PostInfo{} + err = bdb.View(func(tx *bolt.Tx) error { + infoBkt := tx.Bucket([]byte(infoBucketName)) + if e := b.load(infoBkt, req.Locator.URL, &info); e != nil { + return errors.Wrapf(e, "can't load info for %s", req.Locator.URL) + } + return nil + }) + + // set read-only from age and manual bucket + readOnlyAge := req.ReadOnlyAge + info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now()) + if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) { + info.ReadOnly = true + } + return []store.PostInfo{info}, err + } + + if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list) + list := []store.PostInfo{} + err = bdb.View(func(tx *bolt.Tx) error { + postsBkt := tx.Bucket([]byte(postsBucketName)) + + c := postsBkt.Cursor() + n := 0 + for k, _ := c.Last(); k != nil; k, _ = c.Prev() { + n++ + if req.Skip > 0 && n <= req.Skip { + continue + } + postURL := string(k) + infoBkt := tx.Bucket([]byte(infoBucketName)) + info := store.PostInfo{} + if e := b.load(infoBkt, postURL, &info); e != nil { + return errors.Wrapf(e, "can't load info for %s", postURL) + } + list = append(list, info) + if req.Limit > 0 && len(list) >= req.Limit { + break + } + } + return nil + }) + return list, err + } + + return nil, errors.Errorf("invalid info request %+v", req) +} + +// ListFlags get list of flagged keys, like blocked & verified user +// works for full locator (post flags) or with userID +func (b *BoltDB) ListFlags(siteID string, flag Flag) (res []interface{}, err error) { + + bdb, e := b.db(siteID) + if e != nil { + return nil, e + } + + switch flag { + case Verified: + err = bdb.View(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(verifiedBucketName)) + _ = usersBkt.ForEach(func(k, _ []byte) error { + res = append(res, string(k)) + return nil + }) + return nil + }) + return res, err + case Blocked: + err = bdb.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + return bucket.ForEach(func(k []byte, v []byte) error { + ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local) + if errParse != nil { + return errors.Wrap(errParse, "can't parse block ts") + } + if time.Now().Before(ts) { + // get user name from comment user section + userName := "" + req := FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: string(k), Limit: 1} + userComments, errUser := b.Find(req) + if errUser == nil && len(userComments) > 0 { + userName = userComments[0].User.Name + } + res = append(res, store.BlockedUser{ID: string(k), Name: userName, Until: ts}) + } + return nil + }) + }) + return res, err + } + return nil, errors.Errorf("flag %s not listable", flag) +} + +// Delete post(s) by id or by userID +func (b *BoltDB) Delete(req DeleteRequest) error { + + bdb, e := b.db(req.Locator.SiteID) + if e != nil { + return e + } + + switch { + case req.Locator.URL != "" && req.CommentID != "": + return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode) + case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "": + return b.deleteUser(bdb, req.Locator.SiteID, req.UserID) + case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "": + return b.deleteAll(bdb, req.Locator.SiteID) + } + + return errors.Errorf("invalid delete request %+v", req) +} + +// Close boltdb store +func (b *BoltDB) Close() error { + errs := new(multierror.Error) + for site, db := range b.dbs { + err := errors.Wrapf(db.Close(), "can't close site %s", site) + errs = multierror.Append(errs, err) + } + return errs.ErrorOrNil() +} + +// Last returns up to max last comments for given siteID +func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) { + + comments = []store.Comment{} + + if max > lastLimit || max == 0 { + max = lastLimit + } + + bdb, err := b.db(siteID) + if err != nil { + return nil, err + } + + err = bdb.View(func(tx *bolt.Tx) error { + lastBkt := tx.Bucket([]byte(lastBucketName)) + c := lastBkt.Cursor() + + for k, v := c.Last(); k != nil; k, v = c.Prev() { + + if !since.IsZero() { + // stop if reached "since" ts + tsSince := []byte(since.Format(tsNano)) + if bytes.Compare(k, tsSince) <= 0 { + break + } + } + url, commentID, e := b.parseRef(v) + if e != nil { + return e + } + postBkt, e := b.getPostBucket(tx, url) + if e != nil { + return e + } + + comment := store.Comment{} + if e = b.load(postBkt, commentID, &comment); e != nil { + log.Printf("[WARN] can't load comment for %s from store %s", commentID, url) + continue + } + if comment.Deleted { + continue + } + comments = append(comments, comment) + if len(comments) >= max { + break + } + } + return nil + }) + + return comments, err +} + +// userComments extracts all comments for given site and given userID +// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref +func (b *BoltDB) userComments(siteID, userID string, limit, skip int) (comments []store.Comment, err error) { + + comments = []store.Comment{} + commentRefs := []string{} + + bdb, err := b.db(siteID) + if err != nil { + return nil, err + } + + if limit == 0 || limit > userLimit { + limit = userLimit + } + + // get list of references to comments + err = bdb.View(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(userBucketName)) + userIDBkt := usersBkt.Bucket([]byte(userID)) + if userIDBkt == nil { + return errors.Errorf("no comments for user %s in store", userID) + } + + c := userIDBkt.Cursor() + skipComments := 0 + for k, v := c.Last(); k != nil; k, v = c.Prev() { + if len(commentRefs) >= limit { + break + } + if skip > 0 && skipComments < skip { + skipComments++ + continue + } + commentRefs = append(commentRefs, string(v)) + } + return nil + }) + + if err != nil { + return comments, err + } + + // retrieve comments for refs + for _, v := range commentRefs { + url, commentID, errParse := b.parseRef([]byte(v)) + if errParse != nil { + return comments, errors.Wrapf(errParse, "can't parse reference %s", v) + } + if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil { + comments = append(comments, c) + } + } + + return comments, err +} + +func (b *BoltDB) checkFlag(req FlagRequest) (val bool) { + + bdb, err := b.db(req.Locator.SiteID) + if err != nil { + return false + } + + key := req.Locator.URL + if req.UserID != "" { + key = req.UserID + } + + if req.Flag == Blocked { + var blocked bool + _ = bdb.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(blocksBucketName)) + v := bucket.Get([]byte(key)) + if v == nil { + blocked = false + return nil + } + + until, e := time.Parse(tsNano, string(v)) + if e != nil { + blocked = false + return nil + } + blocked = time.Now().Before(until) + return nil + }) + return blocked + } + + _ = bdb.View(func(tx *bolt.Tx) error { + var bucket *bolt.Bucket + if bucket, err = b.flagBucket(tx, req.Flag); err != nil { + return err + } + val = bucket.Get([]byte(key)) != nil + return nil + }) + return val +} + +func (b *BoltDB) setFlag(req FlagRequest) (res bool, err error) { + bdb, e := b.db(req.Locator.SiteID) + if e != nil { + return false, e + } + + key := req.Locator.URL + if req.UserID != "" { + key = req.UserID + } + + err = bdb.Update(func(tx *bolt.Tx) error { + var bucket *bolt.Bucket + if bucket, err = b.flagBucket(tx, req.Flag); err != nil { + return err + } + switch req.Update { + case FlagTrue: + if req.Flag == Blocked { + val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year + if req.TTL > 0 { + val = time.Now().Add(req.TTL).Format(tsNano) + } + if e = bucket.Put([]byte(key), []byte(val)); e != nil { + return errors.Wrapf(e, "failed to put blocked to %s", key) + } + res = true + return nil + } + + if e = bucket.Put([]byte(key), []byte(time.Now().Format(tsNano))); e != nil { + return errors.Wrapf(e, "failed to set flag %s for %s", req.Flag, req.Locator.URL) + } + res = true + return nil + case FlagFalse: + if e = bucket.Delete([]byte(key)); e != nil { + return errors.Wrapf(e, "failed to clean flag %s for %s", req.Flag, req.Locator.URL) + } + res = false + } + return nil + }) + + return res, err +} + +func (b *BoltDB) flagBucket(tx *bolt.Tx, flag Flag) (bkt *bolt.Bucket, err error) { + switch flag { + case ReadOnly: + bkt = tx.Bucket([]byte(readonlyBucketName)) + case Blocked: + bkt = tx.Bucket([]byte(blocksBucketName)) + case Verified: + bkt = tx.Bucket([]byte(verifiedBucketName)) + default: + return nil, errors.Errorf("unsupported flag %v", flag) + } + return bkt, nil +} + +func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID string, mode store.DeleteMode) error { + + return bdb.Update(func(tx *bolt.Tx) error { + + postBkt, e := b.getPostBucket(tx, locator.URL) + if e != nil { + return e + } + + comment := store.Comment{} + if e = b.load(postBkt, commentID, &comment); e != nil { + return errors.Wrapf(e, "can't load key %s from bucket %s", commentID, locator.URL) + } + // set deleted status and clear fields + comment.SetDeleted(mode) + + if e = b.save(postBkt, commentID, comment); e != nil { + return errors.Wrapf(e, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) + } + + // delete from "last" bucket + lastBkt := tx.Bucket([]byte(lastBucketName)) + if e = lastBkt.Delete([]byte(commentID)); e != nil { + return errors.Wrapf(e, "can't delete key %s from bucket %s", commentID, lastBucketName) + } + + // decrement comments count for post url + if _, e = b.count(tx, comment.Locator.URL, -1); e != nil { + return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator) + } + + return nil + }) +} + +// deleteAll removes all top-level buckets for given siteID +func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error { + + // delete all buckets except blocked users + toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName} + + // delete top-level buckets + err := bdb.Update(func(tx *bolt.Tx) error { + for _, bktName := range toDelete { + + if e := tx.DeleteBucket([]byte(bktName)); e != nil { + return errors.Wrapf(e, "failed to delete top level bucket %s", bktName) + } + if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { + return errors.Wrapf(e, "failed to create top level bucket %s", bktName) + } + } + return nil + }) + + return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID) +} + +// deleteUser removes all comments for given user. Everything will be market as deleted +// and user name and userID will be changed to "deleted". Also removes from last and from user buckets. +func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string) error { + bdb, err := b.db(siteID) + if err != nil { + return err + } + + // get list of all comments outside of transaction loop + posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}}) + if err != nil { + return err + } + + type commentInfo struct { + locator store.Locator + commentID string + } + + // get list of commentID for all user's comment + comments := []commentInfo{} + for _, postInfo := range posts { + err = bdb.View(func(tx *bolt.Tx) error { + postsBkt := tx.Bucket([]byte(postsBucketName)) + postBkt := postsBkt.Bucket([]byte(postInfo.URL)) + err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error { + comment := store.Comment{} + if err = json.Unmarshal(commentVal, &comment); err != nil { + return errors.Wrap(err, "failed to unmarshal") + } + if comment.User.ID == userID { + comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID}) + } + return nil + }) + return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL) + }) + if err != nil { + return err + } + } + + log.Printf("[DEBUG] comments for removal=%d", len(comments)) + + // delete collected comments + for _, ci := range comments { + if e := b.deleteComment(bdb, ci.locator, ci.commentID, store.HardDelete); e != nil { + return errors.Wrapf(err, "failed to delete comment %+v", ci) + } + } + + // delete user bucket + err = bdb.Update(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(userBucketName)) + if usersBkt != nil { + if e := usersBkt.DeleteBucket([]byte(userID)); e != nil { + return errors.Wrapf(err, "failed to delete user bucket for %s", userID) + } + } + return nil + }) + + if err != nil { + return errors.Wrap(err, "can't delete user meta") + } + + if len(comments) == 0 { + return errors.Errorf("unknown user %s", userID) + } + + return err +} + +// getPostBucket return bucket with all comments for postURL +func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) { + postsBkt := tx.Bucket([]byte(postsBucketName)) + if postsBkt == nil { + return nil, errors.Errorf("no bucket %s", postsBucketName) + } + res := postsBkt.Bucket([]byte(postURL)) + if res == nil { + return nil, errors.Errorf("no bucket %s in store", postURL) + } + return res, nil +} + +// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post. +func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) { + postsBkt := tx.Bucket([]byte(postsBucketName)) + if postsBkt == nil { + return nil, errors.Errorf("no bucket %s", postsBucketName) + } + res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL)) + if err != nil { + return nil, errors.Wrapf(err, "no bucket %s in store", postURL) + } + return res, nil +} + +func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) { + usersBkt := tx.Bucket([]byte(userBucketName)) + userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID + if e != nil { + return nil, errors.Wrapf(e, "can't get bucket %s", userID) + } + return userIDBkt, nil +} + +// save marshaled value to key for bucket. Should run in update tx +func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) { + if value == nil { + return errors.Errorf("can't save nil value for %s", key) + } + jdata, jerr := json.Marshal(value) + if jerr != nil { + return errors.Wrap(jerr, "can't marshal comment") + } + if err = bkt.Put([]byte(key), jdata); err != nil { + return errors.Wrapf(err, "failed to save key %s", key) + } + return nil +} + +// load and unmarshal json value by key from bucket. Should run in view tx +func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error { + value := bkt.Get([]byte(key)) + if value == nil { + return errors.Errorf("no value for %s", key) + } + + if err := json.Unmarshal(value, &res); err != nil { + return errors.Wrap(err, "failed to unmarshal") + } + return nil +} + +// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor +// it uses separate counts bucket because boltdb Stat call is very slow +func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) { + + infoBkt := tx.Bucket([]byte(infoBucketName)) + + info := store.PostInfo{} + if err := b.load(infoBkt, postURL, &info); err != nil { + info = store.PostInfo{} + } + if val == 0 { // get current count, don't update + return info.Count, nil + } + info.Count += val + + return info.Count, b.save(infoBkt, postURL, &info) +} + +func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) { + infoBkt := tx.Bucket([]byte(infoBucketName)) + info := store.PostInfo{} + if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil { + info = store.PostInfo{ + Count: 0, + URL: comment.Locator.URL, + FirstTS: comment.Timestamp, + LastTS: comment.Timestamp, + } + } + info.Count++ + info.LastTS = comment.Timestamp + return info, b.save(infoBkt, comment.Locator.URL, &info) +} + +func (b *BoltDB) db(siteID string) (*bolt.DB, error) { + if res, ok := b.dbs[siteID]; ok { + return res, nil + } + return nil, errors.Errorf("site %q not found", siteID) +} + +// makeRef creates reference combining url and comment id +func (b *BoltDB) makeRef(comment store.Comment) []byte { + return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID)) +} + +// parseRef gets parts of reference +func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) { + elems := strings.Split(string(val), "!!") + if len(elems) != 2 { + return "", "", errors.Errorf("invalid reference value %s", string(val)) + } + return elems[0], elems[1], nil +} diff --git a/backend/app/store/engine2/bolt_test.go b/backend/app/store/engine2/bolt_test.go new file mode 100644 index 0000000000..02db81100f --- /dev/null +++ b/backend/app/store/engine2/bolt_test.go @@ -0,0 +1,774 @@ +package engine2 + +import ( + "fmt" + "os" + "testing" + "time" + + bolt "github.com/coreos/bbolt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark/backend/app/store" +) + +var testDb = "/tmp/test-remark.db" + +func TestBoltDB_CreateAndFind(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"} + res, err := b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, `some text, link`, res[0].Text) + assert.Equal(t, "user1", res[0].User.ID) + t.Log(res[0].ID) + + _, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) + assert.NotNil(t, err) + assert.Equal(t, "key id-1 already in store", err.Error()) + + req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, Sort: "time"} + _, err = b.Find(req) + assert.EqualError(t, err, `site "radio-t-bad" not found`) + + assert.NoError(t, b.Close()) +} + +func TestBoltDB_CreateFailedReadOnly(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + comment := store.Comment{ + ID: "id-ro", + Text: `some text, link`, + Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + + flagReq := FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagTrue} + v, err := b.Flag(flagReq) + require.NoError(t, err) + assert.Equal(t, true, v) + + _, err = b.Create(comment) + assert.NotNil(t, err) + assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error()) + + flagReq = FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagFalse} + v, err = b.Flag(flagReq) + require.NoError(t, err) + assert.Equal(t, false, v) + + _, err = b.Create(comment) + assert.NoError(t, err) +} + +func TestBoltDB_Get(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"} + res, err := b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 2, len(res), "2 records initially") + + comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) + assert.NoError(t, err) + assert.Equal(t, "some text2", comment.Text) + + comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") + assert.NotNil(t, err) + + _, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID) + assert.EqualError(t, err, `site "bad" not found`) +} + +func TestBoltDB_Update(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"} + res, err := b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 2, len(res), "2 records initially") + + comment := res[0] + comment.Text = "abc 123" + comment.Score = 100 + err = b.Update(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, comment) + assert.NoError(t, err) + + comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + assert.NoError(t, err) + assert.Equal(t, "abc 123", comment.Text) + assert.Equal(t, res[0].ID, comment.ID) + assert.Equal(t, 100, comment.Score) + + err = b.Update(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment) + assert.EqualError(t, err, `site "bad" not found`) + + err = b.Update(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment) + assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`) +} + +func TestBoltDB_FindLast(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"} + res, err := b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "some text2", res[0].Text) + + req.Limit = 1 + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + assert.Equal(t, "some text2", res[0].Text) + + req.Locator.SiteID = "bad" + _, err = b.Find(req) + assert.EqualError(t, err, `site "bad" not found`) +} + +func TestBoltDB_FindLastSince(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local) + req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts} + res, err := b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "some text2", res[0].Text) + + req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local) + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + assert.Equal(t, "some text2", res[0].Text) + + req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local) + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 0, len(res)) +} + +func TestBoltDB_FindForUser(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5} + res, err := b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "some text2", res[0].Text, "sorted by -time") + + req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1} + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 1, len(res), "allow 1 comment") + assert.Equal(t, "some text2", res[0].Text, "sorted by -time") + + req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1} + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 1, len(res), "allow 1 comment") + assert.Equal(t, `some text, link`, res[0].Text, "second comment") + + req = FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1} + _, err = b.Find(req) + assert.EqualError(t, err, `site "bad" not found`) + + req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1} + _, err = b.Find(req) + assert.EqualError(t, err, `no comments for user userZ in store`) +} + +func TestBoltDB_FindForUserPagination(t *testing.T) { + _ = os.Remove(testDb) + b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"}) + require.Nil(t, err) + + defer func() { + require.NoError(t, b.Close()) + _ = os.Remove(testDb) + }() + + c := store.Comment{ + Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + + // write 200 comments + for i := 0; i < 200; i++ { + c.ID = fmt.Sprintf("id-%d", i) + c.Text = fmt.Sprintf("text #%d", i) + c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local) + _, err = b.Create(c) + require.Nil(t, err) + } + + // get all comments + req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"} + res, err := b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 200, len(res)) + assert.Equal(t, "id-199", res[0].ID) + + // seek 0, 5 comments + req.Limit = 5 + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 5, len(res)) + assert.Equal(t, "id-199", res[0].ID) + assert.Equal(t, "id-195", res[4].ID) + + // seek 10, 3 comments + req.Skip, req.Limit = 10, 3 + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 3, len(res)) + assert.Equal(t, "id-189", res[0].ID) + assert.Equal(t, "id-187", res[2].ID) + + // seek 195, ask 10 comments + req.Skip, req.Limit = 195, 10 + res, err = b.Find(req) + assert.NoError(t, err) + assert.Equal(t, 5, len(res)) + assert.Equal(t, "id-4", res[0].ID) + assert.Equal(t, "id-0", res[4].ID) + + // seek 255, ask 10 comments + req.Skip, req.Limit = 255, 10 + res, err = b.Find(req) + assert.NoError(t, err) + assert.Nil(t, err) + assert.Equal(t, 0, len(res)) +} + +func TestBoltDB_CountPost(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}} + c, err := b.Count(req) + assert.NoError(t, err) + assert.Equal(t, 2, c) + + req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}} + c, err = b.Count(req) + assert.NoError(t, err) + assert.Equal(t, 0, c) + + req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}} + _, err = b.Count(req) + assert.EqualError(t, err, `site "bad" not found`) +} + +func TestBoltDB_CountUser(t *testing.T) { + var b, teardown = prep(t) + defer teardown() + + req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"} + c, err := b.Count(req) + assert.NoError(t, err) + assert.Equal(t, 2, c) + + req = FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"} + _, err = b.Count(req) + assert.EqualError(t, err, `site "bad" not found`) + + req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"} + _, err = b.Count(req) + assert.EqualError(t, err, `no comments for user userZ in store for radio-t site`) +} + +func TestBoltDB_InfoPost(t *testing.T) { + b, teardown := prep(t) // two comments for https://radio-t.com + defer teardown() + + ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) } + + // add one more for https://radio-t.com/2 + comment := store.Comment{ + ID: "12345", + Text: `some text, link`, + Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + _, err := b.Create(comment) + assert.NoError(t, err) + + req := InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0} + r, err := b.Info(req) + require.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r) + + req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10} + r, err = b.Info(req) + require.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), + ReadOnly: true}}, r) + + req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0} + r, err = b.Info(req) + require.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r) + + req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0} + _, err = b.Info(req) + require.NotNil(t, err) + + req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0} + _, err = b.Info(req) + require.NotNil(t, err) + + fr := FlagRequest{Flag: ReadOnly, Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: FlagTrue} + _, err = b.Flag(fr) + require.NoError(t, err) + req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0} + r, err = b.Info(req) + require.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), + ReadOnly: true}}, r) +} + +func TestBoltDB_InfoList(t *testing.T) { + b, teardown := prep(t) // two comments for https://radio-t.com + defer teardown() + + // add one more for https://radio-t.com/2 + comment := store.Comment{ + ID: "12345", + Text: `some text, link`, + Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + _, err := b.Create(comment) + assert.Nil(t, err) + + ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) } + + req := InfoRequest{Locator: store.Locator{SiteID: "radio-t"}} + res, err := b.Info(req) + assert.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, + {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) + + req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1} + res, err = b.Info(req) + assert.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, + {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) + + req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1} + res, err = b.Info(req) + assert.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res) + + req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1} + res, err = b.Info(req) + assert.Nil(t, err) + assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) + + req = InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1} + _, err = b.Info(req) + assert.EqualError(t, err, `site "bad" not found`) +} + +func TestBolt_FlagBlockedUser(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + req := FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"} + val, err := b.Flag(req) + assert.NoError(t, err) + assert.False(t, val, "nothing blocked yet") + + req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue} + _, err = b.Flag(req) + assert.NoError(t, err) + val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}) + assert.NoError(t, err) + assert.True(t, val, "user1 blocked") + + req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue} + _, err = b.Flag(req) + assert.NoError(t, err) + val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}) + assert.NoError(t, err) + assert.True(t, val, "user1 still blocked") + + req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagFalse} + _, err = b.Flag(req) + assert.NoError(t, err) + val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}) + assert.NoError(t, err) + assert.False(t, val, "user1 unblocked") + + req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}, UserID: "user1", Update: FlagTrue} + _, err = b.Flag(req) + assert.EqualError(t, err, `site "bad" not found`) + + req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "userX", Update: FlagTrue} + _, err = b.Flag(req) + assert.NoError(t, err, "non-existing user can't be blocked") + + req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"} + val, err = b.Flag(req) + assert.NoError(t, err) + assert.False(t, val, "nothing blocked on wrong site") +} + +func TestBolt_FlagReadOnlyPost(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + req := FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly} + val, err := b.Flag(req) + assert.NoError(t, err) + assert.False(t, val, "nothing ro") + + req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagTrue} + val, err = b.Flag(req) + assert.NoError(t, err) + req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly} + val, err = b.Flag(req) + assert.NoError(t, err) + assert.True(t, val, "url-1 ro") + + req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: ReadOnly} + val, err = b.Flag(req) + assert.NoError(t, err) + assert.False(t, val, "url-2 still writable") + + req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse} + _, err = b.Flag(req) + assert.NoError(t, err) + req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly} + val, err = b.Flag(req) + assert.NoError(t, err) + assert.False(t, val, "url-1 writable") + + req = FlagRequest{Locator: store.Locator{SiteID: "bad", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse} + _, err = b.Flag(req) + assert.EqualError(t, err, `site "bad" not found`) + + req = FlagRequest{Locator: store.Locator{SiteID: "radio-t-bad", URL: "url-1"}, Flag: ReadOnly} + val, err = b.Flag(req) + assert.NoError(t, err) + assert.False(t, val, "nothing ro on wrong site") +} + +func TestBolt_FlagVerified(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + isVerified := func(site, user string) bool { + req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user} + v, err := b.Flag(req) + require.NoError(t, err) + return v + } + + setVerified := func(site, user string, status FlagStatus) error { + req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status} + _, err := b.Flag(req) + return err + } + + assert.False(t, isVerified("radio-t", "u1"), "nothing verified") + + assert.NoError(t, setVerified("radio-t", "u1", FlagTrue)) + assert.True(t, isVerified("radio-t", "u1"), "u1 verified") + + assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified") + assert.NoError(t, setVerified("radio-t", "u1", FlagFalse)) + assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore") + + assert.EqualError(t, setVerified("bad", "u1", FlagTrue), `site "bad" not found`) + assert.NoError(t, setVerified("radio-t", "u1xyz", FlagFalse)) + + assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site") + + assert.NoError(t, setVerified("radio-t", "u1", FlagTrue)) + assert.NoError(t, setVerified("radio-t", "u2", FlagTrue)) + assert.NoError(t, setVerified("radio-t", "u3", FlagFalse)) +} + +func TestBolt_FlagListVerified(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + toIDs := func(inp []interface{}) (res []string) { + res = make([]string, len(inp)) + for i, v := range inp { + vv, ok := v.(string) + require.True(t, ok) + res[i] = vv + } + return res + } + + setVerified := func(site, user string, status FlagStatus) error { + req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status} + _, err := b.Flag(req) + return err + } + + ids, err := b.ListFlags("radio-t", Verified) + assert.NoError(t, err) + assert.Equal(t, []string{}, toIDs(ids), "verified list empty") + + assert.NoError(t, setVerified("radio-t", "u1", FlagTrue)) + assert.NoError(t, setVerified("radio-t", "u2", FlagTrue)) + ids, err = b.ListFlags("radio-t", Verified) + assert.NoError(t, err) + assert.Equal(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids") + + _, err = b.ListFlags("radio-t-bad", Verified) + assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site") +} + +func TestBolt_FlagListBlocked(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + setBlocked := func(site, user string, status FlagStatus, ttl time.Duration) error { + req := FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status, TTL: ttl} + _, err := b.Flag(req) + return err + } + + toBlocked := func(inp []interface{}) (res []store.BlockedUser) { + res = make([]store.BlockedUser, len(inp)) + for i, v := range inp { + vv, ok := v.(store.BlockedUser) + require.True(t, ok) + res[i] = vv + } + return res + } + assert.NoError(t, setBlocked("radio-t", "user1", FlagTrue, 0)) + assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 50*time.Millisecond)) + assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0)) + + vv, err := b.ListFlags("radio-t", Blocked) + assert.NoError(t, err) + + blockedList := toBlocked(vv) + assert.Equal(t, 2, len(blockedList)) + assert.Equal(t, "user1", blockedList[0].ID) + assert.Equal(t, "user2", blockedList[1].ID) + t.Logf("%+v", blockedList) + + // check block expiration + time.Sleep(50 * time.Millisecond) + vv, err = b.ListFlags("radio-t", Blocked) + assert.NoError(t, err) + blockedList = toBlocked(vv) + assert.Equal(t, 1, len(blockedList)) + assert.Equal(t, "user1", blockedList[0].ID) + + _, err = b.ListFlags("bad", Blocked) + assert.EqualError(t, err, `site "bad" not found`) + +} + +func TestBolt_DeleteComment(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"} + res, err := b.Find(reqReq) + assert.NoError(t, err) + assert.Equal(t, 2, len(res), "initially 2 comments") + + count, err := b.Count(reqReq) + require.NoError(t, err) + assert.Equal(t, 2, count, "count=2 initially") + + delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + CommentID: res[0].ID, DeleteMode: store.SoftDelete} + + err = b.Delete(delReq) + assert.NoError(t, err) + + res, err = b.Find(reqReq) + assert.NoError(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "", res[0].Text) + assert.True(t, res[0].Deleted, "marked deleted") + assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) + + assert.Equal(t, "some text2", res[1].Text) + assert.False(t, res[1].Deleted) + + comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10}) + assert.NoError(t, err) + assert.Equal(t, 1, len(comments), "1 in last, 1 removed") + + count, err = b.Count(reqReq) + require.NoError(t, err) + assert.Equal(t, 1, count) + + delReq.CommentID = "123456" + err = b.Delete(delReq) + assert.NotNil(t, err) + + delReq.Locator.SiteID = "bad" + delReq.CommentID = res[0].ID + err = b.Delete(delReq) + assert.EqualError(t, err, `site "bad" not found`) + + delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"} + err = b.Delete(delReq) + assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`) +} + +func TestBolt_DeleteHard(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"} + res, err := b.Find(reqReq) + assert.NoError(t, err) + assert.Equal(t, 2, len(res), "initially 2 comments") + + delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + CommentID: res[0].ID, DeleteMode: store.HardDelete} + err = b.Delete(delReq) + assert.NoError(t, err) + + res, err = b.Find(reqReq) + assert.NoError(t, err) + assert.Equal(t, 2, len(res)) + assert.Equal(t, "", res[0].Text) + assert.True(t, res[0].Deleted, "marked deleted") + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) +} + +func TestBolt_DeleteAll(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + delReq := DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}} + err := b.Delete(delReq) + assert.NoError(t, err) + + comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10}) + assert.NoError(t, err) + assert.Equal(t, 0, len(comments), "nothing left") + + delReq = DeleteRequest{Locator: store.Locator{SiteID: "bad"}} + err = b.Delete(delReq) + assert.EqualError(t, err, `site "bad" not found`) +} + +func TestBoltAdmin_DeleteUser(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}) + require.NoError(t, err) + + comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"}) + assert.NoError(t, err) + assert.Equal(t, 2, len(comments), "2 comments with deleted info") + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User) + assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User) + + c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}}) + assert.NoError(t, err) + assert.Equal(t, 0, c, "0 count") + + _, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5}) + assert.EqualError(t, err, "no comments for user user1 in store") + + comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"}) + assert.Nil(t, err) + assert.Equal(t, 0, len(comments), "nothing left") + + err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"}) + assert.EqualError(t, err, `site "radio-t-bad" not found`) +} + +func TestBoltDB_ref(t *testing.T) { + b := BoltDB{} + comment := store.Comment{ + ID: "12345", + Text: `some text, link`, + Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + res := b.makeRef(comment) + assert.Equal(t, "https://radio-t.com/2!!12345", string(res)) + + url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345")) + assert.NoError(t, err) + assert.Equal(t, "https://radio-t.com/2", url) + assert.Equal(t, "12345", id) + + _, _, err = b.parseRef([]byte("https://radio-t.com/2")) + assert.NotNil(t, err) +} + +func TestBoltDB_NewFailed(t *testing.T) { + _, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"}) + assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory") +} + +// makes new boltdb, put two records +func prep(t *testing.T) (b *BoltDB, teardown func()) { + _ = os.Remove(testDb) + + boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"}) + assert.Nil(t, err) + b = boltStore + + comment := store.Comment{ + ID: "id-1", + Text: `some text, link`, + Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + _, err = b.Create(comment) + assert.Nil(t, err) + + comment = store.Comment{ + ID: "id-2", + Text: "some text2", + Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, + User: store.User{ID: "user1", Name: "user name"}, + } + _, err = b.Create(comment) + assert.Nil(t, err) + + teardown = func() { + require.NoError(t, b.Close()) + _ = os.Remove(testDb) + } + return b, teardown +} diff --git a/backend/app/store/engine2/engine.go b/backend/app/store/engine2/engine.go new file mode 100644 index 0000000000..eb3a6e8c65 --- /dev/null +++ b/backend/app/store/engine2/engine.go @@ -0,0 +1,128 @@ +package engine2 + +// Package engine defines interfaces each supported storage should implement. +// Includes default implementation with boltdb + +import ( + "sort" + "strings" + "time" + + "github.com/umputun/remark/backend/app/store" +) + +// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate +//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go" + +// Interface defines methods provided by low-level storage engine +type Interface interface { + Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id + Update(locator store.Locator, comment store.Comment) error // update comment, mutable parts only + Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id + Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site + Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info + Count(req FindRequest) (int, error) // get count for post or user + Delete(req DeleteRequest) error // delete post(s) by id or by userID + Flag(req FlagRequest) (bool, error) // set and get flags + ListFlags(siteID string, flag Flag) ([]interface{}, error) // get list of flagged keys, like blocked & verified user + Close() error // close storage engine +} + +// FindRequest is the input for all find operations +type FindRequest struct { + Locator store.Locator // lack of URL means site operation + UserID string // presence of UserID treated as user-related find + Sort string // sort order with +/-field syntax + Since time.Time // time limit for found results + Limit, Skip int +} + +// InfoRequest is the input of Info operation used to get meta data about posts +type InfoRequest struct { + Locator store.Locator + Limit, Skip int + ReadOnlyAge int +} + +type DeleteRequest struct { + Locator store.Locator // lack of URL means site operation + CommentID string + UserID string + DeleteMode store.DeleteMode +} + +// Flag defines type of binary attribute +type Flag string + +// FlagStatus represents values of the flag update +type FlagStatus int + +// enum of update values +const ( + FlagNonSet FlagStatus = 0 + FlagTrue FlagStatus = 1 + FlagFalse FlagStatus = -1 +) + +// Enum of all flags +const ( + ReadOnly = Flag("readonly") + Verified = Flag("verified") + Blocked = Flag("blocked") +) + +// FlagRequest is the input for both get/set for flags, like blocked, verified and so on +type FlagRequest struct { + Flag Flag // flag type + Locator store.Locator // post locator + UserID string // for flags setting user status + Update FlagStatus // if FlagNonSet it will be get op, if set will set the value + TTL time.Duration // ttl for time-sensitive flags only, like blocked for some period +} + +const ( + // limits + lastLimit = 1000 + userLimit = 500 +) + +// SortComments is for engines can't sort data internally +func SortComments(comments []store.Comment, sortFld string) []store.Comment { + sort.Slice(comments, func(i, j int) bool { + switch sortFld { + case "+time", "-time", "time", "+active", "-active", "active": + if strings.HasPrefix(sortFld, "-") { + return comments[i].Timestamp.After(comments[j].Timestamp) + } + return comments[i].Timestamp.Before(comments[j].Timestamp) + + case "+score", "-score", "score": + if strings.HasPrefix(sortFld, "-") { + if comments[i].Score == comments[j].Score { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Score > comments[j].Score + } + if comments[i].Score == comments[j].Score { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Score < comments[j].Score + + case "+controversy", "-controversy", "controversy": + if strings.HasPrefix(sortFld, "-") { + if comments[i].Controversy == comments[j].Controversy { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Controversy > comments[j].Controversy + } + if comments[i].Controversy == comments[j].Controversy { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Controversy < comments[j].Controversy + + default: + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + }) + return comments +} diff --git a/backend/app/store/engine2/engine_test.go b/backend/app/store/engine2/engine_test.go new file mode 100644 index 0000000000..a17585b9a6 --- /dev/null +++ b/backend/app/store/engine2/engine_test.go @@ -0,0 +1,55 @@ +package engine2 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/umputun/remark/backend/app/store" +) + +func TestEngine_sortComments(t *testing.T) { + cc := []store.Comment{ + {ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)}, + {ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)}, + {ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)}, + {ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)}, + } + + SortComments(cc, "+time") + assert.Equal(t, "1", cc[0].ID) + assert.Equal(t, "2", cc[1].ID) + assert.Equal(t, "3", cc[2].ID) + assert.Equal(t, "4", cc[3].ID) + + SortComments(cc, "-time") + assert.Equal(t, "4", cc[0].ID) + assert.Equal(t, "3", cc[1].ID) + assert.Equal(t, "2", cc[2].ID) + assert.Equal(t, "1", cc[3].ID) + + SortComments(cc, "score") + assert.Equal(t, "2", cc[0].ID) + assert.Equal(t, "1", cc[1].ID) + assert.Equal(t, "3", cc[2].ID) + assert.Equal(t, "4", cc[3].ID) + + SortComments(cc, "-score") + assert.Equal(t, "3", cc[0].ID) + assert.Equal(t, "4", cc[1].ID) + assert.Equal(t, "1", cc[2].ID) + assert.Equal(t, "2", cc[3].ID) + + SortComments(cc, "controversy") + assert.Equal(t, "1", cc[0].ID) + assert.Equal(t, "4", cc[1].ID) + assert.Equal(t, "2", cc[2].ID) + assert.Equal(t, "3", cc[3].ID) + + SortComments(cc, "-controversy") + assert.Equal(t, "3", cc[0].ID) + assert.Equal(t, "2", cc[1].ID) + assert.Equal(t, "1", cc[2].ID) + assert.Equal(t, "4", cc[3].ID) +} diff --git a/backend/app/store/remote/remote.go b/backend/app/store/remote/remote.go new file mode 100644 index 0000000000..ff805465df --- /dev/null +++ b/backend/app/store/remote/remote.go @@ -0,0 +1,134 @@ +package remote + +import ( + "bytes" + "encoding/json" + "net/http" + "time" + + "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/store" +) + +// Client implements remote engine and delegates all calls to remote http server +type Client struct { + API string + Client http.Client + AuthUser string + AuthPasswd string +} + +// Request encloses method name and all params +type Request struct { + Method string `json:"method"` + Params interface{} `json:"params"` +} + +// Response encloses result and error received from remote server +type Response struct { + Result *json.RawMessage `json:"result,omitempty"` + Error string `json:"error,omitempty"` +} + +// Create comment and return ID +func (r *Client) Create(comment store.Comment) (commentID string, err error) { + + resp, err := r.call("create", comment) + if err != nil { + return "", err + } + + err = json.Unmarshal(*resp.Result, &commentID) + return commentID, err +} + +// Get comment by ID +func (r *Client) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { + resp, err := r.call("get", locator, commentID) + if err != nil { + return store.Comment{}, err + } + + err = json.Unmarshal(*resp.Result, &comment) + return comment, err +} + +// Put updates comment, mutable parts only +func (r *Client) Put(locator store.Locator, comment store.Comment) error { + _, err := r.call("put", locator, comment) + return err +} + +// Find comments for locator +func (r *Client) Find(locator store.Locator, sort string) (comments []store.Comment, err error) { + resp, err := r.call("find", locator, sort) + if err != nil { + return []store.Comment{}, err + } + err = json.Unmarshal(*resp.Result, &comments) + return comments, err +} + +// Last comments for given site, sorted by time +func (r *Client) Last(siteID string, limit int, since time.Time) (comments []store.Comment, err error) { + resp, err := r.call("last", siteID, limit, since) + if err != nil { + return []store.Comment{}, err + } + err = json.Unmarshal(*resp.Result, &comments) + return comments, err +} + +// User get comments by user, sorted by time +func (r *Client) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) { + resp, err := r.call("user", siteID, userID, limit, skip) + if err != nil { + return []store.Comment{}, err + } + err = json.Unmarshal(*resp.Result, &comments) + return comments, err +} + +// UserCount gets comments count by user +func (r *Client) UserCount(siteID, userID string) (count int, err error) { + resp, err := r.call("user_count", siteID, userID) + if err != nil { + return 0, err + } + err = json.Unmarshal(*resp.Result, &count) + return count, err +} + +func (r *Client) call(method string, args ...interface{}) (*Response, error) { + + b, err := json.Marshal(Request{Method: method, Params: args}) + if err != nil { + return nil, errors.Wrapf(err, "marshaling failed for %s", method) + } + + req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b)) + if err != nil { + return nil, errors.Wrapf(err, "failed to make request for %s", method) + } + + req.SetBasicAuth(r.AuthUser, r.AuthPasswd) + resp, err := r.Client.Do(req) + if err != nil { + return nil, errors.Wrapf(err, "remote call failed for %s", method) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, errors.Errorf("bad status %d for %s", resp.StatusCode, method) + } + + cr := Response{} + if err = json.NewDecoder(resp.Body).Decode(&cr); err != nil { + return nil, errors.Wrapf(err, "failed to decode response for %s", method) + } + + if cr.Error != "" { + return nil, errors.New(cr.Error) + } + return &cr, nil +} diff --git a/backend/app/store/remote/remote_test.go b/backend/app/store/remote/remote_test.go new file mode 100644 index 0000000000..d3e56f9db9 --- /dev/null +++ b/backend/app/store/remote/remote_test.go @@ -0,0 +1,133 @@ +package remote + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark/backend/app/store" +) + +func TestClient_Create(t *testing.T) { + ts := testServer(t, `{"method":"create","params":[{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{"result":"12345"}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"}, + Text: "msg"}) + assert.NoError(t, err) + assert.Equal(t, "12345", res) + t.Logf("%v %T", res, res) +} + +func TestClient_Get(t *testing.T) { + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, + `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + res, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.NoError(t, err) + assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res) + t.Logf("%v %T", res, res) +} + +func TestClient_GetWithErrorResult(t *testing.T) { + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, `{"error":"failed"}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.EqualError(t, err, "failed") +} + +func TestClient_GetWithErrorDecode(t *testing.T) { + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, ``) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.EqualError(t, err, "failed to decode response for get: EOF") +} + +func TestClient_GetWithErrorRemote(t *testing.T) { + c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "remote call failed for get:")) +} + +func TestClient_FailedStatus(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + t.Logf("req: %s", string(body)) + w.WriteHeader(400) + })) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.EqualError(t, err, "bad status 400 for get") +} + +func TestClient_Put(t *testing.T) { + ts := testServer(t, `{"method":"put","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + err := c.Put(store.Locator{URL: "http://example.com/url"}, store.Comment{ID: "123", + Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"}, Text: "msg"}) + assert.NoError(t, err) + +} + +func TestClient_Find(t *testing.T) { + ts := testServer(t, `{"method":"find","params":[{"url":"http://example.com/url"},""]}`, + `{"result":[{"text":"1"},{"text":"2"}]}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + res, err := c.Find(store.Locator{URL: "http://example.com/url"}, "") + assert.NoError(t, err) + assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) +} + +func TestClient_Last(t *testing.T) { + ts := testServer(t, `{"method":"last","params":["site1",100,"2019-06-06T19:34:10Z"]}`, + `{"result":[{"text":"1"},{"text":"2"}]}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + res, err := c.Last("site1", 100, time.Date(2019, 6, 6, 19, 34, 10, 0, time.UTC)) + assert.NoError(t, err) + assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) +} + +func TestClient_User(t *testing.T) { + ts := testServer(t, `{"method":"user","params":["site1","u1",100,4]}`, `{"result":[{"text":"1"},{"text":"2"}]}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + res, err := c.User("site1", "u1", 100, 4) + assert.NoError(t, err) + assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) +} + +func testServer(t *testing.T, req, resp string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, req, string(body)) + t.Logf("req: %s", string(body)) + fmt.Fprintf(w, resp) + })) +} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 6719e042e4..9bb5ab42df 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -19,12 +19,13 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/image" ) // DataStore wraps store.Interface with additional methods type DataStore struct { - engine.Interface + Engine engine2.Interface EditDuration time.Duration AdminStore admin.Store MaxCommentSize int @@ -98,12 +99,13 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) }() s.submitImages(comment) - return s.Interface.Create(comment) + return s.Engine.Create(comment) } // Find wraps engine's Find call and alter results if needed func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) { - comments, err := s.Interface.Find(locator, sort) + req := engine2.FindRequest{Locator: locator, Sort: sort} + comments, err := s.Engine.Find(req) if err != nil { return comments, err } @@ -130,19 +132,24 @@ func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([ // Get comment by ID func (s *DataStore) Get(locator store.Locator, commentID string, user store.User) (store.Comment, error) { - c, err := s.Interface.Get(locator, commentID) + c, err := s.Engine.Get(locator, commentID) if err != nil { return store.Comment{}, err } return s.alterComment(c, user), nil } +// Put updates comment, mutable parts only +func (s *DataStore) Put(locator store.Locator, comment store.Comment) error { + return s.Engine.Update(locator, comment) +} + // submitImages initiated delayed commit of all images from the comment uploaded to remark42 func (s *DataStore) submitImages(comment store.Comment) { s.ImageService.Submit(func() []string { c := comment - cc, err := s.Interface.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment + cc, err := s.Engine.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment if err != nil { log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err) return nil @@ -182,14 +189,20 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err return comment, nil } +// DeleteAll removes all data from site +func (s *DataStore) DeleteAll(siteID string) error { + req := engine2.DeleteRequest{Locator: store.Locator{SiteID: siteID}} + return s.Engine.Delete(req) +} + // SetPin pin/un-pin comment as special func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error { - comment, err := s.Interface.Get(locator, commentID) + comment, err := s.Engine.Get(locator, commentID) if err != nil { return err } comment.Pin = status - return s.Put(locator, comment) + return s.Engine.Update(locator, comment) } // Vote for comment by id and locator @@ -199,7 +212,7 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, cLock.Lock() // prevents race on voting defer cLock.Unlock() - comment, err = s.Interface.Get(locator, commentID) + comment, err = s.Engine.Get(locator, commentID) if err != nil { return comment, err } @@ -258,7 +271,7 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, comment.Controversy = s.controversy(s.upsAndDowns(comment)) - return comment, s.Put(locator, comment) + return comment, s.Engine.Update(locator, comment) } // controversy calculates controversial index of votes @@ -287,7 +300,7 @@ type EditRequest struct { // EditComment to edit text and update Edit info func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) { - comment, err = s.Interface.Get(locator, commentID) + comment, err = s.Engine.Get(locator, commentID) if err != nil { return comment, err } @@ -303,7 +316,8 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi if req.Delete { // delete request comment.Deleted = true - return comment, s.Delete(locator, commentID, store.SoftDelete) + delReq := engine2.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: store.SoftDelete} + return comment, s.Engine.Delete(delReq) } if s.RestrictedWordsMatcher != nil && s.RestrictedWordsMatcher.Match(comment.Locator.SiteID, req.Text) { @@ -318,7 +332,7 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi } comment.Sanitize() - err = s.Put(locator, comment) + err = s.Engine.Update(locator, comment) return comment, err } @@ -336,7 +350,8 @@ func (s *DataStore) HasReplies(comment store.Comment) bool { return true } - comments, err := s.Interface.Last(comment.Locator.SiteID, maxLastCommentsReply, time.Time{}) + req := engine2.FindRequest{Locator: store.Locator{SiteID: comment.Locator.SiteID}, Limit: maxLastCommentsReply} + comments, err := s.Engine.Find(req) if err != nil { log.Printf("[WARN] can't get last comments for reply check, %v", err) return false @@ -395,7 +410,7 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s return comment, errors.New("no title extractor") } - comment, err = s.Interface.Get(locator, commentID) + comment, err = s.Engine.Get(locator, commentID) if err != nil { return comment, err } @@ -406,7 +421,7 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s return comment, err } comment.PostTitle = title - err = s.Put(locator, comment) + err = s.Engine.Update(locator, comment) return comment, err } @@ -414,7 +429,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) { res := []store.PostInfo{} for _, p := range postIDs { - if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil { + req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID, URL: p}} + if c, err := s.Engine.Count(req); err == nil { res = append(res, store.PostInfo{URL: p, Count: c}) } } @@ -449,20 +465,126 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool { return false } +// IsReadOnly checks if post read-only +func (s *DataStore) IsReadOnly(locator store.Locator) bool { + req := engine2.FlagRequest{Locator: locator, Flag: engine2.ReadOnly} + ro, err := s.Engine.Flag(req) + return err == nil && ro +} + +// SetReadOnly set/reset read-only flag +func (s *DataStore) SetReadOnly(locator store.Locator, status bool) error { + roStatus := engine2.FlagFalse + if status { + roStatus = engine2.FlagTrue + + } + req := engine2.FlagRequest{Locator: locator, Flag: engine2.ReadOnly, Update: roStatus} + _, err := s.Engine.Flag(req) + return err +} + +// IsVerified checks if user verified +func (s *DataStore) IsVerified(siteID string, userID string) bool { + req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Verified} + ro, err := s.Engine.Flag(req) + return err == nil && ro +} + +// SetVerified set/reset verified status for user +func (s *DataStore) SetVerified(siteID string, userID string, status bool) error { + roStatus := engine2.FlagFalse + if status { + roStatus = engine2.FlagTrue + } + req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Verified, Update: roStatus} + _, err := s.Engine.Flag(req) + return err +} + +// IsBlocked checks if user blocked +func (s *DataStore) IsBlocked(siteID string, userID string) bool { + req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Blocked} + ro, err := s.Engine.Flag(req) + return err == nil && ro +} + +// SetBlock set/reset verified status for user +func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { + roStatus := engine2.FlagFalse + if status { + roStatus = engine2.FlagTrue + } + req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, + Flag: engine2.Blocked, Update: roStatus, TTL: ttl} + _, err := s.Engine.Flag(req) + return err +} + +// Blocked returns list with all blocked users +func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) { + blocked, e := s.Engine.ListFlags(siteID, engine2.Blocked) + if e != nil { + return nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID) + } + for _, v := range blocked { + res = append(res, v.(store.BlockedUser)) + } + return res, nil +} + +// Info get post info +func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) { + req := engine2.InfoRequest{Locator: locator, ReadOnlyAge: readonlyAge} + res, err := s.Engine.Info(req) + if err != nil { + return store.PostInfo{}, err + } + if len(res) == 0 { + return store.PostInfo{}, errors.Errorf("post %+v not found", locator) + } + return res[0], nil +} + +// Delete comment by id +func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { + req := engine2.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode} + return s.Engine.Delete(req) +} + +// DeleteUser removes all comments from user +func (s *DataStore) DeleteUser(siteID string, userID string) error { + req := engine2.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: store.HardDelete} + return s.Engine.Delete(req) +} + +// List of commented posts +func (s *DataStore) List(siteID string, limit int, skip int) ([]store.PostInfo, error) { + req := engine2.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip} + return s.Engine.Info(req) +} + +// Count gets number of comments for the post +func (s *DataStore) Count(locator store.Locator) (int, error) { + req := engine2.FindRequest{Locator: locator} + return s.Engine.Count(req) +} + // Metas returns metadata for users and posts func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) { umetas = []UserMetaData{} pmetas = []PostMetaData{} + // set posts meta - posts, err := s.List(siteID, 0, 0) + posts, err := s.Engine.Info(engine2.InfoRequest{Locator: store.Locator{SiteID: siteID}}) if err != nil { return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID) } + for _, p := range posts { if s.IsReadOnly(store.Locator{SiteID: siteID, URL: p.URL}) { pmetas = append(pmetas, PostMetaData{URL: p.URL, ReadOnly: true}) } - } // set users meta @@ -484,11 +606,12 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe } // process verified users - verified, err := s.Verified(siteID) + verified, err := s.Engine.ListFlags(siteID, engine2.Verified) if err != nil { return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID) } - for _, v := range verified { + for _, vi := range verified { + v := vi.(string) val, ok := m[v] if !ok { val = UserMetaData{ID: v} @@ -531,22 +654,35 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post // User gets comment for given userID on siteID func (s *DataStore) User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error) { - comments, err := s.Interface.User(siteID, userID, limit, skip) + req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Limit: limit, Skip: skip} + comments, err := s.Engine.Find(req) if err != nil { return comments, err } return s.alterComments(comments, user), nil } +// UserCount is comments count by user +func (s *DataStore) UserCount(siteID, userID string) (int, error) { + req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID} + return s.Engine.Count(req) +} + // Last gets last comments for site, cross-post. Limited by count and optional since ts func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error) { - comments, err := s.Interface.Last(siteID, limit, since) + req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Since: since, Sort: "-time"} + comments, err := s.Engine.Find(req) if err != nil { return comments, err } return s.alterComments(comments, user), nil } +// Close store service +func (s *DataStore) Close() error { + return s.Engine.Close() +} + func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) { for _, v := range c.Votes { if v { @@ -583,7 +719,9 @@ func (s *DataStore) alterComments(cc []store.Comment, user store.User) (res []st func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Comment) { - blocked := s.IsBlocked(c.Locator.SiteID, c.User.ID) + blocReq := engine2.FlagRequest{Flag: engine2.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID} + blocked, _ := s.Engine.Flag(blocReq) + // process blocked users if blocked { if !user.Admin { // reset comment to deleted for non-admins @@ -595,7 +733,8 @@ func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Co // set verified status retroactively if !blocked { - c.User.Verified = s.IsVerified(c.Locator.SiteID, c.User.ID) + verifReq := engine2.FlagRequest{Flag: engine2.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID} + c.User.Verified, _ = s.Engine.Flag(verifReq) } // hide info from non-admins From 29ffa577339bb47095eb76b9ef421996537bff5d Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 8 Jun 2019 13:34:10 -0500 Subject: [PATCH 02/24] restore all service coverage with new engine --- backend/app/migrator/disqus_test.go | 6 +- backend/app/migrator/migrator_test.go | 18 +- backend/app/migrator/native_test.go | 6 +- backend/app/migrator/wordpress_test.go | 6 +- backend/app/store/engine2/engine_mock.go | 205 ++++++++++ backend/app/store/service/service.go | 1 + backend/app/store/service/service_test.go | 446 ++++++++++++++++------ 7 files changed, 554 insertions(+), 134 deletions(-) create mode 100644 backend/app/store/engine2/engine_mock.go diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index 5388f9b37f..8de1b33243 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -12,15 +12,15 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/service" ) func TestDisqus_Import(t *testing.T) { defer os.Remove("/tmp/remark-test.db") - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} + dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} d := Disqus{DataStore: &dataStore} size, err := d.Import(strings.NewReader(xmlTestDisqus), "test") assert.Nil(t, err) diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index 4b1dae2e4f..d1026fbdc5 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -12,7 +12,7 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/service" ) @@ -25,9 +25,9 @@ func TestMigrator_ImportDisqus(t *testing.T) { err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600) require.Nil(t, err) - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} + dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/disqus-test.xml", @@ -51,9 +51,9 @@ func TestMigrator_ImportWordPress(t *testing.T) { err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600) require.Nil(t, err) - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} + dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/wordpress-test.xml", @@ -80,9 +80,9 @@ func TestMigrator_ImportNative(t *testing.T) { err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600) require.Nil(t, err) - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) + b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} + dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ DataStore: dataStore, @@ -100,9 +100,9 @@ func TestMigrator_ImportNative(t *testing.T) { func TestMigrator_ImportFailed(t *testing.T) { defer os.Remove("/tmp/remark-test.db") - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") - dataStore := &service.DataStore{Interface: b} + dataStore := &service.DataStore{Engine: b} _, err = ImportComments(ImportParams{ DataStore: dataStore, InputFile: "/tmp/disqus-test.xml", diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go index ed0b0c9738..24837c4940 100644 --- a/backend/app/migrator/native_test.go +++ b/backend/app/migrator/native_test.go @@ -16,7 +16,7 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/service" ) @@ -142,10 +142,10 @@ func TestNative_ImportManyWithError(t *testing.T) { func prep(t *testing.T) *service.DataStore { os.Remove(testDb) - boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) + boltStore, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) - b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")} + b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")} comment := store.Comment{ ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc", diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go index 86a99b9305..3ad8e2b8da 100644 --- a/backend/app/migrator/wordpress_test.go +++ b/backend/app/migrator/wordpress_test.go @@ -11,17 +11,17 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/service" ) func TestWordPress_Import(t *testing.T) { siteID := "testWP" defer func() { _ = os.Remove("/tmp/remark-test.db") }() - b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) + b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) assert.Nil(t, err, "create store") - dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} + dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} wp := WordPress{DataStore: &dataStore} size, err := wp.Import(strings.NewReader(xmlTestWP), siteID) assert.Nil(t, err) diff --git a/backend/app/store/engine2/engine_mock.go b/backend/app/store/engine2/engine_mock.go new file mode 100644 index 0000000000..01232fb39f --- /dev/null +++ b/backend/app/store/engine2/engine_mock.go @@ -0,0 +1,205 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. +package engine2 + +import mock "github.com/stretchr/testify/mock" +import store "github.com/umputun/remark/backend/app/store" + +// MockInterface is an autogenerated mock type for the Interface type +type MockInterface struct { + mock.Mock +} + +// Close provides a mock function with given fields: +func (_m *MockInterface) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Count provides a mock function with given fields: req +func (_m *MockInterface) Count(req FindRequest) (int, error) { + ret := _m.Called(req) + + var r0 int + if rf, ok := ret.Get(0).(func(FindRequest) int); ok { + r0 = rf(req) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(FindRequest) error); ok { + r1 = rf(req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Create provides a mock function with given fields: comment +func (_m *MockInterface) Create(comment store.Comment) (string, error) { + ret := _m.Called(comment) + + var r0 string + if rf, ok := ret.Get(0).(func(store.Comment) string); ok { + r0 = rf(comment) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(store.Comment) error); ok { + r1 = rf(comment) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: req +func (_m *MockInterface) Delete(req DeleteRequest) error { + ret := _m.Called(req) + + var r0 error + if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok { + r0 = rf(req) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Find provides a mock function with given fields: req +func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) { + ret := _m.Called(req) + + var r0 []store.Comment + if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok { + r0 = rf(req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.Comment) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(FindRequest) error); ok { + r1 = rf(req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Flag provides a mock function with given fields: req +func (_m *MockInterface) Flag(req FlagRequest) (bool, error) { + ret := _m.Called(req) + + var r0 bool + if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok { + r0 = rf(req) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(FlagRequest) error); ok { + r1 = rf(req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Get provides a mock function with given fields: locator, commentID +func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) { + ret := _m.Called(locator, commentID) + + var r0 store.Comment + if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok { + r0 = rf(locator, commentID) + } else { + r0 = ret.Get(0).(store.Comment) + } + + var r1 error + if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { + r1 = rf(locator, commentID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Info provides a mock function with given fields: req +func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) { + ret := _m.Called(req) + + var r0 []store.PostInfo + if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok { + r0 = rf(req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.PostInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(InfoRequest) error); ok { + r1 = rf(req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListFlags provides a mock function with given fields: siteID, flag +func (_m *MockInterface) ListFlags(siteID string, flag Flag) ([]interface{}, error) { + ret := _m.Called(siteID, flag) + + var r0 []interface{} + if rf, ok := ret.Get(0).(func(string, Flag) []interface{}); ok { + r0 = rf(siteID, flag) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]interface{}) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, Flag) error); ok { + r1 = rf(siteID, flag) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: locator, comment +func (_m *MockInterface) Update(locator store.Locator, comment store.Comment) error { + ret := _m.Called(locator, comment) + + var r0 error + if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok { + r0 = rf(locator, comment) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 9bb5ab42df..5fdf0d58ac 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -103,6 +103,7 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) } // Find wraps engine's Find call and alter results if needed +// user used to filter results for self vs others func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) { req := engine2.FindRequest{Locator: locator, Sort: sort} comments, err := s.Engine.Find(req) diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index d83d01b952..9c18ca31df 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -21,7 +21,7 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine" + "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/image" ) @@ -30,7 +30,7 @@ var testDb = "/tmp/test-remark.db" func TestService_CreateFromEmpty(t *testing.T) { defer teardown(t) ks := admin.NewStaticKeyStore("secret 123") - b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks} comment := store.Comment{ Text: "text", User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"}, @@ -40,7 +40,7 @@ func TestService_CreateFromEmpty(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "text", res.Text) @@ -54,7 +54,7 @@ func TestService_CreateFromEmpty(t *testing.T) { func TestService_CreateFromPartial(t *testing.T) { defer teardown(t) ks := admin.NewStaticKeyStore("secret 123") - b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks} comment := store.Comment{ Text: "text", Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC), @@ -66,7 +66,7 @@ func TestService_CreateFromPartial(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "text", res.Text) @@ -81,7 +81,7 @@ func TestService_CreateFromPartial(t *testing.T) { func TestService_CreateFromPartialWithTitle(t *testing.T) { defer teardown(t) ks := admin.NewStaticKeyStore("secret 123") - b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks, + b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks, TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})} comment := store.Comment{ Text: "text", @@ -94,7 +94,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "Радио-Т 630 — Радио-Т Подкаст", res.PostTitle) @@ -102,7 +102,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) { comment.PostTitle = "post blah" id, err = b.Create(comment) assert.NoError(t, err) - res, err = b.Interface.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id) + res, err = b.Engine.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "post blah", res.PostTitle, "keep comment title") @@ -131,7 +131,7 @@ func TestService_SetTitle(t *testing.T) { defer tss.Close() ks := admin.NewStaticKeyStore("secret 123") - b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks, + b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks, TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})} comment := store.Comment{ Text: "text", @@ -145,7 +145,7 @@ func TestService_SetTitle(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Interface.Get(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "", res.PostTitle) @@ -157,14 +157,14 @@ func TestService_SetTitle(t *testing.T) { require.NoError(t, err) assert.Equal(t, "post1 blah 123", c.PostTitle) - b = DataStore{Interface: prepStoreEngine(t), AdminStore: ks} + b = DataStore{Engine: prepStoreEngine(t), AdminStore: ks} _, err = b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id) require.EqualError(t, err, "no title extractor") } func TestService_Vote(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} comment := store.Comment{ Text: "text", @@ -174,19 +174,32 @@ func TestService_Vote(t *testing.T) { _, err := b.Create(comment) assert.NoError(t, err) - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) t.Logf("%+v", res[0]) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 3, len(res)) assert.Equal(t, 0, res[0].Score) assert.Equal(t, 0, res[0].Vote) assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially") + // vote +1 as user1 c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 1, c.Score) assert.Equal(t, 1, c.Vote) assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +") + // check result as user1 + c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.User{ID: "user1"}) + assert.NoError(t, err) + assert.Equal(t, 1, c.Score) + assert.Equal(t, 1, c.Vote, "can see own vote result") + assert.Nil(t, c.Votes) + // check result as user2 + c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.User{ID: "user2"}) + assert.NoError(t, err) + assert.Equal(t, 1, c.Score) + assert.Equal(t, 0, c.Vote, "can't see other user vote result") + assert.Nil(t, c.Votes) c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true) assert.NotNil(t, err, "self-voting not allowed") @@ -195,17 +208,28 @@ func TestService_Vote(t *testing.T) { assert.NotNil(t, err, "double-voting rejected") assert.True(t, strings.HasPrefix(err.Error(), "user user1 already voted")) - res, err = b.Interface.Last("radio-t", 0, time.Time{}) - assert.Nil(t, err) + // check in last as user1 + res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user1"}) + assert.NoError(t, err) + t.Logf("%+v", res[0]) assert.Equal(t, 3, len(res)) assert.Equal(t, 1, res[0].Score) assert.Equal(t, 1, res[0].Vote) assert.Equal(t, 0.0, res[0].Controversy) + // check in last as user2 + res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user2"}) + assert.NoError(t, err) + t.Logf("%+v", res[0]) + assert.Equal(t, 3, len(res)) + assert.Equal(t, 1, res[0].Score) + assert.Equal(t, 0, res[0].Vote) + assert.Equal(t, 0.0, res[0].Controversy) + _, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false) - assert.Nil(t, err, "vote reset") - res, err = b.Interface.Last("radio-t", 0, time.Time{}) - assert.Nil(t, err) + assert.NoError(t, err, "vote reset") + res, err = b.Last("radio-t", 0, time.Time{}, store.User{}) + assert.NoError(t, err) assert.Equal(t, 3, len(res)) assert.Equal(t, 0, res[0].Score) assert.Equal(t, 0, res[0].Vote) @@ -214,25 +238,25 @@ func TestService_Vote(t *testing.T) { func TestService_VoteLimit(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2} _, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true) - assert.Nil(t, err) + assert.NoError(t, err) _, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true) - assert.Nil(t, err) + assert.NoError(t, err) _, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user4", true) assert.NotNil(t, err, "vote limit reached") assert.True(t, strings.HasPrefix(err.Error(), "maximum number of votes exceeded for comment id-1")) _, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user4", true) - assert.Nil(t, err) + assert.NoError(t, err) } func TestService_VotesDisabled(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0} _, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true) assert.EqualError(t, err, "maximum number of votes exceeded for comment id-1") @@ -240,7 +264,7 @@ func TestService_VotesDisabled(t *testing.T) { func TestService_VoteAggressive(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} comment := store.Comment{ Text: "text", @@ -250,8 +274,8 @@ func TestService_VoteAggressive(t *testing.T) { _, err := b.Create(comment) assert.NoError(t, err) - res, err := b.Interface.Last("radio-t", 0, time.Time{}) - require.Nil(t, err) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) + require.NoError(t, err) t.Logf("%+v", res[0]) assert.Equal(t, 3, len(res)) assert.Equal(t, 0, res[0].Score) @@ -259,7 +283,7 @@ func TestService_VoteAggressive(t *testing.T) { // add a vote as user2 _, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true) - require.Nil(t, err) + require.NoError(t, err) // crazy vote +1 as user1 var wg sync.WaitGroup @@ -271,13 +295,14 @@ func TestService_VoteAggressive(t *testing.T) { }() } wg.Wait() - res, err = b.Interface.Last("radio-t", 0, time.Time{}) + res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user1"}) require.NoError(t, err) t.Logf("%+v", res[0]) assert.Equal(t, 3, len(res)) assert.Equal(t, 2, res[0].Score, "add single +1") - assert.Equal(t, 2, len(res[0].Votes), "made a single vote") + assert.Equal(t, 1, res[0].Vote, "user1 voted +1") + assert.Equal(t, 0, len(res[0].Votes), "votes hidden") // random +1/-1 result should be [0..2] rand.Seed(time.Now().UnixNano()) @@ -290,7 +315,7 @@ func TestService_VoteAggressive(t *testing.T) { }() } wg.Wait() - res, err = b.Interface.Last("radio-t", 0, time.Time{}) + res, err = b.Last("radio-t", 0, time.Time{}, store.User{}) require.NoError(t, err) assert.Equal(t, 3, len(res)) t.Logf("%+v %d", res[0], res[0].Score) @@ -300,7 +325,7 @@ func TestService_VoteAggressive(t *testing.T) { func TestService_VoteConcurrent(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} comment := store.Comment{ Text: "text", @@ -309,7 +334,7 @@ func TestService_VoteConcurrent(t *testing.T) { } _, err := b.Create(comment) assert.NoError(t, err) - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) require.Nil(t, err) // concurrent vote +1 as multiple users for the same comment @@ -324,35 +349,35 @@ func TestService_VoteConcurrent(t *testing.T) { }() } wg.Wait() - res, err = b.Interface.Last("radio-t", 0, time.Time{}) + res, err = b.Last("radio-t", 0, time.Time{}, store.User{}) require.NoError(t, err) assert.Equal(t, 100, res[0].Score, "should have 100 score") - assert.Equal(t, 100, len(res[0].Votes), "should have 100 votes") + assert.Equal(t, 0, len(res[0].Votes), "should hide votes") assert.Equal(t, 0.0, res[0].Controversy, "should have 0 controversy") } func TestService_VotePositive(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1, PositiveScore: true} _, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false) assert.EqualError(t, err, "minimal score reached for comment id-1") _, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true) - assert.Nil(t, err, "minimal score doesn't affect positive vote") + assert.NoError(t, err, "minimal score doesn't affect positive vote") - b = DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), + b = DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1, PositiveScore: false} c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false) - assert.Nil(t, err, "minimal score ignored") + assert.NoError(t, err, "minimal score ignored") assert.Equal(t, -1, c.Score) assert.Equal(t, 0.0, c.Controversy) } func TestService_VoteControversy(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1} c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user2", false) assert.NoError(t, err) @@ -370,7 +395,7 @@ func TestService_VoteControversy(t *testing.T) { assert.InDelta(t, 1.73, c.Controversy, 0.01) // check if stored - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) require.NoError(t, err) assert.Equal(t, 1, res[0].Score, "should have 1 score") assert.InDelta(t, 1.73, res[0].Controversy, 0.01) @@ -402,81 +427,82 @@ func TestService_Controversy(t *testing.T) { func TestService_Pin(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) t.Logf("%+v", res[0]) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 2, len(res)) assert.Equal(t, false, res[0].Pin) err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true) - assert.Nil(t, err) + assert.NoError(t, err) - c, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) - assert.Nil(t, err) + c, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + assert.NoError(t, err) assert.Equal(t, true, c.Pin) err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false) - assert.Nil(t, err) - c, err = b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) - assert.Nil(t, err) + assert.NoError(t, err) + c, err = b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + assert.NoError(t, err) assert.Equal(t, false, c.Pin) } func TestService_EditComment(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) t.Logf("%+v", res[0]) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 2, len(res)) assert.Nil(t, res[0].Edit) comment, err := b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"}) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, "my edit", comment.Edit.Summary) assert.Equal(t, "xxx", comment.Text) assert.Equal(t, "yyy", comment.Orig) - c, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) - assert.Nil(t, err) + c, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + assert.NoError(t, err) assert.Equal(t, "my edit", c.Edit.Summary) assert.Equal(t, "xxx", c.Text) _, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"}) - assert.Nil(t, err, "allow second edit") + assert.NoError(t, err, "allow second edit") } func TestService_DeleteComment(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) t.Logf("%+v", res[0]) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 2, len(res)) assert.Nil(t, res[0].Edit) _, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Delete: true}) - assert.Nil(t, err) + assert.NoError(t, err) - c, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) - assert.Nil(t, err) + c, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + assert.NoError(t, err) assert.True(t, c.Deleted) t.Logf("%+v", c) } func TestService_EditCommentDurationFailed(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")} + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticKeyStore("secret 123")} - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) t.Logf("%+v", res[0]) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 2, len(res)) assert.Nil(t, res[0].Edit) @@ -489,11 +515,11 @@ func TestService_EditCommentDurationFailed(t *testing.T) { func TestService_EditCommentReplyFailed(t *testing.T) { defer teardown(t) - b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")} - res, err := b.Interface.Last("radio-t", 0, time.Time{}) + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) t.Logf("%+v", res[1]) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 2, len(res)) assert.Nil(t, res[1].Edit) @@ -531,7 +557,7 @@ func TestService_ValidateComment(t *testing.T) { for n, tt := range tbl { e := b.ValidateComment(&tt.inp) if tt.err == nil { - assert.Nil(t, e, "check #%d", n) + assert.NoError(t, e, "check #%d", n) continue } require.NotNil(t, e) @@ -552,15 +578,15 @@ func TestService_Counts(t *testing.T) { User: store.User{ID: "user1", Name: "user name"}, } _, err := b.Create(comment) - assert.Nil(t, err) + assert.NoError(t, err) - svc := DataStore{Interface: b} + svc := DataStore{Engine: b} res, err := svc.Counts("radio-t", []string{"https://radio-t.com/2"}) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res) res, err = svc.Counts("radio-t", []string{"https://radio-t.com", "https://radio-t.com/2", "blah"}) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, []store.PostInfo{ {URL: "https://radio-t.com", Count: 2}, {URL: "https://radio-t.com/2", Count: 1}, @@ -571,7 +597,7 @@ func TestService_Counts(t *testing.T) { func TestService_GetMetas(t *testing.T) { defer teardown(t) // two comments for https://radio-t.com - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")} um, pm, err := b.Metas("radio-t") @@ -602,7 +628,7 @@ func TestService_GetMetas(t *testing.T) { func TestService_SetMetas(t *testing.T) { defer teardown(t) // two comments for https://radio-t.com - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")} umetas := []UserMetaData{} pmetas := []PostMetaData{} @@ -626,7 +652,7 @@ func TestService_SetMetas(t *testing.T) { func TestService_IsAdmin(t *testing.T) { defer teardown(t) // two comments for https://radio-t.com - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} assert.False(t, b.IsAdmin("radio-t", "user1")) @@ -637,7 +663,7 @@ func TestService_HasReplies(t *testing.T) { defer teardown(t) // two comments for https://radio-t.com, no reply - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} comment := store.Comment{ @@ -668,7 +694,7 @@ func TestService_UserReplies(t *testing.T) { defer teardown(t) // two comments for https://radio-t.com, no reply - b := DataStore{Interface: prepStoreEngine(t), + b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} c1 := store.Comment{ @@ -744,7 +770,7 @@ func TestService_Find(t *testing.T) { defer teardown(t) // two comments for https://radio-t.com, no reply - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time", store.User{}) @@ -761,8 +787,8 @@ func TestService_Find(t *testing.T) { Score: 1, Votes: map[string]bool{"id-1": true, "id-2": true, "123456": false}, } - _, err = b.Interface.Create(comment) // create directly with engine, doesn't set Controversy - assert.Nil(t, err) + _, err = b.Engine.Create(comment) // create directly with engine, doesn't set Controversy + assert.NoError(t, err) // make sure Controversy altered res, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "-controversy", store.User{}) @@ -774,6 +800,192 @@ func TestService_Find(t *testing.T) { assert.InDelta(t, 0, res[1].Controversy, 0.01) } +func TestService_Info(t *testing.T) { + defer teardown(t) + + // two comments for https://radio-t.com, no reply + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} + + info, err := b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0) + require.NoError(t, err) + assert.Equal(t, "https://radio-t.com", info.URL) + assert.Equal(t, 2, info.Count) + assert.False(t, info.ReadOnly) + assert.True(t, info.LastTS.After(info.FirstTS)) + + time.Sleep(1 * time.Second) // make post RO in 1sec + info, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 1) + require.NoError(t, err) + assert.Equal(t, "https://radio-t.com", info.URL) + assert.True(t, info.ReadOnly) +} + +func TestService_Delete(t *testing.T) { + defer teardown(t) + + // two comments for https://radio-t.com, no reply + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} + + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) + assert.Equal(t, 2, len(res)) + assert.NoError(t, err) + + err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.SoftDelete) + assert.NoError(t, err) + + res, err = b.Last("radio-t", 0, time.Time{}, store.User{}) + assert.Equal(t, 1, len(res), "one left") + assert.NoError(t, err) +} + +// DeleteUser removes all comments from user +func TestService_DeleteUser(t *testing.T) { + defer teardown(t) + // two comments for https://radio-t.com, no reply + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} + + // add one more for user2 + comment := store.Comment{ + ID: "123456xyz", + Text: `some text, link`, + Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user2", Name: "user name"}, + } + _, err := b.Create(comment) + assert.NoError(t, err) + + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) + assert.Equal(t, 3, len(res), "3 comments initially, for 2 diff users and 2 posts") + assert.NoError(t, err) + + err = b.DeleteUser("radio-t", "user1") + assert.NoError(t, err) + + res, err = b.Last("radio-t", 0, time.Time{}, store.User{}) + assert.Equal(t, 1, len(res), "only one comment left for user2") + assert.NoError(t, err) + assert.Equal(t, "user2", res[0].User.ID) +} + +func TestService_List(t *testing.T) { + defer teardown(t) + // two comments for https://radio-t.com, no reply + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} + + // add one more for user2 + comment := store.Comment{ + ID: "id-3", + Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), + Text: `some text, link`, + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user2", Name: "user name"}, + } + _, err := b.Create(comment) + assert.NoError(t, err) + + res, err := b.List("radio-t", 0, 0) + assert.NoError(t, err) + assert.Equal(t, 2, len(res), "2 posts") + assert.Equal(t, "https://radio-t.com/2", res[0].URL) + assert.Equal(t, 1, res[0].Count) + assert.Equal(t, time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), res[0].FirstTS) + assert.Equal(t, time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), res[0].LastTS) + + assert.Equal(t, "https://radio-t.com", res[1].URL) + assert.Equal(t, 2, res[1].Count) + assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), res[1].FirstTS) + assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), res[1].LastTS) +} + +func TestService_Count(t *testing.T) { + defer teardown(t) + // two comments for https://radio-t.com, no reply + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} + + // add one more for user2 + comment := store.Comment{ + ID: "id-3", + Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), + Text: `some text, link`, + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user2", Name: "user name"}, + } + _, err := b.Create(comment) + assert.NoError(t, err) + + c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) + assert.NoError(t, err) + assert.Equal(t, 2, c) + + c, err = b.Count(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}) + assert.NoError(t, err) + assert.Equal(t, 1, c) + + c, err = b.Count(store.Locator{URL: "https://radio-t.com/3", SiteID: "radio-t"}) + assert.NoError(t, err) + assert.Equal(t, 0, c) +} + +func TestService_UserCount(t *testing.T) { + defer teardown(t) + // two comments for https://radio-t.com, no reply + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} + + // add one more for user2 + comment := store.Comment{ + ID: "id-3", + Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), + Text: `some text, link`, + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user2", Name: "user name"}, + } + _, err := b.Create(comment) + assert.NoError(t, err) + + c, err := b.UserCount("radio-t", "user1") + assert.NoError(t, err) + assert.Equal(t, 2, c) + + c, err = b.UserCount("radio-t", "user2") + assert.NoError(t, err) + assert.Equal(t, 1, c) + + c, err = b.UserCount("radio-t", "userBad") + assert.EqualError(t, err, "no comments for user userBad in store for radio-t site") +} + +func TestService_DeleteAll(t *testing.T) { + defer teardown(t) + // two comments for https://radio-t.com, no reply + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, + AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")} + + // add one more for user2 + comment := store.Comment{ + ID: "id-3", + Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), + Text: `some text, link`, + Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, + User: store.User{ID: "user2", Name: "user name"}, + } + _, err := b.Create(comment) + assert.NoError(t, err) + + err = b.DeleteAll("radio-t") + assert.NoError(t, err) + + res, err := b.Last("radio-t", 0, time.Time{}, store.User{}) + assert.NoError(t, err) + assert.Equal(t, 0, len(res)) +} + func TestService_submitImages(t *testing.T) { defer teardown(t) lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc) @@ -783,7 +995,7 @@ func TestService_submitImages(t *testing.T) { imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50} // two comments for https://radio-t.com - b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond, + b := DataStore{Engine: prepStoreEngine(t), EditDuration: 50 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc} c := store.Comment{ @@ -793,7 +1005,7 @@ func TestService_submitImages(t *testing.T) { Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, User: store.User{ID: "user1", Name: "user name"}, } - _, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages + _, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages assert.NoError(t, err) b.submitImages(c) @@ -803,40 +1015,42 @@ func TestService_submitImages(t *testing.T) { func TestService_alterComment(t *testing.T) { defer teardown(t) - engineMock := engine.MockInterface{} - engineMock.On("IsBlocked", mock.Anything, mock.Anything).Return(false) - engineMock.On("IsVerified", mock.Anything, mock.Anything).Return(false) - svc := DataStore{Interface: &engineMock} - - r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1"}}, store.User{Name: "dev", Admin: false}) - assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: ""}}, r, "ip cleaned") - r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1"}}, store.User{Name: "dev", Admin: true}) - assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "127.0.0.1"}}, r, "ip not cleaned") - - engineMock = engine.MockInterface{} - engineMock.On("IsBlocked", mock.Anything, mock.Anything).Return(false) - engineMock.On("IsVerified", mock.Anything, mock.Anything).Return(true) - svc = DataStore{Interface: &engineMock} - r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", Verified: true}}, - store.User{Name: "dev", Admin: false}) - assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true}}, r, "verified set") - - engineMock = engine.MockInterface{} - engineMock.On("IsBlocked", mock.Anything, mock.Anything).Return(true) - engineMock.On("IsVerified", mock.Anything, mock.Anything).Return(false) - svc = DataStore{Interface: &engineMock} - r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", Verified: true}}, - store.User{Name: "dev", Admin: false}) - assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true, Blocked: true}, Deleted: true}, r, - "blocked") + engineMock := engine2.MockInterface{} + engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Blocked, UserID: "devid"}).Return(false, nil) + engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Verified, UserID: "devid"}).Return(false, nil) + svc := DataStore{Engine: &engineMock} + + r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, + store.User{Name: "dev", ID: "devid", Admin: false}) + assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid"}}, r, "ip cleaned") + r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, + store.User{Name: "dev", ID: "devid", Admin: true}) + assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, r, "ip not cleaned") + + engineMock = engine2.MockInterface{} + engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Blocked, UserID: "devid"}).Return(false, nil) + engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Verified, UserID: "devid"}).Return(true, nil) + svc = DataStore{Engine: &engineMock} + r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}}, + store.User{Name: "dev", ID: "devid", Admin: false}) + assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid", Verified: true}}, r, "verified set") + + engineMock = engine2.MockInterface{} + engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Blocked, UserID: "devid"}).Return(true, nil) + engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Verified, UserID: "devid"}).Return(false, nil) + svc = DataStore{Engine: &engineMock} + r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}}, + store.User{Name: "dev", ID: "devid", Admin: false}) + assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true, Blocked: true, ID: "devid"}, + Deleted: true}, r, "blocked") } // makes new boltdb, put two records -func prepStoreEngine(t *testing.T) engine.Interface { +func prepStoreEngine(t *testing.T) engine2.Interface { _ = os.Remove(testDb) - boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"}) - assert.Nil(t, err) + boltStore, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"}) + assert.NoError(t, err) b := boltStore comment := store.Comment{ @@ -847,7 +1061,7 @@ func prepStoreEngine(t *testing.T) engine.Interface { User: store.User{ID: "user1", Name: "user name"}, } _, err = b.Create(comment) - assert.Nil(t, err) + assert.NoError(t, err) comment = store.Comment{ ID: "id-2", @@ -857,7 +1071,7 @@ func prepStoreEngine(t *testing.T) engine.Interface { User: store.User{ID: "user1", Name: "user name"}, } _, err = b.Create(comment) - assert.Nil(t, err) + assert.NoError(t, err) return b } From fa15fb1c75c253d33175ca85a753e061573a8184 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sat, 8 Jun 2019 21:28:01 -0500 Subject: [PATCH 03/24] make new engine primary, rename package --- backend/app/cmd/server.go | 10 +- backend/app/migrator/disqus_test.go | 4 +- backend/app/migrator/migrator_test.go | 10 +- backend/app/migrator/native_test.go | 4 +- backend/app/migrator/wordpress_test.go | 4 +- backend/app/rest/api/rest_test.go | 4 +- backend/app/store/{engine2 => engine}/bolt.go | 2 +- .../store/{engine2 => engine}/bolt_test.go | 2 +- backend/app/store/engine/engine.go | 87 +++- backend/app/store/engine/engine_mock.go | 313 +++----------- backend/app/store/engine2/engine.go | 128 ------ backend/app/store/engine2/engine_mock.go | 205 --------- .../{engine => engine_old}/bolt_accessor.go | 2 +- .../bolt_accessor_test.go | 2 +- .../{engine => engine_old}/bolt_admin.go | 2 +- .../{engine => engine_old}/bolt_admin_test.go | 2 +- backend/app/store/engine_old/engine.go | 87 ++++ backend/app/store/engine_old/engine_mock.go | 408 ++++++++++++++++++ .../{engine2 => engine_old}/engine_test.go | 2 +- .../app/store/{engine => engine_old}/mongo.go | 2 +- .../{engine => engine_old}/mongo_test.go | 2 +- backend/app/store/service/service.go | 65 ++- backend/app/store/service/service_test.go | 24 +- 23 files changed, 685 insertions(+), 686 deletions(-) rename backend/app/store/{engine2 => engine}/bolt.go (99%) rename backend/app/store/{engine2 => engine}/bolt_test.go (99%) delete mode 100644 backend/app/store/engine2/engine.go delete mode 100644 backend/app/store/engine2/engine_mock.go rename backend/app/store/{engine => engine_old}/bolt_accessor.go (99%) rename backend/app/store/{engine => engine_old}/bolt_accessor_test.go (99%) rename backend/app/store/{engine => engine_old}/bolt_admin.go (99%) rename backend/app/store/{engine => engine_old}/bolt_admin_test.go (99%) create mode 100644 backend/app/store/engine_old/engine.go create mode 100644 backend/app/store/engine_old/engine_mock.go rename backend/app/store/{engine2 => engine_old}/engine_test.go (98%) rename backend/app/store/{engine => engine_old}/mongo.go (99%) rename backend/app/store/{engine => engine_old}/mongo_test.go (99%) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 0ac0e87323..e2eb8ee47c 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -31,7 +31,7 @@ import ( "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine2" + "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/service" ) @@ -401,7 +401,7 @@ func (a *serverApp) activateBackup(ctx context.Context) { } // makeDataStore creates store for all sites -func (s *ServerCommand) makeDataStore() (result engine2.Interface, err error) { +func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { log.Printf("[INFO] make data store, type=%s", s.Store.Type) switch s.Store.Type { @@ -409,11 +409,11 @@ func (s *ServerCommand) makeDataStore() (result engine2.Interface, err error) { if err = makeDirs(s.Store.Bolt.Path); err != nil { return nil, errors.Wrap(err, "failed to create bolt store") } - sites := []engine2.BoltSite{} + sites := []engine.BoltSite{} for _, site := range s.Sites { - sites = append(sites, engine2.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)}) + sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)}) } - result, err = engine2.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...) + result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...) // case "mongo": // mgServer, e := s.makeMongo() // if e != nil { diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go index 8de1b33243..5394e3f301 100644 --- a/backend/app/migrator/disqus_test.go +++ b/backend/app/migrator/disqus_test.go @@ -12,13 +12,13 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine2" + "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) func TestDisqus_Import(t *testing.T) { defer os.Remove("/tmp/remark-test.db") - b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} d := Disqus{DataStore: &dataStore} diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go index d1026fbdc5..4a2151e27a 100644 --- a/backend/app/migrator/migrator_test.go +++ b/backend/app/migrator/migrator_test.go @@ -12,7 +12,7 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine2" + "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) @@ -25,7 +25,7 @@ func TestMigrator_ImportDisqus(t *testing.T) { err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600) require.Nil(t, err) - b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ @@ -51,7 +51,7 @@ func TestMigrator_ImportWordPress(t *testing.T) { err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600) require.Nil(t, err) - b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} size, err := ImportComments(ImportParams{ @@ -80,7 +80,7 @@ func TestMigrator_ImportNative(t *testing.T) { err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600) require.Nil(t, err) - b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"}) require.Nil(t, err, "create store") dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} @@ -100,7 +100,7 @@ func TestMigrator_ImportNative(t *testing.T) { func TestMigrator_ImportFailed(t *testing.T) { defer os.Remove("/tmp/remark-test.db") - b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"}) require.Nil(t, err, "create store") dataStore := &service.DataStore{Engine: b} _, err = ImportComments(ImportParams{ diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go index 24837c4940..85570563b3 100644 --- a/backend/app/migrator/native_test.go +++ b/backend/app/migrator/native_test.go @@ -16,7 +16,7 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine2" + "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) @@ -142,7 +142,7 @@ func TestNative_ImportManyWithError(t *testing.T) { func prep(t *testing.T) *service.DataStore { os.Remove(testDb) - boltStore, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{SiteID: "radio-t", FileName: testDb}) + boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb}) assert.Nil(t, err) b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")} diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go index 3ad8e2b8da..a229b6cbb9 100644 --- a/backend/app/migrator/wordpress_test.go +++ b/backend/app/migrator/wordpress_test.go @@ -11,14 +11,14 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine2" + "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/service" ) func TestWordPress_Import(t *testing.T) { siteID := "testWP" defer func() { _ = os.Remove("/tmp/remark-test.db") }() - b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID}) assert.Nil(t, err, "create store") dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")} diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go index d4fedc3144..469ad4cdf9 100644 --- a/backend/app/rest/api/rest_test.go +++ b/backend/app/rest/api/rest_test.go @@ -31,7 +31,7 @@ import ( "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" adminstore "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine2" + "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/image" "github.com/umputun/remark/backend/app/store/service" ) @@ -291,7 +291,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) { os.RemoveAll("/tmp/ava-remark42") os.RemoveAll("/tmp/pics-remark42") - b, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: testDb, SiteID: "radio-t"}) + b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"}) require.Nil(t, err) memCache, err := cache.NewMemoryCache() diff --git a/backend/app/store/engine2/bolt.go b/backend/app/store/engine/bolt.go similarity index 99% rename from backend/app/store/engine2/bolt.go rename to backend/app/store/engine/bolt.go index 2a5c6937d8..fe48d28639 100644 --- a/backend/app/store/engine2/bolt.go +++ b/backend/app/store/engine/bolt.go @@ -1,4 +1,4 @@ -package engine2 +package engine import ( "bytes" diff --git a/backend/app/store/engine2/bolt_test.go b/backend/app/store/engine/bolt_test.go similarity index 99% rename from backend/app/store/engine2/bolt_test.go rename to backend/app/store/engine/bolt_test.go index 02db81100f..b373c5d687 100644 --- a/backend/app/store/engine2/bolt_test.go +++ b/backend/app/store/engine/bolt_test.go @@ -1,4 +1,4 @@ -package engine2 +package engine import ( "fmt" diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go index 5c6a68271c..4e1688c314 100644 --- a/backend/app/store/engine/engine.go +++ b/backend/app/store/engine/engine.go @@ -1,6 +1,7 @@ +package engine + // Package engine defines interfaces each supported storage should implement. // Includes default implementation with boltdb -package engine import ( "sort" @@ -15,28 +16,68 @@ import ( // Interface defines methods provided by low-level storage engine type Interface interface { - Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id - Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id - Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only - Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator - Last(siteID string, limit int, since time.Time) ([]store.Comment, error) // last comments for given site, sorted by time - User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time - UserCount(siteID, userID string) (int, error) // comments count by user - Count(locator store.Locator) (int, error) // number of comments for the post - List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts - Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info - Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id - DeleteAll(siteID string) error // delete all data from site - DeleteUser(siteID string, userID string) error // remove all comments from user - SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent) - IsBlocked(siteID string, userID string) bool // check if user blocked - Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users - SetReadOnly(locator store.Locator, status bool) error // set/reset read-only flag - IsReadOnly(locator store.Locator) bool // check if post read-only - SetVerified(siteID string, userID string, status bool) error // set/reset verified flag - IsVerified(siteID string, userID string) bool // check verified status - Verified(siteID string) ([]string, error) // list of verified user ids - Close() error // close/stop engine + Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id + Update(locator store.Locator, comment store.Comment) error // update comment, mutable parts only + Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id + Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site + Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info + Count(req FindRequest) (int, error) // get count for post or user + Delete(req DeleteRequest) error // delete post(s) by id or by userID + Flag(req FlagRequest) (bool, error) // set and get flags + ListFlags(siteID string, flag Flag) ([]interface{}, error) // get list of flagged keys, like blocked & verified user + Close() error // close storage engine +} + +// FindRequest is the input for all find operations +type FindRequest struct { + Locator store.Locator // lack of URL means site operation + UserID string // presence of UserID treated as user-related find + Sort string // sort order with +/-field syntax + Since time.Time // time limit for found results + Limit, Skip int +} + +// InfoRequest is the input of Info operation used to get meta data about posts +type InfoRequest struct { + Locator store.Locator + Limit, Skip int + ReadOnlyAge int +} + +type DeleteRequest struct { + Locator store.Locator // lack of URL means site operation + CommentID string + UserID string + DeleteMode store.DeleteMode +} + +// Flag defines type of binary attribute +type Flag string + +// FlagStatus represents values of the flag update +type FlagStatus int + +// enum of update values +const ( + FlagNonSet FlagStatus = 0 + FlagTrue FlagStatus = 1 + FlagFalse FlagStatus = -1 +) + +// Enum of all flags +const ( + ReadOnly = Flag("readonly") + Verified = Flag("verified") + Blocked = Flag("blocked") +) + +// FlagRequest is the input for both get/set for flags, like blocked, verified and so on +type FlagRequest struct { + Flag Flag // flag type + Locator store.Locator // post locator + UserID string // for flags setting user status + Update FlagStatus // if FlagNonSet it will be get op, if set will set the value + TTL time.Duration // ttl for time-sensitive flags only, like blocked for some period } const ( diff --git a/backend/app/store/engine/engine_mock.go b/backend/app/store/engine/engine_mock.go index e274d53858..38496094a0 100644 --- a/backend/app/store/engine/engine_mock.go +++ b/backend/app/store/engine/engine_mock.go @@ -3,36 +3,12 @@ package engine import mock "github.com/stretchr/testify/mock" import store "github.com/umputun/remark/backend/app/store" -import time "time" // MockInterface is an autogenerated mock type for the Interface type type MockInterface struct { mock.Mock } -// Blocked provides a mock function with given fields: siteID -func (_m *MockInterface) Blocked(siteID string) ([]store.BlockedUser, error) { - ret := _m.Called(siteID) - - var r0 []store.BlockedUser - if rf, ok := ret.Get(0).(func(string) []store.BlockedUser); ok { - r0 = rf(siteID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.BlockedUser) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(siteID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // Close provides a mock function with given fields: func (_m *MockInterface) Close() error { ret := _m.Called() @@ -47,20 +23,20 @@ func (_m *MockInterface) Close() error { return r0 } -// Count provides a mock function with given fields: locator -func (_m *MockInterface) Count(locator store.Locator) (int, error) { - ret := _m.Called(locator) +// Count provides a mock function with given fields: req +func (_m *MockInterface) Count(req FindRequest) (int, error) { + ret := _m.Called(req) var r0 int - if rf, ok := ret.Get(0).(func(store.Locator) int); ok { - r0 = rf(locator) + if rf, ok := ret.Get(0).(func(FindRequest) int); ok { + r0 = rf(req) } else { r0 = ret.Get(0).(int) } var r1 error - if rf, ok := ret.Get(1).(func(store.Locator) error); ok { - r1 = rf(locator) + if rf, ok := ret.Get(1).(func(FindRequest) error); ok { + r1 = rf(req) } else { r1 = ret.Error(1) } @@ -89,13 +65,13 @@ func (_m *MockInterface) Create(comment store.Comment) (string, error) { return r0, r1 } -// Delete provides a mock function with given fields: locator, commentID, mode -func (_m *MockInterface) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - ret := _m.Called(locator, commentID, mode) +// Delete provides a mock function with given fields: req +func (_m *MockInterface) Delete(req DeleteRequest) error { + ret := _m.Called(req) var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, string, store.DeleteMode) error); ok { - r0 = rf(locator, commentID, mode) + if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok { + r0 = rf(req) } else { r0 = ret.Error(0) } @@ -103,50 +79,43 @@ func (_m *MockInterface) Delete(locator store.Locator, commentID string, mode st return r0 } -// DeleteAll provides a mock function with given fields: siteID -func (_m *MockInterface) DeleteAll(siteID string) error { - ret := _m.Called(siteID) +// Find provides a mock function with given fields: req +func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) { + ret := _m.Called(req) - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(siteID) + var r0 []store.Comment + if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok { + r0 = rf(req) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.Comment) + } } - return r0 -} - -// DeleteUser provides a mock function with given fields: siteID, userID -func (_m *MockInterface) DeleteUser(siteID string, userID string) error { - ret := _m.Called(siteID, userID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(siteID, userID) + var r1 error + if rf, ok := ret.Get(1).(func(FindRequest) error); ok { + r1 = rf(req) } else { - r0 = ret.Error(0) + r1 = ret.Error(1) } - return r0 + return r0, r1 } -// Find provides a mock function with given fields: locator, sort -func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comment, error) { - ret := _m.Called(locator, sort) +// Flag provides a mock function with given fields: req +func (_m *MockInterface) Flag(req FlagRequest) (bool, error) { + ret := _m.Called(req) - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(store.Locator, string) []store.Comment); ok { - r0 = rf(locator, sort) + var r0 bool + if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok { + r0 = rf(req) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } + r0 = ret.Get(0).(bool) } var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { - r1 = rf(locator, sort) + if rf, ok := ret.Get(1).(func(FlagRequest) error); ok { + r1 = rf(req) } else { r1 = ret.Error(1) } @@ -175,85 +144,22 @@ func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Com return r0, r1 } -// Info provides a mock function with given fields: locator, readonlyAge -func (_m *MockInterface) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) { - ret := _m.Called(locator, readonlyAge) +// Info provides a mock function with given fields: req +func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) { + ret := _m.Called(req) - var r0 store.PostInfo - if rf, ok := ret.Get(0).(func(store.Locator, int) store.PostInfo); ok { - r0 = rf(locator, readonlyAge) - } else { - r0 = ret.Get(0).(store.PostInfo) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, int) error); ok { - r1 = rf(locator, readonlyAge) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// IsBlocked provides a mock function with given fields: siteID, userID -func (_m *MockInterface) IsBlocked(siteID string, userID string) bool { - ret := _m.Called(siteID, userID) - - var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsReadOnly provides a mock function with given fields: locator -func (_m *MockInterface) IsReadOnly(locator store.Locator) bool { - ret := _m.Called(locator) - - var r0 bool - if rf, ok := ret.Get(0).(func(store.Locator) bool); ok { - r0 = rf(locator) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsVerified provides a mock function with given fields: siteID, userID -func (_m *MockInterface) IsVerified(siteID string, userID string) bool { - ret := _m.Called(siteID, userID) - - var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Last provides a mock function with given fields: siteID, limit, since -func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]store.Comment, error) { - ret := _m.Called(siteID, limit, since) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(string, int, time.Time) []store.Comment); ok { - r0 = rf(siteID, limit, since) + var r0 []store.PostInfo + if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok { + r0 = rf(req) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) + r0 = ret.Get(0).([]store.PostInfo) } } var r1 error - if rf, ok := ret.Get(1).(func(string, int, time.Time) error); ok { - r1 = rf(siteID, limit, since) + if rf, ok := ret.Get(1).(func(InfoRequest) error); ok { + r1 = rf(req) } else { r1 = ret.Error(1) } @@ -261,22 +167,22 @@ func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]stor return r0, r1 } -// List provides a mock function with given fields: siteID, limit, skip -func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostInfo, error) { - ret := _m.Called(siteID, limit, skip) +// ListFlags provides a mock function with given fields: siteID, flag +func (_m *MockInterface) ListFlags(siteID string, flag Flag) ([]interface{}, error) { + ret := _m.Called(siteID, flag) - var r0 []store.PostInfo - if rf, ok := ret.Get(0).(func(string, int, int) []store.PostInfo); ok { - r0 = rf(siteID, limit, skip) + var r0 []interface{} + if rf, ok := ret.Get(0).(func(string, Flag) []interface{}); ok { + r0 = rf(siteID, flag) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.PostInfo) + r0 = ret.Get(0).([]interface{}) } } var r1 error - if rf, ok := ret.Get(1).(func(string, int, int) error); ok { - r1 = rf(siteID, limit, skip) + if rf, ok := ret.Get(1).(func(string, Flag) error); ok { + r1 = rf(siteID, flag) } else { r1 = ret.Error(1) } @@ -284,8 +190,8 @@ func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostI return r0, r1 } -// Put provides a mock function with given fields: locator, comment -func (_m *MockInterface) Put(locator store.Locator, comment store.Comment) error { +// Update provides a mock function with given fields: locator, comment +func (_m *MockInterface) Update(locator store.Locator, comment store.Comment) error { ret := _m.Called(locator, comment) var r0 error @@ -297,112 +203,3 @@ func (_m *MockInterface) Put(locator store.Locator, comment store.Comment) error return r0 } - -// SetBlock provides a mock function with given fields: siteID, userID, status, ttl -func (_m *MockInterface) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - ret := _m.Called(siteID, userID, status, ttl) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, bool, time.Duration) error); ok { - r0 = rf(siteID, userID, status, ttl) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetReadOnly provides a mock function with given fields: locator, status -func (_m *MockInterface) SetReadOnly(locator store.Locator, status bool) error { - ret := _m.Called(locator, status) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, bool) error); ok { - r0 = rf(locator, status) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetVerified provides a mock function with given fields: siteID, userID, status -func (_m *MockInterface) SetVerified(siteID string, userID string, status bool) error { - ret := _m.Called(siteID, userID, status) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, bool) error); ok { - r0 = rf(siteID, userID, status) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// User provides a mock function with given fields: siteID, userID, limit, skip -func (_m *MockInterface) User(siteID string, userID string, limit int, skip int) ([]store.Comment, error) { - ret := _m.Called(siteID, userID, limit, skip) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(string, string, int, int) []store.Comment); ok { - r0 = rf(siteID, userID, limit, skip) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string, int, int) error); ok { - r1 = rf(siteID, userID, limit, skip) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// UserCount provides a mock function with given fields: siteID, userID -func (_m *MockInterface) UserCount(siteID string, userID string) (int, error) { - ret := _m.Called(siteID, userID) - - var r0 int - if rf, ok := ret.Get(0).(func(string, string) int); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(siteID, userID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Verified provides a mock function with given fields: siteID -func (_m *MockInterface) Verified(siteID string) ([]string, error) { - ret := _m.Called(siteID) - - var r0 []string - if rf, ok := ret.Get(0).(func(string) []string); ok { - r0 = rf(siteID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(siteID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/backend/app/store/engine2/engine.go b/backend/app/store/engine2/engine.go deleted file mode 100644 index eb3a6e8c65..0000000000 --- a/backend/app/store/engine2/engine.go +++ /dev/null @@ -1,128 +0,0 @@ -package engine2 - -// Package engine defines interfaces each supported storage should implement. -// Includes default implementation with boltdb - -import ( - "sort" - "strings" - "time" - - "github.com/umputun/remark/backend/app/store" -) - -// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate -//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go" - -// Interface defines methods provided by low-level storage engine -type Interface interface { - Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id - Update(locator store.Locator, comment store.Comment) error // update comment, mutable parts only - Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id - Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site - Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info - Count(req FindRequest) (int, error) // get count for post or user - Delete(req DeleteRequest) error // delete post(s) by id or by userID - Flag(req FlagRequest) (bool, error) // set and get flags - ListFlags(siteID string, flag Flag) ([]interface{}, error) // get list of flagged keys, like blocked & verified user - Close() error // close storage engine -} - -// FindRequest is the input for all find operations -type FindRequest struct { - Locator store.Locator // lack of URL means site operation - UserID string // presence of UserID treated as user-related find - Sort string // sort order with +/-field syntax - Since time.Time // time limit for found results - Limit, Skip int -} - -// InfoRequest is the input of Info operation used to get meta data about posts -type InfoRequest struct { - Locator store.Locator - Limit, Skip int - ReadOnlyAge int -} - -type DeleteRequest struct { - Locator store.Locator // lack of URL means site operation - CommentID string - UserID string - DeleteMode store.DeleteMode -} - -// Flag defines type of binary attribute -type Flag string - -// FlagStatus represents values of the flag update -type FlagStatus int - -// enum of update values -const ( - FlagNonSet FlagStatus = 0 - FlagTrue FlagStatus = 1 - FlagFalse FlagStatus = -1 -) - -// Enum of all flags -const ( - ReadOnly = Flag("readonly") - Verified = Flag("verified") - Blocked = Flag("blocked") -) - -// FlagRequest is the input for both get/set for flags, like blocked, verified and so on -type FlagRequest struct { - Flag Flag // flag type - Locator store.Locator // post locator - UserID string // for flags setting user status - Update FlagStatus // if FlagNonSet it will be get op, if set will set the value - TTL time.Duration // ttl for time-sensitive flags only, like blocked for some period -} - -const ( - // limits - lastLimit = 1000 - userLimit = 500 -) - -// SortComments is for engines can't sort data internally -func SortComments(comments []store.Comment, sortFld string) []store.Comment { - sort.Slice(comments, func(i, j int) bool { - switch sortFld { - case "+time", "-time", "time", "+active", "-active", "active": - if strings.HasPrefix(sortFld, "-") { - return comments[i].Timestamp.After(comments[j].Timestamp) - } - return comments[i].Timestamp.Before(comments[j].Timestamp) - - case "+score", "-score", "score": - if strings.HasPrefix(sortFld, "-") { - if comments[i].Score == comments[j].Score { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Score > comments[j].Score - } - if comments[i].Score == comments[j].Score { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Score < comments[j].Score - - case "+controversy", "-controversy", "controversy": - if strings.HasPrefix(sortFld, "-") { - if comments[i].Controversy == comments[j].Controversy { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Controversy > comments[j].Controversy - } - if comments[i].Controversy == comments[j].Controversy { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Controversy < comments[j].Controversy - - default: - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - }) - return comments -} diff --git a/backend/app/store/engine2/engine_mock.go b/backend/app/store/engine2/engine_mock.go deleted file mode 100644 index 01232fb39f..0000000000 --- a/backend/app/store/engine2/engine_mock.go +++ /dev/null @@ -1,205 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. -package engine2 - -import mock "github.com/stretchr/testify/mock" -import store "github.com/umputun/remark/backend/app/store" - -// MockInterface is an autogenerated mock type for the Interface type -type MockInterface struct { - mock.Mock -} - -// Close provides a mock function with given fields: -func (_m *MockInterface) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Count provides a mock function with given fields: req -func (_m *MockInterface) Count(req FindRequest) (int, error) { - ret := _m.Called(req) - - var r0 int - if rf, ok := ret.Get(0).(func(FindRequest) int); ok { - r0 = rf(req) - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func(FindRequest) error); ok { - r1 = rf(req) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Create provides a mock function with given fields: comment -func (_m *MockInterface) Create(comment store.Comment) (string, error) { - ret := _m.Called(comment) - - var r0 string - if rf, ok := ret.Get(0).(func(store.Comment) string); ok { - r0 = rf(comment) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Comment) error); ok { - r1 = rf(comment) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Delete provides a mock function with given fields: req -func (_m *MockInterface) Delete(req DeleteRequest) error { - ret := _m.Called(req) - - var r0 error - if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok { - r0 = rf(req) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Find provides a mock function with given fields: req -func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) { - ret := _m.Called(req) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok { - r0 = rf(req) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(FindRequest) error); ok { - r1 = rf(req) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Flag provides a mock function with given fields: req -func (_m *MockInterface) Flag(req FlagRequest) (bool, error) { - ret := _m.Called(req) - - var r0 bool - if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok { - r0 = rf(req) - } else { - r0 = ret.Get(0).(bool) - } - - var r1 error - if rf, ok := ret.Get(1).(func(FlagRequest) error); ok { - r1 = rf(req) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Get provides a mock function with given fields: locator, commentID -func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) { - ret := _m.Called(locator, commentID) - - var r0 store.Comment - if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok { - r0 = rf(locator, commentID) - } else { - r0 = ret.Get(0).(store.Comment) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { - r1 = rf(locator, commentID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Info provides a mock function with given fields: req -func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) { - ret := _m.Called(req) - - var r0 []store.PostInfo - if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok { - r0 = rf(req) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.PostInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(InfoRequest) error); ok { - r1 = rf(req) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListFlags provides a mock function with given fields: siteID, flag -func (_m *MockInterface) ListFlags(siteID string, flag Flag) ([]interface{}, error) { - ret := _m.Called(siteID, flag) - - var r0 []interface{} - if rf, ok := ret.Get(0).(func(string, Flag) []interface{}); ok { - r0 = rf(siteID, flag) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]interface{}) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, Flag) error); ok { - r1 = rf(siteID, flag) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Update provides a mock function with given fields: locator, comment -func (_m *MockInterface) Update(locator store.Locator, comment store.Comment) error { - ret := _m.Called(locator, comment) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok { - r0 = rf(locator, comment) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine_old/bolt_accessor.go similarity index 99% rename from backend/app/store/engine/bolt_accessor.go rename to backend/app/store/engine_old/bolt_accessor.go index c900d7747c..f48c75da01 100644 --- a/backend/app/store/engine/bolt_accessor.go +++ b/backend/app/store/engine_old/bolt_accessor.go @@ -1,4 +1,4 @@ -package engine +package engine_old import ( "bytes" diff --git a/backend/app/store/engine/bolt_accessor_test.go b/backend/app/store/engine_old/bolt_accessor_test.go similarity index 99% rename from backend/app/store/engine/bolt_accessor_test.go rename to backend/app/store/engine_old/bolt_accessor_test.go index 1bf900d6ba..fa0f638163 100644 --- a/backend/app/store/engine/bolt_accessor_test.go +++ b/backend/app/store/engine_old/bolt_accessor_test.go @@ -1,4 +1,4 @@ -package engine +package engine_old import ( "fmt" diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine_old/bolt_admin.go similarity index 99% rename from backend/app/store/engine/bolt_admin.go rename to backend/app/store/engine_old/bolt_admin.go index 8924f5609f..914346f2ec 100644 --- a/backend/app/store/engine/bolt_admin.go +++ b/backend/app/store/engine_old/bolt_admin.go @@ -1,4 +1,4 @@ -package engine +package engine_old import ( "encoding/json" diff --git a/backend/app/store/engine/bolt_admin_test.go b/backend/app/store/engine_old/bolt_admin_test.go similarity index 99% rename from backend/app/store/engine/bolt_admin_test.go rename to backend/app/store/engine_old/bolt_admin_test.go index 06899b3b9b..a1b4c3c1f8 100644 --- a/backend/app/store/engine/bolt_admin_test.go +++ b/backend/app/store/engine_old/bolt_admin_test.go @@ -1,4 +1,4 @@ -package engine +package engine_old import ( "testing" diff --git a/backend/app/store/engine_old/engine.go b/backend/app/store/engine_old/engine.go new file mode 100644 index 0000000000..2a8dad42da --- /dev/null +++ b/backend/app/store/engine_old/engine.go @@ -0,0 +1,87 @@ +// Package engine defines interfaces each supported storage should implement. +// Includes default implementation with boltdb +package engine_old + +import ( + "sort" + "strings" + "time" + + "github.com/umputun/remark/backend/app/store" +) + +// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate +//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go" + +// Interface defines methods provided by low-level storage engine +type Interface interface { + Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id + Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id + Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only + Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator + Last(siteID string, limit int, since time.Time) ([]store.Comment, error) // last comments for given site, sorted by time + User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time + UserCount(siteID, userID string) (int, error) // comments count by user + Count(locator store.Locator) (int, error) // number of comments for the post + List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts + Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info + Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id + DeleteAll(siteID string) error // delete all data from site + DeleteUser(siteID string, userID string) error // remove all comments from user + SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent) + IsBlocked(siteID string, userID string) bool // check if user blocked + Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users + SetReadOnly(locator store.Locator, status bool) error // set/reset read-only flag + IsReadOnly(locator store.Locator) bool // check if post read-only + SetVerified(siteID string, userID string, status bool) error // set/reset verified flag + IsVerified(siteID string, userID string) bool // check verified status + Verified(siteID string) ([]string, error) // list of verified user ids + Close() error // close/stop engine +} + +const ( + // limits + lastLimit = 1000 + userLimit = 500 +) + +// SortComments is for engines can't sort data internally +func SortComments(comments []store.Comment, sortFld string) []store.Comment { + sort.Slice(comments, func(i, j int) bool { + switch sortFld { + case "+time", "-time", "time", "+active", "-active", "active": + if strings.HasPrefix(sortFld, "-") { + return comments[i].Timestamp.After(comments[j].Timestamp) + } + return comments[i].Timestamp.Before(comments[j].Timestamp) + + case "+score", "-score", "score": + if strings.HasPrefix(sortFld, "-") { + if comments[i].Score == comments[j].Score { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Score > comments[j].Score + } + if comments[i].Score == comments[j].Score { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Score < comments[j].Score + + case "+controversy", "-controversy", "controversy": + if strings.HasPrefix(sortFld, "-") { + if comments[i].Controversy == comments[j].Controversy { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Controversy > comments[j].Controversy + } + if comments[i].Controversy == comments[j].Controversy { + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + return comments[i].Controversy < comments[j].Controversy + + default: + return comments[i].Timestamp.Before(comments[j].Timestamp) + } + }) + return comments +} diff --git a/backend/app/store/engine_old/engine_mock.go b/backend/app/store/engine_old/engine_mock.go new file mode 100644 index 0000000000..879402b15d --- /dev/null +++ b/backend/app/store/engine_old/engine_mock.go @@ -0,0 +1,408 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. +package engine_old + +import mock "github.com/stretchr/testify/mock" +import store "github.com/umputun/remark/backend/app/store" +import time "time" + +// MockInterface is an autogenerated mock type for the Interface type +type MockInterface struct { + mock.Mock +} + +// Blocked provides a mock function with given fields: siteID +func (_m *MockInterface) Blocked(siteID string) ([]store.BlockedUser, error) { + ret := _m.Called(siteID) + + var r0 []store.BlockedUser + if rf, ok := ret.Get(0).(func(string) []store.BlockedUser); ok { + r0 = rf(siteID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.BlockedUser) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(siteID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Close provides a mock function with given fields: +func (_m *MockInterface) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Count provides a mock function with given fields: locator +func (_m *MockInterface) Count(locator store.Locator) (int, error) { + ret := _m.Called(locator) + + var r0 int + if rf, ok := ret.Get(0).(func(store.Locator) int); ok { + r0 = rf(locator) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(store.Locator) error); ok { + r1 = rf(locator) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Create provides a mock function with given fields: comment +func (_m *MockInterface) Create(comment store.Comment) (string, error) { + ret := _m.Called(comment) + + var r0 string + if rf, ok := ret.Get(0).(func(store.Comment) string); ok { + r0 = rf(comment) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(store.Comment) error); ok { + r1 = rf(comment) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: locator, commentID, mode +func (_m *MockInterface) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { + ret := _m.Called(locator, commentID, mode) + + var r0 error + if rf, ok := ret.Get(0).(func(store.Locator, string, store.DeleteMode) error); ok { + r0 = rf(locator, commentID, mode) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteAll provides a mock function with given fields: siteID +func (_m *MockInterface) DeleteAll(siteID string) error { + ret := _m.Called(siteID) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(siteID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteUser provides a mock function with given fields: siteID, userID +func (_m *MockInterface) DeleteUser(siteID string, userID string) error { + ret := _m.Called(siteID, userID) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(siteID, userID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Find provides a mock function with given fields: locator, sort +func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comment, error) { + ret := _m.Called(locator, sort) + + var r0 []store.Comment + if rf, ok := ret.Get(0).(func(store.Locator, string) []store.Comment); ok { + r0 = rf(locator, sort) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.Comment) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { + r1 = rf(locator, sort) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Get provides a mock function with given fields: locator, commentID +func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) { + ret := _m.Called(locator, commentID) + + var r0 store.Comment + if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok { + r0 = rf(locator, commentID) + } else { + r0 = ret.Get(0).(store.Comment) + } + + var r1 error + if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { + r1 = rf(locator, commentID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Info provides a mock function with given fields: locator, readonlyAge +func (_m *MockInterface) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) { + ret := _m.Called(locator, readonlyAge) + + var r0 store.PostInfo + if rf, ok := ret.Get(0).(func(store.Locator, int) store.PostInfo); ok { + r0 = rf(locator, readonlyAge) + } else { + r0 = ret.Get(0).(store.PostInfo) + } + + var r1 error + if rf, ok := ret.Get(1).(func(store.Locator, int) error); ok { + r1 = rf(locator, readonlyAge) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IsBlocked provides a mock function with given fields: siteID, userID +func (_m *MockInterface) IsBlocked(siteID string, userID string) bool { + ret := _m.Called(siteID, userID) + + var r0 bool + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(siteID, userID) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// IsReadOnly provides a mock function with given fields: locator +func (_m *MockInterface) IsReadOnly(locator store.Locator) bool { + ret := _m.Called(locator) + + var r0 bool + if rf, ok := ret.Get(0).(func(store.Locator) bool); ok { + r0 = rf(locator) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// IsVerified provides a mock function with given fields: siteID, userID +func (_m *MockInterface) IsVerified(siteID string, userID string) bool { + ret := _m.Called(siteID, userID) + + var r0 bool + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(siteID, userID) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// Last provides a mock function with given fields: siteID, limit, since +func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]store.Comment, error) { + ret := _m.Called(siteID, limit, since) + + var r0 []store.Comment + if rf, ok := ret.Get(0).(func(string, int, time.Time) []store.Comment); ok { + r0 = rf(siteID, limit, since) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.Comment) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, int, time.Time) error); ok { + r1 = rf(siteID, limit, since) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// List provides a mock function with given fields: siteID, limit, skip +func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostInfo, error) { + ret := _m.Called(siteID, limit, skip) + + var r0 []store.PostInfo + if rf, ok := ret.Get(0).(func(string, int, int) []store.PostInfo); ok { + r0 = rf(siteID, limit, skip) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.PostInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, int, int) error); ok { + r1 = rf(siteID, limit, skip) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Put provides a mock function with given fields: locator, comment +func (_m *MockInterface) Put(locator store.Locator, comment store.Comment) error { + ret := _m.Called(locator, comment) + + var r0 error + if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok { + r0 = rf(locator, comment) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SetBlock provides a mock function with given fields: siteID, userID, status, ttl +func (_m *MockInterface) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { + ret := _m.Called(siteID, userID, status, ttl) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, bool, time.Duration) error); ok { + r0 = rf(siteID, userID, status, ttl) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SetReadOnly provides a mock function with given fields: locator, status +func (_m *MockInterface) SetReadOnly(locator store.Locator, status bool) error { + ret := _m.Called(locator, status) + + var r0 error + if rf, ok := ret.Get(0).(func(store.Locator, bool) error); ok { + r0 = rf(locator, status) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SetVerified provides a mock function with given fields: siteID, userID, status +func (_m *MockInterface) SetVerified(siteID string, userID string, status bool) error { + ret := _m.Called(siteID, userID, status) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, bool) error); ok { + r0 = rf(siteID, userID, status) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// User provides a mock function with given fields: siteID, userID, limit, skip +func (_m *MockInterface) User(siteID string, userID string, limit int, skip int) ([]store.Comment, error) { + ret := _m.Called(siteID, userID, limit, skip) + + var r0 []store.Comment + if rf, ok := ret.Get(0).(func(string, string, int, int) []store.Comment); ok { + r0 = rf(siteID, userID, limit, skip) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]store.Comment) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, string, int, int) error); ok { + r1 = rf(siteID, userID, limit, skip) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// UserCount provides a mock function with given fields: siteID, userID +func (_m *MockInterface) UserCount(siteID string, userID string) (int, error) { + ret := _m.Called(siteID, userID) + + var r0 int + if rf, ok := ret.Get(0).(func(string, string) int); ok { + r0 = rf(siteID, userID) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(siteID, userID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Verified provides a mock function with given fields: siteID +func (_m *MockInterface) Verified(siteID string) ([]string, error) { + ret := _m.Called(siteID) + + var r0 []string + if rf, ok := ret.Get(0).(func(string) []string); ok { + r0 = rf(siteID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(siteID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/backend/app/store/engine2/engine_test.go b/backend/app/store/engine_old/engine_test.go similarity index 98% rename from backend/app/store/engine2/engine_test.go rename to backend/app/store/engine_old/engine_test.go index a17585b9a6..dc04233b74 100644 --- a/backend/app/store/engine2/engine_test.go +++ b/backend/app/store/engine_old/engine_test.go @@ -1,4 +1,4 @@ -package engine2 +package engine_old import ( "testing" diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine_old/mongo.go similarity index 99% rename from backend/app/store/engine/mongo.go rename to backend/app/store/engine_old/mongo.go index 3520711d34..6cd3f3117d 100644 --- a/backend/app/store/engine/mongo.go +++ b/backend/app/store/engine_old/mongo.go @@ -1,4 +1,4 @@ -package engine +package engine_old import ( "time" diff --git a/backend/app/store/engine/mongo_test.go b/backend/app/store/engine_old/mongo_test.go similarity index 99% rename from backend/app/store/engine/mongo_test.go rename to backend/app/store/engine_old/mongo_test.go index 5b3f4b27bd..cb16051c88 100644 --- a/backend/app/store/engine/mongo_test.go +++ b/backend/app/store/engine_old/mongo_test.go @@ -1,4 +1,4 @@ -package engine +package engine_old import ( "fmt" diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 5fdf0d58ac..0087a877b7 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -19,13 +19,12 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" - "github.com/umputun/remark/backend/app/store/engine2" "github.com/umputun/remark/backend/app/store/image" ) // DataStore wraps store.Interface with additional methods type DataStore struct { - Engine engine2.Interface + Engine engine.Interface EditDuration time.Duration AdminStore admin.Store MaxCommentSize int @@ -105,7 +104,7 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error) // Find wraps engine's Find call and alter results if needed // user used to filter results for self vs others func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) { - req := engine2.FindRequest{Locator: locator, Sort: sort} + req := engine.FindRequest{Locator: locator, Sort: sort} comments, err := s.Engine.Find(req) if err != nil { return comments, err @@ -192,7 +191,7 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err // DeleteAll removes all data from site func (s *DataStore) DeleteAll(siteID string) error { - req := engine2.DeleteRequest{Locator: store.Locator{SiteID: siteID}} + req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}} return s.Engine.Delete(req) } @@ -317,7 +316,7 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi if req.Delete { // delete request comment.Deleted = true - delReq := engine2.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: store.SoftDelete} + delReq := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: store.SoftDelete} return comment, s.Engine.Delete(delReq) } @@ -351,7 +350,7 @@ func (s *DataStore) HasReplies(comment store.Comment) bool { return true } - req := engine2.FindRequest{Locator: store.Locator{SiteID: comment.Locator.SiteID}, Limit: maxLastCommentsReply} + req := engine.FindRequest{Locator: store.Locator{SiteID: comment.Locator.SiteID}, Limit: maxLastCommentsReply} comments, err := s.Engine.Find(req) if err != nil { log.Printf("[WARN] can't get last comments for reply check, %v", err) @@ -430,7 +429,7 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) { res := []store.PostInfo{} for _, p := range postIDs { - req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID, URL: p}} + req := engine.FindRequest{Locator: store.Locator{SiteID: siteID, URL: p}} if c, err := s.Engine.Count(req); err == nil { res = append(res, store.PostInfo{URL: p, Count: c}) } @@ -468,63 +467,63 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool { // IsReadOnly checks if post read-only func (s *DataStore) IsReadOnly(locator store.Locator) bool { - req := engine2.FlagRequest{Locator: locator, Flag: engine2.ReadOnly} + req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly} ro, err := s.Engine.Flag(req) return err == nil && ro } // SetReadOnly set/reset read-only flag func (s *DataStore) SetReadOnly(locator store.Locator, status bool) error { - roStatus := engine2.FlagFalse + roStatus := engine.FlagFalse if status { - roStatus = engine2.FlagTrue + roStatus = engine.FlagTrue } - req := engine2.FlagRequest{Locator: locator, Flag: engine2.ReadOnly, Update: roStatus} + req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly, Update: roStatus} _, err := s.Engine.Flag(req) return err } // IsVerified checks if user verified func (s *DataStore) IsVerified(siteID string, userID string) bool { - req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Verified} + req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified} ro, err := s.Engine.Flag(req) return err == nil && ro } // SetVerified set/reset verified status for user func (s *DataStore) SetVerified(siteID string, userID string, status bool) error { - roStatus := engine2.FlagFalse + roStatus := engine.FlagFalse if status { - roStatus = engine2.FlagTrue + roStatus = engine.FlagTrue } - req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Verified, Update: roStatus} + req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified, Update: roStatus} _, err := s.Engine.Flag(req) return err } // IsBlocked checks if user blocked func (s *DataStore) IsBlocked(siteID string, userID string) bool { - req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine2.Blocked} + req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Blocked} ro, err := s.Engine.Flag(req) return err == nil && ro } // SetBlock set/reset verified status for user func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - roStatus := engine2.FlagFalse + roStatus := engine.FlagFalse if status { - roStatus = engine2.FlagTrue + roStatus = engine.FlagTrue } - req := engine2.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, - Flag: engine2.Blocked, Update: roStatus, TTL: ttl} + req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, + Flag: engine.Blocked, Update: roStatus, TTL: ttl} _, err := s.Engine.Flag(req) return err } // Blocked returns list with all blocked users func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) { - blocked, e := s.Engine.ListFlags(siteID, engine2.Blocked) + blocked, e := s.Engine.ListFlags(siteID, engine.Blocked) if e != nil { return nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID) } @@ -536,7 +535,7 @@ func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) // Info get post info func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) { - req := engine2.InfoRequest{Locator: locator, ReadOnlyAge: readonlyAge} + req := engine.InfoRequest{Locator: locator, ReadOnlyAge: readonlyAge} res, err := s.Engine.Info(req) if err != nil { return store.PostInfo{}, err @@ -549,25 +548,25 @@ func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo // Delete comment by id func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - req := engine2.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode} + req := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode} return s.Engine.Delete(req) } // DeleteUser removes all comments from user func (s *DataStore) DeleteUser(siteID string, userID string) error { - req := engine2.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: store.HardDelete} + req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: store.HardDelete} return s.Engine.Delete(req) } // List of commented posts func (s *DataStore) List(siteID string, limit int, skip int) ([]store.PostInfo, error) { - req := engine2.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip} + req := engine.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip} return s.Engine.Info(req) } // Count gets number of comments for the post func (s *DataStore) Count(locator store.Locator) (int, error) { - req := engine2.FindRequest{Locator: locator} + req := engine.FindRequest{Locator: locator} return s.Engine.Count(req) } @@ -577,7 +576,7 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe pmetas = []PostMetaData{} // set posts meta - posts, err := s.Engine.Info(engine2.InfoRequest{Locator: store.Locator{SiteID: siteID}}) + posts, err := s.Engine.Info(engine.InfoRequest{Locator: store.Locator{SiteID: siteID}}) if err != nil { return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID) } @@ -607,7 +606,7 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe } // process verified users - verified, err := s.Engine.ListFlags(siteID, engine2.Verified) + verified, err := s.Engine.ListFlags(siteID, engine.Verified) if err != nil { return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID) } @@ -655,7 +654,7 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post // User gets comment for given userID on siteID func (s *DataStore) User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error) { - req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Limit: limit, Skip: skip} + req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Limit: limit, Skip: skip} comments, err := s.Engine.Find(req) if err != nil { return comments, err @@ -665,13 +664,13 @@ func (s *DataStore) User(siteID, userID string, limit, skip int, user store.User // UserCount is comments count by user func (s *DataStore) UserCount(siteID, userID string) (int, error) { - req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID} + req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID} return s.Engine.Count(req) } // Last gets last comments for site, cross-post. Limited by count and optional since ts func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error) { - req := engine2.FindRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Since: since, Sort: "-time"} + req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Since: since, Sort: "-time"} comments, err := s.Engine.Find(req) if err != nil { return comments, err @@ -720,7 +719,7 @@ func (s *DataStore) alterComments(cc []store.Comment, user store.User) (res []st func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Comment) { - blocReq := engine2.FlagRequest{Flag: engine2.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID} + blocReq := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID} blocked, _ := s.Engine.Flag(blocReq) // process blocked users @@ -734,7 +733,7 @@ func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Co // set verified status retroactively if !blocked { - verifReq := engine2.FlagRequest{Flag: engine2.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID} + verifReq := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID} c.User.Verified, _ = s.Engine.Flag(verifReq) } diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index 9c18ca31df..ef688c2ee5 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -21,7 +21,7 @@ import ( "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" - "github.com/umputun/remark/backend/app/store/engine2" + "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/image" ) @@ -1015,9 +1015,9 @@ func TestService_submitImages(t *testing.T) { func TestService_alterComment(t *testing.T) { defer teardown(t) - engineMock := engine2.MockInterface{} - engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Blocked, UserID: "devid"}).Return(false, nil) - engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Verified, UserID: "devid"}).Return(false, nil) + engineMock := engine.MockInterface{} + engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(false, nil) + engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil) svc := DataStore{Engine: &engineMock} r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, @@ -1027,17 +1027,17 @@ func TestService_alterComment(t *testing.T) { store.User{Name: "dev", ID: "devid", Admin: true}) assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, r, "ip not cleaned") - engineMock = engine2.MockInterface{} - engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Blocked, UserID: "devid"}).Return(false, nil) - engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Verified, UserID: "devid"}).Return(true, nil) + engineMock = engine.MockInterface{} + engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(false, nil) + engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(true, nil) svc = DataStore{Engine: &engineMock} r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}}, store.User{Name: "dev", ID: "devid", Admin: false}) assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid", Verified: true}}, r, "verified set") - engineMock = engine2.MockInterface{} - engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Blocked, UserID: "devid"}).Return(true, nil) - engineMock.On("Flag", engine2.FlagRequest{Flag: engine2.Verified, UserID: "devid"}).Return(false, nil) + engineMock = engine.MockInterface{} + engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(true, nil) + engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil) svc = DataStore{Engine: &engineMock} r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}}, store.User{Name: "dev", ID: "devid", Admin: false}) @@ -1046,10 +1046,10 @@ func TestService_alterComment(t *testing.T) { } // makes new boltdb, put two records -func prepStoreEngine(t *testing.T) engine2.Interface { +func prepStoreEngine(t *testing.T) engine.Interface { _ = os.Remove(testDb) - boltStore, err := engine2.NewBoltDB(bolt.Options{}, engine2.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"}) + boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"}) assert.NoError(t, err) b := boltStore From a6a744810c59f0a2cc4e53e46d6deedf384a10b0 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 9 Jun 2019 01:22:35 -0500 Subject: [PATCH 04/24] remote client adjusted to new engine flavour --- backend/app/store/engine/engine.go | 39 +++++++------- backend/app/store/remote/remote.go | 66 ++++++++++++++++-------- backend/app/store/remote/remote_test.go | 68 +++++++++++++++++++------ 3 files changed, 118 insertions(+), 55 deletions(-) diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go index 4e1688c314..cd618fe2fd 100644 --- a/backend/app/store/engine/engine.go +++ b/backend/app/store/engine/engine.go @@ -22,7 +22,7 @@ type Interface interface { Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info Count(req FindRequest) (int, error) // get count for post or user - Delete(req DeleteRequest) error // delete post(s) by id or by userID + Delete(req DeleteRequest) error // delete post(s) by id or by userID Flag(req FlagRequest) (bool, error) // set and get flags ListFlags(siteID string, flag Flag) ([]interface{}, error) // get list of flagged keys, like blocked & verified user Close() error // close storage engine @@ -30,25 +30,28 @@ type Interface interface { // FindRequest is the input for all find operations type FindRequest struct { - Locator store.Locator // lack of URL means site operation - UserID string // presence of UserID treated as user-related find - Sort string // sort order with +/-field syntax - Since time.Time // time limit for found results - Limit, Skip int + Locator store.Locator `json:"locator"` // lack of URL means site operation + UserID string `json:"user_id,omitempty"` // presence of UserID treated as user-related find + Sort string `json:"sort,omitempty"` // sort order with +/-field syntax + Since time.Time `json:"since,omitempty"` // time limit for found results + Limit int `json:"limit,omitempty"` + Skip int `json:"skip,omitempty"` } // InfoRequest is the input of Info operation used to get meta data about posts type InfoRequest struct { - Locator store.Locator - Limit, Skip int - ReadOnlyAge int + Locator store.Locator `json:"locator"` + Limit int `json:"limit,omitempty"` + Skip int `json:"skip,omitempty"` + ReadOnlyAge int `json:"ro_age,omitempty"` } +// DeleteRequest is the input for all delete operations (comments, sites, users) type DeleteRequest struct { - Locator store.Locator // lack of URL means site operation - CommentID string - UserID string - DeleteMode store.DeleteMode + Locator store.Locator `json:"locator"` // lack of URL means site operation + CommentID string `json:"comment_id,omitempty"` + UserID string `json:"user_id,omitempty"` + DeleteMode store.DeleteMode `json:"del_mode"` } // Flag defines type of binary attribute @@ -73,11 +76,11 @@ const ( // FlagRequest is the input for both get/set for flags, like blocked, verified and so on type FlagRequest struct { - Flag Flag // flag type - Locator store.Locator // post locator - UserID string // for flags setting user status - Update FlagStatus // if FlagNonSet it will be get op, if set will set the value - TTL time.Duration // ttl for time-sensitive flags only, like blocked for some period + Flag Flag `json:"flag"` // flag type + Locator store.Locator `json:"locator"` // post locator + UserID string `json:"user_id,omitempty"` // for flags setting user status + Update FlagStatus `json:"update,omitempty"` // if FlagNonSet it will be get op, if set will set the value + TTL time.Duration `json:"ttl,omitempty"` // ttl for time-sensitive flags only, like blocked for some period } const ( diff --git a/backend/app/store/remote/remote.go b/backend/app/store/remote/remote.go index ff805465df..a39a5249d8 100644 --- a/backend/app/store/remote/remote.go +++ b/backend/app/store/remote/remote.go @@ -4,11 +4,11 @@ import ( "bytes" "encoding/json" "net/http" - "time" "github.com/pkg/errors" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/engine" ) // Client implements remote engine and delegates all calls to remote http server @@ -54,45 +54,55 @@ func (r *Client) Get(locator store.Locator, commentID string) (comment store.Com return comment, err } -// Put updates comment, mutable parts only -func (r *Client) Put(locator store.Locator, comment store.Comment) error { - _, err := r.call("put", locator, comment) +// Update comment, mutable parts only +func (r *Client) Update(locator store.Locator, comment store.Comment) error { + _, err := r.call("update", locator, comment) return err } // Find comments for locator -func (r *Client) Find(locator store.Locator, sort string) (comments []store.Comment, err error) { - resp, err := r.call("find", locator, sort) +func (r *Client) Find(req engine.FindRequest) (comments []store.Comment, err error) { + resp, err := r.call("find", req) if err != nil { - return []store.Comment{}, err + return nil, err } err = json.Unmarshal(*resp.Result, &comments) return comments, err } -// Last comments for given site, sorted by time -func (r *Client) Last(siteID string, limit int, since time.Time) (comments []store.Comment, err error) { - resp, err := r.call("last", siteID, limit, since) +// Info returns post(s) meta info +func (r *Client) Info(req engine.InfoRequest) (info []store.PostInfo, err error) { + resp, err := r.call("info", req) if err != nil { - return []store.Comment{}, err + return nil, err } - err = json.Unmarshal(*resp.Result, &comments) - return comments, err + err = json.Unmarshal(*resp.Result, &info) + return info, err } -// User get comments by user, sorted by time -func (r *Client) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) { - resp, err := r.call("user", siteID, userID, limit, skip) +// Flag sets and gets flags +func (r *Client) Flag(req engine.FlagRequest) (status bool, err error) { + resp, err := r.call("flag", req) if err != nil { - return []store.Comment{}, err + return false, err } - err = json.Unmarshal(*resp.Result, &comments) - return comments, err + err = json.Unmarshal(*resp.Result, &status) + return status, err +} + +// ListFlags get list of flagged keys, like blocked & verified user +func (r *Client) ListFlags(siteID string, flag engine.Flag) (list []interface{}, err error) { + resp, err := r.call("list_flags", siteID, flag) + if err != nil { + return nil, err + } + err = json.Unmarshal(*resp.Result, &list) + return list, err } -// UserCount gets comments count by user -func (r *Client) UserCount(siteID, userID string) (count int, err error) { - resp, err := r.call("user_count", siteID, userID) +// Count gets comments count by user or site +func (r *Client) Count(req engine.FindRequest) (count int, err error) { + resp, err := r.call("count", req) if err != nil { return 0, err } @@ -100,6 +110,18 @@ func (r *Client) UserCount(siteID, userID string) (count int, err error) return count, err } +// Delete post(s) by id or by userID +func (r *Client) Delete(req engine.DeleteRequest) error { + _, err := r.call("delete", req) + return err +} + +// Close storage engine +func (r *Client) Close() error { + _, err := r.call("close") + return err +} + func (r *Client) call(method string, args ...interface{}) (*Response, error) { b, err := json.Marshal(Request{Method: method, Params: args}) diff --git a/backend/app/store/remote/remote_test.go b/backend/app/store/remote/remote_test.go index d3e56f9db9..45814c3ae0 100644 --- a/backend/app/store/remote/remote_test.go +++ b/backend/app/store/remote/remote_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/engine" ) func TestClient_Create(t *testing.T) { @@ -79,47 +80,84 @@ func TestClient_FailedStatus(t *testing.T) { assert.EqualError(t, err, "bad status 400 for get") } -func TestClient_Put(t *testing.T) { - ts := testServer(t, `{"method":"put","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{}`) +func TestClient_Update(t *testing.T) { + ts := testServer(t, `{"method":"update","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} - err := c.Put(store.Locator{URL: "http://example.com/url"}, store.Comment{ID: "123", + err := c.Update(store.Locator{URL: "http://example.com/url"}, store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"}, Text: "msg"}) assert.NoError(t, err) } func TestClient_Find(t *testing.T) { - ts := testServer(t, `{"method":"find","params":[{"url":"http://example.com/url"},""]}`, - `{"result":[{"text":"1"},{"text":"2"}]}`) + ts := testServer(t, `{"method":"find","params":[{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10}]}`, `{"result":[{"text":"1"},{"text":"2"}]}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} - res, err := c.Find(store.Locator{URL: "http://example.com/url"}, "") + res, err := c.Find(engine.FindRequest{Locator: store.Locator{URL: "http://example.com/url"}, Sort: "-time", Limit: 10}) assert.NoError(t, err) assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) } -func TestClient_Last(t *testing.T) { - ts := testServer(t, `{"method":"last","params":["site1",100,"2019-06-06T19:34:10Z"]}`, - `{"result":[{"text":"1"},{"text":"2"}]}`) +func TestClient_Info(t *testing.T) { + ts := testServer(t, `{"method":"info","params":[{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10}]}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} - res, err := c.Last("site1", 100, time.Date(2019, 6, 6, 19, 34, 10, 0, time.UTC)) + res, err := c.Info(engine.InfoRequest{Locator: store.Locator{URL: "http://example.com/url"}, + Limit: 10, Skip: 5, ReadOnlyAge: 10}) assert.NoError(t, err) - assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) + assert.Equal(t, []store.PostInfo{{URL: "u1", Count: 22}, {URL: "u2", Count: 33}}, res) } -func TestClient_User(t *testing.T) { - ts := testServer(t, `{"method":"user","params":["site1","u1",100,4]}`, `{"result":[{"text":"1"},{"text":"2"}]}`) +func TestClient_Flag(t *testing.T) { + ts := testServer(t, `{"method":"flag","params":[{"flag":"verified","locator":{"url":"http://example.com/url"}}]}`, + `{"result":false}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} - res, err := c.User("site1", "u1", 100, 4) + res, err := c.Flag(engine.FlagRequest{Locator: store.Locator{URL: "http://example.com/url"}, Flag: engine.Verified}) + assert.NoError(t, err) + assert.Equal(t, false, res) +} + +func TestClient_ListFlag(t *testing.T) { + ts := testServer(t, `{"method":"list_flags","params":["site_id","blocked"]}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + res, err := c.ListFlags("site_id", engine.Blocked) + assert.NoError(t, err) + assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res) +} + +func TestClient_Count(t *testing.T) { + ts := testServer(t, `{"method":"count","params":[{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"}]}`, + `{"result":11}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + res, err := c.Count(engine.FindRequest{Locator: store.Locator{URL: "http://example.com/url"}}) + assert.NoError(t, err) + assert.Equal(t, 11, res) +} + +func TestClient_Delete(t *testing.T) { + ts := testServer(t, `{"method":"delete","params":[{"locator":{"url":"http://example.com/url"},"del_mode":0}]}`, `{}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + + err := c.Delete(engine.DeleteRequest{Locator: store.Locator{URL: "http://example.com/url"}}) + assert.NoError(t, err) +} + +func TestClient_Close(t *testing.T) { + ts := testServer(t, `{"method":"close","params":null}`, `{}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + err := c.Close() assert.NoError(t, err) - assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) } func testServer(t *testing.T, req, resp string) *httptest.Server { From 625dbff3b1ed2d2a00cb0f3e3701f2998ffe8851 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 9 Jun 2019 01:49:34 -0500 Subject: [PATCH 05/24] split remote to common protocol package and engine implemetation --- backend/app/store/engine/remote.go | 104 ++++++++++++++ backend/app/store/engine/remote_test.go | 171 ++++++++++++++++++++++++ backend/app/store/remote/remote.go | 99 +------------- backend/app/store/remote/remote_test.go | 145 +++----------------- 4 files changed, 295 insertions(+), 224 deletions(-) create mode 100644 backend/app/store/engine/remote.go create mode 100644 backend/app/store/engine/remote_test.go diff --git a/backend/app/store/engine/remote.go b/backend/app/store/engine/remote.go new file mode 100644 index 0000000000..997b9492bd --- /dev/null +++ b/backend/app/store/engine/remote.go @@ -0,0 +1,104 @@ +package engine + +import ( + "encoding/json" + + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/remote" +) + +// Remote implements remote engine and delegates all Calls to remote http server +type Remote struct { + remote.Client +} + +// Create comment and return ID +func (r *Remote) Create(comment store.Comment) (commentID string, err error) { + + resp, err := r.Call("create", comment) + if err != nil { + return "", err + } + + err = json.Unmarshal(*resp.Result, &commentID) + return commentID, err +} + +// Get comment by ID +func (r *Remote) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { + resp, err := r.Call("get", locator, commentID) + if err != nil { + return store.Comment{}, err + } + + err = json.Unmarshal(*resp.Result, &comment) + return comment, err +} + +// Update comment, mutable parts only +func (r *Remote) Update(locator store.Locator, comment store.Comment) error { + _, err := r.Call("update", locator, comment) + return err +} + +// Find comments for locator +func (r *Remote) Find(req FindRequest) (comments []store.Comment, err error) { + resp, err := r.Call("find", req) + if err != nil { + return nil, err + } + err = json.Unmarshal(*resp.Result, &comments) + return comments, err +} + +// Info returns post(s) meta info +func (r *Remote) Info(req InfoRequest) (info []store.PostInfo, err error) { + resp, err := r.Call("info", req) + if err != nil { + return nil, err + } + err = json.Unmarshal(*resp.Result, &info) + return info, err +} + +// Flag sets and gets flags +func (r *Remote) Flag(req FlagRequest) (status bool, err error) { + resp, err := r.Call("flag", req) + if err != nil { + return false, err + } + err = json.Unmarshal(*resp.Result, &status) + return status, err +} + +// ListFlags get list of flagged keys, like blocked & verified user +func (r *Remote) ListFlags(siteID string, flag Flag) (list []interface{}, err error) { + resp, err := r.Call("list_flags", siteID, flag) + if err != nil { + return nil, err + } + err = json.Unmarshal(*resp.Result, &list) + return list, err +} + +// Count gets comments count by user or site +func (r *Remote) Count(req FindRequest) (count int, err error) { + resp, err := r.Call("count", req) + if err != nil { + return 0, err + } + err = json.Unmarshal(*resp.Result, &count) + return count, err +} + +// Delete post(s) by id or by userID +func (r *Remote) Delete(req DeleteRequest) error { + _, err := r.Call("delete", req) + return err +} + +// Close storage engine +func (r *Remote) Close() error { + _, err := r.Call("close") + return err +} diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go new file mode 100644 index 0000000000..91e013f5bf --- /dev/null +++ b/backend/app/store/engine/remote_test.go @@ -0,0 +1,171 @@ +package engine + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark/backend/app/store" + "github.com/umputun/remark/backend/app/store/remote" +) + +func TestClient_Create(t *testing.T) { + ts := testServer(t, `{"method":"create","params":[{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{"result":"12345"}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"}, + Text: "msg"}) + assert.NoError(t, err) + assert.Equal(t, "12345", res) + t.Logf("%v %T", res, res) +} + +func TestClient_Get(t *testing.T) { + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, + `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + res, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.NoError(t, err) + assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res) + t.Logf("%v %T", res, res) +} + +func TestClient_GetWithErrorResult(t *testing.T) { + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, `{"error":"failed"}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.EqualError(t, err, "failed") +} + +func TestClient_GetWithErrorDecode(t *testing.T) { + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, ``) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.EqualError(t, err, "failed to decode response for get: EOF") +} + +func TestClient_GetWithErrorRemote(t *testing.T) { + c := Remote{Client: remote.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "remote Call failed for get:")) +} + +func TestClient_FailedStatus(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + t.Logf("req: %s", string(body)) + w.WriteHeader(400) + })) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + assert.EqualError(t, err, "bad status 400 for get") +} + +func TestClient_Update(t *testing.T) { + ts := testServer(t, `{"method":"update","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + err := c.Update(store.Locator{URL: "http://example.com/url"}, store.Comment{ID: "123", + Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"}, Text: "msg"}) + assert.NoError(t, err) + +} + +func TestClient_Find(t *testing.T) { + ts := testServer(t, `{"method":"find","params":[{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10}]}`, `{"result":[{"text":"1"},{"text":"2"}]}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + res, err := c.Find(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}, Sort: "-time", Limit: 10}) + assert.NoError(t, err) + assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) +} + +func TestClient_Info(t *testing.T) { + ts := testServer(t, `{"method":"info","params":[{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10}]}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + res, err := c.Info(InfoRequest{Locator: store.Locator{URL: "http://example.com/url"}, + Limit: 10, Skip: 5, ReadOnlyAge: 10}) + assert.NoError(t, err) + assert.Equal(t, []store.PostInfo{{URL: "u1", Count: 22}, {URL: "u2", Count: 33}}, res) +} + +func TestClient_Flag(t *testing.T) { + ts := testServer(t, `{"method":"flag","params":[{"flag":"verified","locator":{"url":"http://example.com/url"}}]}`, + `{"result":false}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + res, err := c.Flag(FlagRequest{Locator: store.Locator{URL: "http://example.com/url"}, Flag: Verified}) + assert.NoError(t, err) + assert.Equal(t, false, res) +} + +func TestClient_ListFlag(t *testing.T) { + ts := testServer(t, `{"method":"list_flags","params":["site_id","blocked"]}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + res, err := c.ListFlags("site_id", Blocked) + assert.NoError(t, err) + assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res) +} + +func TestClient_Count(t *testing.T) { + ts := testServer(t, `{"method":"count","params":[{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"}]}`, + `{"result":11}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + res, err := c.Count(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}}) + assert.NoError(t, err) + assert.Equal(t, 11, res) +} + +func TestClient_Delete(t *testing.T) { + ts := testServer(t, `{"method":"delete","params":[{"locator":{"url":"http://example.com/url"},"del_mode":0}]}`, `{}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + err := c.Delete(DeleteRequest{Locator: store.Locator{URL: "http://example.com/url"}}) + assert.NoError(t, err) +} + +func TestClient_Close(t *testing.T) { + ts := testServer(t, `{"method":"close","params":null}`, `{}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + err := c.Close() + assert.NoError(t, err) +} + +func testServer(t *testing.T, req, resp string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, req, string(body)) + t.Logf("req: %s", string(body)) + fmt.Fprintf(w, resp) + })) +} diff --git a/backend/app/store/remote/remote.go b/backend/app/store/remote/remote.go index a39a5249d8..77c688981c 100644 --- a/backend/app/store/remote/remote.go +++ b/backend/app/store/remote/remote.go @@ -6,9 +6,6 @@ import ( "net/http" "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/engine" ) // Client implements remote engine and delegates all calls to remote http server @@ -31,98 +28,8 @@ type Response struct { Error string `json:"error,omitempty"` } -// Create comment and return ID -func (r *Client) Create(comment store.Comment) (commentID string, err error) { - - resp, err := r.call("create", comment) - if err != nil { - return "", err - } - - err = json.Unmarshal(*resp.Result, &commentID) - return commentID, err -} - -// Get comment by ID -func (r *Client) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { - resp, err := r.call("get", locator, commentID) - if err != nil { - return store.Comment{}, err - } - - err = json.Unmarshal(*resp.Result, &comment) - return comment, err -} - -// Update comment, mutable parts only -func (r *Client) Update(locator store.Locator, comment store.Comment) error { - _, err := r.call("update", locator, comment) - return err -} - -// Find comments for locator -func (r *Client) Find(req engine.FindRequest) (comments []store.Comment, err error) { - resp, err := r.call("find", req) - if err != nil { - return nil, err - } - err = json.Unmarshal(*resp.Result, &comments) - return comments, err -} - -// Info returns post(s) meta info -func (r *Client) Info(req engine.InfoRequest) (info []store.PostInfo, err error) { - resp, err := r.call("info", req) - if err != nil { - return nil, err - } - err = json.Unmarshal(*resp.Result, &info) - return info, err -} - -// Flag sets and gets flags -func (r *Client) Flag(req engine.FlagRequest) (status bool, err error) { - resp, err := r.call("flag", req) - if err != nil { - return false, err - } - err = json.Unmarshal(*resp.Result, &status) - return status, err -} - -// ListFlags get list of flagged keys, like blocked & verified user -func (r *Client) ListFlags(siteID string, flag engine.Flag) (list []interface{}, err error) { - resp, err := r.call("list_flags", siteID, flag) - if err != nil { - return nil, err - } - err = json.Unmarshal(*resp.Result, &list) - return list, err -} - -// Count gets comments count by user or site -func (r *Client) Count(req engine.FindRequest) (count int, err error) { - resp, err := r.call("count", req) - if err != nil { - return 0, err - } - err = json.Unmarshal(*resp.Result, &count) - return count, err -} - -// Delete post(s) by id or by userID -func (r *Client) Delete(req engine.DeleteRequest) error { - _, err := r.call("delete", req) - return err -} - -// Close storage engine -func (r *Client) Close() error { - _, err := r.call("close") - return err -} - -func (r *Client) call(method string, args ...interface{}) (*Response, error) { +// Call remote server with given method and arguments +func (r *Client) Call(method string, args ...interface{}) (*Response, error) { b, err := json.Marshal(Request{Method: method, Params: args}) if err != nil { @@ -137,7 +44,7 @@ func (r *Client) call(method string, args ...interface{}) (*Response, error) { req.SetBasicAuth(r.AuthUser, r.AuthPasswd) resp, err := r.Client.Do(req) if err != nil { - return nil, errors.Wrapf(err, "remote call failed for %s", method) + return nil, errors.Wrapf(err, "remote Call failed for %s", method) } defer resp.Body.Close() if resp.StatusCode != 200 { diff --git a/backend/app/store/remote/remote_test.go b/backend/app/store/remote/remote_test.go index 45814c3ae0..11105bef70 100644 --- a/backend/app/store/remote/remote_test.go +++ b/backend/app/store/remote/remote_test.go @@ -1,163 +1,52 @@ package remote import ( + "encoding/json" "fmt" "io/ioutil" "net/http" "net/http/httptest" - "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/engine" ) -func TestClient_Create(t *testing.T) { - ts := testServer(t, `{"method":"create","params":[{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{"result":"12345"}`) +func TestClient_Call(t *testing.T) { + ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"result":"12345"}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} - - res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"}, - Text: "msg"}) + resp, err := c.Call("test", 123, "abc") assert.NoError(t, err) + res := "" + err = json.Unmarshal(*resp.Result, &res) assert.Equal(t, "12345", res) t.Logf("%v %T", res, res) } -func TestClient_Get(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, - `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`) +func TestClient_CallError(t *testing.T) { + ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"error":"some error"}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} - - res, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") - assert.NoError(t, err) - assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res) - t.Logf("%v %T", res, res) + _, err := c.Call("test", 123, "abc") + assert.EqualError(t, err, "some error") } -func TestClient_GetWithErrorResult(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, `{"error":"failed"}`) +func TestClient_CallBadResponse(t *testing.T) { + ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"result":"12345 invalid}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} - - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") - assert.EqualError(t, err, "failed") + _, err := c.Call("test", 123, "abc") + assert.NotNil(t, err) } -func TestClient_GetWithErrorDecode(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, ``) +func TestClient_CallBadRemote(t *testing.T) { + ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"result":"12345"}`) defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") - assert.EqualError(t, err, "failed to decode response for get: EOF") -} - -func TestClient_GetWithErrorRemote(t *testing.T) { c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}} - - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + _, err := c.Call("test", 123) assert.NotNil(t, err) - assert.True(t, strings.Contains(err.Error(), "remote call failed for get:")) -} - -func TestClient_FailedStatus(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) - require.NoError(t, err) - t.Logf("req: %s", string(body)) - w.WriteHeader(400) - })) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") - assert.EqualError(t, err, "bad status 400 for get") -} - -func TestClient_Update(t *testing.T) { - ts := testServer(t, `{"method":"update","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - err := c.Update(store.Locator{URL: "http://example.com/url"}, store.Comment{ID: "123", - Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"}, Text: "msg"}) - assert.NoError(t, err) - -} - -func TestClient_Find(t *testing.T) { - ts := testServer(t, `{"method":"find","params":[{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10}]}`, `{"result":[{"text":"1"},{"text":"2"}]}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - res, err := c.Find(engine.FindRequest{Locator: store.Locator{URL: "http://example.com/url"}, Sort: "-time", Limit: 10}) - assert.NoError(t, err) - assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) -} - -func TestClient_Info(t *testing.T) { - ts := testServer(t, `{"method":"info","params":[{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10}]}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - res, err := c.Info(engine.InfoRequest{Locator: store.Locator{URL: "http://example.com/url"}, - Limit: 10, Skip: 5, ReadOnlyAge: 10}) - assert.NoError(t, err) - assert.Equal(t, []store.PostInfo{{URL: "u1", Count: 22}, {URL: "u2", Count: 33}}, res) -} - -func TestClient_Flag(t *testing.T) { - ts := testServer(t, `{"method":"flag","params":[{"flag":"verified","locator":{"url":"http://example.com/url"}}]}`, - `{"result":false}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - res, err := c.Flag(engine.FlagRequest{Locator: store.Locator{URL: "http://example.com/url"}, Flag: engine.Verified}) - assert.NoError(t, err) - assert.Equal(t, false, res) -} - -func TestClient_ListFlag(t *testing.T) { - ts := testServer(t, `{"method":"list_flags","params":["site_id","blocked"]}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - res, err := c.ListFlags("site_id", engine.Blocked) - assert.NoError(t, err) - assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res) -} - -func TestClient_Count(t *testing.T) { - ts := testServer(t, `{"method":"count","params":[{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"}]}`, - `{"result":11}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - res, err := c.Count(engine.FindRequest{Locator: store.Locator{URL: "http://example.com/url"}}) - assert.NoError(t, err) - assert.Equal(t, 11, res) -} - -func TestClient_Delete(t *testing.T) { - ts := testServer(t, `{"method":"delete","params":[{"locator":{"url":"http://example.com/url"},"del_mode":0}]}`, `{}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - - err := c.Delete(engine.DeleteRequest{Locator: store.Locator{URL: "http://example.com/url"}}) - assert.NoError(t, err) -} - -func TestClient_Close(t *testing.T) { - ts := testServer(t, `{"method":"close","params":null}`, `{}`) - defer ts.Close() - c := Client{API: ts.URL, Client: http.Client{}} - err := c.Close() - assert.NoError(t, err) } func testServer(t *testing.T, req, resp string) *httptest.Server { From fa83c00bfddbbd6b8ff4d0f9ca6135cc130e9228 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 9 Jun 2019 17:17:20 -0500 Subject: [PATCH 06/24] initial ver of remote server --- backend/app/store/remote/client.go | 53 +++++++++ .../remote/{remote_test.go => client_test.go} | 0 backend/app/store/remote/remote.go | 49 +-------- backend/app/store/remote/server.go | 101 ++++++++++++++++++ backend/app/store/remote/server_test.go | 67 ++++++++++++ 5 files changed, 225 insertions(+), 45 deletions(-) create mode 100644 backend/app/store/remote/client.go rename backend/app/store/remote/{remote_test.go => client_test.go} (100%) create mode 100644 backend/app/store/remote/server.go create mode 100644 backend/app/store/remote/server_test.go diff --git a/backend/app/store/remote/client.go b/backend/app/store/remote/client.go new file mode 100644 index 0000000000..1fb8ed2fac --- /dev/null +++ b/backend/app/store/remote/client.go @@ -0,0 +1,53 @@ +package remote + +import ( + "bytes" + "encoding/json" + "net/http" + + "github.com/pkg/errors" +) + +// Client implements remote engine and delegates all calls to remote http server +type Client struct { + API string + Client http.Client + AuthUser string + AuthPasswd string +} + +// Call remote server with given method and arguments +func (r *Client) Call(method string, args ...interface{}) (*Response, error) { + + b, err := json.Marshal(Request{Method: method, Params: args}) + if err != nil { + return nil, errors.Wrapf(err, "marshaling failed for %s", method) + } + + req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b)) + if err != nil { + return nil, errors.Wrapf(err, "failed to make request for %s", method) + } + + if r.AuthUser != "" && r.AuthPasswd != "" { + req.SetBasicAuth(r.AuthUser, r.AuthPasswd) + } + resp, err := r.Client.Do(req) + if err != nil { + return nil, errors.Wrapf(err, "remote call failed for %s", method) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, errors.Errorf("bad status %d for %s", resp.StatusCode, method) + } + + cr := Response{} + if err = json.NewDecoder(resp.Body).Decode(&cr); err != nil { + return nil, errors.Wrapf(err, "failed to decode response for %s", method) + } + + if cr.Error != "" { + return nil, errors.New(cr.Error) + } + return &cr, nil +} diff --git a/backend/app/store/remote/remote_test.go b/backend/app/store/remote/client_test.go similarity index 100% rename from backend/app/store/remote/remote_test.go rename to backend/app/store/remote/client_test.go diff --git a/backend/app/store/remote/remote.go b/backend/app/store/remote/remote.go index 77c688981c..e72a41a91a 100644 --- a/backend/app/store/remote/remote.go +++ b/backend/app/store/remote/remote.go @@ -1,20 +1,13 @@ +// Package remote implements client ans server for RPC-like communication with remote storage. +// The protocol is somewhat simplified version of json-rpc with a single POST call sending +// Request json (method name and the list of parameters) and receiving back json Response with "result" json +// and error string package remote import ( - "bytes" "encoding/json" - "net/http" - - "github.com/pkg/errors" ) -// Client implements remote engine and delegates all calls to remote http server -type Client struct { - API string - Client http.Client - AuthUser string - AuthPasswd string -} // Request encloses method name and all params type Request struct { @@ -27,37 +20,3 @@ type Response struct { Result *json.RawMessage `json:"result,omitempty"` Error string `json:"error,omitempty"` } - -// Call remote server with given method and arguments -func (r *Client) Call(method string, args ...interface{}) (*Response, error) { - - b, err := json.Marshal(Request{Method: method, Params: args}) - if err != nil { - return nil, errors.Wrapf(err, "marshaling failed for %s", method) - } - - req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b)) - if err != nil { - return nil, errors.Wrapf(err, "failed to make request for %s", method) - } - - req.SetBasicAuth(r.AuthUser, r.AuthPasswd) - resp, err := r.Client.Do(req) - if err != nil { - return nil, errors.Wrapf(err, "remote Call failed for %s", method) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return nil, errors.Errorf("bad status %d for %s", resp.StatusCode, method) - } - - cr := Response{} - if err = json.NewDecoder(resp.Body).Decode(&cr); err != nil { - return nil, errors.Wrapf(err, "failed to decode response for %s", method) - } - - if cr.Error != "" { - return nil, errors.New(cr.Error) - } - return &cr, nil -} diff --git a/backend/app/store/remote/server.go b/backend/app/store/remote/server.go new file mode 100644 index 0000000000..bfabef2aef --- /dev/null +++ b/backend/app/store/remote/server.go @@ -0,0 +1,101 @@ +package remote + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "sync" + "time" + + "github.com/go-chi/chi" + "github.com/go-chi/render" + "github.com/pkg/errors" +) + +type Server struct { + CommandURL string + AuthUser string + AuthPasswd string + + funcs struct { + m map[string]ServerFn + once sync.Once + } + + httpServer struct { + *http.Server + sync.Mutex + } +} + +type ServerFn func(params *json.RawMessage) Response + +func (s *Server) Run(port int) error { + if s.funcs.m == nil && len(s.funcs.m) == 0 { + return errors.Errorf("nothing mapped for dispatch, Add has to be called prior to Run") + } + + router := chi.NewRouter() + + type request struct { + Method string `json:"method"` + Params *json.RawMessage `json:"params"` + } + + router.Post(s.CommandURL, func(w http.ResponseWriter, r *http.Request) { + req := request{} + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + fn, ok := s.funcs.m[req.Method] + if !ok { + w.WriteHeader(http.StatusNotImplemented) + return + } + render.JSON(w, r, fn(req.Params)) + }) + + s.httpServer.Lock() + s.httpServer.Server = &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: router, + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + } + s.httpServer.Unlock() + + log.Printf("[INFO] listen on %d", port) + return s.httpServer.ListenAndServe() +} + +func (s *Server) EncodeResponse(resp interface{}) (Response, error) { + v, err := json.Marshal(&resp) + if err != nil { + return Response{}, err + } + raw := json.RawMessage{} + if err = raw.UnmarshalJSON(v); err != nil { + return Response{}, err + } + return Response{Result: &raw}, nil +} + +func (s *Server) Shutdown() error { + s.httpServer.Lock() + defer s.httpServer.Unlock() + if s.httpServer.Server == nil { + return errors.Errorf("http server is not running") + } + return s.httpServer.Shutdown(context.TODO()) +} + +func (s *Server) Add(method string, fn ServerFn) { + s.funcs.once.Do(func() { + s.funcs.m = map[string]ServerFn{} + }) + s.funcs.m[method] = fn +} diff --git a/backend/app/store/remote/server_test.go b/backend/app/store/remote/server_test.go new file mode 100644 index 0000000000..4d6a11dc85 --- /dev/null +++ b/backend/app/store/remote/server_test.go @@ -0,0 +1,67 @@ +package remote + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServer(t *testing.T) { + s := Server{CommandURL: "/v1/cmd"} + + type respData struct { + Res1 string + Res2 bool + } + + s.Add("test", func(params *json.RawMessage) Response { + args := []interface{}{} + if err := json.Unmarshal(*params, &args); err != nil { + return Response{Error: err.Error()} + } + t.Logf("%+v", args) + + assert.Equal(t, 4, len(args)) + assert.Equal(t, "blah", args[0].(string)) + assert.Equal(t, 42., args[1].(float64)) + assert.Equal(t, true, args[2].(bool)) + assert.Equal(t, "", args[3].(time.Time)) + + r, err := s.EncodeResponse(respData{"res blah", true}) + assert.NoError(t, err) + return r + }) + + go func() { s.Run(9091) }() + time.Sleep(10 * time.Millisecond) + + // check with direct http call + clientReq := Request{Method: "test", Params: []interface{}{"blah", 42, true, time.Date(2018, 6, 9, 16, 7, 25, 0, time.UTC)}} + b := bytes.Buffer{} + require.NoError(t, json.NewEncoder(&b).Encode(clientReq)) + resp, err := http.Post("http://127.0.0.1:9091/v1/cmd", "application/json", &b) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + data, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Equal(t, `{"result":{"Res1":"res blah","Res2":true}}`+"\n", string(data)) + + // check with client call + c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} + r, err := c.Call("test", "blah", 42, true, time.Date(2018, 6, 9, 16, 7, 25, 0, time.UTC)) + assert.NoError(t, err) + assert.Equal(t, "", r.Error) + + res := respData{} + err = json.Unmarshal(*r.Result, &res) + assert.Equal(t, respData{Res1: "res blah", Res2: true}, res) + + assert.NoError(t, s.Shutdown()) +} From 3f73a019cc5dfe80b724916d1d275166db8a1fa2 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 9 Jun 2019 19:06:17 -0500 Subject: [PATCH 07/24] implement DeleteUser soft mode to allow mapping for blocked users #341 --- backend/app/rest/api/admin.go | 10 +++---- backend/app/rest/api/admin_test.go | 18 ++++++++++--- backend/app/store/engine/bolt.go | 30 +++++++++++---------- backend/app/store/engine/bolt_test.go | 38 +++++++++++++++++++++++++-- backend/app/store/service/service.go | 4 +-- 5 files changed, 74 insertions(+), 26 deletions(-) diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go index 4a03cac924..39057c1d9a 100644 --- a/backend/app/rest/api/admin.go +++ b/backend/app/rest/api/admin.go @@ -28,7 +28,7 @@ type admin struct { type adminStore interface { Delete(locator store.Locator, commentID string, mode store.DeleteMode) error - DeleteUser(siteID string, userID string) error + DeleteUser(siteID string, userID string, mode store.DeleteMode) error User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error) IsBlocked(siteID string, userID string) bool SetBlock(siteID string, userID string, status bool, ttl time.Duration) error @@ -64,7 +64,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID) - if err := a.dataService.DeleteUser(siteID, userID); err != nil { + if err := a.dataService.DeleteUser(siteID, userID, store.HardDelete); err != nil { rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal) return } @@ -109,7 +109,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) { return } - if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil { + if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess) return } @@ -145,9 +145,9 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) { return } - // delete comments for blocked user + // delete comments for blocked user. if blockStatus { - if err := a.dataService.DeleteUser(siteID, userID); err != nil { + if err := a.dataService.DeleteUser(siteID, userID, store.SoftDelete); err != nil { log.Printf("[WARN] can't delete comments for blocked user %s on site %s, %v", userID, siteID, err) } } diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index 796ab2e98a..a7346f7ceb 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -353,13 +353,23 @@ func TestAdmin_Block(t *testing.T) { assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1")) assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2")) - } func TestAdmin_BlockedList(t *testing.T) { - ts, _, teardown := startupT(t) + ts, srv, teardown := startupT(t) defer teardown() + c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}} + c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t", + URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2 name", ID: "user2"}} + + // write comments for user1 and user2 + _, err := srv.DataService.Create(c1) + assert.Nil(t, err) + _, err = srv.DataService.Create(c2) + assert.Nil(t, err) + // block user1 req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil) @@ -386,8 +396,10 @@ func TestAdmin_BlockedList(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 2, len(users), "two users blocked") assert.Equal(t, "user1", users[0].ID) + assert.Equal(t, "user1 name", users[0].Name) assert.Equal(t, "user2", users[1].ID) - + assert.Equal(t, "user2 name", users[1].Name) + t.Logf("%+v", users) time.Sleep(50 * time.Millisecond) req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil) diff --git a/backend/app/store/engine/bolt.go b/backend/app/store/engine/bolt.go index fe48d28639..e4aa256737 100644 --- a/backend/app/store/engine/bolt.go +++ b/backend/app/store/engine/bolt.go @@ -378,7 +378,7 @@ func (b *BoltDB) Delete(req DeleteRequest) error { case req.Locator.URL != "" && req.CommentID != "": return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode) case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "": - return b.deleteUser(bdb, req.Locator.SiteID, req.UserID) + return b.deleteUser(bdb, req.Locator.SiteID, req.UserID, req.DeleteMode) case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "": return b.deleteAll(bdb, req.Locator.SiteID) } @@ -673,7 +673,7 @@ func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error { // deleteUser removes all comments for given user. Everything will be market as deleted // and user name and userID will be changed to "deleted". Also removes from last and from user buckets. -func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string) error { +func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error { bdb, err := b.db(siteID) if err != nil { return err @@ -717,24 +717,26 @@ func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string) error { // delete collected comments for _, ci := range comments { - if e := b.deleteComment(bdb, ci.locator, ci.commentID, store.HardDelete); e != nil { + if e := b.deleteComment(bdb, ci.locator, ci.commentID, mode); e != nil { return errors.Wrapf(err, "failed to delete comment %+v", ci) } } - // delete user bucket - err = bdb.Update(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(userBucketName)) - if usersBkt != nil { - if e := usersBkt.DeleteBucket([]byte(userID)); e != nil { - return errors.Wrapf(err, "failed to delete user bucket for %s", userID) + // delete user bucket in hard mode + if mode == store.HardDelete { + err = bdb.Update(func(tx *bolt.Tx) error { + usersBkt := tx.Bucket([]byte(userBucketName)) + if usersBkt != nil { + if e := usersBkt.DeleteBucket([]byte(userID)); e != nil { + return errors.Wrapf(err, "failed to delete user bucket for %s", userID) + } } - } - return nil - }) + return nil + }) - if err != nil { - return errors.Wrap(err, "can't delete user meta") + if err != nil { + return errors.Wrap(err, "can't delete user meta") + } } if len(comments) == 0 { diff --git a/backend/app/store/engine/bolt_test.go b/backend/app/store/engine/bolt_test.go index b373c5d687..502fcb8dee 100644 --- a/backend/app/store/engine/bolt_test.go +++ b/backend/app/store/engine/bolt_test.go @@ -683,12 +683,12 @@ func TestBolt_DeleteAll(t *testing.T) { assert.EqualError(t, err, `site "bad" not found`) } -func TestBoltAdmin_DeleteUser(t *testing.T) { +func TestBoltAdmin_DeleteUserHard(t *testing.T) { b, teardown := prep(t) defer teardown() - err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}) + err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete}) require.NoError(t, err) comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"}) @@ -712,6 +712,40 @@ func TestBoltAdmin_DeleteUser(t *testing.T) { assert.EqualError(t, err, `site "radio-t-bad" not found`) } +func TestBoltAdmin_DeleteUserSoft(t *testing.T) { + + b, teardown := prep(t) + defer teardown() + + err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete}) + require.NoError(t, err) + + comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"}) + assert.NoError(t, err) + assert.Equal(t, 2, len(comments), "2 comments with deleted info") + assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User) + assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User) + + c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}}) + assert.NoError(t, err) + assert.Equal(t, 0, c, "0 count") + + comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5}) + assert.NoError(t, err, "no comments for user user1 in store") + assert.Equal(t, 2, len(comments), "2 comments with deleted info") + assert.True(t, comments[0].Deleted) + assert.True(t, comments[1].Deleted) + assert.Equal(t, "", comments[0].Text) + assert.Equal(t, "", comments[1].Text) + + comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"}) + assert.NoError(t, err) + assert.Equal(t, 0, len(comments), "nothing left") + + err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"}) + assert.EqualError(t, err, `site "radio-t-bad" not found`) +} + func TestBoltDB_ref(t *testing.T) { b := BoltDB{} comment := store.Comment{ diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 0087a877b7..7a0acc8821 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -553,8 +553,8 @@ func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.D } // DeleteUser removes all comments from user -func (s *DataStore) DeleteUser(siteID string, userID string) error { - req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: store.HardDelete} +func (s *DataStore) DeleteUser(siteID string, userID string, mode store.DeleteMode) error { + req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: mode} return s.Engine.Delete(req) } From 05a3871ceb7e064aae2fdf3495dcb4b8fd24ef03 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Jun 2019 01:36:45 -0500 Subject: [PATCH 08/24] remote server covered with tests --- backend/app/store/remote/client.go | 20 ++- backend/app/store/remote/client_test.go | 30 ++++- backend/app/store/remote/remote.go | 5 +- backend/app/store/remote/server.go | 89 ++++++++++---- backend/app/store/remote/server_test.go | 154 ++++++++++++++++++++++-- 5 files changed, 256 insertions(+), 42 deletions(-) diff --git a/backend/app/store/remote/client.go b/backend/app/store/remote/client.go index 1fb8ed2fac..9e5eb796e7 100644 --- a/backend/app/store/remote/client.go +++ b/backend/app/store/remote/client.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "net/http" + "reflect" + "sync/atomic" "github.com/pkg/errors" ) @@ -14,20 +16,32 @@ type Client struct { Client http.Client AuthUser string AuthPasswd string + + id uint64 } // Call remote server with given method and arguments func (r *Client) Call(method string, args ...interface{}) (*Response, error) { - b, err := json.Marshal(Request{Method: method, Params: args}) - if err != nil { - return nil, errors.Wrapf(err, "marshaling failed for %s", method) + var b []byte + var err error + if len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Struct { + b, err = json.Marshal(Request{Method: method, Params: args[0], ID: atomic.AddUint64(&r.id, 1)}) + if err != nil { + return nil, errors.Wrapf(err, "marshaling failed for %s", method) + } + } else { + b, err = json.Marshal(Request{Method: method, Params: args, ID: atomic.AddUint64(&r.id, 1)}) + if err != nil { + return nil, errors.Wrapf(err, "marshaling failed for %s", method) + } } req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b)) if err != nil { return nil, errors.Wrapf(err, "failed to make request for %s", method) } + req.Header.Set("Content-Type", "application/json; charset=utf-8") if r.AuthUser != "" && r.AuthPasswd != "" { req.SetBasicAuth(r.AuthUser, r.AuthPasswd) diff --git a/backend/app/store/remote/client_test.go b/backend/app/store/remote/client_test.go index 11105bef70..98a2c79473 100644 --- a/backend/app/store/remote/client_test.go +++ b/backend/app/store/remote/client_test.go @@ -14,7 +14,7 @@ import ( ) func TestClient_Call(t *testing.T) { - ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"result":"12345"}`) + ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} resp, err := c.Call("test", 123, "abc") @@ -25,8 +25,30 @@ func TestClient_Call(t *testing.T) { t.Logf("%v %T", res, res) } +func TestClient_CallWithObject(t *testing.T) { + ts := testServer(t, `{"method":"test","params":{"F1":123,"F2":"abc","F3":"2019-06-09T23:03:55Z"},"id":1}`, `{"result":"12345"}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + obj := struct { + F1 int + F2 string + F3 time.Time + }{ + F1: 123, + F2: "abc", + F3: time.Date(2019, 6, 9, 23, 3, 55, 0, time.UTC), + } + + resp, err := c.Call("test", obj) + assert.NoError(t, err) + res := "" + err = json.Unmarshal(*resp.Result, &res) + assert.Equal(t, "12345", res) + t.Logf("%v %T", res, res) +} + func TestClient_CallError(t *testing.T) { - ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"error":"some error"}`) + ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"error":"some error"}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} _, err := c.Call("test", 123, "abc") @@ -34,7 +56,7 @@ func TestClient_CallError(t *testing.T) { } func TestClient_CallBadResponse(t *testing.T) { - ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"result":"12345 invalid}`) + ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345 invalid}`) defer ts.Close() c := Client{API: ts.URL, Client: http.Client{}} _, err := c.Call("test", 123, "abc") @@ -42,7 +64,7 @@ func TestClient_CallBadResponse(t *testing.T) { } func TestClient_CallBadRemote(t *testing.T) { - ts := testServer(t, `{"method":"test","params":[123,"abc"]}`, `{"result":"12345"}`) + ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`) defer ts.Close() c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}} _, err := c.Call("test", 123) diff --git a/backend/app/store/remote/remote.go b/backend/app/store/remote/remote.go index e72a41a91a..e766efcb83 100644 --- a/backend/app/store/remote/remote.go +++ b/backend/app/store/remote/remote.go @@ -8,15 +8,16 @@ import ( "encoding/json" ) - // Request encloses method name and all params type Request struct { Method string `json:"method"` - Params interface{} `json:"params"` + Params interface{} `json:"params,omitempty"` + ID uint64 `json:"id"` } // Response encloses result and error received from remote server type Response struct { Result *json.RawMessage `json:"result,omitempty"` Error string `json:"error,omitempty"` + ID uint64 `json:"id"` } diff --git a/backend/app/store/remote/server.go b/backend/app/store/remote/server.go index bfabef2aef..4e70ad01cd 100644 --- a/backend/app/store/remote/server.go +++ b/backend/app/store/remote/server.go @@ -4,20 +4,28 @@ import ( "context" "encoding/json" "fmt" - "log" "net/http" "sync" "time" + "github.com/didip/tollbooth" + "github.com/didip/tollbooth_chi" "github.com/go-chi/chi" + "github.com/go-chi/chi/middleware" "github.com/go-chi/render" + log "github.com/go-pkgz/lgr" + R "github.com/go-pkgz/rest" + "github.com/go-pkgz/rest/logger" "github.com/pkg/errors" ) +// Server is json-rpc server with an optional basic auth type Server struct { - CommandURL string + API string AuthUser string AuthPasswd string + Version string + AppName string funcs struct { m map[string]ServerFn @@ -30,33 +38,25 @@ type Server struct { } } -type ServerFn func(params *json.RawMessage) Response +// ServerFn handler registered for each method with Add +type ServerFn func(id uint64, params json.RawMessage) Response +// Run http server on given port func (s *Server) Run(port int) error { + if s.funcs.m == nil && len(s.funcs.m) == 0 { return errors.Errorf("nothing mapped for dispatch, Add has to be called prior to Run") } router := chi.NewRouter() + router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default())) + router.Use(R.AppInfo(s.AppName, "umputun", s.Version), R.Ping) + logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[INFO]")).Handler + router.Use(middleware.Timeout(5 * time.Second)) + router.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache) + router.Use(s.basicAuth) - type request struct { - Method string `json:"method"` - Params *json.RawMessage `json:"params"` - } - - router.Post(s.CommandURL, func(w http.ResponseWriter, r *http.Request) { - req := request{} - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - w.WriteHeader(http.StatusBadRequest) - return - } - fn, ok := s.funcs.m[req.Method] - if !ok { - w.WriteHeader(http.StatusNotImplemented) - return - } - render.JSON(w, r, fn(req.Params)) - }) + router.Post(s.API, s.handler) s.httpServer.Lock() s.httpServer.Server = &http.Server{ @@ -72,18 +72,23 @@ func (s *Server) Run(port int) error { return s.httpServer.ListenAndServe() } -func (s *Server) EncodeResponse(resp interface{}) (Response, error) { +// EncodeResponse convert anything to Response +func (s *Server) EncodeResponse(id uint64, resp interface{}, e error) (Response, error) { v, err := json.Marshal(&resp) if err != nil { return Response{}, err } + if e != nil { + return Response{ID: id, Result: nil, Error: e.Error()}, nil + } raw := json.RawMessage{} if err = raw.UnmarshalJSON(v); err != nil { return Response{}, err } - return Response{Result: &raw}, nil + return Response{ID: id, Result: &raw}, nil } +// Shutdown http server func (s *Server) Shutdown() error { s.httpServer.Lock() defer s.httpServer.Unlock() @@ -93,9 +98,47 @@ func (s *Server) Shutdown() error { return s.httpServer.Shutdown(context.TODO()) } +// Add method handler func (s *Server) Add(method string, fn ServerFn) { s.funcs.once.Do(func() { s.funcs.m = map[string]ServerFn{} }) s.funcs.m[method] = fn } + +func (s *Server) handler(w http.ResponseWriter, r *http.Request) { + req := struct { + ID uint64 `json:"id"` + Method string `json:"method"` + Params *json.RawMessage `json:"params"` + }{} + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + fn, ok := s.funcs.m[req.Method] + if !ok { + w.WriteHeader(http.StatusNotImplemented) + return + } + render.JSON(w, r, fn(req.ID, *req.Params)) +} + +func (s *Server) basicAuth(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + if s.AuthUser == "" || s.AuthPasswd == "" { + h.ServeHTTP(w, r) + return + } + + user, pass, ok := r.BasicAuth() + if user != s.AuthUser || pass != s.AuthPasswd || !ok { + w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) + http.Error(w, "Unauthorized.", http.StatusUnauthorized) + return + } + h.ServeHTTP(w, r) + }) +} diff --git a/backend/app/store/remote/server_test.go b/backend/app/store/remote/server_test.go index 4d6a11dc85..ae0f38a0ce 100644 --- a/backend/app/store/remote/server_test.go +++ b/backend/app/store/remote/server_test.go @@ -5,35 +5,36 @@ import ( "encoding/json" "io/ioutil" "net/http" + "net/http/httptest" "testing" "time" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestServer(t *testing.T) { - s := Server{CommandURL: "/v1/cmd"} +func TestServerPrimitiveTypes(t *testing.T) { + s := Server{API: "/v1/cmd"} type respData struct { Res1 string Res2 bool } - s.Add("test", func(params *json.RawMessage) Response { + s.Add("test", func(id uint64, params json.RawMessage) Response { args := []interface{}{} - if err := json.Unmarshal(*params, &args); err != nil { + if err := json.Unmarshal(params, &args); err != nil { return Response{Error: err.Error()} } t.Logf("%+v", args) - assert.Equal(t, 4, len(args)) + assert.Equal(t, 3, len(args)) assert.Equal(t, "blah", args[0].(string)) assert.Equal(t, 42., args[1].(float64)) assert.Equal(t, true, args[2].(bool)) - assert.Equal(t, "", args[3].(time.Time)) - r, err := s.EncodeResponse(respData{"res blah", true}) + r, err := s.EncodeResponse(id, respData{"res blah", true}, nil) assert.NoError(t, err) return r }) @@ -42,7 +43,7 @@ func TestServer(t *testing.T) { time.Sleep(10 * time.Millisecond) // check with direct http call - clientReq := Request{Method: "test", Params: []interface{}{"blah", 42, true, time.Date(2018, 6, 9, 16, 7, 25, 0, time.UTC)}} + clientReq := Request{Method: "test", Params: []interface{}{"blah", 42, true}, ID: 123} b := bytes.Buffer{} require.NoError(t, json.NewEncoder(&b).Encode(clientReq)) resp, err := http.Post("http://127.0.0.1:9091/v1/cmd", "application/json", &b) @@ -51,17 +52,150 @@ func TestServer(t *testing.T) { assert.Equal(t, 200, resp.StatusCode) data, err := ioutil.ReadAll(resp.Body) assert.NoError(t, err) - assert.Equal(t, `{"result":{"Res1":"res blah","Res2":true}}`+"\n", string(data)) + assert.Equal(t, `{"result":{"Res1":"res blah","Res2":true},"id":123}`+"\n", string(data)) // check with client call c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} - r, err := c.Call("test", "blah", 42, true, time.Date(2018, 6, 9, 16, 7, 25, 0, time.UTC)) + r, err := c.Call("test", "blah", 42, true) assert.NoError(t, err) assert.Equal(t, "", r.Error) res := respData{} err = json.Unmarshal(*r.Result, &res) assert.Equal(t, respData{Res1: "res blah", Res2: true}, res) + assert.Equal(t, uint64(1), r.ID) + assert.NoError(t, s.Shutdown()) +} + +func TestServerWithObject(t *testing.T) { + s := Server{API: "/v1/cmd"} + + type respData struct { + Res1 string + Res2 bool + } + + type reqData struct { + Time time.Time + F1 string + F2 time.Duration + } + + s.Add("test", func(id uint64, params json.RawMessage) Response { + arg := reqData{} + if err := json.Unmarshal(params, &arg); err != nil { + return Response{Error: err.Error()} + } + t.Logf("%+v", arg) + + r, err := s.EncodeResponse(id, respData{"res blah", true}, nil) + assert.NoError(t, err) + return r + }) + + go func() { s.Run(9091) }() + time.Sleep(10 * time.Millisecond) + + c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} + r, err := c.Call("test", reqData{Time: time.Now(), F1: "sawert", F2: time.Minute}) + assert.NoError(t, err) + assert.Equal(t, "", r.Error) + + res := respData{} + err = json.Unmarshal(*r.Result, &res) + assert.Equal(t, respData{Res1: "res blah", Res2: true}, res) + + assert.NoError(t, s.Shutdown()) +} + +func TestServerMethodNotImplemented(t *testing.T) { + s := Server{} + ts := httptest.NewServer(http.HandlerFunc(s.handler)) + defer ts.Close() + s.Add("test", func(id uint64, params json.RawMessage) Response { + return Response{} + }) + + r := Request{Method: "blah"} + buf := bytes.Buffer{} + assert.NoError(t, json.NewEncoder(&buf).Encode(r)) + resp, err := http.Post(ts.URL, "application/json", &buf) + require.NoError(t, err) + assert.Equal(t, http.StatusNotImplemented, resp.StatusCode) + + assert.EqualError(t, s.Shutdown(), "http server is not running") +} + +func TestServerWithAuth(t *testing.T) { + s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"} + + s.Add("test", func(id uint64, params json.RawMessage) Response { + args := []interface{}{} + if err := json.Unmarshal(params, &args); err != nil { + return Response{Error: err.Error()} + } + t.Logf("%+v", args) + + assert.Equal(t, 3, len(args)) + assert.Equal(t, "blah", args[0].(string)) + assert.Equal(t, 42., args[1].(float64)) + assert.Equal(t, true, args[2].(bool)) + + r, err := s.EncodeResponse(id, "res blah", nil) + assert.NoError(t, err) + return r + }) + + go func() { s.Run(9091) }() + time.Sleep(10 * time.Millisecond) + + c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"} + r, err := c.Call("test", "blah", 42, true) + assert.NoError(t, err) + assert.Equal(t, "", r.Error) + val := "" + err = json.Unmarshal(*r.Result, &val) + assert.NoError(t, err) + assert.Equal(t, "res blah", val) + + c = Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} + _, err = c.Call("test", "blah", 42, true) + assert.EqualError(t, err, "bad status 401 for test") assert.NoError(t, s.Shutdown()) } + +func TestServerErrReturn(t *testing.T) { + s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"} + + s.Add("test", func(id uint64, params json.RawMessage) Response { + args := []interface{}{} + if err := json.Unmarshal(params, &args); err != nil { + return Response{Error: err.Error()} + } + t.Logf("%+v", args) + + assert.Equal(t, 3, len(args)) + assert.Equal(t, "blah", args[0].(string)) + assert.Equal(t, 42., args[1].(float64)) + assert.Equal(t, true, args[2].(bool)) + + r, err := s.EncodeResponse(id, "res blah", errors.New("some error")) + assert.NoError(t, err) + return r + }) + + go func() { s.Run(9091) }() + time.Sleep(10 * time.Millisecond) + + c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"} + _, err := c.Call("test", "blah", 42, true) + assert.EqualError(t, err, "some error") + + assert.NoError(t, s.Shutdown()) +} + +func TestServerNoHandlers(t *testing.T) { + s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"} + assert.EqualError(t, s.Run(9091), "nothing mapped for dispatch, Add has to be called prior to Run") +} From 465571ab39b8b3661239a60cc26366b06f608d28 Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Jun 2019 01:56:37 -0500 Subject: [PATCH 09/24] fix remote client tests with id --- backend/app/rest/api/admin_test.go | 1 - backend/app/store/engine/remote_test.go | 29 ++++++++++++----------- backend/app/store/service/service_test.go | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go index a7346f7ceb..9d7ee71c3a 100644 --- a/backend/app/rest/api/admin_test.go +++ b/backend/app/rest/api/admin_test.go @@ -411,7 +411,6 @@ func TestAdmin_BlockedList(t *testing.T) { err = json.NewDecoder(res.Body).Decode(&users) assert.Nil(t, err) assert.Equal(t, 1, len(users), "one user left blocked") - } func TestAdmin_ReadOnly(t *testing.T) { diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go index 91e013f5bf..eadb7fff32 100644 --- a/backend/app/store/engine/remote_test.go +++ b/backend/app/store/engine/remote_test.go @@ -17,7 +17,8 @@ import ( ) func TestClient_Create(t *testing.T) { - ts := testServer(t, `{"method":"create","params":[{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{"result":"12345"}`) + ts := testServer(t, `{"method":"create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, + `{"result":"12345","id":1}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -29,7 +30,7 @@ func TestClient_Create(t *testing.T) { } func TestClient_Get(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"],"id":1}`, `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -41,7 +42,7 @@ func TestClient_Get(t *testing.T) { } func TestClient_GetWithErrorResult(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, `{"error":"failed"}`) + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"],"id":1}`, `{"error":"failed"}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -50,7 +51,7 @@ func TestClient_GetWithErrorResult(t *testing.T) { } func TestClient_GetWithErrorDecode(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"]}`, ``) + ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"],"id":1}`, ``) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -63,7 +64,7 @@ func TestClient_GetWithErrorRemote(t *testing.T) { _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") assert.NotNil(t, err) - assert.True(t, strings.Contains(err.Error(), "remote Call failed for get:")) + assert.True(t, strings.Contains(err.Error(), "remote call failed for get:"), err.Error()) } func TestClient_FailedStatus(t *testing.T) { @@ -81,7 +82,7 @@ func TestClient_FailedStatus(t *testing.T) { } func TestClient_Update(t *testing.T) { - ts := testServer(t, `{"method":"update","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}]}`, `{}`) + ts := testServer(t, `{"method":"update","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}],"id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -92,7 +93,7 @@ func TestClient_Update(t *testing.T) { } func TestClient_Find(t *testing.T) { - ts := testServer(t, `{"method":"find","params":[{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10}]}`, `{"result":[{"text":"1"},{"text":"2"}]}`) + ts := testServer(t, `{"method":"find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -102,7 +103,7 @@ func TestClient_Find(t *testing.T) { } func TestClient_Info(t *testing.T) { - ts := testServer(t, `{"method":"info","params":[{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10}]}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) + ts := testServer(t, `{"method":"info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -113,7 +114,7 @@ func TestClient_Info(t *testing.T) { } func TestClient_Flag(t *testing.T) { - ts := testServer(t, `{"method":"flag","params":[{"flag":"verified","locator":{"url":"http://example.com/url"}}]}`, + ts := testServer(t, `{"method":"flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, `{"result":false}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -124,7 +125,8 @@ func TestClient_Flag(t *testing.T) { } func TestClient_ListFlag(t *testing.T) { - ts := testServer(t, `{"method":"list_flags","params":["site_id","blocked"]}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) + ts := testServer(t, `{"method":"list_flags","params":["site_id","blocked"],"id":1}`, + `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} res, err := c.ListFlags("site_id", Blocked) @@ -133,8 +135,7 @@ func TestClient_ListFlag(t *testing.T) { } func TestClient_Count(t *testing.T) { - ts := testServer(t, `{"method":"count","params":[{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"}]}`, - `{"result":11}`) + ts := testServer(t, `{"method":"count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -144,7 +145,7 @@ func TestClient_Count(t *testing.T) { } func TestClient_Delete(t *testing.T) { - ts := testServer(t, `{"method":"delete","params":[{"locator":{"url":"http://example.com/url"},"del_mode":0}]}`, `{}`) + ts := testServer(t, `{"method":"delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -153,7 +154,7 @@ func TestClient_Delete(t *testing.T) { } func TestClient_Close(t *testing.T) { - ts := testServer(t, `{"method":"close","params":null}`, `{}`) + ts := testServer(t, `{"method":"close","params":null,"id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} err := c.Close() diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index ef688c2ee5..f9f17a8da8 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -862,7 +862,7 @@ func TestService_DeleteUser(t *testing.T) { assert.Equal(t, 3, len(res), "3 comments initially, for 2 diff users and 2 posts") assert.NoError(t, err) - err = b.DeleteUser("radio-t", "user1") + err = b.DeleteUser("radio-t", "user1", store.HardDelete) assert.NoError(t, err) res, err = b.Last("radio-t", 0, time.Time{}, store.User{}) From 0722e64cd27442d327074ba684b3746b6342a88d Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 10 Jun 2019 23:26:10 -0500 Subject: [PATCH 10/24] switch list flag to struct param --- backend/app/store/engine/bolt.go | 10 +++++----- backend/app/store/engine/bolt_test.go | 12 ++++++------ backend/app/store/engine/engine.go | 2 +- backend/app/store/engine/engine_mock.go | 14 +++++++------- backend/app/store/engine/remote.go | 4 ++-- backend/app/store/engine/remote_test.go | 4 ++-- backend/app/store/service/service.go | 4 ++-- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/backend/app/store/engine/bolt.go b/backend/app/store/engine/bolt.go index e4aa256737..7fbf804850 100644 --- a/backend/app/store/engine/bolt.go +++ b/backend/app/store/engine/bolt.go @@ -322,14 +322,14 @@ func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) { // ListFlags get list of flagged keys, like blocked & verified user // works for full locator (post flags) or with userID -func (b *BoltDB) ListFlags(siteID string, flag Flag) (res []interface{}, err error) { +func (b *BoltDB) ListFlags(req FlagRequest) (res []interface{}, err error) { - bdb, e := b.db(siteID) + bdb, e := b.db(req.Locator.SiteID) if e != nil { return nil, e } - switch flag { + switch req.Flag { case Verified: err = bdb.View(func(tx *bolt.Tx) error { usersBkt := tx.Bucket([]byte(verifiedBucketName)) @@ -351,7 +351,7 @@ func (b *BoltDB) ListFlags(siteID string, flag Flag) (res []interface{}, err err if time.Now().Before(ts) { // get user name from comment user section userName := "" - req := FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: string(k), Limit: 1} + req := FindRequest{Locator: store.Locator{SiteID: req.Locator.SiteID}, UserID: string(k), Limit: 1} userComments, errUser := b.Find(req) if errUser == nil && len(userComments) > 0 { userName = userComments[0].User.Name @@ -363,7 +363,7 @@ func (b *BoltDB) ListFlags(siteID string, flag Flag) (res []interface{}, err err }) return res, err } - return nil, errors.Errorf("flag %s not listable", flag) + return nil, errors.Errorf("flag %s not listable", req.Flag) } // Delete post(s) by id or by userID diff --git a/backend/app/store/engine/bolt_test.go b/backend/app/store/engine/bolt_test.go index 502fcb8dee..5bbeecf49e 100644 --- a/backend/app/store/engine/bolt_test.go +++ b/backend/app/store/engine/bolt_test.go @@ -530,17 +530,17 @@ func TestBolt_FlagListVerified(t *testing.T) { return err } - ids, err := b.ListFlags("radio-t", Verified) + ids, err := b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}}) assert.NoError(t, err) assert.Equal(t, []string{}, toIDs(ids), "verified list empty") assert.NoError(t, setVerified("radio-t", "u1", FlagTrue)) assert.NoError(t, setVerified("radio-t", "u2", FlagTrue)) - ids, err = b.ListFlags("radio-t", Verified) + ids, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}}) assert.NoError(t, err) assert.Equal(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids") - _, err = b.ListFlags("radio-t-bad", Verified) + _, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t-bad"}}) assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site") } @@ -568,7 +568,7 @@ func TestBolt_FlagListBlocked(t *testing.T) { assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 50*time.Millisecond)) assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0)) - vv, err := b.ListFlags("radio-t", Blocked) + vv, err := b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}}) assert.NoError(t, err) blockedList := toBlocked(vv) @@ -579,13 +579,13 @@ func TestBolt_FlagListBlocked(t *testing.T) { // check block expiration time.Sleep(50 * time.Millisecond) - vv, err = b.ListFlags("radio-t", Blocked) + vv, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}}) assert.NoError(t, err) blockedList = toBlocked(vv) assert.Equal(t, 1, len(blockedList)) assert.Equal(t, "user1", blockedList[0].ID) - _, err = b.ListFlags("bad", Blocked) + _, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}}) assert.EqualError(t, err, `site "bad" not found`) } diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go index cd618fe2fd..db46d646a4 100644 --- a/backend/app/store/engine/engine.go +++ b/backend/app/store/engine/engine.go @@ -24,7 +24,7 @@ type Interface interface { Count(req FindRequest) (int, error) // get count for post or user Delete(req DeleteRequest) error // delete post(s) by id or by userID Flag(req FlagRequest) (bool, error) // set and get flags - ListFlags(siteID string, flag Flag) ([]interface{}, error) // get list of flagged keys, like blocked & verified user + ListFlags(req FlagRequest) ([]interface{}, error) // get list of flagged keys, like blocked & verified user Close() error // close storage engine } diff --git a/backend/app/store/engine/engine_mock.go b/backend/app/store/engine/engine_mock.go index 38496094a0..2c1437f7ff 100644 --- a/backend/app/store/engine/engine_mock.go +++ b/backend/app/store/engine/engine_mock.go @@ -167,13 +167,13 @@ func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) { return r0, r1 } -// ListFlags provides a mock function with given fields: siteID, flag -func (_m *MockInterface) ListFlags(siteID string, flag Flag) ([]interface{}, error) { - ret := _m.Called(siteID, flag) +// ListFlags provides a mock function with given fields: req +func (_m *MockInterface) ListFlags(req FlagRequest) ([]interface{}, error) { + ret := _m.Called(req) var r0 []interface{} - if rf, ok := ret.Get(0).(func(string, Flag) []interface{}); ok { - r0 = rf(siteID, flag) + if rf, ok := ret.Get(0).(func(FlagRequest) []interface{}); ok { + r0 = rf(req) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]interface{}) @@ -181,8 +181,8 @@ func (_m *MockInterface) ListFlags(siteID string, flag Flag) ([]interface{}, err } var r1 error - if rf, ok := ret.Get(1).(func(string, Flag) error); ok { - r1 = rf(siteID, flag) + if rf, ok := ret.Get(1).(func(FlagRequest) error); ok { + r1 = rf(req) } else { r1 = ret.Error(1) } diff --git a/backend/app/store/engine/remote.go b/backend/app/store/engine/remote.go index 997b9492bd..6ae521024e 100644 --- a/backend/app/store/engine/remote.go +++ b/backend/app/store/engine/remote.go @@ -72,8 +72,8 @@ func (r *Remote) Flag(req FlagRequest) (status bool, err error) { } // ListFlags get list of flagged keys, like blocked & verified user -func (r *Remote) ListFlags(siteID string, flag Flag) (list []interface{}, err error) { - resp, err := r.Call("list_flags", siteID, flag) +func (r *Remote) ListFlags(req FlagRequest) (list []interface{}, err error) { + resp, err := r.Call("list_flags", req) if err != nil { return nil, err } diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go index eadb7fff32..eae1830095 100644 --- a/backend/app/store/engine/remote_test.go +++ b/backend/app/store/engine/remote_test.go @@ -125,11 +125,11 @@ func TestClient_Flag(t *testing.T) { } func TestClient_ListFlag(t *testing.T) { - ts := testServer(t, `{"method":"list_flags","params":["site_id","blocked"],"id":1}`, + ts := testServer(t, `{"method":"list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} - res, err := c.ListFlags("site_id", Blocked) + res, err := c.ListFlags(FlagRequest{Locator: store.Locator{SiteID: "site_id"}, Flag: Blocked}) assert.NoError(t, err) assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res) } diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 7a0acc8821..32e34ef6cc 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -523,7 +523,7 @@ func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time // Blocked returns list with all blocked users func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) { - blocked, e := s.Engine.ListFlags(siteID, engine.Blocked) + blocked, e := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Blocked}) if e != nil { return nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID) } @@ -606,7 +606,7 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe } // process verified users - verified, err := s.Engine.ListFlags(siteID, engine.Verified) + verified, err := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Verified}) if err != nil { return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID) } From 2ed7068ae157fd8056ee0b9e7af1c7019cccfcb7 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 18 Jun 2019 16:49:21 -0500 Subject: [PATCH 11/24] error responses of json api with json, time out on remote server close --- backend/app/store/remote/server.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/app/store/remote/server.go b/backend/app/store/remote/server.go index 4e70ad01cd..c44e4e1a8c 100644 --- a/backend/app/store/remote/server.go +++ b/backend/app/store/remote/server.go @@ -17,6 +17,8 @@ import ( R "github.com/go-pkgz/rest" "github.com/go-pkgz/rest/logger" "github.com/pkg/errors" + + "github.com/umputun/remark/backend/app/rest" ) // Server is json-rpc server with an optional basic auth @@ -39,11 +41,14 @@ type Server struct { } // ServerFn handler registered for each method with Add +// Implementations provided by consumer and define response logic. type ServerFn func(id uint64, params json.RawMessage) Response // Run http server on given port func (s *Server) Run(port int) error { - + if s.AuthUser == "" || s.AuthPasswd == "" { + log.Print("[WARN] extension server runs without auth") + } if s.funcs.m == nil && len(s.funcs.m) == 0 { return errors.Errorf("nothing mapped for dispatch, Add has to be called prior to Run") } @@ -95,7 +100,9 @@ func (s *Server) Shutdown() error { if s.httpServer.Server == nil { return errors.Errorf("http server is not running") } - return s.httpServer.Shutdown(context.TODO()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return s.httpServer.Shutdown(ctx) } // Add method handler @@ -114,12 +121,12 @@ func (s *Server) handler(w http.ResponseWriter, r *http.Request) { }{} if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - w.WriteHeader(http.StatusBadRequest) + rest.SendErrorJSON(w, r, http.StatusBadRequest, err, req.Method, 0) return } fn, ok := s.funcs.m[req.Method] if !ok { - w.WriteHeader(http.StatusNotImplemented) + rest.SendErrorJSON(w, r, http.StatusNotImplemented, errors.New("unsupported method"), req.Method, 0) return } render.JSON(w, r, fn(req.ID, *req.Params)) @@ -136,7 +143,7 @@ func (s *Server) basicAuth(h http.Handler) http.Handler { user, pass, ok := r.BasicAuth() if user != s.AuthUser || pass != s.AuthPasswd || !ok { w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) - http.Error(w, "Unauthorized.", http.StatusUnauthorized) + http.Error(w, "Unauthorized", http.StatusUnauthorized) return } h.ServeHTTP(w, r) From ab73d825e97b500680618f1fc0e00fab97b543b6 Mon Sep 17 00:00:00 2001 From: Umputun Date: Wed, 19 Jun 2019 22:29:24 -0500 Subject: [PATCH 12/24] change remote interface to struct requests --- backend/app/store/engine/bolt.go | 24 +++++----- backend/app/store/engine/bolt_test.go | 28 +++++++++--- backend/app/store/engine/engine.go | 26 +++++++---- backend/app/store/engine/engine_mock.go | 24 +++++----- backend/app/store/engine/remote.go | 24 +++++----- backend/app/store/engine/remote_test.go | 56 +++++++++++++---------- backend/app/store/service/service.go | 31 +++++++------ backend/app/store/service/service_test.go | 25 ++++++---- 8 files changed, 139 insertions(+), 99 deletions(-) diff --git a/backend/app/store/engine/bolt.go b/backend/app/store/engine/bolt.go index 7fbf804850..2fc7e02fa1 100644 --- a/backend/app/store/engine/bolt.go +++ b/backend/app/store/engine/bolt.go @@ -137,19 +137,19 @@ func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { } // Get returns comment for locator.URL and commentID string -func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { +func (b *BoltDB) Get(req GetRequest) (comment store.Comment, err error) { - bdb, err := b.db(locator.SiteID) + bdb, err := b.db(req.Locator.SiteID) if err != nil { return comment, err } err = bdb.View(func(tx *bolt.Tx) error { - bucket, e := b.getPostBucket(tx, locator.URL) + bucket, e := b.getPostBucket(tx, req.Locator.URL) if e != nil { return e } - return b.load(bucket, commentID, &comment) + return b.load(bucket, req.CommentID, &comment) }) return comment, err } @@ -204,9 +204,10 @@ func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) { } // Update for locator.URL with mutable part of comment -func (b *BoltDB) Update(locator store.Locator, comment store.Comment) error { +func (b *BoltDB) Update(comment store.Comment) error { - if curComment, err := b.Get(locator, comment.ID); err == nil { + getReq := GetRequest{Locator: comment.Locator, CommentID: comment.ID} + if curComment, err := b.Get(getReq); err == nil { // preserve immutable fields comment.ParentID = curComment.ParentID comment.Locator = curComment.Locator @@ -214,13 +215,13 @@ func (b *BoltDB) Update(locator store.Locator, comment store.Comment) error { comment.User = curComment.User } - bdb, err := b.db(locator.SiteID) + bdb, err := b.db(comment.Locator.SiteID) if err != nil { return err } return bdb.Update(func(tx *bolt.Tx) error { - bucket, e := b.getPostBucket(tx, locator.URL) + bucket, e := b.getPostBucket(tx, comment.Locator.URL) if e != nil { return e } @@ -351,8 +352,8 @@ func (b *BoltDB) ListFlags(req FlagRequest) (res []interface{}, err error) { if time.Now().Before(ts) { // get user name from comment user section userName := "" - req := FindRequest{Locator: store.Locator{SiteID: req.Locator.SiteID}, UserID: string(k), Limit: 1} - userComments, errUser := b.Find(req) + findReq := FindRequest{Locator: store.Locator{SiteID: req.Locator.SiteID}, UserID: string(k), Limit: 1} + userComments, errUser := b.Find(findReq) if errUser == nil && len(userComments) > 0 { userName = userComments[0].User.Name } @@ -500,7 +501,8 @@ func (b *BoltDB) userComments(siteID, userID string, limit, skip int) (comments if errParse != nil { return comments, errors.Wrapf(errParse, "can't parse reference %s", v) } - if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil { + getReq := GetRequest{Locator: store.Locator{SiteID: siteID, URL: url}, CommentID: commentID} + if c, errRef := b.Get(getReq); errRef == nil { comments = append(comments, c) } } diff --git a/backend/app/store/engine/bolt_test.go b/backend/app/store/engine/bolt_test.go index 5bbeecf49e..588fb2e9c4 100644 --- a/backend/app/store/engine/bolt_test.go +++ b/backend/app/store/engine/bolt_test.go @@ -19,6 +19,10 @@ func TestBoltDB_CreateAndFind(t *testing.T) { var b, teardown = prep(t) defer teardown() + var bb Interface + bb = b + _ = bb + req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"} res, err := b.Find(req) assert.NoError(t, err) @@ -77,14 +81,14 @@ func TestBoltDB_Get(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 2, len(res), "2 records initially") - comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) + comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)) assert.NoError(t, err) assert.Equal(t, "some text2", comment.Text) - comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") + comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")) assert.NotNil(t, err) - _, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID) + _, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)) assert.EqualError(t, err, `site "bad" not found`) } @@ -100,19 +104,22 @@ func TestBoltDB_Update(t *testing.T) { comment := res[0] comment.Text = "abc 123" comment.Score = 100 - err = b.Update(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, comment) + err = b.Update(comment) assert.NoError(t, err) - comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)) assert.NoError(t, err) assert.Equal(t, "abc 123", comment.Text) assert.Equal(t, res[0].ID, comment.ID) assert.Equal(t, 100, comment.Score) - err = b.Update(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment) + comment.Locator.SiteID = "bad" + err = b.Update(comment) assert.EqualError(t, err, `site "bad" not found`) - err = b.Update(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment) + comment.Locator.SiteID="radio-t" + comment.Locator.URL="https://radio-t.com-bad" + err = b.Update(comment) assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`) } @@ -806,3 +813,10 @@ func prep(t *testing.T) (b *BoltDB, teardown func()) { } return b, teardown } + +func getReq(locator store.Locator, commentID string) GetRequest { + return GetRequest{ + Locator: locator, + CommentID: commentID, + } +} \ No newline at end of file diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go index db46d646a4..5d30d7ee23 100644 --- a/backend/app/store/engine/engine.go +++ b/backend/app/store/engine/engine.go @@ -16,16 +16,22 @@ import ( // Interface defines methods provided by low-level storage engine type Interface interface { - Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id - Update(locator store.Locator, comment store.Comment) error // update comment, mutable parts only - Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id - Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site - Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info - Count(req FindRequest) (int, error) // get count for post or user - Delete(req DeleteRequest) error // delete post(s) by id or by userID - Flag(req FlagRequest) (bool, error) // set and get flags - ListFlags(req FlagRequest) ([]interface{}, error) // get list of flagged keys, like blocked & verified user - Close() error // close storage engine + Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id + Update(comment store.Comment) error // update comment, mutable parts only + Get(req GetRequest) (store.Comment, error) // get comment by id + Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site + Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info + Count(req FindRequest) (int, error) // get count for post or user + Delete(req DeleteRequest) error // delete post(s) by id or by userID + Flag(req FlagRequest) (bool, error) // set and get flags + ListFlags(req FlagRequest) ([]interface{}, error) // get list of flagged keys, like blocked & verified user + Close() error // close storage engine +} + +// GetRequest is the input for Get func +type GetRequest struct { + Locator store.Locator `json:"locator"` + CommentID string `json:"comment_id"` } // FindRequest is the input for all find operations diff --git a/backend/app/store/engine/engine_mock.go b/backend/app/store/engine/engine_mock.go index 2c1437f7ff..7549c92d5d 100644 --- a/backend/app/store/engine/engine_mock.go +++ b/backend/app/store/engine/engine_mock.go @@ -123,20 +123,20 @@ func (_m *MockInterface) Flag(req FlagRequest) (bool, error) { return r0, r1 } -// Get provides a mock function with given fields: locator, commentID -func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) { - ret := _m.Called(locator, commentID) +// Get provides a mock function with given fields: req +func (_m *MockInterface) Get(req GetRequest) (store.Comment, error) { + ret := _m.Called(req) var r0 store.Comment - if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok { - r0 = rf(locator, commentID) + if rf, ok := ret.Get(0).(func(GetRequest) store.Comment); ok { + r0 = rf(req) } else { r0 = ret.Get(0).(store.Comment) } var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { - r1 = rf(locator, commentID) + if rf, ok := ret.Get(1).(func(GetRequest) error); ok { + r1 = rf(req) } else { r1 = ret.Error(1) } @@ -190,13 +190,13 @@ func (_m *MockInterface) ListFlags(req FlagRequest) ([]interface{}, error) { return r0, r1 } -// Update provides a mock function with given fields: locator, comment -func (_m *MockInterface) Update(locator store.Locator, comment store.Comment) error { - ret := _m.Called(locator, comment) +// Update provides a mock function with given fields: comment +func (_m *MockInterface) Update(comment store.Comment) error { + ret := _m.Called(comment) var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok { - r0 = rf(locator, comment) + if rf, ok := ret.Get(0).(func(store.Comment) error); ok { + r0 = rf(comment) } else { r0 = ret.Error(0) } diff --git a/backend/app/store/engine/remote.go b/backend/app/store/engine/remote.go index 6ae521024e..fe3f1b700e 100644 --- a/backend/app/store/engine/remote.go +++ b/backend/app/store/engine/remote.go @@ -15,7 +15,7 @@ type Remote struct { // Create comment and return ID func (r *Remote) Create(comment store.Comment) (commentID string, err error) { - resp, err := r.Call("create", comment) + resp, err := r.Call("store.create", comment) if err != nil { return "", err } @@ -25,8 +25,8 @@ func (r *Remote) Create(comment store.Comment) (commentID string, err error) { } // Get comment by ID -func (r *Remote) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { - resp, err := r.Call("get", locator, commentID) +func (r *Remote) Get(req GetRequest) (comment store.Comment, err error) { + resp, err := r.Call("store.get", req) if err != nil { return store.Comment{}, err } @@ -36,14 +36,14 @@ func (r *Remote) Get(locator store.Locator, commentID string) (comment store.Com } // Update comment, mutable parts only -func (r *Remote) Update(locator store.Locator, comment store.Comment) error { - _, err := r.Call("update", locator, comment) +func (r *Remote) Update(comment store.Comment) error { + _, err := r.Call("store.update", comment) return err } // Find comments for locator func (r *Remote) Find(req FindRequest) (comments []store.Comment, err error) { - resp, err := r.Call("find", req) + resp, err := r.Call("store.find", req) if err != nil { return nil, err } @@ -53,7 +53,7 @@ func (r *Remote) Find(req FindRequest) (comments []store.Comment, err error) { // Info returns post(s) meta info func (r *Remote) Info(req InfoRequest) (info []store.PostInfo, err error) { - resp, err := r.Call("info", req) + resp, err := r.Call("store.info", req) if err != nil { return nil, err } @@ -63,7 +63,7 @@ func (r *Remote) Info(req InfoRequest) (info []store.PostInfo, err error) { // Flag sets and gets flags func (r *Remote) Flag(req FlagRequest) (status bool, err error) { - resp, err := r.Call("flag", req) + resp, err := r.Call("store.flag", req) if err != nil { return false, err } @@ -73,7 +73,7 @@ func (r *Remote) Flag(req FlagRequest) (status bool, err error) { // ListFlags get list of flagged keys, like blocked & verified user func (r *Remote) ListFlags(req FlagRequest) (list []interface{}, err error) { - resp, err := r.Call("list_flags", req) + resp, err := r.Call("store.list_flags", req) if err != nil { return nil, err } @@ -83,7 +83,7 @@ func (r *Remote) ListFlags(req FlagRequest) (list []interface{}, err error) { // Count gets comments count by user or site func (r *Remote) Count(req FindRequest) (count int, err error) { - resp, err := r.Call("count", req) + resp, err := r.Call("store.count", req) if err != nil { return 0, err } @@ -93,12 +93,12 @@ func (r *Remote) Count(req FindRequest) (count int, err error) { // Delete post(s) by id or by userID func (r *Remote) Delete(req DeleteRequest) error { - _, err := r.Call("delete", req) + _, err := r.Call("store.delete", req) return err } // Close storage engine func (r *Remote) Close() error { - _, err := r.Call("close") + _, err := r.Call("store.close") return err } diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go index eae1830095..965367ea66 100644 --- a/backend/app/store/engine/remote_test.go +++ b/backend/app/store/engine/remote_test.go @@ -17,11 +17,14 @@ import ( ) func TestClient_Create(t *testing.T) { - ts := testServer(t, `{"method":"create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, + ts := testServer(t, `{"method":"store.create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":"12345","id":1}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + var eng Interface = &c + _ = eng + res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"}, Text: "msg"}) assert.NoError(t, err) @@ -30,41 +33,44 @@ func TestClient_Create(t *testing.T) { } func TestClient_Get(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"],"id":1}`, - `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`) + ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} - res, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"} + res, err := c.Get(req) assert.NoError(t, err) assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res) t.Logf("%v %T", res, res) } func TestClient_GetWithErrorResult(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"],"id":1}`, `{"error":"failed"}`) + ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"error":"failed"}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"} + _, err := c.Get(req) assert.EqualError(t, err, "failed") } func TestClient_GetWithErrorDecode(t *testing.T) { - ts := testServer(t, `{"method":"get","params":[{"url":"http://example.com/url"},"site"],"id":1}`, ``) + ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, ``) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") - assert.EqualError(t, err, "failed to decode response for get: EOF") + req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"} + _, err := c.Get(req) + assert.EqualError(t, err, "failed to decode response for store.get: EOF") } func TestClient_GetWithErrorRemote(t *testing.T) { c := Remote{Client: remote.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}} - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") + req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"} + _, err := c.Get(req) assert.NotNil(t, err) - assert.True(t, strings.Contains(err.Error(), "remote call failed for get:"), err.Error()) + assert.True(t, strings.Contains(err.Error(), "remote call failed for store.get:"), err.Error()) } func TestClient_FailedStatus(t *testing.T) { @@ -77,23 +83,24 @@ func TestClient_FailedStatus(t *testing.T) { defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} - _, err := c.Get(store.Locator{URL: "http://example.com/url"}, "site") - assert.EqualError(t, err, "bad status 400 for get") + req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"} + _, err := c.Get(req) + assert.EqualError(t, err, "bad status 400 for store.get") } func TestClient_Update(t *testing.T) { - ts := testServer(t, `{"method":"update","params":[{"url":"http://example.com/url"},{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"}],"id":1}`, `{}`) + ts := testServer(t, `{"method":"store.update","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} - err := c.Update(store.Locator{URL: "http://example.com/url"}, store.Comment{ID: "123", - Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"}, Text: "msg"}) + err := c.Update(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"}, + Text: "msg"}) assert.NoError(t, err) } func TestClient_Find(t *testing.T) { - ts := testServer(t, `{"method":"find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`) + ts := testServer(t, `{"method":"store.find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -103,7 +110,7 @@ func TestClient_Find(t *testing.T) { } func TestClient_Info(t *testing.T) { - ts := testServer(t, `{"method":"info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) + ts := testServer(t, `{"method":"store.info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -114,8 +121,7 @@ func TestClient_Info(t *testing.T) { } func TestClient_Flag(t *testing.T) { - ts := testServer(t, `{"method":"flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, - `{"result":false}`) + ts := testServer(t, `{"method":"store.flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, `{"result":false}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -125,8 +131,7 @@ func TestClient_Flag(t *testing.T) { } func TestClient_ListFlag(t *testing.T) { - ts := testServer(t, `{"method":"list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, - `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) + ts := testServer(t, `{"method":"store.list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} res, err := c.ListFlags(FlagRequest{Locator: store.Locator{SiteID: "site_id"}, Flag: Blocked}) @@ -135,7 +140,7 @@ func TestClient_ListFlag(t *testing.T) { } func TestClient_Count(t *testing.T) { - ts := testServer(t, `{"method":"count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`) + ts := testServer(t, `{"method":"store.count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -145,7 +150,8 @@ func TestClient_Count(t *testing.T) { } func TestClient_Delete(t *testing.T) { - ts := testServer(t, `{"method":"delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`, `{}`) + ts := testServer(t, `{"method":"store.delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`, + `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -154,7 +160,7 @@ func TestClient_Delete(t *testing.T) { } func TestClient_Close(t *testing.T) { - ts := testServer(t, `{"method":"close","params":null,"id":1}`, `{}`) + ts := testServer(t, `{"method":"store.close","params":null,"id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} err := c.Close() diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 32e34ef6cc..376e92d4dc 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -132,7 +132,7 @@ func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([ // Get comment by ID func (s *DataStore) Get(locator store.Locator, commentID string, user store.User) (store.Comment, error) { - c, err := s.Engine.Get(locator, commentID) + c, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID}) if err != nil { return store.Comment{}, err } @@ -141,7 +141,8 @@ func (s *DataStore) Get(locator store.Locator, commentID string, user store.User // Put updates comment, mutable parts only func (s *DataStore) Put(locator store.Locator, comment store.Comment) error { - return s.Engine.Update(locator, comment) + comment.Locator = locator + return s.Engine.Update(comment) } // submitImages initiated delayed commit of all images from the comment uploaded to remark42 @@ -149,7 +150,8 @@ func (s *DataStore) submitImages(comment store.Comment) { s.ImageService.Submit(func() []string { c := comment - cc, err := s.Engine.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment + // this can be called after last edit, we have to retrieve fresh comment + cc, err := s.Engine.Get(engine.GetRequest{Locator: c.Locator, CommentID: c.ID}) if err != nil { log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err) return nil @@ -197,12 +199,13 @@ func (s *DataStore) DeleteAll(siteID string) error { // SetPin pin/un-pin comment as special func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error { - comment, err := s.Engine.Get(locator, commentID) + comment, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID}) if err != nil { return err } comment.Pin = status - return s.Engine.Update(locator, comment) + comment.Locator = locator + return s.Engine.Update(comment) } // Vote for comment by id and locator @@ -212,7 +215,7 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, cLock.Lock() // prevents race on voting defer cLock.Unlock() - comment, err = s.Engine.Get(locator, commentID) + comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID}) if err != nil { return comment, err } @@ -270,8 +273,8 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, } comment.Controversy = s.controversy(s.upsAndDowns(comment)) - - return comment, s.Engine.Update(locator, comment) + comment.Locator = locator + return comment, s.Engine.Update(comment) } // controversy calculates controversial index of votes @@ -300,7 +303,7 @@ type EditRequest struct { // EditComment to edit text and update Edit info func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) { - comment, err = s.Engine.Get(locator, commentID) + comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID}) if err != nil { return comment, err } @@ -330,9 +333,10 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi Timestamp: time.Now(), Summary: req.Summary, } - + comment.Locator = locator comment.Sanitize() - err = s.Engine.Update(locator, comment) + + err = s.Engine.Update(comment) return comment, err } @@ -410,7 +414,7 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s return comment, errors.New("no title extractor") } - comment, err = s.Engine.Get(locator, commentID) + comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID}) if err != nil { return comment, err } @@ -421,7 +425,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s return comment, err } comment.PostTitle = title - err = s.Engine.Update(locator, comment) + comment.Locator = locator + err = s.Engine.Update(comment) return comment, err } diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index f9f17a8da8..d9eb5012c7 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -40,7 +40,7 @@ func TestService_CreateFromEmpty(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "text", res.Text) @@ -66,7 +66,7 @@ func TestService_CreateFromPartial(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "text", res.Text) @@ -94,7 +94,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "Радио-Т 630 — Радио-Т Подкаст", res.PostTitle) @@ -102,7 +102,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) { comment.PostTitle = "post blah" id, err = b.Create(comment) assert.NoError(t, err) - res, err = b.Engine.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id) + res, err = b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "post blah", res.PostTitle, "keep comment title") @@ -145,7 +145,7 @@ func TestService_SetTitle(t *testing.T) { assert.NoError(t, err) assert.True(t, id != "", id) - res, err := b.Engine.Get(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id) + res, err := b.Engine.Get(getReq(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)) assert.NoError(t, err) t.Logf("%+v", res) assert.Equal(t, "", res.PostTitle) @@ -438,13 +438,13 @@ func TestService_Pin(t *testing.T) { err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true) assert.NoError(t, err) - c, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)) assert.NoError(t, err) assert.Equal(t, true, c.Pin) err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false) assert.NoError(t, err) - c, err = b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + c, err = b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)) assert.NoError(t, err) assert.Equal(t, false, c.Pin) } @@ -466,7 +466,7 @@ func TestService_EditComment(t *testing.T) { assert.Equal(t, "xxx", comment.Text) assert.Equal(t, "yyy", comment.Orig) - c, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)) assert.NoError(t, err) assert.Equal(t, "my edit", c.Edit.Summary) assert.Equal(t, "xxx", c.Text) @@ -489,7 +489,7 @@ func TestService_DeleteComment(t *testing.T) { _, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Delete: true}) assert.NoError(t, err) - c, err := b.Engine.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID) + c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)) assert.NoError(t, err) assert.True(t, c.Deleted) t.Logf("%+v", c) @@ -1079,3 +1079,10 @@ func prepStoreEngine(t *testing.T) engine.Interface { func teardown(_ *testing.T) { _ = os.Remove(testDb) } + +func getReq(locator store.Locator, commentID string) engine.GetRequest { + return engine.GetRequest{ + Locator: locator, + CommentID: commentID, + } +} \ No newline at end of file From 21cd81e3b627084ef92999a96928b69c09047831 Mon Sep 17 00:00:00 2001 From: Umputun Date: Thu, 20 Jun 2019 22:39:00 -0500 Subject: [PATCH 13/24] add remote implementation of admin store --- backend/app/cmd/avatar.go | 18 +----- backend/app/cmd/server.go | 49 +++++++--------- backend/app/cmd/server_test.go | 4 +- backend/app/rest/api/rest.go | 7 ++- backend/app/store/admin/admin.go | 30 +++++----- backend/app/store/admin/admin_test.go | 48 ++-------------- backend/app/store/admin/mongo.go | 56 ------------------- backend/app/store/admin/remote.go | 55 ++++++++++++++++++ backend/app/store/admin/remote_test.go | 74 +++++++++++++++++++++++++ backend/app/store/engine/bolt.go | 2 +- backend/app/store/engine/remote_test.go | 28 +++++----- backend/app/store/service/service.go | 6 +- 12 files changed, 196 insertions(+), 181 deletions(-) delete mode 100644 backend/app/store/admin/mongo.go create mode 100644 backend/app/store/admin/remote.go create mode 100644 backend/app/store/admin/remote_test.go diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go index 5c7e9a92ba..e0075113c2 100644 --- a/backend/app/cmd/avatar.go +++ b/backend/app/cmd/avatar.go @@ -2,14 +2,12 @@ package cmd import ( "path" - "time" bolt "github.com/coreos/bbolt" log "github.com/go-pkgz/lgr" "github.com/pkg/errors" "github.com/go-pkgz/auth/avatar" - "github.com/go-pkgz/mongo" ) // AvatarCommand set of flags and command for avatar migration @@ -18,7 +16,7 @@ import ( type AvatarCommand struct { AvatarSrc AvatarGroup `group:"src" namespace:"src"` AvatarDst AvatarGroup `group:"dst" namespace:"dst"` - Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` + // Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` migrator AvatarMigrator CommonOpts @@ -78,13 +76,6 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) { return nil, err } return avatar.NewLocalFS(gr.FS.Path), nil - case "mongo": - mgServer, err := ac.makeMongo() - if err != nil { - return nil, errors.Wrap(err, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "") - return avatar.NewGridFS(conn), nil case "bolt": if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil { return nil, err @@ -93,10 +84,3 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) { } return nil, errors.Errorf("unsupported avatar store type %s", gr.Type) } - -func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) { - if ac.Mongo.URL == "" { - return nil, errors.New("no mongo URL provided") - } - return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second) -} diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index e5c46bb4c7..8029340034 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -22,7 +22,6 @@ import ( "github.com/go-pkgz/auth/avatar" "github.com/go-pkgz/auth/provider" "github.com/go-pkgz/auth/token" - "github.com/go-pkgz/mongo" "github.com/go-pkgz/rest/cache" "github.com/umputun/remark/backend/app/migrator" @@ -41,7 +40,6 @@ type ServerCommand struct { Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"` Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"` Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"` - Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` Admin AdminGroup `group:"admin" namespace:"admin" env-namespace:"ADMIN"` Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"` Image ImageGroup `group:"image" namespace:"image" env-namespace:"IMAGE"` @@ -89,11 +87,15 @@ type AuthGroup struct { // StoreGroup defines options group for store params type StoreGroup struct { - Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"` + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"remote" default:"bolt"` Bolt struct { Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` + Remote struct { + API string `long:"api" env:"API" description:"remote extension api url"` + TimeOut time.Duration `long:"timeout" env:"TIMEOUT" description:"http timeout"` + } `group:"remote" namespace:"remote" env-namespace:"REMOTE"` } // ImageGroup defines options group for store pictures @@ -134,12 +136,6 @@ type CacheGroup struct { } `group:"max" namespace:"max" env-namespace:"MAX"` } -// MongoGroup holds all mongo params, used by store, avatar and cache -type MongoGroup struct { - URL string `long:"url" env:"URL" description:"mongo url"` - DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"` -} - // AdminGroup defines options group for admin params type AdminGroup struct { Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"` @@ -436,13 +432,13 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { return nil, err } return avatar.NewLocalFS(s.Avatar.FS.Path), nil - case "mongo": - mgServer, err := s.makeMongo() - if err != nil { - return nil, errors.Wrap(err, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") - return avatar.NewGridFS(conn), nil + // case "mongo": + // mgServer, err := s.makeMongo() + // if err != nil { + // return nil, errors.Wrap(err, "failed to create mongo server") + // } + // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") + // return avatar.NewGridFS(conn), nil case "bolt": if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil { return nil, err @@ -485,13 +481,13 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) { } } return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil - case "mongo": - mgServer, e := s.makeMongo() - if e != nil { - return nil, errors.Wrap(e, "failed to create mongo server") - } - conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin") - return admin.NewMongoStore(conn, s.SharedSecret), nil + // case "mongo": + // mgServer, e := s.makeMongo() + // if e != nil { + // return nil, errors.Wrap(e, "failed to create mongo server") + // } + // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin") + // return admin.NewMongoStore(conn, s.SharedSecret), nil default: return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type) } @@ -517,13 +513,6 @@ func (s *ServerCommand) makeCache() (cache.LoadingCache, error) { return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type) } -func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) { - if s.Mongo.URL == "" { - return nil, errors.New("no mongo URL provided") - } - return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second) -} - func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) { providers := 0 diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 3f0a60c54b..344225be6a 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -55,7 +55,9 @@ func TestServerApp(t *testing.T) { body, _ = ioutil.ReadAll(resp.Body) t.Log(string(body)) - assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email") + email, err := app.dataService.AdminStore.Email("") + assert.NoError(t, err) + assert.Equal(t, "admin@demo.remark42.com", email, "default admin email") app.Wait() } diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index b5e1c30751..30d037d748 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -377,6 +377,9 @@ func (s *Rest) updateLimiter() float64 { func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) { siteID := r.URL.Query().Get("site") + admins, _ := s.DataService.AdminStore.Admins(siteID) + emails, _ := s.DataService.AdminStore.Email(siteID) + cnf := struct { Version string `json:"version"` EditDuration int `json:"edit_duration"` @@ -393,8 +396,8 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) { Version: s.Version, EditDuration: int(s.DataService.EditDuration.Seconds()), MaxCommentSize: s.DataService.MaxCommentSize, - Admins: s.DataService.AdminStore.Admins(siteID), - AdminEmail: s.DataService.AdminStore.Email(siteID), + Admins: admins, + AdminEmail: emails, LowScore: s.ScoreThresholds.Low, CriticalScore: s.ScoreThresholds.Critical, PositiveScore: s.DataService.PositiveScore, diff --git a/backend/app/store/admin/admin.go b/backend/app/store/admin/admin.go index d0b5069253..8a6fc6692f 100644 --- a/backend/app/store/admin/admin.go +++ b/backend/app/store/admin/admin.go @@ -10,25 +10,17 @@ import ( // Store defines interface returning admins info for given site type Store interface { Key() (key string, err error) - Admins(siteID string) (ids []string) - Email(siteID string) (email string) + Admins(siteID string) (ids []string, err error) + Email(siteID string) (email string, err error) } -// StaticStore implements keys.Store with a single, predefined key +// StaticStore implements keys.Store with a single set of admins and email for all sites type StaticStore struct { admins []string email string key string } -// Key returns static key for all sites, allows empty site -func (s *StaticStore) Key() (key string, err error) { - if s.key == "" { - return "", errors.New("empty key for static key store") - } - return s.key, nil -} - // NewStaticStore makes StaticStore instance with given key func NewStaticStore(key string, admins []string, email string) *StaticStore { log.Printf("[DEBUG] admin users %+v, email %s", admins, email) @@ -40,12 +32,20 @@ func NewStaticKeyStore(key string) *StaticStore { return &StaticStore{key: key, admins: []string{}, email: ""} } +// Key returns static key, same for all sites +func (s *StaticStore) Key() (key string, err error) { + if s.key == "" { + return "", errors.New("empty key for static key store") + } + return s.key, nil +} + // Admins returns static list of admin's ids, the same for all sites -func (s *StaticStore) Admins(string) (ids []string) { - return s.admins +func (s *StaticStore) Admins(string) (ids []string, err error) { + return s.admins, nil } // Email gets static email address -func (s *StaticStore) Email(string) (email string) { - return s.email +func (s *StaticStore) Email(string) (email string, err error) { + return s.email, nil } diff --git a/backend/app/store/admin/admin_test.go b/backend/app/store/admin/admin_test.go index 2d587f3ae0..6ce4517879 100644 --- a/backend/app/store/admin/admin_test.go +++ b/backend/app/store/admin/admin_test.go @@ -3,10 +3,7 @@ package admin import ( "testing" - "github.com/globalsign/mgo" - "github.com/go-pkgz/mongo" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestStaticStore_Get(t *testing.T) { @@ -16,48 +13,11 @@ func TestStaticStore_Get(t *testing.T) { assert.NoError(t, err, "valid store") assert.Equal(t, "key123", k, "valid site") - a := ks.Admins("any") - assert.Equal(t, []string{"123", "xyz"}, a) - - email := ks.Email("blah") - assert.Equal(t, "aa@example.com", email) -} - -func TestMongoStore_Get(t *testing.T) { - conn, err := mongo.MakeTestConnection(t) - require.NoError(t, err) - var ms Store = NewMongoStore(conn, "secret") - - recs := []mongoRec{ - {"site1", []string{"i11", "i12"}, "e1"}, - {"site2", []string{"i21", "i22"}, "e2"}, - } - err = conn.WithCollection(func(coll *mgo.Collection) error { - if e1 := coll.Insert(recs[0]); e1 != nil { - return e1 - } - return coll.Insert(recs[1]) - }) - require.NoError(t, err) - - admins := ms.Admins("site1") - assert.Equal(t, []string{"i11", "i12"}, admins) - email := ms.Email("site1") - assert.Equal(t, "e1", email) - key, err := ms.Key() + a, err := ks.Admins("any") assert.NoError(t, err) - assert.Equal(t, "secret", key) + assert.Equal(t, []string{"123", "xyz"}, a) - admins = ms.Admins("site2") - assert.Equal(t, []string{"i21", "i22"}, admins) - email = ms.Email("site2") - assert.Equal(t, "e2", email) - key, err = ms.Key() + email, err := ks.Email("blah") assert.NoError(t, err) - assert.Equal(t, "secret", key) - - admins = ms.Admins("no-site-in-db") - assert.Equal(t, []string{}, admins) - email = ms.Email("no-site-in-db") - assert.Equal(t, "", email) + assert.Equal(t, "aa@example.com", email) } diff --git a/backend/app/store/admin/mongo.go b/backend/app/store/admin/mongo.go deleted file mode 100644 index 1f19bc328c..0000000000 --- a/backend/app/store/admin/mongo.go +++ /dev/null @@ -1,56 +0,0 @@ -package admin - -import ( - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - log "github.com/go-pkgz/lgr" - - "github.com/go-pkgz/mongo" -) - -// MongoStore implements admin.Store with mongo backend -type MongoStore struct { - connection *mongo.Connection - key string -} - -type mongoRec struct { - SiteID string `bson:"site"` - IDs []string `bson:"admin_ids"` - Email string `bson:"admin_email"` -} - -// NewMongoStore makes admin Store for mongo's connection -func NewMongoStore(conn *mongo.Connection, key string) *MongoStore { - log.Printf("[DEBUG] make mongo admin store with %+v", conn) - return &MongoStore{connection: conn, key: key} -} - -// Key executes find by siteID and returns substructure with secret key -func (m *MongoStore) Key() (key string, err error) { - return m.key, nil -} - -// Admins executes find by siteID and returns admins ids -func (m *MongoStore) Admins(siteID string) (ids []string) { - resp := mongoRec{} - err := m.connection.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID}).One(&resp) - }) - if err != nil { - return []string{} - } - return resp.IDs -} - -// Email executes find by siteID and returns admin's email -func (m *MongoStore) Email(siteID string) (email string) { - resp := mongoRec{} - err := m.connection.WithCollection(func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID}).One(&resp) - }) - if err != nil { - return "" - } - return resp.Email -} diff --git a/backend/app/store/admin/remote.go b/backend/app/store/admin/remote.go new file mode 100644 index 0000000000..1ba1805c16 --- /dev/null +++ b/backend/app/store/admin/remote.go @@ -0,0 +1,55 @@ +/* + * Copyright 2019 Umputun. All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ + +package admin + +import ( + "encoding/json" + + "github.com/umputun/remark/backend/app/store/remote" +) + +// Remote implements remote engine and delegates all Calls to remote http server +type Remote struct { + remote.Client +} + +// Key returns the key, same for all sites +func (r *Remote) Key() (key string, err error) { + resp, err := r.Call("admin.key") + if err != nil { + return "", err + } + + err = json.Unmarshal(*resp.Result, &key) + return key, err +} + +// Admins returns list of admin's ids for given site +func (r *Remote) Admins(siteID string) (ids []string, err error) { + resp, err := r.Call("admin.admins", siteID) + if err != nil { + return []string{}, err + } + + if err = json.Unmarshal(*resp.Result, &ids); err != nil { + return []string{}, err + } + return ids, nil +} + +// Email gets email address for given site +func (r *Remote) Email(siteID string) (email string, err error) { + resp, err := r.Call("admin.email", siteID) + if err != nil { + return "", err + } + + if err = json.Unmarshal(*resp.Result, &email); err != nil { + return "", err + } + return email, nil +} diff --git a/backend/app/store/admin/remote_test.go b/backend/app/store/admin/remote_test.go new file mode 100644 index 0000000000..d1940f081c --- /dev/null +++ b/backend/app/store/admin/remote_test.go @@ -0,0 +1,74 @@ +/* + * Copyright 2019 Umputun. All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ + +package admin + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/remark/backend/app/store/remote" +) + +func TestRemote_Key(t *testing.T) { + ts := testServer(t, `{"method":"admin.key","params":null,"id":1}`, + `{"result":"12345","id":1}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.Key() + assert.NoError(t, err) + assert.Equal(t, "12345", res) + t.Logf("%v %T", res, res) +} + +func TestRemote_Admins(t *testing.T) { + ts := testServer(t, `{"method":"admin.admins","params":["site-1"],"id":1}`, + `{"result":["id1","id2"],"id":1}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.Admins("site-1") + assert.NoError(t, err) + assert.Equal(t, []string{"id1", "id2"}, res) + t.Logf("%v %T", res, res) +} + +func TestRemote_Email(t *testing.T) { + ts := testServer(t, `{"method":"admin.email","params":["site-1"],"id":1}`, + `{"result":"bbb@example.com","id":1}`) + defer ts.Close() + c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} + + var a Store = &c + _ = a + + res, err := c.Email("site-1") + assert.NoError(t, err) + assert.Equal(t, "bbb@example.com", res) + t.Logf("%v %T", res, res) +} +func testServer(t *testing.T, req, resp string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, req, string(body)) + t.Logf("req: %s", string(body)) + fmt.Fprintf(w, resp) + })) +} diff --git a/backend/app/store/engine/bolt.go b/backend/app/store/engine/bolt.go index 2fc7e02fa1..f94660c9ac 100644 --- a/backend/app/store/engine/bolt.go +++ b/backend/app/store/engine/bolt.go @@ -4,11 +4,11 @@ import ( "bytes" "encoding/json" "fmt" - "log" "strings" "time" bolt "github.com/coreos/bbolt" + log "github.com/go-pkgz/lgr" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go index 965367ea66..7ac33254eb 100644 --- a/backend/app/store/engine/remote_test.go +++ b/backend/app/store/engine/remote_test.go @@ -16,7 +16,7 @@ import ( "github.com/umputun/remark/backend/app/store/remote" ) -func TestClient_Create(t *testing.T) { +func TestRemote_Create(t *testing.T) { ts := testServer(t, `{"method":"store.create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":"12345","id":1}`) defer ts.Close() @@ -32,7 +32,7 @@ func TestClient_Create(t *testing.T) { t.Logf("%v %T", res, res) } -func TestClient_Get(t *testing.T) { +func TestRemote_Get(t *testing.T) { ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -44,7 +44,7 @@ func TestClient_Get(t *testing.T) { t.Logf("%v %T", res, res) } -func TestClient_GetWithErrorResult(t *testing.T) { +func TestRemote_GetWithErrorResult(t *testing.T) { ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"error":"failed"}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -54,7 +54,7 @@ func TestClient_GetWithErrorResult(t *testing.T) { assert.EqualError(t, err, "failed") } -func TestClient_GetWithErrorDecode(t *testing.T) { +func TestRemote_GetWithErrorDecode(t *testing.T) { ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, ``) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -64,7 +64,7 @@ func TestClient_GetWithErrorDecode(t *testing.T) { assert.EqualError(t, err, "failed to decode response for store.get: EOF") } -func TestClient_GetWithErrorRemote(t *testing.T) { +func TestRemote_GetWithErrorRemote(t *testing.T) { c := Remote{Client: remote.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}} req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"} @@ -73,7 +73,7 @@ func TestClient_GetWithErrorRemote(t *testing.T) { assert.True(t, strings.Contains(err.Error(), "remote call failed for store.get:"), err.Error()) } -func TestClient_FailedStatus(t *testing.T) { +func TestRemote_FailedStatus(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) require.NoError(t, err) @@ -88,7 +88,7 @@ func TestClient_FailedStatus(t *testing.T) { assert.EqualError(t, err, "bad status 400 for store.get") } -func TestClient_Update(t *testing.T) { +func TestRemote_Update(t *testing.T) { ts := testServer(t, `{"method":"store.update","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -99,7 +99,7 @@ func TestClient_Update(t *testing.T) { } -func TestClient_Find(t *testing.T) { +func TestRemote_Find(t *testing.T) { ts := testServer(t, `{"method":"store.find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -109,7 +109,7 @@ func TestClient_Find(t *testing.T) { assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res) } -func TestClient_Info(t *testing.T) { +func TestRemote_Info(t *testing.T) { ts := testServer(t, `{"method":"store.info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -120,7 +120,7 @@ func TestClient_Info(t *testing.T) { assert.Equal(t, []store.PostInfo{{URL: "u1", Count: 22}, {URL: "u2", Count: 33}}, res) } -func TestClient_Flag(t *testing.T) { +func TestRemote_Flag(t *testing.T) { ts := testServer(t, `{"method":"store.flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, `{"result":false}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -130,7 +130,7 @@ func TestClient_Flag(t *testing.T) { assert.Equal(t, false, res) } -func TestClient_ListFlag(t *testing.T) { +func TestRemote_ListFlag(t *testing.T) { ts := testServer(t, `{"method":"store.list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -139,7 +139,7 @@ func TestClient_ListFlag(t *testing.T) { assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res) } -func TestClient_Count(t *testing.T) { +func TestRemote_Count(t *testing.T) { ts := testServer(t, `{"method":"store.count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} @@ -149,7 +149,7 @@ func TestClient_Count(t *testing.T) { assert.Equal(t, 11, res) } -func TestClient_Delete(t *testing.T) { +func TestRemote_Delete(t *testing.T) { ts := testServer(t, `{"method":"store.delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`, `{}`) defer ts.Close() @@ -159,7 +159,7 @@ func TestClient_Delete(t *testing.T) { assert.NoError(t, err) } -func TestClient_Close(t *testing.T) { +func TestRemote_Close(t *testing.T) { ts := testServer(t, `{"method":"store.close","params":null,"id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 376e92d4dc..5a537bc661 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -462,7 +462,11 @@ func (s *DataStore) ValidateComment(c *store.Comment) error { // IsAdmin checks if usesID in the list of admins func (s *DataStore) IsAdmin(siteID string, userID string) bool { - for _, a := range s.AdminStore.Admins(siteID) { + admins, err := s.AdminStore.Admins(siteID) + if err != nil { + return false + } + for _, a := range admins { if a == userID { return true } From 247e1e22201c9f05a523fa9c4340be838824b483 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 21 Jun 2019 00:06:23 -0500 Subject: [PATCH 14/24] fix empty (no args) remote calls --- backend/app/store/remote/client.go | 11 +++++++++-- backend/app/store/remote/client_test.go | 12 ++++++++++++ backend/app/store/remote/server.go | 8 +++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/backend/app/store/remote/client.go b/backend/app/store/remote/client.go index 9e5eb796e7..5317ed451c 100644 --- a/backend/app/store/remote/client.go +++ b/backend/app/store/remote/client.go @@ -25,12 +25,19 @@ func (r *Client) Call(method string, args ...interface{}) (*Response, error) { var b []byte var err error - if len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Struct { + + switch { + case args == nil || len(args) == 0: + b, err = json.Marshal(Request{Method: method, ID: atomic.AddUint64(&r.id, 1)}) + if err != nil { + return nil, errors.Wrapf(err, "marshaling failed for %s", method) + } + case len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Struct: b, err = json.Marshal(Request{Method: method, Params: args[0], ID: atomic.AddUint64(&r.id, 1)}) if err != nil { return nil, errors.Wrapf(err, "marshaling failed for %s", method) } - } else { + default: b, err = json.Marshal(Request{Method: method, Params: args, ID: atomic.AddUint64(&r.id, 1)}) if err != nil { return nil, errors.Wrapf(err, "marshaling failed for %s", method) diff --git a/backend/app/store/remote/client_test.go b/backend/app/store/remote/client_test.go index 98a2c79473..5979903166 100644 --- a/backend/app/store/remote/client_test.go +++ b/backend/app/store/remote/client_test.go @@ -47,6 +47,18 @@ func TestClient_CallWithObject(t *testing.T) { t.Logf("%v %T", res, res) } +func TestClient_CallWithNoParams(t *testing.T) { + ts := testServer(t, `{"method":"test","id":1}`, `{"result":"12345"}`) + defer ts.Close() + c := Client{API: ts.URL, Client: http.Client{}} + resp, err := c.Call("test") + assert.NoError(t, err) + res := "" + err = json.Unmarshal(*resp.Result, &res) + assert.Equal(t, "12345", res) + t.Logf("%v %T", res, res) +} + func TestClient_CallError(t *testing.T) { ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"error":"some error"}`) defer ts.Close() diff --git a/backend/app/store/remote/server.go b/backend/app/store/remote/server.go index c44e4e1a8c..49082af35c 100644 --- a/backend/app/store/remote/server.go +++ b/backend/app/store/remote/server.go @@ -129,7 +129,13 @@ func (s *Server) handler(w http.ResponseWriter, r *http.Request) { rest.SendErrorJSON(w, r, http.StatusNotImplemented, errors.New("unsupported method"), req.Method, 0) return } - render.JSON(w, r, fn(req.ID, *req.Params)) + + params := json.RawMessage{} + if req.Params != nil { + params = *req.Params + } + + render.JSON(w, r, fn(req.ID, params)) } func (s *Server) basicAuth(h http.Handler) http.Handler { From e83a4a293cfbdfe1ae923895478195a917a857c0 Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 23 Jun 2019 22:46:23 -0500 Subject: [PATCH 15/24] support remote server group handler --- backend/app/store/remote/client.go | 2 +- backend/app/store/remote/server.go | 22 ++++++++++ backend/app/store/remote/server_test.go | 56 +++++++++++++++++++++---- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/backend/app/store/remote/client.go b/backend/app/store/remote/client.go index 5317ed451c..e200e03fa0 100644 --- a/backend/app/store/remote/client.go +++ b/backend/app/store/remote/client.go @@ -59,7 +59,7 @@ func (r *Client) Call(method string, args ...interface{}) (*Response, error) { } defer resp.Body.Close() if resp.StatusCode != 200 { - return nil, errors.Errorf("bad status %d for %s", resp.StatusCode, method) + return nil, errors.Errorf("bad status %s for %s", resp.Status, method) } cr := Response{} diff --git a/backend/app/store/remote/server.go b/backend/app/store/remote/server.go index 49082af35c..ba1b59dcc6 100644 --- a/backend/app/store/remote/server.go +++ b/backend/app/store/remote/server.go @@ -40,6 +40,9 @@ type Server struct { } } +// Encoder is a function to encode call's result to Response +type Encoder func(id uint64, resp interface{}, e error) (Response, error) + // ServerFn handler registered for each method with Add // Implementations provided by consumer and define response logic. type ServerFn func(id uint64, params json.RawMessage) Response @@ -107,10 +110,29 @@ func (s *Server) Shutdown() error { // Add method handler func (s *Server) Add(method string, fn ServerFn) { + s.httpServer.Lock() + defer s.httpServer.Unlock() + if s.httpServer.Server != nil { + log.Printf("[WARN] ignored method %s, can't be added to activated server", method) + return + } + s.funcs.once.Do(func() { s.funcs.m = map[string]ServerFn{} }) + s.funcs.m[method] = fn + log.Printf("[INFO] add handler for %s", method) +} + +// HandlersGroup alias for map of handlers +type HandlersGroup map[string]ServerFn + +// Group of handlers with common prefix +func (s *Server) Group(prefix string, m HandlersGroup) { + for k, v := range m { + s.Add(prefix+"."+k, v) + } } func (s *Server) handler(w http.ResponseWriter, r *http.Request) { diff --git a/backend/app/store/remote/server_test.go b/backend/app/store/remote/server_test.go index ae0f38a0ce..49342c99ca 100644 --- a/backend/app/store/remote/server_test.go +++ b/backend/app/store/remote/server_test.go @@ -40,6 +40,7 @@ func TestServerPrimitiveTypes(t *testing.T) { }) go func() { s.Run(9091) }() + defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) // check with direct http call @@ -64,7 +65,6 @@ func TestServerPrimitiveTypes(t *testing.T) { err = json.Unmarshal(*r.Result, &res) assert.Equal(t, respData{Res1: "res blah", Res2: true}, res) assert.Equal(t, uint64(1), r.ID) - assert.NoError(t, s.Shutdown()) } func TestServerWithObject(t *testing.T) { @@ -94,6 +94,7 @@ func TestServerWithObject(t *testing.T) { }) go func() { s.Run(9091) }() + defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} @@ -104,8 +105,6 @@ func TestServerWithObject(t *testing.T) { res := respData{} err = json.Unmarshal(*r.Result, &res) assert.Equal(t, respData{Res1: "res blah", Res2: true}, res) - - assert.NoError(t, s.Shutdown()) } func TestServerMethodNotImplemented(t *testing.T) { @@ -148,6 +147,7 @@ func TestServerWithAuth(t *testing.T) { go func() { s.Run(9091) }() time.Sleep(10 * time.Millisecond) + defer func() { assert.NoError(t, s.Shutdown()) }() c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"} r, err := c.Call("test", "blah", 42, true) @@ -160,9 +160,7 @@ func TestServerWithAuth(t *testing.T) { c = Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} _, err = c.Call("test", "blah", 42, true) - assert.EqualError(t, err, "bad status 401 for test") - - assert.NoError(t, s.Shutdown()) + assert.EqualError(t, err, "bad status 401 Unauthorized for test") } func TestServerErrReturn(t *testing.T) { @@ -186,13 +184,57 @@ func TestServerErrReturn(t *testing.T) { }) go func() { s.Run(9091) }() + defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"} _, err := c.Call("test", "blah", 42, true) assert.EqualError(t, err, "some error") +} - assert.NoError(t, s.Shutdown()) +func TestServerGroup(t *testing.T) { + s := Server{API: "/v1/cmd"} + s.Group("pre", HandlersGroup{ + "fn1": func(id uint64, params json.RawMessage) Response { + return Response{} + }, + "fn2": func(id uint64, params json.RawMessage) Response { + return Response{} + }, + }) + go func() { s.Run(9091) }() + defer func() { assert.NoError(t, s.Shutdown()) }() + time.Sleep(10 * time.Millisecond) + + c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} + _, err := c.Call("fn1") + assert.EqualError(t, err, "bad status 501 Not Implemented for fn1") + + _, err = c.Call("pre.fn1") + assert.NoError(t, err) + _, err = c.Call("pre.fn2") + assert.NoError(t, err) +} + +func TestServerAddLate(t *testing.T) { + s := Server{API: "/v1/cmd"} + s.Add("fn1", func(id uint64, params json.RawMessage) Response { + return Response{} + }) + go func() { s.Run(9091) }() + defer func() { assert.NoError(t, s.Shutdown()) }() + time.Sleep(10 * time.Millisecond) + + // too late, ignored after run + s.Add("fn2", func(id uint64, params json.RawMessage) Response { + return Response{} + }) + + c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}} + _, err := c.Call("fn1") + assert.NoError(t, err) + _, err = c.Call("fn2") + assert.EqualError(t, err, "bad status 501 Not Implemented for fn2") } func TestServerNoHandlers(t *testing.T) { From 2b7ddcb6e551a8292ff96d61367153e60402d761 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 14:08:26 -0500 Subject: [PATCH 16/24] adjust remote tests, remove legacy mongo tests --- backend/app/cmd/avatar_test.go | 22 ++++++---------------- backend/app/store/admin/remote_test.go | 2 +- backend/app/store/engine/remote_test.go | 4 ++-- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/backend/app/cmd/avatar_test.go b/backend/app/cmd/avatar_test.go index e0df7193be..203f246c82 100644 --- a/backend/app/cmd/avatar_test.go +++ b/backend/app/cmd/avatar_test.go @@ -22,23 +22,13 @@ func TestAvatar_Execute(t *testing.T) { } defer os.RemoveAll("/tmp/ava-test") - // from fs to mongo + // from fs to bolt cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}} cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) p := flags.NewParser(&cmd, flags.Default) - _, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo", - "--mongo.url=" + mongoURL, "--mongo.db=test_remark"}) - require.Nil(t, err) - err = cmd.Execute(nil) - assert.NoError(t, err) - - // from fs to bolt - cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}} - cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) - p = flags.NewParser(&cmd, flags.Default) - _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt", + _, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt", "--dst.bolt.file=/tmp/ava-test.db"}) - require.Nil(t, err) + require.NoError(t, err) err = cmd.Execute(nil) assert.NoError(t, err) @@ -46,9 +36,9 @@ func TestAvatar_Execute(t *testing.T) { cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}} cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) p = flags.NewParser(&cmd, flags.Default) - _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo", - "--mongo.url=" + mongoURL, "--mongo.db=test_remark"}) - require.Nil(t, err) + _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt", + "--dst.bolt.file=/tmp/ava-test.db"}) + require.NoError(t, err) err = cmd.Execute(nil) assert.Error(t, err, "failed blah") } diff --git a/backend/app/store/admin/remote_test.go b/backend/app/store/admin/remote_test.go index d1940f081c..e39fdfc7ee 100644 --- a/backend/app/store/admin/remote_test.go +++ b/backend/app/store/admin/remote_test.go @@ -20,7 +20,7 @@ import ( ) func TestRemote_Key(t *testing.T) { - ts := testServer(t, `{"method":"admin.key","params":null,"id":1}`, + ts := testServer(t, `{"method":"admin.key","id":1}`, `{"result":"12345","id":1}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go index 7ac33254eb..7b206e437d 100644 --- a/backend/app/store/engine/remote_test.go +++ b/backend/app/store/engine/remote_test.go @@ -85,7 +85,7 @@ func TestRemote_FailedStatus(t *testing.T) { req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"} _, err := c.Get(req) - assert.EqualError(t, err, "bad status 400 for store.get") + assert.EqualError(t, err, "bad status 400 Bad Request for store.get") } func TestRemote_Update(t *testing.T) { @@ -160,7 +160,7 @@ func TestRemote_Delete(t *testing.T) { } func TestRemote_Close(t *testing.T) { - ts := testServer(t, `{"method":"store.close","params":null,"id":1}`, `{}`) + ts := testServer(t, `{"method":"store.close","id":1}`, `{}`) defer ts.Close() c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}} err := c.Close() From db597734a2ef6243f01d7687313a41d3b617c748 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 14:08:44 -0500 Subject: [PATCH 17/24] add remote selection to store and admin --- backend/app/cmd/server.go | 62 +++++++++++++++------------------- backend/app/cmd/server_test.go | 57 ------------------------------- 2 files changed, 28 insertions(+), 91 deletions(-) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 8029340034..6976593c65 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -32,6 +32,7 @@ import ( "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/image" + "github.com/umputun/remark/backend/app/store/remote" "github.com/umputun/remark/backend/app/store/service" ) @@ -92,10 +93,7 @@ type StoreGroup struct { Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"` Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"` } `group:"bolt" namespace:"bolt" env-namespace:"BOLT"` - Remote struct { - API string `long:"api" env:"API" description:"remote extension api url"` - TimeOut time.Duration `long:"timeout" env:"TIMEOUT" description:"http timeout"` - } `group:"remote" namespace:"remote" env-namespace:"REMOTE"` + Remote RemoteGroup `group:"remote" namespace:"remote" env-namespace:"REMOTE"` } // ImageGroup defines options group for store pictures @@ -138,11 +136,12 @@ type CacheGroup struct { // AdminGroup defines options group for admin params type AdminGroup struct { - Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"` + Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"remote" default:"shared"` Shared struct { Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","` Email string `long:"email" env:"EMAIL" default:"" description:"admin email"` } `group:"shared" namespace:"shared" env-namespace:"SHARED"` + Remote RemoteGroup `group:"remote" namespace:"remote" env-namespace:"REMOTE"` } // NotifyGroup defines options for notification @@ -174,6 +173,14 @@ type StreamGroup struct { MaxActive int `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"` } +// RemoteGroup defines options for remote modules (plugins) +type RemoteGroup struct { + API string `long:"api" env:"API" description:"remote extension api url"` + TimeOut time.Duration `long:"timeout" env:"TIMEOUT" description:"http timeout"` + AuthUser string `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"` + AuthPassword string `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"` +} + // serverApp holds all active objects type serverApp struct { *ServerCommand @@ -410,13 +417,14 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) { sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)}) } result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...) - // case "mongo": - // mgServer, e := s.makeMongo() - // if e != nil { - // return result, errors.Wrap(e, "failed to create mongo server") - // } - // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") - // result, err = engine.NewMongo(conn, 500, 100*time.Millisecond) + case "remote": + r := &engine.Remote{Client: remote.Client{ + API: s.Store.Remote.API, + Client: http.Client{Timeout: s.Store.Remote.TimeOut}, + AuthUser: s.Store.Remote.AuthUser, + AuthPasswd: s.Store.Remote.AuthPassword, + }} + return r, nil default: return nil, errors.Errorf("unsupported store type %s", s.Store.Type) } @@ -432,13 +440,6 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) { return nil, err } return avatar.NewLocalFS(s.Avatar.FS.Path), nil - // case "mongo": - // mgServer, err := s.makeMongo() - // if err != nil { - // return nil, errors.Wrap(err, "failed to create mongo server") - // } - // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "") - // return avatar.NewGridFS(conn), nil case "bolt": if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil { return nil, err @@ -481,13 +482,14 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) { } } return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil - // case "mongo": - // mgServer, e := s.makeMongo() - // if e != nil { - // return nil, errors.Wrap(e, "failed to create mongo server") - // } - // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin") - // return admin.NewMongoStore(conn, s.SharedSecret), nil + case "remote": + r := &admin.Remote{Client: remote.Client{ + API: s.Admin.Remote.API, + Client: http.Client{Timeout: s.Admin.Remote.TimeOut}, + AuthUser: s.Admin.Remote.AuthUser, + AuthPasswd: s.Admin.Remote.AuthPassword, + }} + return r, nil default: return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type) } @@ -499,14 +501,6 @@ func (s *ServerCommand) makeCache() (cache.LoadingCache, error) { case "mem": return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), cache.MaxKeys(s.Cache.Max.Items)) - // case "mongo": - // mgServer, err := s.makeMongo() - // if err != nil { - // return nil, errors.Wrap(err, "failed to create mongo server") - // } - // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache") - // return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value), - // cache.MaxKeys(s.Cache.Max.Items)) case "none": return &cache.Nop{}, nil } diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 344225be6a..ec108cd23d 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -14,10 +14,8 @@ import ( "time" "github.com/dgrijalva/jwt-go" - "github.com/globalsign/mgo" "github.com/go-pkgz/auth/token" log "github.com/go-pkgz/lgr" - "github.com/go-pkgz/mongo" "github.com/jessevdk/go-flags" "github.com/stretchr/testify/assert" @@ -131,61 +129,6 @@ func TestServerApp_AnonMode(t *testing.T) { app.Wait() } -func TestServerApp_WithMongo(t *testing.T) { - - mongoURL := os.Getenv("MONGO_TEST") - if mongoURL == "" { - mongoURL = "mongodb://localhost:27017/test" - } - if mongoURL == "skip" { - t.Skip("skip mongo app test") - } - - opts := ServerCommand{} - opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) - - // prepare options - p := flags.NewParser(&opts, flags.Default) - _, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo", - "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"}) - require.Nil(t, err) - opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" - opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp" - - // create app - app, err := opts.newServerApp() - require.Nil(t, err) - - defer func() { - s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second) - assert.NoError(t, e) - conn := mongo.NewConnection(s, "test_remark", "") - _ = conn.WithDB(func(dbase *mgo.Database) error { - assert.NoError(t, dbase.DropDatabase()) - return nil - }) - }() - - ctx, cancel := context.WithCancel(context.Background()) - go func() { - time.Sleep(5 * time.Second) - log.Print("[TEST] terminate app") - cancel() - }() - go func() { _ = app.run(ctx) }() - time.Sleep(100 * time.Millisecond) // let server start - - // send ping - resp, err := http.Get("http://localhost:12345/api/v1/ping") - require.Nil(t, err) - defer resp.Body.Close() - assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, "pong", string(body)) - - app.Wait() -} func TestServerApp_WithSSL(t *testing.T) { opts := ServerCommand{} From 41e95f9d844be68ebc2678d72c71d56ce5b3251f Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 15:28:03 -0500 Subject: [PATCH 18/24] fix recreation of bdb in delete --- backend/app/store/engine/bolt.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/app/store/engine/bolt.go b/backend/app/store/engine/bolt.go index f94660c9ac..54abdad0f0 100644 --- a/backend/app/store/engine/bolt.go +++ b/backend/app/store/engine/bolt.go @@ -676,10 +676,6 @@ func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error { // deleteUser removes all comments for given user. Everything will be market as deleted // and user name and userID will be changed to "deleted". Also removes from last and from user buckets. func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error { - bdb, err := b.db(siteID) - if err != nil { - return err - } // get list of all comments outside of transaction loop posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}}) From 4810b017a9968c28125cfb766e9fa6e8f424e382 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 15:29:04 -0500 Subject: [PATCH 19/24] larger limit for remote srv throttler --- backend/app/store/remote/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/store/remote/server.go b/backend/app/store/remote/server.go index ba1b59dcc6..12dd48f10c 100644 --- a/backend/app/store/remote/server.go +++ b/backend/app/store/remote/server.go @@ -61,7 +61,7 @@ func (s *Server) Run(port int) error { router.Use(R.AppInfo(s.AppName, "umputun", s.Version), R.Ping) logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[INFO]")).Handler router.Use(middleware.Timeout(5 * time.Second)) - router.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache) + router.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(1000, nil)), middleware.NoCache) router.Use(s.basicAuth) router.Post(s.API, s.handler) From 8309dd8b1517fcbf1df8a7504d727cd85a6a7474 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 15:29:50 -0500 Subject: [PATCH 20/24] clear mongo leftovers --- .dockerignore | 3 +- .gitignore | 1 + .travis.yml | 6 - Dockerfile | 7 +- backend/app/cmd/avatar.go | 1 - backend/app/cmd/avatar_test.go | 9 +- backend/app/cmd/server.go | 8 +- backend/app/store/engine_old/bolt_accessor.go | 550 ---------------- .../store/engine_old/bolt_accessor_test.go | 406 ------------ backend/app/store/engine_old/bolt_admin.go | 339 ---------- .../app/store/engine_old/bolt_admin_test.go | 246 ------- backend/app/store/engine_old/engine.go | 87 --- backend/app/store/engine_old/engine_mock.go | 408 ------------ backend/app/store/engine_old/engine_test.go | 55 -- backend/app/store/engine_old/mongo.go | 380 ----------- backend/app/store/engine_old/mongo_test.go | 604 ------------------ backend/app/store/remote/client.go | 2 +- backend/go.mod | 3 +- 18 files changed, 11 insertions(+), 3104 deletions(-) delete mode 100644 backend/app/store/engine_old/bolt_accessor.go delete mode 100644 backend/app/store/engine_old/bolt_accessor_test.go delete mode 100644 backend/app/store/engine_old/bolt_admin.go delete mode 100644 backend/app/store/engine_old/bolt_admin_test.go delete mode 100644 backend/app/store/engine_old/engine.go delete mode 100644 backend/app/store/engine_old/engine_mock.go delete mode 100644 backend/app/store/engine_old/engine_test.go delete mode 100644 backend/app/store/engine_old/mongo.go delete mode 100644 backend/app/store/engine_old/mongo_test.go diff --git a/.dockerignore b/.dockerignore index 37da97c59f..c016742cea 100644 --- a/.dockerignore +++ b/.dockerignore @@ -20,4 +20,5 @@ debug.test *.prof *.test remark42 -/backend/var/ \ No newline at end of file +/backend/var/ +compose-private-backend.yml \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3eb5f67e70..f6821858c2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ debug.test remark42 /bin/ /backend/var/ +compose-private-backend.yml diff --git a/.travis.yml b/.travis.yml index 01016812cc..9cb6efc16d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ install: - docker-compose --version script: - - docker run -d --name=mongo mongo:3.6 && sleep 3 - - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo) - - echo "running mongo on $MONGO_TEST" - docker build --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN --build-arg CI=$CI @@ -19,7 +16,4 @@ script: --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG --build-arg TRAVIS_TAG=$TRAVIS_TAG - --build-arg MONGO_TEST=$MONGO_TEST . - - docker rm -f mongo - \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6724f22375..63582ca554 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ ARG DRONE_BRANCH ARG DRONE_PULL_REQUEST ARG SKIP_BACKEND_TEST -ARG MONGO_TEST ADD backend /build/backend ADD .git /build/.git @@ -29,18 +28,14 @@ ENV GOFLAGS="-mod=vendor" # run tests RUN \ - if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ cd app && \ if [ -z "$SKIP_BACKEND_TEST" ] ; then \ - go test -covermode=count -coverprofile=/profile.cov_tmp ./... && \ + go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \ cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \ else echo "skip backend test" ; fi -RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts - # linters RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \ - if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \ golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \ --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \ --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \ diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go index e0075113c2..b867648304 100644 --- a/backend/app/cmd/avatar.go +++ b/backend/app/cmd/avatar.go @@ -16,7 +16,6 @@ import ( type AvatarCommand struct { AvatarSrc AvatarGroup `group:"src" namespace:"src"` AvatarDst AvatarGroup `group:"dst" namespace:"dst"` - // Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"` migrator AvatarMigrator CommonOpts diff --git a/backend/app/cmd/avatar_test.go b/backend/app/cmd/avatar_test.go index 203f246c82..e31a53b86f 100644 --- a/backend/app/cmd/avatar_test.go +++ b/backend/app/cmd/avatar_test.go @@ -13,13 +13,6 @@ import ( func TestAvatar_Execute(t *testing.T) { - mongoURL := os.Getenv("MONGO_TEST") - if mongoURL == "" { - mongoURL = "mongodb://localhost:27017/test" - } - if mongoURL == "skip" { - t.Skip("skip mongo app test") - } defer os.RemoveAll("/tmp/ava-test") // from fs to bolt @@ -37,7 +30,7 @@ func TestAvatar_Execute(t *testing.T) { cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"}) p = flags.NewParser(&cmd, flags.Default) _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt", - "--dst.bolt.file=/tmp/ava-test.db"}) + "--dst.bolt.file=/tmp/ava-test2.db"}) require.NoError(t, err) err = cmd.Execute(nil) assert.Error(t, err, "failed blah") diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 6976593c65..40aeb435ad 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -98,7 +98,7 @@ type StoreGroup struct { // ImageGroup defines options group for store pictures type ImageGroup struct { - Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"` + Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"` FS struct { Path string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"` Staging string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"` @@ -114,7 +114,7 @@ type ImageGroup struct { // AvatarGroup defines options group for avatar params type AvatarGroup struct { - Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"` + Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"` FS struct { Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"` } `group:"fs" namespace:"fs" env-namespace:"FS"` @@ -126,7 +126,7 @@ type AvatarGroup struct { // CacheGroup defines options group for cache params type CacheGroup struct { - Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"` + Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"` Max struct { Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"` Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"` @@ -176,7 +176,7 @@ type StreamGroup struct { // RemoteGroup defines options for remote modules (plugins) type RemoteGroup struct { API string `long:"api" env:"API" description:"remote extension api url"` - TimeOut time.Duration `long:"timeout" env:"TIMEOUT" description:"http timeout"` + TimeOut time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"` AuthUser string `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"` AuthPassword string `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"` } diff --git a/backend/app/store/engine_old/bolt_accessor.go b/backend/app/store/engine_old/bolt_accessor.go deleted file mode 100644 index f48c75da01..0000000000 --- a/backend/app/store/engine_old/bolt_accessor.go +++ /dev/null @@ -1,550 +0,0 @@ -package engine_old - -import ( - "bytes" - "encoding/json" - "fmt" - "strings" - "time" - - bolt "github.com/coreos/bbolt" - log "github.com/go-pkgz/lgr" - "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe. -// there are 5 types of top-level buckets: -// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment -// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID -// value is not full comment but a reference combined from post-url+commentID -// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value -// is a nested bucket named userID with kv as ts:reference -// - blocking info sits in "block" bucket. Key is userID, value - ts -// - counts per post to keep number of comments. Key is post url, value - count -// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts -type BoltDB struct { - dbs map[string]*bolt.DB -} - -const ( - // top level buckets - postsBucketName = "posts" - lastBucketName = "last" - userBucketName = "users" - blocksBucketName = "block" - infoBucketName = "info" - readonlyBucketName = "readonly" - verifiedBucketName = "verified" - - tsNano = "2006-01-02T15:04:05.000000000Z07:00" -) - -// BoltSite defines single site param -type BoltSite struct { - FileName string // full path to boltdb - SiteID string // ID to access given site -} - -// NewBoltDB makes persistent boltdb-based store -func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) { - log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options) - result := BoltDB{dbs: make(map[string]*bolt.DB)} - for _, site := range sites { - db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second} - if err != nil { - return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName) - } - - // make top-level buckets - topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, - infoBucketName, readonlyBucketName, verifiedBucketName} - err = db.Update(func(tx *bolt.Tx) error { - for _, bktName := range topBuckets { - if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { - return errors.Wrapf(e, "failed to create top level bucket %s", bktName) - } - } - return nil - }) - - if err != nil { - return nil, errors.Wrap(err, "failed to create top level bucket)") - } - - result.dbs[site.SiteID] = db - log.Printf("[DEBUG] bolt store created for %s", site.SiteID) - } - return &result, nil -} - -// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket -func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) { - - bdb, err := b.db(comment.Locator.SiteID) - if err != nil { - return "", err - } - - if b.IsReadOnly(comment.Locator) { - return "", errors.Errorf("post %s is read-only", comment.Locator.URL) - } - - err = bdb.Update(func(tx *bolt.Tx) error { - - postBkt, e := b.makePostBucket(tx, comment.Locator.URL) - if e != nil { - return e - } - - // check if key already in store, reject doubles - if postBkt.Get([]byte(comment.ID)) != nil { - return errors.Errorf("key %s already in store", comment.ID) - } - - // serialize comment to json []byte for bolt and save - if e = b.save(postBkt, comment.ID, comment); e != nil { - return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL) - } - - ref := b.makeRef(comment) - - // add reference to comment to "last" bucket - lastBkt := tx.Bucket([]byte(lastBucketName)) - commentTs := []byte(comment.Timestamp.Format(tsNano)) - e = lastBkt.Put(commentTs, ref) - if e != nil { - return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName) - } - - // add reference to commentID to "users" bucket - userBkt, e := b.getUserBucket(tx, comment.User.ID) - if e != nil { - return errors.Wrapf(e, "can't get bucket %s", comment.User.ID) - } - // put into individual user's bucket with ts as a key - if e = userBkt.Put(commentTs, ref); e != nil { - return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID) - } - - // set info with the count for post url - if _, e = b.setInfo(tx, comment); e != nil { - return errors.Wrapf(e, "failed to set info for %s", comment.Locator) - } - return nil - }) - - return comment.ID, err -} - -// Find returns all comments for post and sorts results -func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) { - comments = []store.Comment{} - - bdb, err := b.db(locator.SiteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - - bucket, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - - return bucket.ForEach(func(k, v []byte) error { - comment := store.Comment{} - if e = json.Unmarshal(v, &comment); e != nil { - return errors.Wrap(e, "failed to unmarshal") - } - comments = append(comments, comment) - return nil - }) - }) - - comments = SortComments(comments, sortFld) - return comments, err -} - -// Last returns up to max last comments for given siteID -func (b *BoltDB) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) { - - comments = []store.Comment{} - - if max > lastLimit || max == 0 { - max = lastLimit - } - - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - lastBkt := tx.Bucket([]byte(lastBucketName)) - c := lastBkt.Cursor() - - for k, v := c.Last(); k != nil; k, v = c.Prev() { - - if !since.IsZero() { - // stop if reached "since" ts - tsSince := []byte(since.Format(tsNano)) - if bytes.Compare(k, tsSince) <= 0 { - break - } - } - url, commentID, e := b.parseRef(v) - if e != nil { - return e - } - postBkt, e := b.getPostBucket(tx, url) - if e != nil { - return e - } - - comment := store.Comment{} - if e = b.load(postBkt, commentID, &comment); e != nil { - log.Printf("[WARN] can't load comment for %s from store %s", commentID, url) - continue - } - if comment.Deleted { - continue - } - comments = append(comments, comment) - if len(comments) >= max { - break - } - } - return nil - }) - - return comments, err -} - -// Count returns number of comments for locator -func (b *BoltDB) Count(locator store.Locator) (count int, err error) { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return 0, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - var e error - count, e = b.count(tx, locator.URL, 0) - return e - }) - - return count, err -} - -// List returns list of all commented posts with counters -// uses count bucket to get number of comments -func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) { - - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - postsBkt := tx.Bucket([]byte(postsBucketName)) - - c := postsBkt.Cursor() - n := 0 - for k, _ := c.Last(); k != nil; k, _ = c.Prev() { - n++ - if skip > 0 && n <= skip { - continue - } - postURL := string(k) - infoBkt := tx.Bucket([]byte(infoBucketName)) - info := store.PostInfo{} - if e := b.load(infoBkt, postURL, &info); e != nil { - return errors.Wrapf(e, "can't load info for %s", postURL) - } - list = append(list, info) - if limit > 0 && len(list) >= limit { - break - } - } - return nil - }) - - return list, err -} - -// Info returns time range and count for locator -func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) { - bdb, err := b.db(locator.SiteID) - if err != nil { - return store.PostInfo{}, err - } - - info := store.PostInfo{} - err = bdb.View(func(tx *bolt.Tx) error { - infoBkt := tx.Bucket([]byte(infoBucketName)) - if e := b.load(infoBkt, locator.URL, &info); e != nil { - return errors.Wrapf(e, "can't load info for %s", locator.URL) - } - return nil - }) - - // set read-only from age and manual bucket - info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now()) - if b.IsReadOnly(locator) { - info.ReadOnly = true - } - return info, err -} - -// User extracts all comments for given site and given userID -// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref -func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) { - - comments = []store.Comment{} - commentRefs := []string{} - - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - if limit == 0 || limit > userLimit { - limit = userLimit - } - - // get list of references to comments - err = bdb.View(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(userBucketName)) - userIDBkt := usersBkt.Bucket([]byte(userID)) - if userIDBkt == nil { - return errors.Errorf("no comments for user %s in store", userID) - } - - c := userIDBkt.Cursor() - skipComments := 0 - for k, v := c.Last(); k != nil; k, v = c.Prev() { - if len(commentRefs) >= limit { - break - } - if skip > 0 && skipComments < skip { - skipComments++ - continue - } - commentRefs = append(commentRefs, string(v)) - } - return nil - }) - - if err != nil { - return comments, err - } - - // retrieve comments for refs - for _, v := range commentRefs { - url, commentID, errParse := b.parseRef([]byte(v)) - if errParse != nil { - return comments, errors.Wrapf(errParse, "can't parse reference %s", v) - } - if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil { - comments = append(comments, c) - } - } - - return comments, err -} - -// UserCount returns number of comments for user -func (b *BoltDB) UserCount(siteID, userID string) (int, error) { - bdb, err := b.db(siteID) - if err != nil { - return 0, err - } - count := 0 - err = bdb.View(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(userBucketName)) - userIDBkt := usersBkt.Bucket([]byte(userID)) - if userIDBkt == nil { - return errors.Errorf("no comments for user %s in store", userID) - } - stats := userIDBkt.Stats() - count = stats.KeyN - return nil - }) - return count, err -} - -// Get returns comment for locator.URL and commentID string -func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return comment, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - bucket, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - return b.load(bucket, commentID, &comment) - }) - return comment, err -} - -// Put updates comment for locator.URL with mutable part of comment -func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error { - - if curComment, err := b.Get(locator, comment.ID); err == nil { - // preserve immutable fields - comment.ParentID = curComment.ParentID - comment.Locator = curComment.Locator - comment.Timestamp = curComment.Timestamp - comment.User = curComment.User - } - - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - return b.save(bucket, comment.ID, comment) - }) -} - -// Close boltdb store -func (b *BoltDB) Close() error { - errs := new(multierror.Error) - for site, db := range b.dbs { - err := errors.Wrapf(db.Close(), "can't close site %s", site) - errs = multierror.Append(errs, err) - } - return errs.ErrorOrNil() -} - -// getPostBucket return bucket with all comments for postURL -func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) { - postsBkt := tx.Bucket([]byte(postsBucketName)) - if postsBkt == nil { - return nil, errors.Errorf("no bucket %s", postsBucketName) - } - res := postsBkt.Bucket([]byte(postURL)) - if res == nil { - return nil, errors.Errorf("no bucket %s in store", postURL) - } - return res, nil -} - -// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post. -func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) { - postsBkt := tx.Bucket([]byte(postsBucketName)) - if postsBkt == nil { - return nil, errors.Errorf("no bucket %s", postsBucketName) - } - res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL)) - if err != nil { - return nil, errors.Wrapf(err, "no bucket %s in store", postURL) - } - return res, nil -} - -func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) { - usersBkt := tx.Bucket([]byte(userBucketName)) - userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID - if e != nil { - return nil, errors.Wrapf(e, "can't get bucket %s", userID) - } - return userIDBkt, nil -} - -// save marshaled value to key for bucket. Should run in update tx -func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) { - if value == nil { - return errors.Errorf("can't save nil value for %s", key) - } - jdata, jerr := json.Marshal(value) - if jerr != nil { - return errors.Wrap(jerr, "can't marshal comment") - } - if err = bkt.Put([]byte(key), jdata); err != nil { - return errors.Wrapf(err, "failed to save key %s", key) - } - return nil -} - -// load and unmarshal json value by key from bucket. Should run in view tx -func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error { - value := bkt.Get([]byte(key)) - if value == nil { - return errors.Errorf("no value for %s", key) - } - - if err := json.Unmarshal(value, &res); err != nil { - return errors.Wrap(err, "failed to unmarshal") - } - return nil -} - -// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor -// it uses separate counts bucket because boltdb Stat call is very slow -func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) { - - infoBkt := tx.Bucket([]byte(infoBucketName)) - - info := store.PostInfo{} - if err := b.load(infoBkt, postURL, &info); err != nil { - info = store.PostInfo{} - } - if val == 0 { // get current count, don't update - return info.Count, nil - } - info.Count += val - - return info.Count, b.save(infoBkt, postURL, &info) -} - -func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) { - infoBkt := tx.Bucket([]byte(infoBucketName)) - info := store.PostInfo{} - if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil { - info = store.PostInfo{ - Count: 0, - URL: comment.Locator.URL, - FirstTS: comment.Timestamp, - LastTS: comment.Timestamp, - } - } - info.Count++ - info.LastTS = comment.Timestamp - return info, b.save(infoBkt, comment.Locator.URL, &info) -} - -func (b *BoltDB) db(siteID string) (*bolt.DB, error) { - if res, ok := b.dbs[siteID]; ok { - return res, nil - } - return nil, errors.Errorf("site %q not found", siteID) -} - -// makeRef creates reference combining url and comment id -func (b *BoltDB) makeRef(comment store.Comment) []byte { - return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID)) -} - -// parseRef gets parts of reference -func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) { - elems := strings.Split(string(val), "!!") - if len(elems) != 2 { - return "", "", errors.Errorf("invalid reference value %s", string(val)) - } - return elems[0], elems[1], nil -} diff --git a/backend/app/store/engine_old/bolt_accessor_test.go b/backend/app/store/engine_old/bolt_accessor_test.go deleted file mode 100644 index fa0f638163..0000000000 --- a/backend/app/store/engine_old/bolt_accessor_test.go +++ /dev/null @@ -1,406 +0,0 @@ -package engine_old - -import ( - "fmt" - "os" - "testing" - "time" - - bolt "github.com/coreos/bbolt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" -) - -var testDb = "test-remark.db" - -func TestBoltDB_CreateAndFind(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, `some text, link`, res[0].Text) - assert.Equal(t, "user1", res[0].User.ID) - t.Log(res[0].ID) - - _, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) - assert.NotNil(t, err) - assert.Equal(t, "key id-1 already in store", err.Error()) - - _, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time") - assert.EqualError(t, err, `site "radio-t-bad" not found`) - - assert.NoError(t, b.Close()) -} - -func TestBoltDB_CreateReadOnly(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - comment := store.Comment{ - ID: "id-ro", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - err := b.SetReadOnly(comment.Locator, true) - require.Nil(t, err) - - _, err = b.Create(comment) - assert.NotNil(t, err) - assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error()) - - err = b.SetReadOnly(comment.Locator, false) - require.Nil(t, err) - _, err = b.Create(comment) - assert.Nil(t, err) -} - -func TestBoltDB_Get(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) - assert.Nil(t, err) - assert.Equal(t, "some text2", comment.Text) - - comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") - assert.NotNil(t, err) - - _, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_Put(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment := res[0] - comment.Text = "abc 123" - comment.Score = 100 - err = b.Put(loc, comment) - assert.Nil(t, err) - - comment, err = b.Get(loc, res[0].ID) - assert.Nil(t, err) - assert.Equal(t, "abc 123", comment.Text) - assert.Equal(t, res[0].ID, comment.ID) - assert.Equal(t, 100, comment.Score) - - err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment) - assert.EqualError(t, err, `site "bad" not found`) - - err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment) - assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`) -} - -func TestBoltDB_Last(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.Last("radio-t", 0, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = b.Last("radio-t", 1, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - _, err = b.Last("bad", 0, time.Time{}) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_LastSince(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local) - res, err := b.Last("radio-t", 0, ts) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - ts = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local) - res, err = b.Last("radio-t", 0, ts) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - ts = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local) - res, err = b.Last("radio-t", 0, ts) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestBoltDB_Count(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 2, c) - - c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c) - - _, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_List(t *testing.T) { - b, teardown := prep(t) // two comments for https://radio-t.com - defer teardown() - - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := b.Create(comment) - assert.Nil(t, err) - - ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) } - - res, err := b.List("radio-t", 0, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, - res) - - res, err = b.List("radio-t", -1, -1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - res, err = b.List("radio-t", 1, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res) - - res, err = b.List("radio-t", 1, 1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - _, err = b.List("bad", 1, 1) - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltDB_Info(t *testing.T) { - b, teardown := prep(t) // two comments for https://radio-t.com - defer teardown() - - ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) } - - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := b.Create(comment) - assert.Nil(t, err) - - r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r) - - r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) - - r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r) - - _, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0) - require.NotNil(t, err) - - _, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0) - require.NotNil(t, err) - - err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true) - require.Nil(t, err) - r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) - -} - -func TestBoltDB_GetForUser(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - res, err := b.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = b.User("radio-t", "user1", 1, 0) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = b.User("radio-t", "user1", 1, 1) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, `some text, link`, res[0].Text, "second comment") - - _, err = b.User("bad", "user1", 1, 0) - assert.EqualError(t, err, `site "bad" not found`) - - _, err = b.User("radio-t", "userZ", 1, 0) - assert.EqualError(t, err, `no comments for user userZ in store`) -} - -func TestBoltDB_GetForUserPagination(t *testing.T) { - os.Remove(testDb) - b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"}) - require.Nil(t, err) - - defer func() { - require.NoError(t, b.Close()) - os.Remove(testDb) - }() - - c := store.Comment{ - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - - // write 200 comments - for i := 0; i < 200; i++ { - c.ID = fmt.Sprintf("id-%d", i) - c.Text = fmt.Sprintf("text #%d", i) - c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local) - _, err = b.Create(c) - require.Nil(t, err) - } - - // get all comments - res, err := b.User("radio-t", "user1", 0, 0) - assert.Nil(t, err) - assert.Equal(t, 200, len(res)) - assert.Equal(t, "id-199", res[0].ID) - - // seek 0, 5 comments - res, err = b.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-199", res[0].ID) - assert.Equal(t, "id-195", res[4].ID) - - // seek 10, 3 comments - res, err = b.User("radio-t", "user1", 3, 10) - assert.Nil(t, err) - assert.Equal(t, 3, len(res)) - assert.Equal(t, "id-189", res[0].ID) - assert.Equal(t, "id-187", res[2].ID) - - // seek 195, ask 10 comments - res, err = b.User("radio-t", "user1", 10, 195) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-4", res[0].ID) - assert.Equal(t, "id-0", res[4].ID) - - // seek 255, ask 10 comments - res, err = b.User("radio-t", "user1", 10, 255) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestBoltDB_GetForUserCounter(t *testing.T) { - var b, teardown = prep(t) - defer teardown() - - count, err := b.UserCount("radio-t", "user1") - assert.Nil(t, err) - assert.Equal(t, 2, count) - - _, err = b.UserCount("bad", "user1") - assert.EqualError(t, err, `site "bad" not found`) - - _, err = b.UserCount("radio-t", "userZ") - assert.EqualError(t, err, `no comments for user userZ in store`) -} - -func TestBoltDB_Ref(t *testing.T) { - b := BoltDB{} - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - res := b.makeRef(comment) - assert.Equal(t, "https://radio-t.com/2!!12345", string(res)) - - url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345")) - assert.Nil(t, err) - assert.Equal(t, "https://radio-t.com/2", url) - assert.Equal(t, "12345", id) - - _, _, err = b.parseRef([]byte("https://radio-t.com/2")) - assert.NotNil(t, err) -} - -func TestBoltDB_New(t *testing.T) { - _, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"}) - assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory") -} - -// makes new boltdb, put two records -func prep(t *testing.T) (b *BoltDB, teardown func()) { - os.Remove(testDb) - - boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"}) - assert.Nil(t, err) - b = boltStore - - comment := store.Comment{ - ID: "id-1", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err = b.Create(comment) - assert.Nil(t, err) - - comment = store.Comment{ - ID: "id-2", - Text: "some text2", - Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err = b.Create(comment) - assert.Nil(t, err) - - teardown = func() { - require.NoError(t, b.Close()) - os.Remove(testDb) - } - return b, teardown -} diff --git a/backend/app/store/engine_old/bolt_admin.go b/backend/app/store/engine_old/bolt_admin.go deleted file mode 100644 index 914346f2ec..0000000000 --- a/backend/app/store/engine_old/bolt_admin.go +++ /dev/null @@ -1,339 +0,0 @@ -package engine_old - -import ( - "encoding/json" - "time" - - bolt "github.com/coreos/bbolt" - log "github.com/go-pkgz/lgr" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// Delete removes comment, by locator from the store. -// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. -// From last bucket removed for real. -func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - - postBkt, e := b.getPostBucket(tx, locator.URL) - if e != nil { - return e - } - - comment := store.Comment{} - if err = b.load(postBkt, commentID, &comment); err != nil { - return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL) - } - // set deleted status and clear fields - comment.SetDeleted(mode) - - if err = b.save(postBkt, commentID, comment); err != nil { - return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL) - } - - // delete from "last" bucket - lastBkt := tx.Bucket([]byte(lastBucketName)) - if err = lastBkt.Delete([]byte(commentID)); err != nil { - return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName) - } - - // decrement comments count for post url - if _, e = b.count(tx, comment.Locator.URL, -1); e != nil { - return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator) - } - - return nil - }) -} - -// DeleteAll removes all top-level buckets for given siteID -func (b *BoltDB) DeleteAll(siteID string) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - // delete all buckets except blocked users - toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName} - - // delete top-level buckets - err = bdb.Update(func(tx *bolt.Tx) error { - for _, bktName := range toDelete { - - if e := tx.DeleteBucket([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to delete top level bucket %s", bktName) - } - if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil { - return errors.Wrapf(err, "failed to create top level bucket %s", bktName) - } - } - return nil - }) - - return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID) -} - -// DeleteUser removes all comments for given user. Everything will be market as deleted -// and user name and userID will be changed to "deleted". Also removes from last and from user buckets. -func (b *BoltDB) DeleteUser(siteID string, userID string) error { - bdb, err := b.db(siteID) - if err != nil { - return err - } - - // get list of all comments outside of transaction loop - posts, err := b.List(siteID, 0, 0) - if err != nil { - return err - } - - type commentInfo struct { - locator store.Locator - commentID string - } - - // get list of commentID for all user's comment - comments := []commentInfo{} - for _, postInfo := range posts { - err = bdb.View(func(tx *bolt.Tx) error { - postsBkt := tx.Bucket([]byte(postsBucketName)) - postBkt := postsBkt.Bucket([]byte(postInfo.URL)) - err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error { - comment := store.Comment{} - if err = json.Unmarshal(commentVal, &comment); err != nil { - return errors.Wrap(err, "failed to unmarshal") - } - if comment.User.ID == userID { - comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID}) - } - return nil - }) - return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL) - }) - if err != nil { - return err - } - } - - log.Printf("[DEBUG] comments for removal=%d", len(comments)) - - // delete collected comments - for _, ci := range comments { - if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil { - return errors.Wrapf(err, "failed to delete comment %+v", ci) - } - } - - // delete user bucket - err = bdb.Update(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(userBucketName)) - if usersBkt != nil { - if e := usersBkt.DeleteBucket([]byte(userID)); e != nil { - return errors.Wrapf(err, "failed to delete user bucket for %s", userID) - } - } - return nil - }) - - if err != nil { - return errors.Wrap(err, "can't delete user meta") - } - - if len(comments) == 0 { - return errors.Errorf("unknown user %s", userID) - } - - return err -} - -// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent -// block uses blocksBucketName with key=userID and val=TTL+now -func (b *BoltDB) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - - bdb, err := b.db(siteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - switch status { - case true: - val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year - if ttl > 0 { - val = time.Now().Add(ttl).Format(tsNano) - } - if e := bucket.Put([]byte(userID), []byte(val)); e != nil { - return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName) - } - case false: - if e := bucket.Delete([]byte(userID)); e != nil { - return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName) - } - } - return nil - }) -} - -// IsBlocked checks if user blocked -func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) { - - bdb, err := b.db(siteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - val := bucket.Get([]byte(userID)) - if val == nil { - blocked = false - return nil - } - - until, e := time.Parse(tsNano, string(val)) - if e != nil { - blocked = false - return nil - } - blocked = time.Now().Before(until) - return nil - }) - return blocked -} - -// Blocked get lists of blocked users for given site -// bucket uses userID: -func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) { - users = []store.BlockedUser{} - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - - err = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(blocksBucketName)) - return bucket.ForEach(func(k []byte, v []byte) error { - ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local) - if errParse != nil { - return errors.Wrap(errParse, "can't parse block ts") - } - if time.Now().Before(ts) { - // get user name from comment user section - userName := "" - userComments, errUser := b.User(siteID, string(k), 1, 0) - if errUser == nil && len(userComments) > 0 { - userName = userComments[0].User.Name - } - users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts}) - } - return nil - }) - }) - - return users, err -} - -// SetReadOnly makes post read-only or reset the ro flag -func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error { - bdb, err := b.db(locator.SiteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(readonlyBucketName)) - switch status { - case true: - if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil { - return errors.Wrapf(e, "failed to set ro for %s", locator.URL) - } - case false: - if e := bucket.Delete([]byte(locator.URL)); e != nil { - return errors.Wrapf(e, "failed to clean ro for %s", locator.URL) - } - } - return nil - }) -} - -// IsReadOnly checks if post in RO mode -func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) { - - bdb, err := b.db(locator.SiteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(readonlyBucketName)) - ro = bucket.Get([]byte(locator.URL)) != nil - return nil - }) - return ro -} - -// SetVerified makes user verified or reset the flag -func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error { - bdb, err := b.db(siteID) - if err != nil { - return err - } - - return bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(verifiedBucketName)) - switch status { - case true: - if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil { - return errors.Wrapf(e, "failed to set verified status for %s", userID) - } - case false: - if e := bucket.Delete([]byte(userID)); e != nil { - return errors.Wrapf(e, "failed to clean verified status for %s", userID) - } - } - return nil - }) -} - -// IsVerified checks if user verified -func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) { - - bdb, err := b.db(siteID) - if err != nil { - return false - } - - _ = bdb.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(verifiedBucketName)) - verified = bucket.Get([]byte(userID)) != nil - return nil - }) - return verified -} - -// Verified returns list of verified userIDs -func (b *BoltDB) Verified(siteID string) (ids []string, err error) { - bdb, err := b.db(siteID) - if err != nil { - return nil, err - } - err = bdb.View(func(tx *bolt.Tx) error { - usersBkt := tx.Bucket([]byte(verifiedBucketName)) - _ = usersBkt.ForEach(func(k, _ []byte) error { - ids = append(ids, string(k)) - return nil - }) - return nil - }) - return ids, err -} diff --git a/backend/app/store/engine_old/bolt_admin_test.go b/backend/app/store/engine_old/bolt_admin_test.go deleted file mode 100644 index a1b4c3c1f8..0000000000 --- a/backend/app/store/engine_old/bolt_admin_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package engine_old - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" -) - -func TestBoltAdmin_Delete(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - count, err := b.Count(loc) - require.NoError(t, err) - assert.Equal(t, 2, count, "count=2 initially") - - err = b.Delete(loc, res[0].ID, store.SoftDelete) - assert.Nil(t, err) - - res, err = b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - - assert.Equal(t, "some text2", res[1].Text) - assert.False(t, res[1].Deleted) - - comments, err := b.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(comments), "1 in last, 1 removed") - - count, err = b.Count(loc) - require.NoError(t, err) - assert.Equal(t, 1, count) - - err = b.Delete(loc, "123456", store.SoftDelete) - assert.NotNil(t, err) - - loc.SiteID = "bad" - err = b.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `site "bad" not found`) - - loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"} - err = b.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`) -} - -func TestBoltAdmin_DeleteHard(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = b.Delete(loc, res[0].ID, store.HardDelete) - assert.Nil(t, err) - - res, err = b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) -} - -func TestBoltAdmin_DeleteAll(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = b.DeleteAll("radio-t") - assert.Nil(t, err) - - comments, err := b.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") - - c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") - - err = b.DeleteAll("bad") - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltAdmin_DeleteUser(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - err := b.DeleteUser("radio-t", "user1") - require.NoError(t, err) - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := b.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "2 comments with deleted info") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User) - - c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") - - _, err = b.User("radio-t", "user1", 5, 0) - assert.EqualError(t, err, "no comments for user user1 in store") - - comments, err := b.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") - - err = b.DeleteUser("radio-t-bad", "user1") - assert.EqualError(t, err, `site "radio-t-bad" not found`) -} - -func TestBoltAdmin_BlockUser(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked") - - assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0)) - assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked") - - assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked") - - assert.NoError(t, b.SetBlock("radio-t", "user1", false, 0)) - assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked") - - assert.EqualError(t, b.SetBlock("bad", "user1", true, 0), `site "bad" not found`) - assert.NoError(t, b.SetBlock("radio-t", "userX", false, 0)) - - assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site") -} - -func TestBoltAdmin_BlockUserWithTTL(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked") - assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond)) - assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked") - time.Sleep(50 * time.Millisecond) - assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically") -} - -func TestBoltAdmin_BlockList(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0)) - assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond)) - assert.NoError(t, b.SetBlock("radio-t", "user3", false, 0)) - - ids, err := b.Blocked("radio-t") - assert.NoError(t, err) - - assert.Equal(t, 2, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - assert.Equal(t, "user2", ids[1].ID) - t.Logf("%+v", ids) - - time.Sleep(50 * time.Millisecond) - ids, err = b.Blocked("radio-t") - assert.NoError(t, err) - assert.Equal(t, 1, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - - _, err = b.Blocked("bad") - assert.EqualError(t, err, `site "bad" not found`) -} - -func TestBoltAdmin_ReadOnly(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro") - - assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true)) - assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro") - - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable") - - assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false)) - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable") - - assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`) - assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false)) - - assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site") -} - -func TestBoltAdmin_Verified(t *testing.T) { - - b, teardown := prep(t) - defer teardown() - - assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified") - - assert.NoError(t, b.SetVerified("radio-t", "u1", true)) - assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified") - - assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified") - assert.NoError(t, b.SetVerified("radio-t", "u1", false)) - assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore") - - assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`) - assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false)) - - assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site") - - assert.NoError(t, b.SetVerified("radio-t", "u1", true)) - assert.NoError(t, b.SetVerified("radio-t", "u2", true)) - assert.NoError(t, b.SetVerified("radio-t", "u3", false)) - - ids, err := b.Verified("radio-t") - assert.NoError(t, err) - assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids") - - _, err = b.Verified("radio-t-bad") - assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site") -} diff --git a/backend/app/store/engine_old/engine.go b/backend/app/store/engine_old/engine.go deleted file mode 100644 index 2a8dad42da..0000000000 --- a/backend/app/store/engine_old/engine.go +++ /dev/null @@ -1,87 +0,0 @@ -// Package engine defines interfaces each supported storage should implement. -// Includes default implementation with boltdb -package engine_old - -import ( - "sort" - "strings" - "time" - - "github.com/umputun/remark/backend/app/store" -) - -// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate -//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go" - -// Interface defines methods provided by low-level storage engine -type Interface interface { - Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id - Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id - Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only - Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator - Last(siteID string, limit int, since time.Time) ([]store.Comment, error) // last comments for given site, sorted by time - User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time - UserCount(siteID, userID string) (int, error) // comments count by user - Count(locator store.Locator) (int, error) // number of comments for the post - List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts - Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info - Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id - DeleteAll(siteID string) error // delete all data from site - DeleteUser(siteID string, userID string) error // remove all comments from user - SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent) - IsBlocked(siteID string, userID string) bool // check if user blocked - Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users - SetReadOnly(locator store.Locator, status bool) error // set/reset read-only flag - IsReadOnly(locator store.Locator) bool // check if post read-only - SetVerified(siteID string, userID string, status bool) error // set/reset verified flag - IsVerified(siteID string, userID string) bool // check verified status - Verified(siteID string) ([]string, error) // list of verified user ids - Close() error // close/stop engine -} - -const ( - // limits - lastLimit = 1000 - userLimit = 500 -) - -// SortComments is for engines can't sort data internally -func SortComments(comments []store.Comment, sortFld string) []store.Comment { - sort.Slice(comments, func(i, j int) bool { - switch sortFld { - case "+time", "-time", "time", "+active", "-active", "active": - if strings.HasPrefix(sortFld, "-") { - return comments[i].Timestamp.After(comments[j].Timestamp) - } - return comments[i].Timestamp.Before(comments[j].Timestamp) - - case "+score", "-score", "score": - if strings.HasPrefix(sortFld, "-") { - if comments[i].Score == comments[j].Score { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Score > comments[j].Score - } - if comments[i].Score == comments[j].Score { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Score < comments[j].Score - - case "+controversy", "-controversy", "controversy": - if strings.HasPrefix(sortFld, "-") { - if comments[i].Controversy == comments[j].Controversy { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Controversy > comments[j].Controversy - } - if comments[i].Controversy == comments[j].Controversy { - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - return comments[i].Controversy < comments[j].Controversy - - default: - return comments[i].Timestamp.Before(comments[j].Timestamp) - } - }) - return comments -} diff --git a/backend/app/store/engine_old/engine_mock.go b/backend/app/store/engine_old/engine_mock.go deleted file mode 100644 index 879402b15d..0000000000 --- a/backend/app/store/engine_old/engine_mock.go +++ /dev/null @@ -1,408 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. -package engine_old - -import mock "github.com/stretchr/testify/mock" -import store "github.com/umputun/remark/backend/app/store" -import time "time" - -// MockInterface is an autogenerated mock type for the Interface type -type MockInterface struct { - mock.Mock -} - -// Blocked provides a mock function with given fields: siteID -func (_m *MockInterface) Blocked(siteID string) ([]store.BlockedUser, error) { - ret := _m.Called(siteID) - - var r0 []store.BlockedUser - if rf, ok := ret.Get(0).(func(string) []store.BlockedUser); ok { - r0 = rf(siteID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.BlockedUser) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(siteID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Close provides a mock function with given fields: -func (_m *MockInterface) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Count provides a mock function with given fields: locator -func (_m *MockInterface) Count(locator store.Locator) (int, error) { - ret := _m.Called(locator) - - var r0 int - if rf, ok := ret.Get(0).(func(store.Locator) int); ok { - r0 = rf(locator) - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator) error); ok { - r1 = rf(locator) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Create provides a mock function with given fields: comment -func (_m *MockInterface) Create(comment store.Comment) (string, error) { - ret := _m.Called(comment) - - var r0 string - if rf, ok := ret.Get(0).(func(store.Comment) string); ok { - r0 = rf(comment) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Comment) error); ok { - r1 = rf(comment) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Delete provides a mock function with given fields: locator, commentID, mode -func (_m *MockInterface) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - ret := _m.Called(locator, commentID, mode) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, string, store.DeleteMode) error); ok { - r0 = rf(locator, commentID, mode) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteAll provides a mock function with given fields: siteID -func (_m *MockInterface) DeleteAll(siteID string) error { - ret := _m.Called(siteID) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(siteID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteUser provides a mock function with given fields: siteID, userID -func (_m *MockInterface) DeleteUser(siteID string, userID string) error { - ret := _m.Called(siteID, userID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Find provides a mock function with given fields: locator, sort -func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comment, error) { - ret := _m.Called(locator, sort) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(store.Locator, string) []store.Comment); ok { - r0 = rf(locator, sort) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { - r1 = rf(locator, sort) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Get provides a mock function with given fields: locator, commentID -func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) { - ret := _m.Called(locator, commentID) - - var r0 store.Comment - if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok { - r0 = rf(locator, commentID) - } else { - r0 = ret.Get(0).(store.Comment) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok { - r1 = rf(locator, commentID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Info provides a mock function with given fields: locator, readonlyAge -func (_m *MockInterface) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) { - ret := _m.Called(locator, readonlyAge) - - var r0 store.PostInfo - if rf, ok := ret.Get(0).(func(store.Locator, int) store.PostInfo); ok { - r0 = rf(locator, readonlyAge) - } else { - r0 = ret.Get(0).(store.PostInfo) - } - - var r1 error - if rf, ok := ret.Get(1).(func(store.Locator, int) error); ok { - r1 = rf(locator, readonlyAge) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// IsBlocked provides a mock function with given fields: siteID, userID -func (_m *MockInterface) IsBlocked(siteID string, userID string) bool { - ret := _m.Called(siteID, userID) - - var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsReadOnly provides a mock function with given fields: locator -func (_m *MockInterface) IsReadOnly(locator store.Locator) bool { - ret := _m.Called(locator) - - var r0 bool - if rf, ok := ret.Get(0).(func(store.Locator) bool); ok { - r0 = rf(locator) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsVerified provides a mock function with given fields: siteID, userID -func (_m *MockInterface) IsVerified(siteID string, userID string) bool { - ret := _m.Called(siteID, userID) - - var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Last provides a mock function with given fields: siteID, limit, since -func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]store.Comment, error) { - ret := _m.Called(siteID, limit, since) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(string, int, time.Time) []store.Comment); ok { - r0 = rf(siteID, limit, since) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int, time.Time) error); ok { - r1 = rf(siteID, limit, since) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// List provides a mock function with given fields: siteID, limit, skip -func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostInfo, error) { - ret := _m.Called(siteID, limit, skip) - - var r0 []store.PostInfo - if rf, ok := ret.Get(0).(func(string, int, int) []store.PostInfo); ok { - r0 = rf(siteID, limit, skip) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.PostInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int, int) error); ok { - r1 = rf(siteID, limit, skip) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Put provides a mock function with given fields: locator, comment -func (_m *MockInterface) Put(locator store.Locator, comment store.Comment) error { - ret := _m.Called(locator, comment) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok { - r0 = rf(locator, comment) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetBlock provides a mock function with given fields: siteID, userID, status, ttl -func (_m *MockInterface) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - ret := _m.Called(siteID, userID, status, ttl) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, bool, time.Duration) error); ok { - r0 = rf(siteID, userID, status, ttl) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetReadOnly provides a mock function with given fields: locator, status -func (_m *MockInterface) SetReadOnly(locator store.Locator, status bool) error { - ret := _m.Called(locator, status) - - var r0 error - if rf, ok := ret.Get(0).(func(store.Locator, bool) error); ok { - r0 = rf(locator, status) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SetVerified provides a mock function with given fields: siteID, userID, status -func (_m *MockInterface) SetVerified(siteID string, userID string, status bool) error { - ret := _m.Called(siteID, userID, status) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, bool) error); ok { - r0 = rf(siteID, userID, status) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// User provides a mock function with given fields: siteID, userID, limit, skip -func (_m *MockInterface) User(siteID string, userID string, limit int, skip int) ([]store.Comment, error) { - ret := _m.Called(siteID, userID, limit, skip) - - var r0 []store.Comment - if rf, ok := ret.Get(0).(func(string, string, int, int) []store.Comment); ok { - r0 = rf(siteID, userID, limit, skip) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.Comment) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string, int, int) error); ok { - r1 = rf(siteID, userID, limit, skip) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// UserCount provides a mock function with given fields: siteID, userID -func (_m *MockInterface) UserCount(siteID string, userID string) (int, error) { - ret := _m.Called(siteID, userID) - - var r0 int - if rf, ok := ret.Get(0).(func(string, string) int); ok { - r0 = rf(siteID, userID) - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(siteID, userID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Verified provides a mock function with given fields: siteID -func (_m *MockInterface) Verified(siteID string) ([]string, error) { - ret := _m.Called(siteID) - - var r0 []string - if rf, ok := ret.Get(0).(func(string) []string); ok { - r0 = rf(siteID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(siteID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/backend/app/store/engine_old/engine_test.go b/backend/app/store/engine_old/engine_test.go deleted file mode 100644 index dc04233b74..0000000000 --- a/backend/app/store/engine_old/engine_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package engine_old - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/umputun/remark/backend/app/store" -) - -func TestEngine_sortComments(t *testing.T) { - cc := []store.Comment{ - {ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)}, - {ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)}, - {ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)}, - {ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)}, - } - - SortComments(cc, "+time") - assert.Equal(t, "1", cc[0].ID) - assert.Equal(t, "2", cc[1].ID) - assert.Equal(t, "3", cc[2].ID) - assert.Equal(t, "4", cc[3].ID) - - SortComments(cc, "-time") - assert.Equal(t, "4", cc[0].ID) - assert.Equal(t, "3", cc[1].ID) - assert.Equal(t, "2", cc[2].ID) - assert.Equal(t, "1", cc[3].ID) - - SortComments(cc, "score") - assert.Equal(t, "2", cc[0].ID) - assert.Equal(t, "1", cc[1].ID) - assert.Equal(t, "3", cc[2].ID) - assert.Equal(t, "4", cc[3].ID) - - SortComments(cc, "-score") - assert.Equal(t, "3", cc[0].ID) - assert.Equal(t, "4", cc[1].ID) - assert.Equal(t, "1", cc[2].ID) - assert.Equal(t, "2", cc[3].ID) - - SortComments(cc, "controversy") - assert.Equal(t, "1", cc[0].ID) - assert.Equal(t, "4", cc[1].ID) - assert.Equal(t, "2", cc[2].ID) - assert.Equal(t, "3", cc[3].ID) - - SortComments(cc, "-controversy") - assert.Equal(t, "3", cc[0].ID) - assert.Equal(t, "2", cc[1].ID) - assert.Equal(t, "1", cc[2].ID) - assert.Equal(t, "4", cc[3].ID) -} diff --git a/backend/app/store/engine_old/mongo.go b/backend/app/store/engine_old/mongo.go deleted file mode 100644 index 6cd3f3117d..0000000000 --- a/backend/app/store/engine_old/mongo.go +++ /dev/null @@ -1,380 +0,0 @@ -package engine_old - -import ( - "time" - - "github.com/globalsign/mgo" - "github.com/globalsign/mgo/bson" - "github.com/go-pkgz/mongo" - "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" - - "github.com/umputun/remark/backend/app/store" -) - -// Mongo implements engine interface -type Mongo struct { - conn *mongo.Connection - postWriter mongo.BufferedWriter -} - -const ( - mongoPosts = "posts" - mongoMetaPosts = "meta_posts" - mongoMetaUsers = "meta_users" -) - -type metaPost struct { - ID string `bson:"_id"` // url - SiteID string `bson:"site"` - ReadOnly bool `bson:"read_only"` -} - -type metaUser struct { - ID string `bson:"_id"` // user_id - SiteID string `bson:"site"` - Verified bool `bson:"verified"` - Blocked bool `bson:"blocked"` - BlockedUntil time.Time `bson:"blocked_until"` -} - -// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off. -// flushDuration triggers automatic flush (write from buffer), 0 disables it and will flush as buffer size reached. -// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time. -func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) { - writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration) - result := Mongo{conn: conn, postWriter: writer} - err := result.prepare() - return &result, errors.Wrap(err, "failed to prepare mongo") -} - -// Create new comment, write can be buffered and delayed. -func (m *Mongo) Create(comment store.Comment) (commentID string, err error) { - // err = m.postWriter.Write(comment) - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - return coll.Insert(&comment) - }) - return comment.ID, err -} - -// Find returns all comments for post and sorts results -func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) { - comments = []store.Comment{} - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL} - return coll.Find(query).Sort(sortFld).All(&comments) - }) - return comments, err -} - -// Get returns comment for locator.URL and commentID string -func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) { - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL} - return coll.Find(query).One(&comment) - }) - return comment, err -} - -// Put updates comment for locator.URL with mutable part of comment -func (m *Mongo) Put(locator store.Locator, comment store.Comment) error { - return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL}, - bson.M{"$set": bson.M{ - "text": comment.Text, - "orig": comment.Orig, - "score": comment.Score, - "votes": comment.Votes, - "pin": comment.Pin, - "deleted": comment.Deleted, - }}) - }) -} - -// Last returns up to max last comments for given siteID -func (m *Mongo) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) { - comments = []store.Comment{} - if max > lastLimit || max == 0 { - max = lastLimit - } - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": siteID, "delete": false} - if !since.IsZero() { - query["time"] = bson.M{"$gt": since} - } - return coll.Find(query).Sort("-time").Limit(max).All(&comments) - }) - return comments, err -} - -// Count returns number of comments for locator -func (m *Mongo) Count(locator store.Locator) (count int, err error) { - - e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false} - count, err = coll.Find(query).Count() - return err - }) - return count, e -} - -// List returns list of all commented posts with counters -func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) { - list = []store.PostInfo{} - - if limit <= 0 { - limit = 1000 - } - if skip < 0 { - skip = 0 - } - - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - pipeline := coll.Pipe([]bson.M{ - {"$match": bson.M{"locator.site": siteID}}, - {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}}, - {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1}, - "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}}, - {"$skip": skip}, - {"$limit": limit}, - }) - return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed") - }) - return list, errors.Wrap(err, "can't get list") -} - -// Info returns time range and count for locator -func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) { - list := []store.PostInfo{} - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - pipeline := coll.Pipe([]bson.M{ - {"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}}, - {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}}, - {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1}, - "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}}, - }) - return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed") - }) - if err != nil { - return info, err - } - if len(list) == 0 { - return info, errors.Errorf("can't load info for %s", locator.URL) - } - info = list[0] - // set read-only from age and manual bucket - info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now()) - if m.IsReadOnly(locator) { - info.ReadOnly = true - } - return info, nil -} - -// User extracts all comments for given site and given userID -func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) { - comments = []store.Comment{} - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - query := bson.M{"locator.site": siteID, "user.id": userID} - return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments) - }) - return comments, errors.Wrapf(err, "can't get comments for user %s", userID) -} - -// UserCount returns number of comments for user -func (m *Mongo) UserCount(siteID, userID string) (count int, err error) { - err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - var e error - count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count() - return e - }) - return count, errors.Wrapf(err, "can't get comments count for user %s", userID) -} - -// SetReadOnly makes post read-only or reset the ro flag -func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) { - return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error { - _, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}}) - return e - }) -} - -// IsReadOnly checks if post in RO -func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) { - meta := metaPost{} - err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta) - }) - return err == nil && meta.ReadOnly -} - -// SetVerified makes user verified or reset the flag -func (m *Mongo) SetVerified(siteID string, userID string, status bool) error { - return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}}) - return e - }) -} - -// IsVerified checks if user verified -func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) { - meta := metaUser{} - err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta) - }) - return err == nil && meta.Verified -} - -// Verified returns list of verified user IDs -func (m *Mongo) Verified(siteID string) (ids []string, err error) { - metas := []metaUser{} - err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas) - }) - if err != nil { - return nil, err - } - for _, meta := range metas { - ids = append(ids, meta.ID) - } - return ids, nil -} - -// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent -// block uses blocksBucketName with key=userID and val=TTL+now -func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error { - - until := time.Time{} - if status { - until = time.Now().AddDate(100, 0, 0) // permanent is 50year - if ttl > 0 { - until = time.Now().Add(ttl) - } - } - return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, - bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}}) - return errors.Wrapf(e, "failed to set block for %s", userID) - }) -} - -// IsBlocked checks if user blocked -func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) { - meta := metaUser{} - err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta) - }) - return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now()) -} - -// Blocked get lists of blocked users for given site -func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) { - users = []store.BlockedUser{} - metas := []metaUser{} - err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - return coll.Find(bson.M{"site": siteID, - "blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas) - }) - if err != nil { - return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID) - } - - for _, mu := range metas { - blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil} - if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 { - blockedUser.Name = ucc[0].User.Name - } - users = append(users, blockedUser) - } - return users, nil -} - -// Delete removes comment, by locator from the store. -// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies. -func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error { - comment := store.Comment{} - err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment) - if e != nil { - return e - } - comment.SetDeleted(mode) - return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment) - }) - return errors.Wrapf(err, "can't delete %s", commentID) -} - -// DeleteAll removes all info about siteID -func (m *Mongo) DeleteAll(siteID string) error { - err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - _, e := coll.RemoveAll(bson.M{"locator.site": siteID}) - return e - }) - return errors.Wrapf(err, "can't delete site %s", siteID) -} - -// DeleteUser removes all comments for given user. Everything will be market as deleted -// and user name and userID will be changed to "deleted". -func (m *Mongo) DeleteUser(siteID string, userID string) error { - comments := []store.Comment{} - return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments) - if e != nil { - return e - } - for _, c := range comments { - if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil { - return e - } - } - return nil - }) -} - -// Close boltdb store -func (m *Mongo) Close() error { - if m.postWriter != nil { - return m.postWriter.Close() - } - return nil -} - -// prepare collections with all indexes -func (m *Mongo) prepare() error { - errs := new(multierror.Error) - e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error { - errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time")) - errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time")) - errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time")) - errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score")) - return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts) - }) - if e != nil { - return e - } - - e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error { - errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site")) - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only")) - return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts) - }) - if e != nil { - return e - } - - return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error { - errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site")) - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked")) - errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified")) - return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers) - }) -} - -func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query { - if limit <= 0 { - limit = 1000 - } - if skip < 0 { - skip = 0 - } - return q.Skip(skip).Limit(limit) -} diff --git a/backend/app/store/engine_old/mongo_test.go b/backend/app/store/engine_old/mongo_test.go deleted file mode 100644 index cb16051c88..0000000000 --- a/backend/app/store/engine_old/mongo_test.go +++ /dev/null @@ -1,604 +0,0 @@ -package engine_old - -import ( - "fmt" - "math/rand" - "testing" - "time" - - "github.com/go-pkgz/mongo" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/umputun/remark/backend/app/store" -) - -func TestMongo_CreateAndFind(t *testing.T) { - var m Interface - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - require.Equal(t, 2, len(res)) - assert.Equal(t, `some text, link`, res[0].Text) - assert.Equal(t, "user1", res[0].User.ID) - t.Log(res[0].ID) - - _, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) - assert.NotNil(t, err, "reject dup") - - id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}}) - assert.Nil(t, err) - assert.Equal(t, "id-3", id) - res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time") - assert.Nil(t, err) - require.Equal(t, 1, len(res)) - - assert.NoError(t, m.Close()) -} - -func TestMongo_Get(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID) - assert.Nil(t, err) - assert.Equal(t, "some text2", comment.Text) - - comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567") - assert.NotNil(t, err, "not found") -} - -func TestMongo_Put(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - - comment := res[0] - comment.Text = "abc 123" - comment.Score = 100 - err = m.Put(loc, comment) - assert.Nil(t, err) - - comment, err = m.Get(loc, res[0].ID) - assert.Nil(t, err) - assert.Equal(t, "abc 123", comment.Text) - assert.Equal(t, res[0].ID, comment.ID) - assert.Equal(t, 100, comment.Score) - - err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment) - assert.EqualError(t, err, `not found`) - - err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment) - assert.EqualError(t, err, `not found`) -} - -func TestMongo_Last(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.Last("radio-t", 0, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) - - res, err = m.Last("radio-t", 1, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(res)) - assert.Equal(t, "some text2", res[0].Text) -} - -func TestMongo_Count(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 2, c) - - c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c) -} - -func TestMongo_List(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := m.Create(comment) - assert.Nil(t, err) - - ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) } - - res, err := m.List("radio-t", 0, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, - res) - - res, err = m.List("radio-t", -1, -1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}, - {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - res, err = m.List("radio-t", 1, 0) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res) - - res, err = m.List("radio-t", 1, 1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res) - - res, err = m.List("bad", 1, 1) - assert.Nil(t, err) - assert.Equal(t, []store.PostInfo{}, res) -} - -func TestMongo_Info(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) } - - // add one more for https://radio-t.com/2 - comment := store.Comment{ - ID: "12345", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - _, err := m.Create(comment) - assert.Nil(t, err) - - r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r) - - r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) - - r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r) - - _, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0) - require.NotNil(t, err) - - _, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0) - require.NotNil(t, err) - - err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true) - require.Nil(t, err) - r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0) - require.Nil(t, err) - assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r) -} - -func TestMongo_ReadOnly(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro") - - assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true)) - assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro") - - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable") - - assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false)) - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable") - - assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"") - assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false)) - - assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site") -} - -func TestMongo_Verified(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified") - - assert.NoError(t, m.SetVerified("radio-t", "u1", true)) - assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified") - - assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified") - assert.NoError(t, m.SetVerified("radio-t", "u1", false)) - assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore") - - assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`) - assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false)) - - assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site") - - assert.NoError(t, m.SetVerified("radio-t", "u1", true)) - assert.NoError(t, m.SetVerified("radio-t", "u2", true)) - assert.NoError(t, m.SetVerified("radio-t", "u3", false)) - - ids, err := m.Verified("radio-t") - assert.NoError(t, err) - assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids") - - ids, err = m.Verified("radio-t-bad") - assert.NoError(t, err) - assert.Equal(t, 0, len(ids)) -} - -func TestMongo_GetForUser(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - res, err := m.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = m.User("radio-t", "user1", 1, 0) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, "some text2", res[0].Text, "sorted by -time") - - res, err = m.User("radio-t", "user1", 1, 1) - assert.Nil(t, err) - assert.Equal(t, 1, len(res), "allow 1 comment") - assert.Equal(t, `some text, link`, res[0].Text, "second comment") - - res, err = m.User("bad", "user1", 1, 0) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestMongo_GetForUserPagination(t *testing.T) { - m, skip := prepMongo(t, false) - if skip { - return - } - c := store.Comment{ - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - - // write 200 comments - for i := 0; i < 200; i++ { - c.ID = fmt.Sprintf("id-%d", i) - c.Text = fmt.Sprintf("text #%d", i) - c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local) - _, err := m.Create(c) - require.Nil(t, err, c.ID) - } - - // get all comments - res, err := m.User("radio-t", "user1", 0, 0) - assert.Nil(t, err) - assert.Equal(t, 200, len(res)) - assert.Equal(t, "id-199", res[0].ID) - - // seek 0, 5 comments - res, err = m.User("radio-t", "user1", 5, 0) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-199", res[0].ID) - assert.Equal(t, "id-195", res[4].ID) - - // seek 10, 3 comments - res, err = m.User("radio-t", "user1", 3, 10) - assert.Nil(t, err) - assert.Equal(t, 3, len(res)) - assert.Equal(t, "id-189", res[0].ID) - assert.Equal(t, "id-187", res[2].ID) - - // seek 195, ask 10 comments - res, err = m.User("radio-t", "user1", 10, 195) - assert.Nil(t, err) - assert.Equal(t, 5, len(res)) - assert.Equal(t, "id-4", res[0].ID) - assert.Equal(t, "id-0", res[4].ID) - - // seek 255, ask 10 comments - res, err = m.User("radio-t", "user1", 10, 255) - assert.Nil(t, err) - assert.Equal(t, 0, len(res)) -} - -func TestMongo_BlockUser(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked") - - assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0)) - assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked") - - assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked") - - assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0)) - assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked") - - assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`) - assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0)) - - assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site") -} - -func TestMongo_BlockUserWithTTL(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked") - assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond)) - assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked") - time.Sleep(500 * time.Millisecond) - assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically") -} - -func TestMongo_GetForUserCounter(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - count, err := m.UserCount("radio-t", "user1") - assert.Nil(t, err) - assert.Equal(t, 2, count) - - count, err = m.UserCount("bad", "user1") - assert.Nil(t, err) - assert.Equal(t, 0, count) -} - -func TestMongo_BlockList(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0)) - assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond)) - assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0)) - - ids, err := m.Blocked("radio-t") - assert.NoError(t, err) - - assert.Equal(t, 2, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - assert.Equal(t, "user2", ids[1].ID) - t.Logf("%+v", ids) - - time.Sleep(500 * time.Millisecond) - ids, err = m.Blocked("radio-t") - assert.NoError(t, err) - assert.Equal(t, 1, len(ids)) - assert.Equal(t, "user1", ids[0].ID) - - ids, err = m.Blocked("bad") - assert.NoError(t, err) - assert.Equal(t, 0, len(ids)) -} - -func TestMongo_Delete(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = m.Delete(loc, res[0].ID, store.SoftDelete) - assert.Nil(t, err) - - res, err = m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - - assert.Equal(t, "some text2", res[1].Text) - assert.False(t, res[1].Deleted) - - comments, err := m.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(comments), "1 in last, 1 removed") - - err = m.Delete(loc, "123456", store.SoftDelete) - assert.NotNil(t, err) - - loc.SiteID = "bad" - err = m.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `can't delete id-1: not found`) - - loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"} - err = m.Delete(loc, res[0].ID, store.SoftDelete) - assert.EqualError(t, err, `can't delete id-1: not found`) -} - -func TestMongo_DeleteHard(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = m.Delete(loc, res[0].ID, store.HardDelete) - assert.Nil(t, err) - - res, err = m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, "", res[0].Text) - assert.True(t, res[0].Deleted, "marked deleted") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) -} - -func TestMongo_DeleteAll(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "initially 2 comments") - - err = m.DeleteAll("radio-t") - assert.Nil(t, err) - - comments, err := m.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") - - c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") -} - -func TestMongo_DeleteUser(t *testing.T) { - m, skip := prepMongo(t, true) // adds two comments - if skip { - return - } - err := m.DeleteUser("radio-t", "user1") - require.NoError(t, err) - - loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"} - res, err := m.Find(loc, "time") - assert.Nil(t, err) - assert.Equal(t, 2, len(res), "2 comments with deleted info") - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User) - assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User) - - c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}) - assert.Nil(t, err) - assert.Equal(t, 0, c, "0 count") - - cc, err := m.User("radio-t", "user1", 5, 0) - assert.Nil(t, err, "no comments for user user1 in store") - assert.Equal(t, 0, len(cc), "no comments for user user1 in store") - - comments, err := m.Last("radio-t", 10, time.Time{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(comments), "nothing left") -} - -func TestMongo_Parallel(t *testing.T) { - var m Interface - var skip bool - m, skip = prepMongoBuffered(t) // buffered engine, no comments - if skip { - return - } - go func() { - for i := 0; i < 100; i++ { - _, err := m.Create(store.Comment{ - ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}) - require.Nil(t, err) - time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond) - } - }() - - for { - time.Sleep(10 * time.Millisecond) - res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time") - assert.Nil(t, err) - if len(res) == 100 { - break - } - } -} - -func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) { - conn, err := mongo.MakeTestConnection(t) - if err != nil { - return nil, true - } - mongo.RemoveTestCollection(t, conn) - - m, err := NewMongo(conn, 1, 0*time.Microsecond) - require.Nil(t, err) - - mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers) - comment := store.Comment{ - ID: "id-1", - Text: `some text, link`, - Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - if writeRecs { - _, err = m.Create(comment) - assert.Nil(t, err) - } - - comment = store.Comment{ - ID: "id-2", - Text: "some text2", - Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), - Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, - User: store.User{ID: "user1", Name: "user name"}, - } - if writeRecs { - _, err = m.Create(comment) - assert.Nil(t, err) - } - - return m, false -} - -func prepMongoBuffered(t *testing.T) (*Mongo, bool) { - conn, err := mongo.MakeTestConnection(t) - if err != nil { - return nil, true - } - mongo.RemoveTestCollection(t, conn) - - m, err := NewMongo(conn, 10, 10*time.Millisecond) - mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers) - - require.Nil(t, err) - return m, false -} diff --git a/backend/app/store/remote/client.go b/backend/app/store/remote/client.go index e200e03fa0..dd033926a0 100644 --- a/backend/app/store/remote/client.go +++ b/backend/app/store/remote/client.go @@ -27,7 +27,7 @@ func (r *Client) Call(method string, args ...interface{}) (*Response, error) { var err error switch { - case args == nil || len(args) == 0: + case len(args) == 0: b, err = json.Marshal(Request{Method: method, ID: atomic.AddUint64(&r.id, 1)}) if err != nil { return nil, errors.Wrapf(err, "marshaling failed for %s", method) diff --git a/backend/go.mod b/backend/go.mod index d00e249f7f..8fccd3d50e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -11,14 +11,13 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/didip/tollbooth v4.0.0+incompatible github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d - github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/go-chi/chi v4.0.2+incompatible github.com/go-chi/cors v1.0.0 github.com/go-chi/render v1.0.1 github.com/go-pkgz/auth v0.5.2 github.com/go-pkgz/lcw v0.3.1 github.com/go-pkgz/lgr v0.6.2 - github.com/go-pkgz/mongo v1.1.2 + github.com/go-pkgz/mongo v1.1.2 // indirect github.com/go-pkgz/repeater v1.1.2 github.com/go-pkgz/rest v1.4.1 github.com/go-pkgz/syncs v1.1.1 From dc5e27aa7fde66d93ac85b97351707eece2e6a1f Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 15:34:11 -0500 Subject: [PATCH 21/24] clean drone build --- .drone.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.drone.yml b/.drone.yml index 57d7f3f65b..9bf95034a6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,10 +7,6 @@ pipeline: build: image: golang:1.12-alpine commands: - - sleep 5 - - nslookup mongo - - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo - - cat backend/.mongo - cd backend/app - go build -v ./... @@ -94,8 +90,3 @@ pipeline: secrets: [ email_username, email_password ] when: status: [ changed, failure ] - -services: - mongo: - image: mongo:3.6 - command: [ --smallfiles ] From 4727e37b8bb04ee1ef00f493d5cc143766ceed56 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 19:35:21 -0500 Subject: [PATCH 22/24] lint: test warnings --- backend/app/cmd/server.go | 2 +- backend/app/{store => }/remote/client.go | 0 backend/app/{store => }/remote/client_test.go | 3 +++ backend/app/{store => }/remote/remote.go | 0 backend/app/{store => }/remote/server.go | 0 backend/app/{store => }/remote/server_test.go | 14 +++++++------- backend/app/store/admin/remote.go | 2 +- backend/app/store/admin/remote_test.go | 2 +- backend/app/store/engine/bolt_test.go | 12 ++++++------ backend/app/store/engine/remote.go | 2 +- backend/app/store/engine/remote_test.go | 2 +- backend/app/store/service/service_test.go | 6 +++--- 12 files changed, 24 insertions(+), 21 deletions(-) rename backend/app/{store => }/remote/client.go (100%) rename backend/app/{store => }/remote/client_test.go (97%) rename backend/app/{store => }/remote/remote.go (100%) rename backend/app/{store => }/remote/server.go (100%) rename backend/app/{store => }/remote/server_test.go (96%) diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 40aeb435ad..ced3ac781e 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -26,13 +26,13 @@ import ( "github.com/umputun/remark/backend/app/migrator" "github.com/umputun/remark/backend/app/notify" + "github.com/umputun/remark/backend/app/remote" "github.com/umputun/remark/backend/app/rest/api" "github.com/umputun/remark/backend/app/rest/proxy" "github.com/umputun/remark/backend/app/store" "github.com/umputun/remark/backend/app/store/admin" "github.com/umputun/remark/backend/app/store/engine" "github.com/umputun/remark/backend/app/store/image" - "github.com/umputun/remark/backend/app/store/remote" "github.com/umputun/remark/backend/app/store/service" ) diff --git a/backend/app/store/remote/client.go b/backend/app/remote/client.go similarity index 100% rename from backend/app/store/remote/client.go rename to backend/app/remote/client.go diff --git a/backend/app/store/remote/client_test.go b/backend/app/remote/client_test.go similarity index 97% rename from backend/app/store/remote/client_test.go rename to backend/app/remote/client_test.go index 5979903166..717b26ca03 100644 --- a/backend/app/store/remote/client_test.go +++ b/backend/app/remote/client_test.go @@ -21,6 +21,7 @@ func TestClient_Call(t *testing.T) { assert.NoError(t, err) res := "" err = json.Unmarshal(*resp.Result, &res) + assert.NoError(t, err) assert.Equal(t, "12345", res) t.Logf("%v %T", res, res) } @@ -43,6 +44,7 @@ func TestClient_CallWithObject(t *testing.T) { assert.NoError(t, err) res := "" err = json.Unmarshal(*resp.Result, &res) + assert.NoError(t, err) assert.Equal(t, "12345", res) t.Logf("%v %T", res, res) } @@ -55,6 +57,7 @@ func TestClient_CallWithNoParams(t *testing.T) { assert.NoError(t, err) res := "" err = json.Unmarshal(*resp.Result, &res) + assert.NoError(t, err) assert.Equal(t, "12345", res) t.Logf("%v %T", res, res) } diff --git a/backend/app/store/remote/remote.go b/backend/app/remote/remote.go similarity index 100% rename from backend/app/store/remote/remote.go rename to backend/app/remote/remote.go diff --git a/backend/app/store/remote/server.go b/backend/app/remote/server.go similarity index 100% rename from backend/app/store/remote/server.go rename to backend/app/remote/server.go diff --git a/backend/app/store/remote/server_test.go b/backend/app/remote/server_test.go similarity index 96% rename from backend/app/store/remote/server_test.go rename to backend/app/remote/server_test.go index 49342c99ca..a2e87e8714 100644 --- a/backend/app/store/remote/server_test.go +++ b/backend/app/remote/server_test.go @@ -39,8 +39,8 @@ func TestServerPrimitiveTypes(t *testing.T) { return r }) - go func() { s.Run(9091) }() - defer func() { assert.NoError(t, s.Shutdown()) }() + go func() { _ = s.Run(9091) }() + defer func() { s.Shutdown() }() time.Sleep(10 * time.Millisecond) // check with direct http call @@ -93,7 +93,7 @@ func TestServerWithObject(t *testing.T) { return r }) - go func() { s.Run(9091) }() + go func() { _ = s.Run(9091) }() defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) @@ -145,7 +145,7 @@ func TestServerWithAuth(t *testing.T) { return r }) - go func() { s.Run(9091) }() + go func() { _ = s.Run(9091) }() time.Sleep(10 * time.Millisecond) defer func() { assert.NoError(t, s.Shutdown()) }() @@ -183,7 +183,7 @@ func TestServerErrReturn(t *testing.T) { return r }) - go func() { s.Run(9091) }() + go func() { _ = s.Run(9091) }() defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) @@ -202,7 +202,7 @@ func TestServerGroup(t *testing.T) { return Response{} }, }) - go func() { s.Run(9091) }() + go func() { _ = s.Run(9091) }() defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) @@ -221,7 +221,7 @@ func TestServerAddLate(t *testing.T) { s.Add("fn1", func(id uint64, params json.RawMessage) Response { return Response{} }) - go func() { s.Run(9091) }() + go func() { _ = s.Run(9091) }() defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) diff --git a/backend/app/store/admin/remote.go b/backend/app/store/admin/remote.go index 1ba1805c16..6cef97c6f1 100644 --- a/backend/app/store/admin/remote.go +++ b/backend/app/store/admin/remote.go @@ -9,7 +9,7 @@ package admin import ( "encoding/json" - "github.com/umputun/remark/backend/app/store/remote" + "github.com/umputun/remark/backend/app/remote" ) // Remote implements remote engine and delegates all Calls to remote http server diff --git a/backend/app/store/admin/remote_test.go b/backend/app/store/admin/remote_test.go index e39fdfc7ee..4b67db0d03 100644 --- a/backend/app/store/admin/remote_test.go +++ b/backend/app/store/admin/remote_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/umputun/remark/backend/app/store/remote" + "github.com/umputun/remark/backend/app/remote" ) func TestRemote_Key(t *testing.T) { diff --git a/backend/app/store/engine/bolt_test.go b/backend/app/store/engine/bolt_test.go index 588fb2e9c4..5a284ef3e7 100644 --- a/backend/app/store/engine/bolt_test.go +++ b/backend/app/store/engine/bolt_test.go @@ -19,8 +19,7 @@ func TestBoltDB_CreateAndFind(t *testing.T) { var b, teardown = prep(t) defer teardown() - var bb Interface - bb = b + var bb Interface = b _ = bb req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"} @@ -117,8 +116,8 @@ func TestBoltDB_Update(t *testing.T) { err = b.Update(comment) assert.EqualError(t, err, `site "bad" not found`) - comment.Locator.SiteID="radio-t" - comment.Locator.URL="https://radio-t.com-bad" + comment.Locator.SiteID = "radio-t" + comment.Locator.URL = "https://radio-t.com-bad" err = b.Update(comment) assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`) } @@ -451,6 +450,7 @@ func TestBolt_FlagReadOnlyPost(t *testing.T) { req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagTrue} val, err = b.Flag(req) assert.NoError(t, err) + assert.Equal(t, true, val) req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly} val, err = b.Flag(req) assert.NoError(t, err) @@ -816,7 +816,7 @@ func prep(t *testing.T) (b *BoltDB, teardown func()) { func getReq(locator store.Locator, commentID string) GetRequest { return GetRequest{ - Locator: locator, + Locator: locator, CommentID: commentID, } -} \ No newline at end of file +} diff --git a/backend/app/store/engine/remote.go b/backend/app/store/engine/remote.go index fe3f1b700e..f9b8b16b38 100644 --- a/backend/app/store/engine/remote.go +++ b/backend/app/store/engine/remote.go @@ -3,8 +3,8 @@ package engine import ( "encoding/json" + "github.com/umputun/remark/backend/app/remote" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/remote" ) // Remote implements remote engine and delegates all Calls to remote http server diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go index 7b206e437d..459e370522 100644 --- a/backend/app/store/engine/remote_test.go +++ b/backend/app/store/engine/remote_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/umputun/remark/backend/app/remote" "github.com/umputun/remark/backend/app/store" - "github.com/umputun/remark/backend/app/store/remote" ) func TestRemote_Create(t *testing.T) { diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index d9eb5012c7..8d6a4c0e58 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -957,7 +957,7 @@ func TestService_UserCount(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 1, c) - c, err = b.UserCount("radio-t", "userBad") + _, err = b.UserCount("radio-t", "userBad") assert.EqualError(t, err, "no comments for user userBad in store for radio-t site") } @@ -1082,7 +1082,7 @@ func teardown(_ *testing.T) { func getReq(locator store.Locator, commentID string) engine.GetRequest { return engine.GetRequest{ - Locator: locator, + Locator: locator, CommentID: commentID, } -} \ No newline at end of file +} From be86a729f190989c76049adbfee34f7bf343d3f3 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 19:42:12 -0500 Subject: [PATCH 23/24] lint: more test warns --- backend/app/remote/server_test.go | 4 +++- backend/app/store/service/service_test.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/app/remote/server_test.go b/backend/app/remote/server_test.go index a2e87e8714..39665bf779 100644 --- a/backend/app/remote/server_test.go +++ b/backend/app/remote/server_test.go @@ -40,7 +40,7 @@ func TestServerPrimitiveTypes(t *testing.T) { }) go func() { _ = s.Run(9091) }() - defer func() { s.Shutdown() }() + defer func() { assert.NoError(t, s.Shutdown()) }() time.Sleep(10 * time.Millisecond) // check with direct http call @@ -63,6 +63,7 @@ func TestServerPrimitiveTypes(t *testing.T) { res := respData{} err = json.Unmarshal(*r.Result, &res) + assert.NoError(t, err) assert.Equal(t, respData{Res1: "res blah", Res2: true}, res) assert.Equal(t, uint64(1), r.ID) } @@ -104,6 +105,7 @@ func TestServerWithObject(t *testing.T) { res := respData{} err = json.Unmarshal(*r.Result, &res) + assert.NoError(t, err) assert.Equal(t, respData{Res1: "res blah", Res2: true}, res) } diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go index 8d6a4c0e58..e19b659df5 100644 --- a/backend/app/store/service/service_test.go +++ b/backend/app/store/service/service_test.go @@ -764,6 +764,8 @@ func TestService_UserReplies(t *testing.T) { cc, u, err = b.UserReplies("radio-t", "uxxx", 10, time.Hour) assert.NoError(t, err) assert.Equal(t, 0, len(cc), "0 replies to uxxx") + assert.Equal(t, "", u) + } func TestService_Find(t *testing.T) { From ea5fbca3fcecee3f1b95720e782cc3346ea0fdef Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 25 Jun 2019 19:55:55 -0500 Subject: [PATCH 24/24] add test for app with remote plugins --- backend/app/cmd/server_test.go | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index ec108cd23d..d8dc29ba83 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -186,6 +186,45 @@ func TestServerApp_WithSSL(t *testing.T) { app.Wait() } +func TestServerApp_WithRemote(t *testing.T) { + + opts := ServerCommand{} + opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"}) + + // prepare options + p := flags.NewParser(&opts, flags.Default) + _, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", + "--store.type=remote", "--store.remote.api=http://127.0.0.1", + "--port=12345", "--admin.type=remote", "--admin.remote.api=http://127.0.0.1", "--avatar.fs.path=/tmp"}) + require.Nil(t, err) + opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid" + opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp" + + // create app + app, err := opts.newServerApp() + require.Nil(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Second) + log.Print("[TEST] terminate app") + cancel() + }() + go func() { _ = app.run(ctx) }() + time.Sleep(100 * time.Millisecond) // let server start + + // send ping + resp, err := http.Get("http://localhost:12345/api/v1/ping") + require.Nil(t, err) + defer resp.Body.Close() + assert.Equal(t, 200, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, "pong", string(body)) + + app.Wait() +} + func TestServerApp_Failed(t *testing.T) { opts := ServerCommand{} opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})