diff --git a/.dockerignore b/.dockerignore
index 37da97c59f..c016742cea 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -20,4 +20,5 @@ debug.test
*.prof
*.test
remark42
-/backend/var/
\ No newline at end of file
+/backend/var/
+compose-private-backend.yml
\ No newline at end of file
diff --git a/.drone.yml b/.drone.yml
index 57d7f3f65b..9bf95034a6 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -7,10 +7,6 @@ pipeline:
build:
image: golang:1.12-alpine
commands:
- - sleep 5
- - nslookup mongo
- - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
- - cat backend/.mongo
- cd backend/app
- go build -v ./...
@@ -94,8 +90,3 @@ pipeline:
secrets: [ email_username, email_password ]
when:
status: [ changed, failure ]
-
-services:
- mongo:
- image: mongo:3.6
- command: [ --smallfiles ]
diff --git a/.gitignore b/.gitignore
index 3eb5f67e70..f6821858c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ debug.test
remark42
/bin/
/backend/var/
+compose-private-backend.yml
diff --git a/.travis.yml b/.travis.yml
index 01016812cc..9cb6efc16d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,9 +3,6 @@ install:
- docker-compose --version
script:
- - docker run -d --name=mongo mongo:3.6 && sleep 3
- - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
- - echo "running mongo on $MONGO_TEST"
- docker build
--build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
--build-arg CI=$CI
@@ -19,7 +16,4 @@ script:
--build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
--build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
--build-arg TRAVIS_TAG=$TRAVIS_TAG
- --build-arg MONGO_TEST=$MONGO_TEST
.
- - docker rm -f mongo
-
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 6724f22375..63582ca554 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,7 +19,6 @@ ARG DRONE_BRANCH
ARG DRONE_PULL_REQUEST
ARG SKIP_BACKEND_TEST
-ARG MONGO_TEST
ADD backend /build/backend
ADD .git /build/.git
@@ -29,18 +28,14 @@ ENV GOFLAGS="-mod=vendor"
# run tests
RUN \
- if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
cd app && \
if [ -z "$SKIP_BACKEND_TEST" ] ; then \
- go test -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+ go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \
cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
else echo "skip backend test" ; fi
-RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-
# linters
RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
- if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
--enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
--enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go
index 5c7e9a92ba..b867648304 100644
--- a/backend/app/cmd/avatar.go
+++ b/backend/app/cmd/avatar.go
@@ -2,14 +2,12 @@ package cmd
import (
"path"
- "time"
bolt "github.com/coreos/bbolt"
log "github.com/go-pkgz/lgr"
"github.com/pkg/errors"
"github.com/go-pkgz/auth/avatar"
- "github.com/go-pkgz/mongo"
)
// AvatarCommand set of flags and command for avatar migration
@@ -18,7 +16,6 @@ import (
type AvatarCommand struct {
AvatarSrc AvatarGroup `group:"src" namespace:"src"`
AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
- Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
migrator AvatarMigrator
CommonOpts
@@ -78,13 +75,6 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
return nil, err
}
return avatar.NewLocalFS(gr.FS.Path), nil
- case "mongo":
- mgServer, err := ac.makeMongo()
- if err != nil {
- return nil, errors.Wrap(err, "failed to create mongo server")
- }
- conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
- return avatar.NewGridFS(conn), nil
case "bolt":
if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
return nil, err
@@ -93,10 +83,3 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
}
return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
}
-
-func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
- if ac.Mongo.URL == "" {
- return nil, errors.New("no mongo URL provided")
- }
- return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
-}
diff --git a/backend/app/cmd/avatar_test.go b/backend/app/cmd/avatar_test.go
index e0df7193be..e31a53b86f 100644
--- a/backend/app/cmd/avatar_test.go
+++ b/backend/app/cmd/avatar_test.go
@@ -13,32 +13,15 @@ import (
func TestAvatar_Execute(t *testing.T) {
- mongoURL := os.Getenv("MONGO_TEST")
- if mongoURL == "" {
- mongoURL = "mongodb://localhost:27017/test"
- }
- if mongoURL == "skip" {
- t.Skip("skip mongo app test")
- }
defer os.RemoveAll("/tmp/ava-test")
- // from fs to mongo
+ // from fs to bolt
cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p := flags.NewParser(&cmd, flags.Default)
- _, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
- "--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
- require.Nil(t, err)
- err = cmd.Execute(nil)
- assert.NoError(t, err)
-
- // from fs to bolt
- cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
- cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
- p = flags.NewParser(&cmd, flags.Default)
- _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+ _, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
"--dst.bolt.file=/tmp/ava-test.db"})
- require.Nil(t, err)
+ require.NoError(t, err)
err = cmd.Execute(nil)
assert.NoError(t, err)
@@ -46,9 +29,9 @@ func TestAvatar_Execute(t *testing.T) {
cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
p = flags.NewParser(&cmd, flags.Default)
- _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
- "--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
- require.Nil(t, err)
+ _, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+ "--dst.bolt.file=/tmp/ava-test2.db"})
+ require.NoError(t, err)
err = cmd.Execute(nil)
assert.Error(t, err, "failed blah")
}
diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index 0213e388c6..ced3ac781e 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -22,11 +22,11 @@ import (
"github.com/go-pkgz/auth/avatar"
"github.com/go-pkgz/auth/provider"
"github.com/go-pkgz/auth/token"
- "github.com/go-pkgz/mongo"
"github.com/go-pkgz/rest/cache"
"github.com/umputun/remark/backend/app/migrator"
"github.com/umputun/remark/backend/app/notify"
+ "github.com/umputun/remark/backend/app/remote"
"github.com/umputun/remark/backend/app/rest/api"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
@@ -41,7 +41,6 @@ type ServerCommand struct {
Store StoreGroup `group:"store" namespace:"store" env-namespace:"STORE"`
Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
Cache CacheGroup `group:"cache" namespace:"cache" env-namespace:"CACHE"`
- Mongo MongoGroup `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
Admin AdminGroup `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
Image ImageGroup `group:"image" namespace:"image" env-namespace:"IMAGE"`
@@ -89,16 +88,17 @@ type AuthGroup struct {
// StoreGroup defines options group for store params
type StoreGroup struct {
- Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
+ Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"remote" default:"bolt"`
Bolt struct {
Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+ Remote RemoteGroup `group:"remote" namespace:"remote" env-namespace:"REMOTE"`
}
// ImageGroup defines options group for store pictures
type ImageGroup struct {
- Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+ Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
Staging string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -114,7 +114,7 @@ type ImageGroup struct {
// AvatarGroup defines options group for avatar params
type AvatarGroup struct {
- Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+ Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"`
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
@@ -126,7 +126,7 @@ type AvatarGroup struct {
// CacheGroup defines options group for cache params
type CacheGroup struct {
- Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
+ Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"`
Max struct {
Items int `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
Value int `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
@@ -134,19 +134,14 @@ type CacheGroup struct {
} `group:"max" namespace:"max" env-namespace:"MAX"`
}
-// MongoGroup holds all mongo params, used by store, avatar and cache
-type MongoGroup struct {
- URL string `long:"url" env:"URL" description:"mongo url"`
- DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
-}
-
// AdminGroup defines options group for admin params
type AdminGroup struct {
- Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
+ Type string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"remote" default:"shared"`
Shared struct {
Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
Email string `long:"email" env:"EMAIL" default:"" description:"admin email"`
} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
+ Remote RemoteGroup `group:"remote" namespace:"remote" env-namespace:"REMOTE"`
}
// NotifyGroup defines options for notification
@@ -178,6 +173,14 @@ type StreamGroup struct {
MaxActive int `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
}
+// RemoteGroup defines options for remote modules (plugins)
+type RemoteGroup struct {
+ API string `long:"api" env:"API" description:"remote extension api url"`
+ TimeOut time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"`
+ AuthUser string `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"`
+ AuthPassword string `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
+}
+
// serverApp holds all active objects
type serverApp struct {
*ServerCommand
@@ -249,7 +252,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
dataService := &service.DataStore{
- Interface: storeEngine,
+ Engine: storeEngine,
EditDuration: s.EditDuration,
AdminStore: adminStore,
MaxCommentSize: s.MaxCommentSize,
@@ -414,13 +417,14 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
- case "mongo":
- mgServer, e := s.makeMongo()
- if e != nil {
- return result, errors.Wrap(e, "failed to create mongo server")
- }
- conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
- result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+ case "remote":
+ r := &engine.Remote{Client: remote.Client{
+ API: s.Store.Remote.API,
+ Client: http.Client{Timeout: s.Store.Remote.TimeOut},
+ AuthUser: s.Store.Remote.AuthUser,
+ AuthPasswd: s.Store.Remote.AuthPassword,
+ }}
+ return r, nil
default:
return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
}
@@ -436,13 +440,6 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
return nil, err
}
return avatar.NewLocalFS(s.Avatar.FS.Path), nil
- case "mongo":
- mgServer, err := s.makeMongo()
- if err != nil {
- return nil, errors.Wrap(err, "failed to create mongo server")
- }
- conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
- return avatar.NewGridFS(conn), nil
case "bolt":
if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
return nil, err
@@ -485,13 +482,14 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
}
}
return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
- case "mongo":
- mgServer, e := s.makeMongo()
- if e != nil {
- return nil, errors.Wrap(e, "failed to create mongo server")
- }
- conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
- return admin.NewMongoStore(conn, s.SharedSecret), nil
+ case "remote":
+ r := &admin.Remote{Client: remote.Client{
+ API: s.Admin.Remote.API,
+ Client: http.Client{Timeout: s.Admin.Remote.TimeOut},
+ AuthUser: s.Admin.Remote.AuthUser,
+ AuthPasswd: s.Admin.Remote.AuthPassword,
+ }}
+ return r, nil
default:
return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
}
@@ -503,27 +501,12 @@ func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
case "mem":
return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
cache.MaxKeys(s.Cache.Max.Items))
- // case "mongo":
- // mgServer, err := s.makeMongo()
- // if err != nil {
- // return nil, errors.Wrap(err, "failed to create mongo server")
- // }
- // conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
- // return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
- // cache.MaxKeys(s.Cache.Max.Items))
case "none":
return &cache.Nop{}, nil
}
return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
}
-func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
- if s.Mongo.URL == "" {
- return nil, errors.New("no mongo URL provided")
- }
- return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
-}
-
func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
providers := 0
diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go
index 3f0a60c54b..d8dc29ba83 100644
--- a/backend/app/cmd/server_test.go
+++ b/backend/app/cmd/server_test.go
@@ -14,10 +14,8 @@ import (
"time"
"github.com/dgrijalva/jwt-go"
- "github.com/globalsign/mgo"
"github.com/go-pkgz/auth/token"
log "github.com/go-pkgz/lgr"
- "github.com/go-pkgz/mongo"
"github.com/jessevdk/go-flags"
"github.com/stretchr/testify/assert"
@@ -55,7 +53,9 @@ func TestServerApp(t *testing.T) {
body, _ = ioutil.ReadAll(resp.Body)
t.Log(string(body))
- assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
+ email, err := app.dataService.AdminStore.Email("")
+ assert.NoError(t, err)
+ assert.Equal(t, "admin@demo.remark42.com", email, "default admin email")
app.Wait()
}
@@ -129,61 +129,6 @@ func TestServerApp_AnonMode(t *testing.T) {
app.Wait()
}
-func TestServerApp_WithMongo(t *testing.T) {
-
- mongoURL := os.Getenv("MONGO_TEST")
- if mongoURL == "" {
- mongoURL = "mongodb://localhost:27017/test"
- }
- if mongoURL == "skip" {
- t.Skip("skip mongo app test")
- }
-
- opts := ServerCommand{}
- opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
-
- // prepare options
- p := flags.NewParser(&opts, flags.Default)
- _, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo",
- "--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
- require.Nil(t, err)
- opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
- opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
-
- // create app
- app, err := opts.newServerApp()
- require.Nil(t, err)
-
- defer func() {
- s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second)
- assert.NoError(t, e)
- conn := mongo.NewConnection(s, "test_remark", "")
- _ = conn.WithDB(func(dbase *mgo.Database) error {
- assert.NoError(t, dbase.DropDatabase())
- return nil
- })
- }()
-
- ctx, cancel := context.WithCancel(context.Background())
- go func() {
- time.Sleep(5 * time.Second)
- log.Print("[TEST] terminate app")
- cancel()
- }()
- go func() { _ = app.run(ctx) }()
- time.Sleep(100 * time.Millisecond) // let server start
-
- // send ping
- resp, err := http.Get("http://localhost:12345/api/v1/ping")
- require.Nil(t, err)
- defer resp.Body.Close()
- assert.Equal(t, 200, resp.StatusCode)
- body, err := ioutil.ReadAll(resp.Body)
- assert.Nil(t, err)
- assert.Equal(t, "pong", string(body))
-
- app.Wait()
-}
func TestServerApp_WithSSL(t *testing.T) {
opts := ServerCommand{}
@@ -241,6 +186,45 @@ func TestServerApp_WithSSL(t *testing.T) {
app.Wait()
}
+func TestServerApp_WithRemote(t *testing.T) {
+
+ opts := ServerCommand{}
+ opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+ // prepare options
+ p := flags.NewParser(&opts, flags.Default)
+ _, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
+ "--store.type=remote", "--store.remote.api=http://127.0.0.1",
+ "--port=12345", "--admin.type=remote", "--admin.remote.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+ require.Nil(t, err)
+ opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
+ opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
+
+ // create app
+ app, err := opts.newServerApp()
+ require.Nil(t, err)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ time.Sleep(5 * time.Second)
+ log.Print("[TEST] terminate app")
+ cancel()
+ }()
+ go func() { _ = app.run(ctx) }()
+ time.Sleep(100 * time.Millisecond) // let server start
+
+ // send ping
+ resp, err := http.Get("http://localhost:12345/api/v1/ping")
+ require.Nil(t, err)
+ defer resp.Body.Close()
+ assert.Equal(t, 200, resp.StatusCode)
+ body, err := ioutil.ReadAll(resp.Body)
+ assert.Nil(t, err)
+ assert.Equal(t, "pong", string(body))
+
+ app.Wait()
+}
+
func TestServerApp_Failed(t *testing.T) {
opts := ServerCommand{}
opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
diff --git a/backend/app/migrator/disqus_test.go b/backend/app/migrator/disqus_test.go
index 5388f9b37f..5394e3f301 100644
--- a/backend/app/migrator/disqus_test.go
+++ b/backend/app/migrator/disqus_test.go
@@ -20,7 +20,7 @@ func TestDisqus_Import(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
- dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+ dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
d := Disqus{DataStore: &dataStore}
size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
assert.Nil(t, err)
diff --git a/backend/app/migrator/migrator_test.go b/backend/app/migrator/migrator_test.go
index 4b1dae2e4f..4a2151e27a 100644
--- a/backend/app/migrator/migrator_test.go
+++ b/backend/app/migrator/migrator_test.go
@@ -27,7 +27,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
- dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+ dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "/tmp/disqus-test.xml",
@@ -53,7 +53,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
- dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+ dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "/tmp/wordpress-test.xml",
@@ -82,7 +82,7 @@ func TestMigrator_ImportNative(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
require.Nil(t, err, "create store")
- dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+ dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
size, err := ImportComments(ImportParams{
DataStore: dataStore,
@@ -102,7 +102,7 @@ func TestMigrator_ImportFailed(t *testing.T) {
defer os.Remove("/tmp/remark-test.db")
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
require.Nil(t, err, "create store")
- dataStore := &service.DataStore{Interface: b}
+ dataStore := &service.DataStore{Engine: b}
_, err = ImportComments(ImportParams{
DataStore: dataStore,
InputFile: "/tmp/disqus-test.xml",
diff --git a/backend/app/migrator/native_test.go b/backend/app/migrator/native_test.go
index ed0b0c9738..85570563b3 100644
--- a/backend/app/migrator/native_test.go
+++ b/backend/app/migrator/native_test.go
@@ -145,7 +145,7 @@ func prep(t *testing.T) *service.DataStore {
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
assert.Nil(t, err)
- b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+ b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
comment := store.Comment{
ID: "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
diff --git a/backend/app/migrator/wordpress_test.go b/backend/app/migrator/wordpress_test.go
index 86a99b9305..a229b6cbb9 100644
--- a/backend/app/migrator/wordpress_test.go
+++ b/backend/app/migrator/wordpress_test.go
@@ -21,7 +21,7 @@ func TestWordPress_Import(t *testing.T) {
b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
assert.Nil(t, err, "create store")
- dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+ dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
wp := WordPress{DataStore: &dataStore}
size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
assert.Nil(t, err)
diff --git a/backend/app/remote/client.go b/backend/app/remote/client.go
new file mode 100644
index 0000000000..dd033926a0
--- /dev/null
+++ b/backend/app/remote/client.go
@@ -0,0 +1,74 @@
+package remote
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "reflect"
+ "sync/atomic"
+
+ "github.com/pkg/errors"
+)
+
+// Client implements remote engine and delegates all calls to remote http server
+type Client struct {
+ API string
+ Client http.Client
+ AuthUser string
+ AuthPasswd string
+
+ id uint64
+}
+
+// Call remote server with given method and arguments
+func (r *Client) Call(method string, args ...interface{}) (*Response, error) {
+
+ var b []byte
+ var err error
+
+ switch {
+ case len(args) == 0:
+ b, err = json.Marshal(Request{Method: method, ID: atomic.AddUint64(&r.id, 1)})
+ if err != nil {
+ return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+ }
+ case len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Struct:
+ b, err = json.Marshal(Request{Method: method, Params: args[0], ID: atomic.AddUint64(&r.id, 1)})
+ if err != nil {
+ return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+ }
+ default:
+ b, err = json.Marshal(Request{Method: method, Params: args, ID: atomic.AddUint64(&r.id, 1)})
+ if err != nil {
+ return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+ }
+ }
+
+ req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b))
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to make request for %s", method)
+ }
+ req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+ if r.AuthUser != "" && r.AuthPasswd != "" {
+ req.SetBasicAuth(r.AuthUser, r.AuthPasswd)
+ }
+ resp, err := r.Client.Do(req)
+ if err != nil {
+ return nil, errors.Wrapf(err, "remote call failed for %s", method)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, errors.Errorf("bad status %s for %s", resp.Status, method)
+ }
+
+ cr := Response{}
+ if err = json.NewDecoder(resp.Body).Decode(&cr); err != nil {
+ return nil, errors.Wrapf(err, "failed to decode response for %s", method)
+ }
+
+ if cr.Error != "" {
+ return nil, errors.New(cr.Error)
+ }
+ return &cr, nil
+}
diff --git a/backend/app/remote/client_test.go b/backend/app/remote/client_test.go
new file mode 100644
index 0000000000..717b26ca03
--- /dev/null
+++ b/backend/app/remote/client_test.go
@@ -0,0 +1,97 @@
+package remote
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestClient_Call(t *testing.T) {
+ ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+ resp, err := c.Call("test", 123, "abc")
+ assert.NoError(t, err)
+ res := ""
+ err = json.Unmarshal(*resp.Result, &res)
+ assert.NoError(t, err)
+ assert.Equal(t, "12345", res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallWithObject(t *testing.T) {
+ ts := testServer(t, `{"method":"test","params":{"F1":123,"F2":"abc","F3":"2019-06-09T23:03:55Z"},"id":1}`, `{"result":"12345"}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+ obj := struct {
+ F1 int
+ F2 string
+ F3 time.Time
+ }{
+ F1: 123,
+ F2: "abc",
+ F3: time.Date(2019, 6, 9, 23, 3, 55, 0, time.UTC),
+ }
+
+ resp, err := c.Call("test", obj)
+ assert.NoError(t, err)
+ res := ""
+ err = json.Unmarshal(*resp.Result, &res)
+ assert.NoError(t, err)
+ assert.Equal(t, "12345", res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallWithNoParams(t *testing.T) {
+ ts := testServer(t, `{"method":"test","id":1}`, `{"result":"12345"}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+ resp, err := c.Call("test")
+ assert.NoError(t, err)
+ res := ""
+ err = json.Unmarshal(*resp.Result, &res)
+ assert.NoError(t, err)
+ assert.Equal(t, "12345", res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallError(t *testing.T) {
+ ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"error":"some error"}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+ _, err := c.Call("test", 123, "abc")
+ assert.EqualError(t, err, "some error")
+}
+
+func TestClient_CallBadResponse(t *testing.T) {
+ ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345 invalid}`)
+ defer ts.Close()
+ c := Client{API: ts.URL, Client: http.Client{}}
+ _, err := c.Call("test", 123, "abc")
+ assert.NotNil(t, err)
+}
+
+func TestClient_CallBadRemote(t *testing.T) {
+ ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
+ defer ts.Close()
+ c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}
+ _, err := c.Call("test", 123)
+ assert.NotNil(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := ioutil.ReadAll(r.Body)
+ require.NoError(t, err)
+ assert.Equal(t, req, string(body))
+ t.Logf("req: %s", string(body))
+ fmt.Fprintf(w, resp)
+ }))
+}
diff --git a/backend/app/remote/remote.go b/backend/app/remote/remote.go
new file mode 100644
index 0000000000..e766efcb83
--- /dev/null
+++ b/backend/app/remote/remote.go
@@ -0,0 +1,23 @@
+// Package remote implements client ans server for RPC-like communication with remote storage.
+// The protocol is somewhat simplified version of json-rpc with a single POST call sending
+// Request json (method name and the list of parameters) and receiving back json Response with "result" json
+// and error string
+package remote
+
+import (
+ "encoding/json"
+)
+
+// Request encloses method name and all params
+type Request struct {
+ Method string `json:"method"`
+ Params interface{} `json:"params,omitempty"`
+ ID uint64 `json:"id"`
+}
+
+// Response encloses result and error received from remote server
+type Response struct {
+ Result *json.RawMessage `json:"result,omitempty"`
+ Error string `json:"error,omitempty"`
+ ID uint64 `json:"id"`
+}
diff --git a/backend/app/remote/server.go b/backend/app/remote/server.go
new file mode 100644
index 0000000000..12dd48f10c
--- /dev/null
+++ b/backend/app/remote/server.go
@@ -0,0 +1,179 @@
+package remote
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "sync"
+ "time"
+
+ "github.com/didip/tollbooth"
+ "github.com/didip/tollbooth_chi"
+ "github.com/go-chi/chi"
+ "github.com/go-chi/chi/middleware"
+ "github.com/go-chi/render"
+ log "github.com/go-pkgz/lgr"
+ R "github.com/go-pkgz/rest"
+ "github.com/go-pkgz/rest/logger"
+ "github.com/pkg/errors"
+
+ "github.com/umputun/remark/backend/app/rest"
+)
+
+// Server is json-rpc server with an optional basic auth
+type Server struct {
+ API string
+ AuthUser string
+ AuthPasswd string
+ Version string
+ AppName string
+
+ funcs struct {
+ m map[string]ServerFn
+ once sync.Once
+ }
+
+ httpServer struct {
+ *http.Server
+ sync.Mutex
+ }
+}
+
+// Encoder is a function to encode call's result to Response
+type Encoder func(id uint64, resp interface{}, e error) (Response, error)
+
+// ServerFn handler registered for each method with Add
+// Implementations provided by consumer and define response logic.
+type ServerFn func(id uint64, params json.RawMessage) Response
+
+// Run http server on given port
+func (s *Server) Run(port int) error {
+ if s.AuthUser == "" || s.AuthPasswd == "" {
+ log.Print("[WARN] extension server runs without auth")
+ }
+ if s.funcs.m == nil && len(s.funcs.m) == 0 {
+ return errors.Errorf("nothing mapped for dispatch, Add has to be called prior to Run")
+ }
+
+ router := chi.NewRouter()
+ router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
+ router.Use(R.AppInfo(s.AppName, "umputun", s.Version), R.Ping)
+ logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[INFO]")).Handler
+ router.Use(middleware.Timeout(5 * time.Second))
+ router.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(1000, nil)), middleware.NoCache)
+ router.Use(s.basicAuth)
+
+ router.Post(s.API, s.handler)
+
+ s.httpServer.Lock()
+ s.httpServer.Server = &http.Server{
+ Addr: fmt.Sprintf(":%d", port),
+ Handler: router,
+ ReadHeaderTimeout: 5 * time.Second,
+ WriteTimeout: 10 * time.Second,
+ IdleTimeout: 30 * time.Second,
+ }
+ s.httpServer.Unlock()
+
+ log.Printf("[INFO] listen on %d", port)
+ return s.httpServer.ListenAndServe()
+}
+
+// EncodeResponse convert anything to Response
+func (s *Server) EncodeResponse(id uint64, resp interface{}, e error) (Response, error) {
+ v, err := json.Marshal(&resp)
+ if err != nil {
+ return Response{}, err
+ }
+ if e != nil {
+ return Response{ID: id, Result: nil, Error: e.Error()}, nil
+ }
+ raw := json.RawMessage{}
+ if err = raw.UnmarshalJSON(v); err != nil {
+ return Response{}, err
+ }
+ return Response{ID: id, Result: &raw}, nil
+}
+
+// Shutdown http server
+func (s *Server) Shutdown() error {
+ s.httpServer.Lock()
+ defer s.httpServer.Unlock()
+ if s.httpServer.Server == nil {
+ return errors.Errorf("http server is not running")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ return s.httpServer.Shutdown(ctx)
+}
+
+// Add method handler
+func (s *Server) Add(method string, fn ServerFn) {
+ s.httpServer.Lock()
+ defer s.httpServer.Unlock()
+ if s.httpServer.Server != nil {
+ log.Printf("[WARN] ignored method %s, can't be added to activated server", method)
+ return
+ }
+
+ s.funcs.once.Do(func() {
+ s.funcs.m = map[string]ServerFn{}
+ })
+
+ s.funcs.m[method] = fn
+ log.Printf("[INFO] add handler for %s", method)
+}
+
+// HandlersGroup alias for map of handlers
+type HandlersGroup map[string]ServerFn
+
+// Group of handlers with common prefix
+func (s *Server) Group(prefix string, m HandlersGroup) {
+ for k, v := range m {
+ s.Add(prefix+"."+k, v)
+ }
+}
+
+func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
+ req := struct {
+ ID uint64 `json:"id"`
+ Method string `json:"method"`
+ Params *json.RawMessage `json:"params"`
+ }{}
+
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ rest.SendErrorJSON(w, r, http.StatusBadRequest, err, req.Method, 0)
+ return
+ }
+ fn, ok := s.funcs.m[req.Method]
+ if !ok {
+ rest.SendErrorJSON(w, r, http.StatusNotImplemented, errors.New("unsupported method"), req.Method, 0)
+ return
+ }
+
+ params := json.RawMessage{}
+ if req.Params != nil {
+ params = *req.Params
+ }
+
+ render.JSON(w, r, fn(req.ID, params))
+}
+
+func (s *Server) basicAuth(h http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+ if s.AuthUser == "" || s.AuthPasswd == "" {
+ h.ServeHTTP(w, r)
+ return
+ }
+
+ user, pass, ok := r.BasicAuth()
+ if user != s.AuthUser || pass != s.AuthPasswd || !ok {
+ w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+ http.Error(w, "Unauthorized", http.StatusUnauthorized)
+ return
+ }
+ h.ServeHTTP(w, r)
+ })
+}
diff --git a/backend/app/remote/server_test.go b/backend/app/remote/server_test.go
new file mode 100644
index 0000000000..39665bf779
--- /dev/null
+++ b/backend/app/remote/server_test.go
@@ -0,0 +1,245 @@
+package remote
+
+import (
+ "bytes"
+ "encoding/json"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestServerPrimitiveTypes(t *testing.T) {
+ s := Server{API: "/v1/cmd"}
+
+ type respData struct {
+ Res1 string
+ Res2 bool
+ }
+
+ s.Add("test", func(id uint64, params json.RawMessage) Response {
+ args := []interface{}{}
+ if err := json.Unmarshal(params, &args); err != nil {
+ return Response{Error: err.Error()}
+ }
+ t.Logf("%+v", args)
+
+ assert.Equal(t, 3, len(args))
+ assert.Equal(t, "blah", args[0].(string))
+ assert.Equal(t, 42., args[1].(float64))
+ assert.Equal(t, true, args[2].(bool))
+
+ r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
+ assert.NoError(t, err)
+ return r
+ })
+
+ go func() { _ = s.Run(9091) }()
+ defer func() { assert.NoError(t, s.Shutdown()) }()
+ time.Sleep(10 * time.Millisecond)
+
+ // check with direct http call
+ clientReq := Request{Method: "test", Params: []interface{}{"blah", 42, true}, ID: 123}
+ b := bytes.Buffer{}
+ require.NoError(t, json.NewEncoder(&b).Encode(clientReq))
+ resp, err := http.Post("http://127.0.0.1:9091/v1/cmd", "application/json", &b)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+ assert.Equal(t, 200, resp.StatusCode)
+ data, err := ioutil.ReadAll(resp.Body)
+ assert.NoError(t, err)
+ assert.Equal(t, `{"result":{"Res1":"res blah","Res2":true},"id":123}`+"\n", string(data))
+
+ // check with client call
+ c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+ r, err := c.Call("test", "blah", 42, true)
+ assert.NoError(t, err)
+ assert.Equal(t, "", r.Error)
+
+ res := respData{}
+ err = json.Unmarshal(*r.Result, &res)
+ assert.NoError(t, err)
+ assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
+ assert.Equal(t, uint64(1), r.ID)
+}
+
+func TestServerWithObject(t *testing.T) {
+ s := Server{API: "/v1/cmd"}
+
+ type respData struct {
+ Res1 string
+ Res2 bool
+ }
+
+ type reqData struct {
+ Time time.Time
+ F1 string
+ F2 time.Duration
+ }
+
+ s.Add("test", func(id uint64, params json.RawMessage) Response {
+ arg := reqData{}
+ if err := json.Unmarshal(params, &arg); err != nil {
+ return Response{Error: err.Error()}
+ }
+ t.Logf("%+v", arg)
+
+ r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
+ assert.NoError(t, err)
+ return r
+ })
+
+ go func() { _ = s.Run(9091) }()
+ defer func() { assert.NoError(t, s.Shutdown()) }()
+ time.Sleep(10 * time.Millisecond)
+
+ c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+ r, err := c.Call("test", reqData{Time: time.Now(), F1: "sawert", F2: time.Minute})
+ assert.NoError(t, err)
+ assert.Equal(t, "", r.Error)
+
+ res := respData{}
+ err = json.Unmarshal(*r.Result, &res)
+ assert.NoError(t, err)
+ assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
+}
+
+func TestServerMethodNotImplemented(t *testing.T) {
+ s := Server{}
+ ts := httptest.NewServer(http.HandlerFunc(s.handler))
+ defer ts.Close()
+ s.Add("test", func(id uint64, params json.RawMessage) Response {
+ return Response{}
+ })
+
+ r := Request{Method: "blah"}
+ buf := bytes.Buffer{}
+ assert.NoError(t, json.NewEncoder(&buf).Encode(r))
+ resp, err := http.Post(ts.URL, "application/json", &buf)
+ require.NoError(t, err)
+ assert.Equal(t, http.StatusNotImplemented, resp.StatusCode)
+
+ assert.EqualError(t, s.Shutdown(), "http server is not running")
+}
+
+func TestServerWithAuth(t *testing.T) {
+ s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+
+ s.Add("test", func(id uint64, params json.RawMessage) Response {
+ args := []interface{}{}
+ if err := json.Unmarshal(params, &args); err != nil {
+ return Response{Error: err.Error()}
+ }
+ t.Logf("%+v", args)
+
+ assert.Equal(t, 3, len(args))
+ assert.Equal(t, "blah", args[0].(string))
+ assert.Equal(t, 42., args[1].(float64))
+ assert.Equal(t, true, args[2].(bool))
+
+ r, err := s.EncodeResponse(id, "res blah", nil)
+ assert.NoError(t, err)
+ return r
+ })
+
+ go func() { _ = s.Run(9091) }()
+ time.Sleep(10 * time.Millisecond)
+ defer func() { assert.NoError(t, s.Shutdown()) }()
+
+ c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
+ r, err := c.Call("test", "blah", 42, true)
+ assert.NoError(t, err)
+ assert.Equal(t, "", r.Error)
+ val := ""
+ err = json.Unmarshal(*r.Result, &val)
+ assert.NoError(t, err)
+ assert.Equal(t, "res blah", val)
+
+ c = Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+ _, err = c.Call("test", "blah", 42, true)
+ assert.EqualError(t, err, "bad status 401 Unauthorized for test")
+}
+
+func TestServerErrReturn(t *testing.T) {
+ s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+
+ s.Add("test", func(id uint64, params json.RawMessage) Response {
+ args := []interface{}{}
+ if err := json.Unmarshal(params, &args); err != nil {
+ return Response{Error: err.Error()}
+ }
+ t.Logf("%+v", args)
+
+ assert.Equal(t, 3, len(args))
+ assert.Equal(t, "blah", args[0].(string))
+ assert.Equal(t, 42., args[1].(float64))
+ assert.Equal(t, true, args[2].(bool))
+
+ r, err := s.EncodeResponse(id, "res blah", errors.New("some error"))
+ assert.NoError(t, err)
+ return r
+ })
+
+ go func() { _ = s.Run(9091) }()
+ defer func() { assert.NoError(t, s.Shutdown()) }()
+ time.Sleep(10 * time.Millisecond)
+
+ c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
+ _, err := c.Call("test", "blah", 42, true)
+ assert.EqualError(t, err, "some error")
+}
+
+func TestServerGroup(t *testing.T) {
+ s := Server{API: "/v1/cmd"}
+ s.Group("pre", HandlersGroup{
+ "fn1": func(id uint64, params json.RawMessage) Response {
+ return Response{}
+ },
+ "fn2": func(id uint64, params json.RawMessage) Response {
+ return Response{}
+ },
+ })
+ go func() { _ = s.Run(9091) }()
+ defer func() { assert.NoError(t, s.Shutdown()) }()
+ time.Sleep(10 * time.Millisecond)
+
+ c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+ _, err := c.Call("fn1")
+ assert.EqualError(t, err, "bad status 501 Not Implemented for fn1")
+
+ _, err = c.Call("pre.fn1")
+ assert.NoError(t, err)
+ _, err = c.Call("pre.fn2")
+ assert.NoError(t, err)
+}
+
+func TestServerAddLate(t *testing.T) {
+ s := Server{API: "/v1/cmd"}
+ s.Add("fn1", func(id uint64, params json.RawMessage) Response {
+ return Response{}
+ })
+ go func() { _ = s.Run(9091) }()
+ defer func() { assert.NoError(t, s.Shutdown()) }()
+ time.Sleep(10 * time.Millisecond)
+
+ // too late, ignored after run
+ s.Add("fn2", func(id uint64, params json.RawMessage) Response {
+ return Response{}
+ })
+
+ c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+ _, err := c.Call("fn1")
+ assert.NoError(t, err)
+ _, err = c.Call("fn2")
+ assert.EqualError(t, err, "bad status 501 Not Implemented for fn2")
+}
+
+func TestServerNoHandlers(t *testing.T) {
+ s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+ assert.EqualError(t, s.Run(9091), "nothing mapped for dispatch, Add has to be called prior to Run")
+}
diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go
index 4a03cac924..39057c1d9a 100644
--- a/backend/app/rest/api/admin.go
+++ b/backend/app/rest/api/admin.go
@@ -28,7 +28,7 @@ type admin struct {
type adminStore interface {
Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
- DeleteUser(siteID string, userID string) error
+ DeleteUser(siteID string, userID string, mode store.DeleteMode) error
User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
IsBlocked(siteID string, userID string) bool
SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
@@ -64,7 +64,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
- if err := a.dataService.DeleteUser(siteID, userID); err != nil {
+ if err := a.dataService.DeleteUser(siteID, userID, store.HardDelete); err != nil {
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
return
}
@@ -109,7 +109,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
return
}
- if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+ if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
return
}
@@ -145,9 +145,9 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
return
}
- // delete comments for blocked user
+ // delete comments for blocked user.
if blockStatus {
- if err := a.dataService.DeleteUser(siteID, userID); err != nil {
+ if err := a.dataService.DeleteUser(siteID, userID, store.SoftDelete); err != nil {
log.Printf("[WARN] can't delete comments for blocked user %s on site %s, %v", userID, siteID, err)
}
}
diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go
index 796ab2e98a..9d7ee71c3a 100644
--- a/backend/app/rest/api/admin_test.go
+++ b/backend/app/rest/api/admin_test.go
@@ -353,13 +353,23 @@ func TestAdmin_Block(t *testing.T) {
assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1"))
assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2"))
-
}
func TestAdmin_BlockedList(t *testing.T) {
- ts, _, teardown := startupT(t)
+ ts, srv, teardown := startupT(t)
defer teardown()
+ c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+ URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+ c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+ URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2 name", ID: "user2"}}
+
+ // write comments for user1 and user2
+ _, err := srv.DataService.Create(c1)
+ assert.Nil(t, err)
+ _, err = srv.DataService.Create(c2)
+ assert.Nil(t, err)
+
// block user1
req, err := http.NewRequest(http.MethodPut,
fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
@@ -386,8 +396,10 @@ func TestAdmin_BlockedList(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 2, len(users), "two users blocked")
assert.Equal(t, "user1", users[0].ID)
+ assert.Equal(t, "user1 name", users[0].Name)
assert.Equal(t, "user2", users[1].ID)
-
+ assert.Equal(t, "user2 name", users[1].Name)
+ t.Logf("%+v", users)
time.Sleep(50 * time.Millisecond)
req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
@@ -399,7 +411,6 @@ func TestAdmin_BlockedList(t *testing.T) {
err = json.NewDecoder(res.Body).Decode(&users)
assert.Nil(t, err)
assert.Equal(t, 1, len(users), "one user left blocked")
-
}
func TestAdmin_ReadOnly(t *testing.T) {
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index b5e1c30751..30d037d748 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -377,6 +377,9 @@ func (s *Rest) updateLimiter() float64 {
func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
siteID := r.URL.Query().Get("site")
+ admins, _ := s.DataService.AdminStore.Admins(siteID)
+ emails, _ := s.DataService.AdminStore.Email(siteID)
+
cnf := struct {
Version string `json:"version"`
EditDuration int `json:"edit_duration"`
@@ -393,8 +396,8 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
Version: s.Version,
EditDuration: int(s.DataService.EditDuration.Seconds()),
MaxCommentSize: s.DataService.MaxCommentSize,
- Admins: s.DataService.AdminStore.Admins(siteID),
- AdminEmail: s.DataService.AdminStore.Email(siteID),
+ Admins: admins,
+ AdminEmail: emails,
LowScore: s.ScoreThresholds.Low,
CriticalScore: s.ScoreThresholds.Critical,
PositiveScore: s.DataService.PositiveScore,
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index a0e8566a6b..469ad4cdf9 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -301,7 +301,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
dataStore := &service.DataStore{
- Interface: b,
+ Engine: b,
EditDuration: 5 * time.Minute,
MaxCommentSize: 4000,
AdminStore: adminStore,
diff --git a/backend/app/store/admin/admin.go b/backend/app/store/admin/admin.go
index d0b5069253..8a6fc6692f 100644
--- a/backend/app/store/admin/admin.go
+++ b/backend/app/store/admin/admin.go
@@ -10,25 +10,17 @@ import (
// Store defines interface returning admins info for given site
type Store interface {
Key() (key string, err error)
- Admins(siteID string) (ids []string)
- Email(siteID string) (email string)
+ Admins(siteID string) (ids []string, err error)
+ Email(siteID string) (email string, err error)
}
-// StaticStore implements keys.Store with a single, predefined key
+// StaticStore implements keys.Store with a single set of admins and email for all sites
type StaticStore struct {
admins []string
email string
key string
}
-// Key returns static key for all sites, allows empty site
-func (s *StaticStore) Key() (key string, err error) {
- if s.key == "" {
- return "", errors.New("empty key for static key store")
- }
- return s.key, nil
-}
-
// NewStaticStore makes StaticStore instance with given key
func NewStaticStore(key string, admins []string, email string) *StaticStore {
log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
@@ -40,12 +32,20 @@ func NewStaticKeyStore(key string) *StaticStore {
return &StaticStore{key: key, admins: []string{}, email: ""}
}
+// Key returns static key, same for all sites
+func (s *StaticStore) Key() (key string, err error) {
+ if s.key == "" {
+ return "", errors.New("empty key for static key store")
+ }
+ return s.key, nil
+}
+
// Admins returns static list of admin's ids, the same for all sites
-func (s *StaticStore) Admins(string) (ids []string) {
- return s.admins
+func (s *StaticStore) Admins(string) (ids []string, err error) {
+ return s.admins, nil
}
// Email gets static email address
-func (s *StaticStore) Email(string) (email string) {
- return s.email
+func (s *StaticStore) Email(string) (email string, err error) {
+ return s.email, nil
}
diff --git a/backend/app/store/admin/admin_test.go b/backend/app/store/admin/admin_test.go
index 2d587f3ae0..6ce4517879 100644
--- a/backend/app/store/admin/admin_test.go
+++ b/backend/app/store/admin/admin_test.go
@@ -3,10 +3,7 @@ package admin
import (
"testing"
- "github.com/globalsign/mgo"
- "github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
)
func TestStaticStore_Get(t *testing.T) {
@@ -16,48 +13,11 @@ func TestStaticStore_Get(t *testing.T) {
assert.NoError(t, err, "valid store")
assert.Equal(t, "key123", k, "valid site")
- a := ks.Admins("any")
- assert.Equal(t, []string{"123", "xyz"}, a)
-
- email := ks.Email("blah")
- assert.Equal(t, "aa@example.com", email)
-}
-
-func TestMongoStore_Get(t *testing.T) {
- conn, err := mongo.MakeTestConnection(t)
- require.NoError(t, err)
- var ms Store = NewMongoStore(conn, "secret")
-
- recs := []mongoRec{
- {"site1", []string{"i11", "i12"}, "e1"},
- {"site2", []string{"i21", "i22"}, "e2"},
- }
- err = conn.WithCollection(func(coll *mgo.Collection) error {
- if e1 := coll.Insert(recs[0]); e1 != nil {
- return e1
- }
- return coll.Insert(recs[1])
- })
- require.NoError(t, err)
-
- admins := ms.Admins("site1")
- assert.Equal(t, []string{"i11", "i12"}, admins)
- email := ms.Email("site1")
- assert.Equal(t, "e1", email)
- key, err := ms.Key()
+ a, err := ks.Admins("any")
assert.NoError(t, err)
- assert.Equal(t, "secret", key)
+ assert.Equal(t, []string{"123", "xyz"}, a)
- admins = ms.Admins("site2")
- assert.Equal(t, []string{"i21", "i22"}, admins)
- email = ms.Email("site2")
- assert.Equal(t, "e2", email)
- key, err = ms.Key()
+ email, err := ks.Email("blah")
assert.NoError(t, err)
- assert.Equal(t, "secret", key)
-
- admins = ms.Admins("no-site-in-db")
- assert.Equal(t, []string{}, admins)
- email = ms.Email("no-site-in-db")
- assert.Equal(t, "", email)
+ assert.Equal(t, "aa@example.com", email)
}
diff --git a/backend/app/store/admin/mongo.go b/backend/app/store/admin/mongo.go
deleted file mode 100644
index 1f19bc328c..0000000000
--- a/backend/app/store/admin/mongo.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package admin
-
-import (
- "github.com/globalsign/mgo"
- "github.com/globalsign/mgo/bson"
- log "github.com/go-pkgz/lgr"
-
- "github.com/go-pkgz/mongo"
-)
-
-// MongoStore implements admin.Store with mongo backend
-type MongoStore struct {
- connection *mongo.Connection
- key string
-}
-
-type mongoRec struct {
- SiteID string `bson:"site"`
- IDs []string `bson:"admin_ids"`
- Email string `bson:"admin_email"`
-}
-
-// NewMongoStore makes admin Store for mongo's connection
-func NewMongoStore(conn *mongo.Connection, key string) *MongoStore {
- log.Printf("[DEBUG] make mongo admin store with %+v", conn)
- return &MongoStore{connection: conn, key: key}
-}
-
-// Key executes find by siteID and returns substructure with secret key
-func (m *MongoStore) Key() (key string, err error) {
- return m.key, nil
-}
-
-// Admins executes find by siteID and returns admins ids
-func (m *MongoStore) Admins(siteID string) (ids []string) {
- resp := mongoRec{}
- err := m.connection.WithCollection(func(coll *mgo.Collection) error {
- return coll.Find(bson.M{"site": siteID}).One(&resp)
- })
- if err != nil {
- return []string{}
- }
- return resp.IDs
-}
-
-// Email executes find by siteID and returns admin's email
-func (m *MongoStore) Email(siteID string) (email string) {
- resp := mongoRec{}
- err := m.connection.WithCollection(func(coll *mgo.Collection) error {
- return coll.Find(bson.M{"site": siteID}).One(&resp)
- })
- if err != nil {
- return ""
- }
- return resp.Email
-}
diff --git a/backend/app/store/admin/remote.go b/backend/app/store/admin/remote.go
new file mode 100644
index 0000000000..6cef97c6f1
--- /dev/null
+++ b/backend/app/store/admin/remote.go
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+ "encoding/json"
+
+ "github.com/umputun/remark/backend/app/remote"
+)
+
+// Remote implements remote engine and delegates all Calls to remote http server
+type Remote struct {
+ remote.Client
+}
+
+// Key returns the key, same for all sites
+func (r *Remote) Key() (key string, err error) {
+ resp, err := r.Call("admin.key")
+ if err != nil {
+ return "", err
+ }
+
+ err = json.Unmarshal(*resp.Result, &key)
+ return key, err
+}
+
+// Admins returns list of admin's ids for given site
+func (r *Remote) Admins(siteID string) (ids []string, err error) {
+ resp, err := r.Call("admin.admins", siteID)
+ if err != nil {
+ return []string{}, err
+ }
+
+ if err = json.Unmarshal(*resp.Result, &ids); err != nil {
+ return []string{}, err
+ }
+ return ids, nil
+}
+
+// Email gets email address for given site
+func (r *Remote) Email(siteID string) (email string, err error) {
+ resp, err := r.Call("admin.email", siteID)
+ if err != nil {
+ return "", err
+ }
+
+ if err = json.Unmarshal(*resp.Result, &email); err != nil {
+ return "", err
+ }
+ return email, nil
+}
diff --git a/backend/app/store/admin/remote_test.go b/backend/app/store/admin/remote_test.go
new file mode 100644
index 0000000000..4b67db0d03
--- /dev/null
+++ b/backend/app/store/admin/remote_test.go
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/remote"
+)
+
+func TestRemote_Key(t *testing.T) {
+ ts := testServer(t, `{"method":"admin.key","id":1}`,
+ `{"result":"12345","id":1}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ var a Store = &c
+ _ = a
+
+ res, err := c.Key()
+ assert.NoError(t, err)
+ assert.Equal(t, "12345", res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Admins(t *testing.T) {
+ ts := testServer(t, `{"method":"admin.admins","params":["site-1"],"id":1}`,
+ `{"result":["id1","id2"],"id":1}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ var a Store = &c
+ _ = a
+
+ res, err := c.Admins("site-1")
+ assert.NoError(t, err)
+ assert.Equal(t, []string{"id1", "id2"}, res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Email(t *testing.T) {
+ ts := testServer(t, `{"method":"admin.email","params":["site-1"],"id":1}`,
+ `{"result":"bbb@example.com","id":1}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ var a Store = &c
+ _ = a
+
+ res, err := c.Email("site-1")
+ assert.NoError(t, err)
+ assert.Equal(t, "bbb@example.com", res)
+ t.Logf("%v %T", res, res)
+}
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := ioutil.ReadAll(r.Body)
+ require.NoError(t, err)
+ assert.Equal(t, req, string(body))
+ t.Logf("req: %s", string(body))
+ fmt.Fprintf(w, resp)
+ }))
+}
diff --git a/backend/app/store/engine/bolt.go b/backend/app/store/engine/bolt.go
new file mode 100644
index 0000000000..54abdad0f0
--- /dev/null
+++ b/backend/app/store/engine/bolt.go
@@ -0,0 +1,863 @@
+package engine
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "time"
+
+ bolt "github.com/coreos/bbolt"
+ log "github.com/go-pkgz/lgr"
+ "github.com/hashicorp/go-multierror"
+ "github.com/pkg/errors"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
+// there are 5 types of top-level buckets:
+// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
+// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
+// value is not full comment but a reference combined from post-url+commentID
+// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
+// is a nested bucket named userID with kv as ts:reference
+// - blocking info sits in "block" bucket. Key is userID, value - ts
+// - counts per post to keep number of comments. Key is post url, value - count
+// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
+type BoltDB struct {
+ dbs map[string]*bolt.DB
+}
+
+const (
+ // top level buckets
+ postsBucketName = "posts"
+ lastBucketName = "last"
+ userBucketName = "users"
+ blocksBucketName = "block"
+ infoBucketName = "info"
+ readonlyBucketName = "readonly"
+ verifiedBucketName = "verified"
+
+ tsNano = "2006-01-02T15:04:05.000000000Z07:00"
+)
+
+// BoltSite defines single site param
+type BoltSite struct {
+ FileName string // full path to boltdb
+ SiteID string // ID of given site
+}
+
+// NewBoltDB makes persistent boltdb-based store. For each site new boltdb file created
+func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
+ log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
+ result := BoltDB{dbs: make(map[string]*bolt.DB)}
+ for _, site := range sites {
+ db, err := bolt.Open(site.FileName, 0600, &options)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
+ }
+
+ // make top-level buckets
+ topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, infoBucketName,
+ readonlyBucketName, verifiedBucketName}
+ err = db.Update(func(tx *bolt.Tx) error {
+ for _, bktName := range topBuckets {
+ if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+ return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+ }
+ }
+ return nil
+ })
+
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to create top level bucket)")
+ }
+
+ result.dbs[site.SiteID] = db
+ log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
+ }
+ return &result, nil
+}
+
+// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
+func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
+
+ bdb, err := b.db(comment.Locator.SiteID)
+ if err != nil {
+ return "", err
+ }
+
+ if b.checkFlag(FlagRequest{Locator: comment.Locator, Flag: ReadOnly}) {
+ return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+ }
+
+ err = bdb.Update(func(tx *bolt.Tx) (err error) {
+ var postBkt, lastBkt, userBkt *bolt.Bucket
+
+ if postBkt, err = b.makePostBucket(tx, comment.Locator.URL); err != nil {
+ return err
+ }
+ // check if key already in store, reject doubles
+ if postBkt.Get([]byte(comment.ID)) != nil {
+ return errors.Errorf("key %s already in store", comment.ID)
+ }
+
+ // serialize comment to json []byte for bolt and save
+ if err = b.save(postBkt, comment.ID, comment); err != nil {
+ return errors.Wrapf(err, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
+ }
+
+ ref := b.makeRef(comment) // reference combines url and comment id
+
+ // add reference to comment to "last" bucket
+ lastBkt = tx.Bucket([]byte(lastBucketName))
+ commentTs := []byte(comment.Timestamp.Format(tsNano))
+ if err = lastBkt.Put(commentTs, ref); err != nil {
+ return errors.Wrapf(err, "can't put reference %s to %s", ref, lastBucketName)
+ }
+
+ // add reference to commentID to "users" bucket
+ if userBkt, err = b.getUserBucket(tx, comment.User.ID); err != nil {
+ return errors.Wrapf(err, "can't get bucket %s", comment.User.ID)
+ }
+ // put into individual user's bucket with ts as a key
+ if err = userBkt.Put(commentTs, ref); err != nil {
+ return errors.Wrapf(err, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
+ }
+
+ // set info with the count for post url
+ if _, err = b.setInfo(tx, comment); err != nil {
+ return errors.Wrapf(err, "failed to set info for %s", comment.Locator)
+ }
+ return nil
+ })
+
+ return comment.ID, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (b *BoltDB) Get(req GetRequest) (comment store.Comment, err error) {
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return comment, err
+ }
+
+ err = bdb.View(func(tx *bolt.Tx) error {
+ bucket, e := b.getPostBucket(tx, req.Locator.URL)
+ if e != nil {
+ return e
+ }
+ return b.load(bucket, req.CommentID, &comment)
+ })
+ return comment, err
+}
+
+// Find returns all comments for given request and sorts results
+func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
+ comments = []store.Comment{}
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return nil, err
+ }
+
+ switch {
+ case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+ err = bdb.View(func(tx *bolt.Tx) error {
+
+ bucket, e := b.getPostBucket(tx, req.Locator.URL)
+ if e != nil {
+ return e
+ }
+
+ return bucket.ForEach(func(k, v []byte) error {
+ comment := store.Comment{}
+ if e = json.Unmarshal(v, &comment); e != nil {
+ return errors.Wrap(e, "failed to unmarshal")
+ }
+ comments = append(comments, comment)
+ return nil
+ })
+ })
+ case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+ comments, err = b.lastComments(req.Locator.SiteID, req.Limit, req.Since)
+ case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+ comments, err = b.userComments(req.Locator.SiteID, req.UserID, req.Limit, req.Skip)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+ return SortComments(comments, req.Sort), nil
+}
+
+// Flag sets and gets flag values
+func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) {
+ if req.Update == FlagNonSet { // read flag value, no update requested
+ return b.checkFlag(req), nil
+ }
+
+ // write flag value
+ return b.setFlag(req)
+}
+
+// Update for locator.URL with mutable part of comment
+func (b *BoltDB) Update(comment store.Comment) error {
+
+ getReq := GetRequest{Locator: comment.Locator, CommentID: comment.ID}
+ if curComment, err := b.Get(getReq); err == nil {
+ // preserve immutable fields
+ comment.ParentID = curComment.ParentID
+ comment.Locator = curComment.Locator
+ comment.Timestamp = curComment.Timestamp
+ comment.User = curComment.User
+ }
+
+ bdb, err := b.db(comment.Locator.SiteID)
+ if err != nil {
+ return err
+ }
+
+ return bdb.Update(func(tx *bolt.Tx) error {
+ bucket, e := b.getPostBucket(tx, comment.Locator.URL)
+ if e != nil {
+ return e
+ }
+ return b.save(bucket, comment.ID, comment)
+ })
+}
+
+// Count returns number of comments for post or user
+func (b *BoltDB) Count(req FindRequest) (count int, err error) {
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return 0, err
+ }
+
+ if req.Locator.URL != "" { // comment's count for post
+ err = bdb.View(func(tx *bolt.Tx) error {
+ var e error
+ count, e = b.count(tx, req.Locator.URL, 0)
+ return e
+ })
+ return count, err
+ }
+
+ if req.UserID != "" { // comment's count for user
+ err = bdb.View(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ userIDBkt := usersBkt.Bucket([]byte(req.UserID))
+ if userIDBkt == nil {
+ return errors.Errorf("no comments for user %s in store for %s site", req.UserID, req.Locator.SiteID)
+ }
+ stats := userIDBkt.Stats()
+ count = stats.KeyN
+ return nil
+ })
+ return count, err
+ }
+
+ return 0, errors.Errorf("invalid count request %+v", req)
+}
+
+// Info get post(s) meta info
+func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) {
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return []store.PostInfo{}, err
+ }
+
+ if req.Locator.URL != "" { // post info
+ info := store.PostInfo{}
+ err = bdb.View(func(tx *bolt.Tx) error {
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+ if e := b.load(infoBkt, req.Locator.URL, &info); e != nil {
+ return errors.Wrapf(e, "can't load info for %s", req.Locator.URL)
+ }
+ return nil
+ })
+
+ // set read-only from age and manual bucket
+ readOnlyAge := req.ReadOnlyAge
+ info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
+ if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
+ info.ReadOnly = true
+ }
+ return []store.PostInfo{info}, err
+ }
+
+ if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+ list := []store.PostInfo{}
+ err = bdb.View(func(tx *bolt.Tx) error {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+
+ c := postsBkt.Cursor()
+ n := 0
+ for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
+ n++
+ if req.Skip > 0 && n <= req.Skip {
+ continue
+ }
+ postURL := string(k)
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+ info := store.PostInfo{}
+ if e := b.load(infoBkt, postURL, &info); e != nil {
+ return errors.Wrapf(e, "can't load info for %s", postURL)
+ }
+ list = append(list, info)
+ if req.Limit > 0 && len(list) >= req.Limit {
+ break
+ }
+ }
+ return nil
+ })
+ return list, err
+ }
+
+ return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (b *BoltDB) ListFlags(req FlagRequest) (res []interface{}, err error) {
+
+ bdb, e := b.db(req.Locator.SiteID)
+ if e != nil {
+ return nil, e
+ }
+
+ switch req.Flag {
+ case Verified:
+ err = bdb.View(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(verifiedBucketName))
+ _ = usersBkt.ForEach(func(k, _ []byte) error {
+ res = append(res, string(k))
+ return nil
+ })
+ return nil
+ })
+ return res, err
+ case Blocked:
+ err = bdb.View(func(tx *bolt.Tx) error {
+ bucket := tx.Bucket([]byte(blocksBucketName))
+ return bucket.ForEach(func(k []byte, v []byte) error {
+ ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+ if errParse != nil {
+ return errors.Wrap(errParse, "can't parse block ts")
+ }
+ if time.Now().Before(ts) {
+ // get user name from comment user section
+ userName := ""
+ findReq := FindRequest{Locator: store.Locator{SiteID: req.Locator.SiteID}, UserID: string(k), Limit: 1}
+ userComments, errUser := b.Find(findReq)
+ if errUser == nil && len(userComments) > 0 {
+ userName = userComments[0].User.Name
+ }
+ res = append(res, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
+ }
+ return nil
+ })
+ })
+ return res, err
+ }
+ return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// Delete post(s) by id or by userID
+func (b *BoltDB) Delete(req DeleteRequest) error {
+
+ bdb, e := b.db(req.Locator.SiteID)
+ if e != nil {
+ return e
+ }
+
+ switch {
+ case req.Locator.URL != "" && req.CommentID != "":
+ return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode)
+ case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "":
+ return b.deleteUser(bdb, req.Locator.SiteID, req.UserID, req.DeleteMode)
+ case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "":
+ return b.deleteAll(bdb, req.Locator.SiteID)
+ }
+
+ return errors.Errorf("invalid delete request %+v", req)
+}
+
+// Close boltdb store
+func (b *BoltDB) Close() error {
+ errs := new(multierror.Error)
+ for site, db := range b.dbs {
+ err := errors.Wrapf(db.Close(), "can't close site %s", site)
+ errs = multierror.Append(errs, err)
+ }
+ return errs.ErrorOrNil()
+}
+
+// Last returns up to max last comments for given siteID
+func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
+
+ comments = []store.Comment{}
+
+ if max > lastLimit || max == 0 {
+ max = lastLimit
+ }
+
+ bdb, err := b.db(siteID)
+ if err != nil {
+ return nil, err
+ }
+
+ err = bdb.View(func(tx *bolt.Tx) error {
+ lastBkt := tx.Bucket([]byte(lastBucketName))
+ c := lastBkt.Cursor()
+
+ for k, v := c.Last(); k != nil; k, v = c.Prev() {
+
+ if !since.IsZero() {
+ // stop if reached "since" ts
+ tsSince := []byte(since.Format(tsNano))
+ if bytes.Compare(k, tsSince) <= 0 {
+ break
+ }
+ }
+ url, commentID, e := b.parseRef(v)
+ if e != nil {
+ return e
+ }
+ postBkt, e := b.getPostBucket(tx, url)
+ if e != nil {
+ return e
+ }
+
+ comment := store.Comment{}
+ if e = b.load(postBkt, commentID, &comment); e != nil {
+ log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
+ continue
+ }
+ if comment.Deleted {
+ continue
+ }
+ comments = append(comments, comment)
+ if len(comments) >= max {
+ break
+ }
+ }
+ return nil
+ })
+
+ return comments, err
+}
+
+// userComments extracts all comments for given site and given userID
+// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
+func (b *BoltDB) userComments(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
+
+ comments = []store.Comment{}
+ commentRefs := []string{}
+
+ bdb, err := b.db(siteID)
+ if err != nil {
+ return nil, err
+ }
+
+ if limit == 0 || limit > userLimit {
+ limit = userLimit
+ }
+
+ // get list of references to comments
+ err = bdb.View(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ userIDBkt := usersBkt.Bucket([]byte(userID))
+ if userIDBkt == nil {
+ return errors.Errorf("no comments for user %s in store", userID)
+ }
+
+ c := userIDBkt.Cursor()
+ skipComments := 0
+ for k, v := c.Last(); k != nil; k, v = c.Prev() {
+ if len(commentRefs) >= limit {
+ break
+ }
+ if skip > 0 && skipComments < skip {
+ skipComments++
+ continue
+ }
+ commentRefs = append(commentRefs, string(v))
+ }
+ return nil
+ })
+
+ if err != nil {
+ return comments, err
+ }
+
+ // retrieve comments for refs
+ for _, v := range commentRefs {
+ url, commentID, errParse := b.parseRef([]byte(v))
+ if errParse != nil {
+ return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
+ }
+ getReq := GetRequest{Locator: store.Locator{SiteID: siteID, URL: url}, CommentID: commentID}
+ if c, errRef := b.Get(getReq); errRef == nil {
+ comments = append(comments, c)
+ }
+ }
+
+ return comments, err
+}
+
+func (b *BoltDB) checkFlag(req FlagRequest) (val bool) {
+
+ bdb, err := b.db(req.Locator.SiteID)
+ if err != nil {
+ return false
+ }
+
+ key := req.Locator.URL
+ if req.UserID != "" {
+ key = req.UserID
+ }
+
+ if req.Flag == Blocked {
+ var blocked bool
+ _ = bdb.View(func(tx *bolt.Tx) error {
+ bucket := tx.Bucket([]byte(blocksBucketName))
+ v := bucket.Get([]byte(key))
+ if v == nil {
+ blocked = false
+ return nil
+ }
+
+ until, e := time.Parse(tsNano, string(v))
+ if e != nil {
+ blocked = false
+ return nil
+ }
+ blocked = time.Now().Before(until)
+ return nil
+ })
+ return blocked
+ }
+
+ _ = bdb.View(func(tx *bolt.Tx) error {
+ var bucket *bolt.Bucket
+ if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+ return err
+ }
+ val = bucket.Get([]byte(key)) != nil
+ return nil
+ })
+ return val
+}
+
+func (b *BoltDB) setFlag(req FlagRequest) (res bool, err error) {
+ bdb, e := b.db(req.Locator.SiteID)
+ if e != nil {
+ return false, e
+ }
+
+ key := req.Locator.URL
+ if req.UserID != "" {
+ key = req.UserID
+ }
+
+ err = bdb.Update(func(tx *bolt.Tx) error {
+ var bucket *bolt.Bucket
+ if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+ return err
+ }
+ switch req.Update {
+ case FlagTrue:
+ if req.Flag == Blocked {
+ val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
+ if req.TTL > 0 {
+ val = time.Now().Add(req.TTL).Format(tsNano)
+ }
+ if e = bucket.Put([]byte(key), []byte(val)); e != nil {
+ return errors.Wrapf(e, "failed to put blocked to %s", key)
+ }
+ res = true
+ return nil
+ }
+
+ if e = bucket.Put([]byte(key), []byte(time.Now().Format(tsNano))); e != nil {
+ return errors.Wrapf(e, "failed to set flag %s for %s", req.Flag, req.Locator.URL)
+ }
+ res = true
+ return nil
+ case FlagFalse:
+ if e = bucket.Delete([]byte(key)); e != nil {
+ return errors.Wrapf(e, "failed to clean flag %s for %s", req.Flag, req.Locator.URL)
+ }
+ res = false
+ }
+ return nil
+ })
+
+ return res, err
+}
+
+func (b *BoltDB) flagBucket(tx *bolt.Tx, flag Flag) (bkt *bolt.Bucket, err error) {
+ switch flag {
+ case ReadOnly:
+ bkt = tx.Bucket([]byte(readonlyBucketName))
+ case Blocked:
+ bkt = tx.Bucket([]byte(blocksBucketName))
+ case Verified:
+ bkt = tx.Bucket([]byte(verifiedBucketName))
+ default:
+ return nil, errors.Errorf("unsupported flag %v", flag)
+ }
+ return bkt, nil
+}
+
+func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID string, mode store.DeleteMode) error {
+
+ return bdb.Update(func(tx *bolt.Tx) error {
+
+ postBkt, e := b.getPostBucket(tx, locator.URL)
+ if e != nil {
+ return e
+ }
+
+ comment := store.Comment{}
+ if e = b.load(postBkt, commentID, &comment); e != nil {
+ return errors.Wrapf(e, "can't load key %s from bucket %s", commentID, locator.URL)
+ }
+ // set deleted status and clear fields
+ comment.SetDeleted(mode)
+
+ if e = b.save(postBkt, commentID, comment); e != nil {
+ return errors.Wrapf(e, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
+ }
+
+ // delete from "last" bucket
+ lastBkt := tx.Bucket([]byte(lastBucketName))
+ if e = lastBkt.Delete([]byte(commentID)); e != nil {
+ return errors.Wrapf(e, "can't delete key %s from bucket %s", commentID, lastBucketName)
+ }
+
+ // decrement comments count for post url
+ if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
+ return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
+ }
+
+ return nil
+ })
+}
+
+// deleteAll removes all top-level buckets for given siteID
+func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
+
+ // delete all buckets except blocked users
+ toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
+
+ // delete top-level buckets
+ err := bdb.Update(func(tx *bolt.Tx) error {
+ for _, bktName := range toDelete {
+
+ if e := tx.DeleteBucket([]byte(bktName)); e != nil {
+ return errors.Wrapf(e, "failed to delete top level bucket %s", bktName)
+ }
+ if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+ return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+ }
+ }
+ return nil
+ })
+
+ return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
+}
+
+// deleteUser removes all comments for given user. Everything will be market as deleted
+// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
+func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error {
+
+ // get list of all comments outside of transaction loop
+ posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}})
+ if err != nil {
+ return err
+ }
+
+ type commentInfo struct {
+ locator store.Locator
+ commentID string
+ }
+
+ // get list of commentID for all user's comment
+ comments := []commentInfo{}
+ for _, postInfo := range posts {
+ err = bdb.View(func(tx *bolt.Tx) error {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+ postBkt := postsBkt.Bucket([]byte(postInfo.URL))
+ err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
+ comment := store.Comment{}
+ if err = json.Unmarshal(commentVal, &comment); err != nil {
+ return errors.Wrap(err, "failed to unmarshal")
+ }
+ if comment.User.ID == userID {
+ comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
+ }
+ return nil
+ })
+ return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
+ })
+ if err != nil {
+ return err
+ }
+ }
+
+ log.Printf("[DEBUG] comments for removal=%d", len(comments))
+
+ // delete collected comments
+ for _, ci := range comments {
+ if e := b.deleteComment(bdb, ci.locator, ci.commentID, mode); e != nil {
+ return errors.Wrapf(err, "failed to delete comment %+v", ci)
+ }
+ }
+
+ // delete user bucket in hard mode
+ if mode == store.HardDelete {
+ err = bdb.Update(func(tx *bolt.Tx) error {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ if usersBkt != nil {
+ if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
+ return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
+ }
+ }
+ return nil
+ })
+
+ if err != nil {
+ return errors.Wrap(err, "can't delete user meta")
+ }
+ }
+
+ if len(comments) == 0 {
+ return errors.Errorf("unknown user %s", userID)
+ }
+
+ return err
+}
+
+// getPostBucket return bucket with all comments for postURL
+func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+ if postsBkt == nil {
+ return nil, errors.Errorf("no bucket %s", postsBucketName)
+ }
+ res := postsBkt.Bucket([]byte(postURL))
+ if res == nil {
+ return nil, errors.Errorf("no bucket %s in store", postURL)
+ }
+ return res, nil
+}
+
+// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
+func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+ postsBkt := tx.Bucket([]byte(postsBucketName))
+ if postsBkt == nil {
+ return nil, errors.Errorf("no bucket %s", postsBucketName)
+ }
+ res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
+ if err != nil {
+ return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
+ }
+ return res, nil
+}
+
+func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
+ usersBkt := tx.Bucket([]byte(userBucketName))
+ userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
+ if e != nil {
+ return nil, errors.Wrapf(e, "can't get bucket %s", userID)
+ }
+ return userIDBkt, nil
+}
+
+// save marshaled value to key for bucket. Should run in update tx
+func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) {
+ if value == nil {
+ return errors.Errorf("can't save nil value for %s", key)
+ }
+ jdata, jerr := json.Marshal(value)
+ if jerr != nil {
+ return errors.Wrap(jerr, "can't marshal comment")
+ }
+ if err = bkt.Put([]byte(key), jdata); err != nil {
+ return errors.Wrapf(err, "failed to save key %s", key)
+ }
+ return nil
+}
+
+// load and unmarshal json value by key from bucket. Should run in view tx
+func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error {
+ value := bkt.Get([]byte(key))
+ if value == nil {
+ return errors.Errorf("no value for %s", key)
+ }
+
+ if err := json.Unmarshal(value, &res); err != nil {
+ return errors.Wrap(err, "failed to unmarshal")
+ }
+ return nil
+}
+
+// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
+// it uses separate counts bucket because boltdb Stat call is very slow
+func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
+
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+
+ info := store.PostInfo{}
+ if err := b.load(infoBkt, postURL, &info); err != nil {
+ info = store.PostInfo{}
+ }
+ if val == 0 { // get current count, don't update
+ return info.Count, nil
+ }
+ info.Count += val
+
+ return info.Count, b.save(infoBkt, postURL, &info)
+}
+
+func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
+ infoBkt := tx.Bucket([]byte(infoBucketName))
+ info := store.PostInfo{}
+ if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil {
+ info = store.PostInfo{
+ Count: 0,
+ URL: comment.Locator.URL,
+ FirstTS: comment.Timestamp,
+ LastTS: comment.Timestamp,
+ }
+ }
+ info.Count++
+ info.LastTS = comment.Timestamp
+ return info, b.save(infoBkt, comment.Locator.URL, &info)
+}
+
+func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
+ if res, ok := b.dbs[siteID]; ok {
+ return res, nil
+ }
+ return nil, errors.Errorf("site %q not found", siteID)
+}
+
+// makeRef creates reference combining url and comment id
+func (b *BoltDB) makeRef(comment store.Comment) []byte {
+ return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
+}
+
+// parseRef gets parts of reference
+func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
+ elems := strings.Split(string(val), "!!")
+ if len(elems) != 2 {
+ return "", "", errors.Errorf("invalid reference value %s", string(val))
+ }
+ return elems[0], elems[1], nil
+}
diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go
deleted file mode 100644
index 1a555937ed..0000000000
--- a/backend/app/store/engine/bolt_accessor.go
+++ /dev/null
@@ -1,550 +0,0 @@
-package engine
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "strings"
- "time"
-
- bolt "github.com/coreos/bbolt"
- log "github.com/go-pkgz/lgr"
- "github.com/hashicorp/go-multierror"
- "github.com/pkg/errors"
-
- "github.com/umputun/remark/backend/app/store"
-)
-
-// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
-// there are 5 types of top-level buckets:
-// - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
-// - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
-// value is not full comment but a reference combined from post-url+commentID
-// - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
-// is a nested bucket named userID with kv as ts:reference
-// - blocking info sits in "block" bucket. Key is userID, value - ts
-// - counts per post to keep number of comments. Key is post url, value - count
-// - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
-type BoltDB struct {
- dbs map[string]*bolt.DB
-}
-
-const (
- // top level buckets
- postsBucketName = "posts"
- lastBucketName = "last"
- userBucketName = "users"
- blocksBucketName = "block"
- infoBucketName = "info"
- readonlyBucketName = "readonly"
- verifiedBucketName = "verified"
-
- tsNano = "2006-01-02T15:04:05.000000000Z07:00"
-)
-
-// BoltSite defines single site param
-type BoltSite struct {
- FileName string // full path to boltdb
- SiteID string // ID to access given site
-}
-
-// NewBoltDB makes persistent boltdb-based store
-func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
- log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
- result := BoltDB{dbs: make(map[string]*bolt.DB)}
- for _, site := range sites {
- db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second}
- if err != nil {
- return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
- }
-
- // make top-level buckets
- topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName,
- infoBucketName, readonlyBucketName, verifiedBucketName}
- err = db.Update(func(tx *bolt.Tx) error {
- for _, bktName := range topBuckets {
- if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
- return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
- }
- }
- return nil
- })
-
- if err != nil {
- return nil, errors.Wrap(err, "failed to create top level bucket)")
- }
-
- result.dbs[site.SiteID] = db
- log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
- }
- return &result, nil
-}
-
-// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
-func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
-
- bdb, err := b.db(comment.Locator.SiteID)
- if err != nil {
- return "", err
- }
-
- if b.IsReadOnly(comment.Locator) {
- return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
- }
-
- err = bdb.Update(func(tx *bolt.Tx) error {
-
- postBkt, e := b.makePostBucket(tx, comment.Locator.URL)
- if e != nil {
- return e
- }
-
- // check if key already in store, reject doubles
- if postBkt.Get([]byte(comment.ID)) != nil {
- return errors.Errorf("key %s already in store", comment.ID)
- }
-
- // serialize comment to json []byte for bolt and save
- if e = b.save(postBkt, []byte(comment.ID), comment); e != nil {
- return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
- }
-
- ref := b.makeRef(comment)
-
- // add reference to comment to "last" bucket
- lastBkt := tx.Bucket([]byte(lastBucketName))
- commentTs := []byte(comment.Timestamp.Format(tsNano))
- e = lastBkt.Put(commentTs, ref)
- if e != nil {
- return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName)
- }
-
- // add reference to commentID to "users" bucket
- userBkt, e := b.getUserBucket(tx, comment.User.ID)
- if e != nil {
- return errors.Wrapf(e, "can't get bucket %s", comment.User.ID)
- }
- // put into individual user's bucket with ts as a key
- if e = userBkt.Put(commentTs, ref); e != nil {
- return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
- }
-
- // set info with the count for post url
- if _, e = b.setInfo(tx, comment); e != nil {
- return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
- }
- return nil
- })
-
- return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
- comments = []store.Comment{}
-
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return nil, err
- }
-
- err = bdb.View(func(tx *bolt.Tx) error {
-
- bucket, e := b.getPostBucket(tx, locator.URL)
- if e != nil {
- return e
- }
-
- return bucket.ForEach(func(k, v []byte) error {
- comment := store.Comment{}
- if e = json.Unmarshal(v, &comment); e != nil {
- return errors.Wrap(e, "failed to unmarshal")
- }
- comments = append(comments, comment)
- return nil
- })
- })
-
- comments = SortComments(comments, sortFld)
- return comments, err
-}
-
-// Last returns up to max last comments for given siteID
-func (b *BoltDB) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
-
- comments = []store.Comment{}
-
- if max > lastLimit || max == 0 {
- max = lastLimit
- }
-
- bdb, err := b.db(siteID)
- if err != nil {
- return nil, err
- }
-
- err = bdb.View(func(tx *bolt.Tx) error {
- lastBkt := tx.Bucket([]byte(lastBucketName))
- c := lastBkt.Cursor()
-
- for k, v := c.Last(); k != nil; k, v = c.Prev() {
-
- if !since.IsZero() {
- // stop if reached "since" ts
- tsSince := []byte(since.Format(tsNano))
- if bytes.Compare(k, tsSince) <= 0 {
- break
- }
- }
- url, commentID, e := b.parseRef(v)
- if e != nil {
- return e
- }
- postBkt, e := b.getPostBucket(tx, url)
- if e != nil {
- return e
- }
-
- comment := store.Comment{}
- if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
- log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
- continue
- }
- if comment.Deleted {
- continue
- }
- comments = append(comments, comment)
- if len(comments) >= max {
- break
- }
- }
- return nil
- })
-
- return comments, err
-}
-
-// Count returns number of comments for locator
-func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
-
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return 0, err
- }
-
- err = bdb.View(func(tx *bolt.Tx) error {
- var e error
- count, e = b.count(tx, locator.URL, 0)
- return e
- })
-
- return count, err
-}
-
-// List returns list of all commented posts with counters
-// uses count bucket to get number of comments
-func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-
- bdb, err := b.db(siteID)
- if err != nil {
- return nil, err
- }
-
- err = bdb.View(func(tx *bolt.Tx) error {
- postsBkt := tx.Bucket([]byte(postsBucketName))
-
- c := postsBkt.Cursor()
- n := 0
- for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
- n++
- if skip > 0 && n <= skip {
- continue
- }
- postURL := string(k)
- infoBkt := tx.Bucket([]byte(infoBucketName))
- info := store.PostInfo{}
- if e := b.load(infoBkt, []byte(postURL), &info); e != nil {
- return errors.Wrapf(e, "can't load info for %s", postURL)
- }
- list = append(list, info)
- if limit > 0 && len(list) >= limit {
- break
- }
- }
- return nil
- })
-
- return list, err
-}
-
-// Info returns time range and count for locator
-func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) {
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return store.PostInfo{}, err
- }
-
- info := store.PostInfo{}
- err = bdb.View(func(tx *bolt.Tx) error {
- infoBkt := tx.Bucket([]byte(infoBucketName))
- if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil {
- return errors.Wrapf(e, "can't load info for %s", locator.URL)
- }
- return nil
- })
-
- // set read-only from age and manual bucket
- info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
- if b.IsReadOnly(locator) {
- info.ReadOnly = true
- }
- return info, err
-}
-
-// User extracts all comments for given site and given userID
-// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
-func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-
- comments = []store.Comment{}
- commentRefs := []string{}
-
- bdb, err := b.db(siteID)
- if err != nil {
- return nil, err
- }
-
- if limit == 0 || limit > userLimit {
- limit = userLimit
- }
-
- // get list of references to comments
- err = bdb.View(func(tx *bolt.Tx) error {
- usersBkt := tx.Bucket([]byte(userBucketName))
- userIDBkt := usersBkt.Bucket([]byte(userID))
- if userIDBkt == nil {
- return errors.Errorf("no comments for user %s in store", userID)
- }
-
- c := userIDBkt.Cursor()
- skipComments := 0
- for k, v := c.Last(); k != nil; k, v = c.Prev() {
- if len(commentRefs) >= limit {
- break
- }
- if skip > 0 && skipComments < skip {
- skipComments++
- continue
- }
- commentRefs = append(commentRefs, string(v))
- }
- return nil
- })
-
- if err != nil {
- return comments, err
- }
-
- // retrieve comments for refs
- for _, v := range commentRefs {
- url, commentID, errParse := b.parseRef([]byte(v))
- if errParse != nil {
- return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
- }
- if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
- comments = append(comments, c)
- }
- }
-
- return comments, err
-}
-
-// UserCount returns number of comments for user
-func (b *BoltDB) UserCount(siteID, userID string) (int, error) {
- bdb, err := b.db(siteID)
- if err != nil {
- return 0, err
- }
- count := 0
- err = bdb.View(func(tx *bolt.Tx) error {
- usersBkt := tx.Bucket([]byte(userBucketName))
- userIDBkt := usersBkt.Bucket([]byte(userID))
- if userIDBkt == nil {
- return errors.Errorf("no comments for user %s in store", userID)
- }
- stats := userIDBkt.Stats()
- count = stats.KeyN
- return nil
- })
- return count, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return comment, err
- }
-
- err = bdb.View(func(tx *bolt.Tx) error {
- bucket, e := b.getPostBucket(tx, locator.URL)
- if e != nil {
- return e
- }
- return b.load(bucket, []byte(commentID), &comment)
- })
- return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
-
- if curComment, err := b.Get(locator, comment.ID); err == nil {
- // preserve immutable fields
- comment.ParentID = curComment.ParentID
- comment.Locator = curComment.Locator
- comment.Timestamp = curComment.Timestamp
- comment.User = curComment.User
- }
-
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return err
- }
-
- return bdb.Update(func(tx *bolt.Tx) error {
- bucket, e := b.getPostBucket(tx, locator.URL)
- if e != nil {
- return e
- }
- return b.save(bucket, []byte(comment.ID), comment)
- })
-}
-
-// Close boltdb store
-func (b *BoltDB) Close() error {
- errs := new(multierror.Error)
- for site, db := range b.dbs {
- err := errors.Wrapf(db.Close(), "can't close site %s", site)
- errs = multierror.Append(errs, err)
- }
- return errs.ErrorOrNil()
-}
-
-// getPostBucket return bucket with all comments for postURL
-func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
- postsBkt := tx.Bucket([]byte(postsBucketName))
- if postsBkt == nil {
- return nil, errors.Errorf("no bucket %s", postsBucketName)
- }
- res := postsBkt.Bucket([]byte(postURL))
- if res == nil {
- return nil, errors.Errorf("no bucket %s in store", postURL)
- }
- return res, nil
-}
-
-// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
-func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
- postsBkt := tx.Bucket([]byte(postsBucketName))
- if postsBkt == nil {
- return nil, errors.Errorf("no bucket %s", postsBucketName)
- }
- res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
- if err != nil {
- return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
- }
- return res, nil
-}
-
-func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
- usersBkt := tx.Bucket([]byte(userBucketName))
- userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
- if e != nil {
- return nil, errors.Wrapf(e, "can't get bucket %s", userID)
- }
- return userIDBkt, nil
-}
-
-// save marshaled value to key for bucket. Should run in update tx
-func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) {
- if value == nil {
- return errors.Errorf("can't save nil value for %s", key)
- }
- jdata, jerr := json.Marshal(value)
- if jerr != nil {
- return errors.Wrap(jerr, "can't marshal comment")
- }
- if err = bkt.Put(key, jdata); err != nil {
- return errors.Wrapf(err, "failed to save key %s", key)
- }
- return nil
-}
-
-// load and unmarshal json value by key from bucket. Should run in view tx
-func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error {
- value := bkt.Get(key)
- if value == nil {
- return errors.Errorf("no value for %s", key)
- }
-
- if err := json.Unmarshal(value, &res); err != nil {
- return errors.Wrap(err, "failed to unmarshal")
- }
- return nil
-}
-
-// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
-// it uses separate counts bucket because boltdb Stat call is very slow
-func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
-
- infoBkt := tx.Bucket([]byte(infoBucketName))
-
- info := store.PostInfo{}
- if err := b.load(infoBkt, []byte(postURL), &info); err != nil {
- info = store.PostInfo{}
- }
- if val == 0 { // get current count, don't update
- return info.Count, nil
- }
- info.Count += val
-
- return info.Count, b.save(infoBkt, []byte(postURL), &info)
-}
-
-func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
- infoBkt := tx.Bucket([]byte(infoBucketName))
- info := store.PostInfo{}
- if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil {
- info = store.PostInfo{
- Count: 0,
- URL: comment.Locator.URL,
- FirstTS: comment.Timestamp,
- LastTS: comment.Timestamp,
- }
- }
- info.Count++
- info.LastTS = comment.Timestamp
- return info, b.save(infoBkt, []byte(comment.Locator.URL), &info)
-}
-
-func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
- if res, ok := b.dbs[siteID]; ok {
- return res, nil
- }
- return nil, errors.Errorf("site %q not found", siteID)
-}
-
-// makeRef creates reference combining url and comment id
-func (b *BoltDB) makeRef(comment store.Comment) []byte {
- return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
-}
-
-// parseRef gets parts of reference
-func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
- elems := strings.Split(string(val), "!!")
- if len(elems) != 2 {
- return "", "", errors.Errorf("invalid reference value %s", string(val))
- }
- return elems[0], elems[1], nil
-}
diff --git a/backend/app/store/engine/bolt_accessor_test.go b/backend/app/store/engine/bolt_accessor_test.go
deleted file mode 100644
index 1bf900d6ba..0000000000
--- a/backend/app/store/engine/bolt_accessor_test.go
+++ /dev/null
@@ -1,406 +0,0 @@
-package engine
-
-import (
- "fmt"
- "os"
- "testing"
- "time"
-
- bolt "github.com/coreos/bbolt"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/umputun/remark/backend/app/store"
-)
-
-var testDb = "test-remark.db"
-
-func TestBoltDB_CreateAndFind(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, `some text, link`, res[0].Text)
- assert.Equal(t, "user1", res[0].User.ID)
- t.Log(res[0].ID)
-
- _, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
- assert.NotNil(t, err)
- assert.Equal(t, "key id-1 already in store", err.Error())
-
- _, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
- assert.EqualError(t, err, `site "radio-t-bad" not found`)
-
- assert.NoError(t, b.Close())
-}
-
-func TestBoltDB_CreateReadOnly(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- comment := store.Comment{
- ID: "id-ro",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- err := b.SetReadOnly(comment.Locator, true)
- require.Nil(t, err)
-
- _, err = b.Create(comment)
- assert.NotNil(t, err)
- assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
-
- err = b.SetReadOnly(comment.Locator, false)
- require.Nil(t, err)
- _, err = b.Create(comment)
- assert.Nil(t, err)
-}
-
-func TestBoltDB_Get(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
-
- comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
- assert.Nil(t, err)
- assert.Equal(t, "some text2", comment.Text)
-
- comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
- assert.NotNil(t, err)
-
- _, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
- assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Put(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := b.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
-
- comment := res[0]
- comment.Text = "abc 123"
- comment.Score = 100
- err = b.Put(loc, comment)
- assert.Nil(t, err)
-
- comment, err = b.Get(loc, res[0].ID)
- assert.Nil(t, err)
- assert.Equal(t, "abc 123", comment.Text)
- assert.Equal(t, res[0].ID, comment.ID)
- assert.Equal(t, 100, comment.Score)
-
- err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
- assert.EqualError(t, err, `site "bad" not found`)
-
- err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
- assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
-}
-
-func TestBoltDB_Last(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- res, err := b.Last("radio-t", 0, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-
- res, err = b.Last("radio-t", 1, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-
- _, err = b.Last("bad", 0, time.Time{})
- assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_LastSince(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
- res, err := b.Last("radio-t", 0, ts)
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-
- ts = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
- res, err = b.Last("radio-t", 0, ts)
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-
- ts = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
- res, err = b.Last("radio-t", 0, ts)
- assert.Nil(t, err)
- assert.Equal(t, 0, len(res))
-}
-
-func TestBoltDB_Count(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 2, c)
-
- c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 0, c)
-
- _, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"})
- assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_List(t *testing.T) {
- b, teardown := prep(t) // two comments for https://radio-t.com
- defer teardown()
-
- // add one more for https://radio-t.com/2
- comment := store.Comment{
- ID: "12345",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- _, err := b.Create(comment)
- assert.Nil(t, err)
-
- ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
-
- res, err := b.List("radio-t", 0, 0)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
- {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
- res)
-
- res, err = b.List("radio-t", -1, -1)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
- {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
- res, err = b.List("radio-t", 1, 0)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
- res, err = b.List("radio-t", 1, 1)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
- _, err = b.List("bad", 1, 1)
- assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Info(t *testing.T) {
- b, teardown := prep(t) // two comments for https://radio-t.com
- defer teardown()
-
- ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
-
- // add one more for https://radio-t.com/2
- comment := store.Comment{
- ID: "12345",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- _, err := b.Create(comment)
- assert.Nil(t, err)
-
- r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
- r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
- r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
- _, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
- require.NotNil(t, err)
-
- _, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
- require.NotNil(t, err)
-
- err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
- require.Nil(t, err)
- r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-}
-
-func TestBoltDB_GetForUser(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- res, err := b.User("radio-t", "user1", 5, 0)
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
- res, err = b.User("radio-t", "user1", 1, 0)
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res), "allow 1 comment")
- assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
- res, err = b.User("radio-t", "user1", 1, 1)
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res), "allow 1 comment")
- assert.Equal(t, `some text, link`, res[0].Text, "second comment")
-
- _, err = b.User("bad", "user1", 1, 0)
- assert.EqualError(t, err, `site "bad" not found`)
-
- _, err = b.User("radio-t", "userZ", 1, 0)
- assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_GetForUserPagination(t *testing.T) {
- os.Remove(testDb)
- b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
- require.Nil(t, err)
-
- defer func() {
- require.NoError(t, b.Close())
- os.Remove(testDb)
- }()
-
- c := store.Comment{
- Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
-
- // write 200 comments
- for i := 0; i < 200; i++ {
- c.ID = fmt.Sprintf("id-%d", i)
- c.Text = fmt.Sprintf("text #%d", i)
- c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
- _, err = b.Create(c)
- require.Nil(t, err)
- }
-
- // get all comments
- res, err := b.User("radio-t", "user1", 0, 0)
- assert.Nil(t, err)
- assert.Equal(t, 200, len(res))
- assert.Equal(t, "id-199", res[0].ID)
-
- // seek 0, 5 comments
- res, err = b.User("radio-t", "user1", 5, 0)
- assert.Nil(t, err)
- assert.Equal(t, 5, len(res))
- assert.Equal(t, "id-199", res[0].ID)
- assert.Equal(t, "id-195", res[4].ID)
-
- // seek 10, 3 comments
- res, err = b.User("radio-t", "user1", 3, 10)
- assert.Nil(t, err)
- assert.Equal(t, 3, len(res))
- assert.Equal(t, "id-189", res[0].ID)
- assert.Equal(t, "id-187", res[2].ID)
-
- // seek 195, ask 10 comments
- res, err = b.User("radio-t", "user1", 10, 195)
- assert.Nil(t, err)
- assert.Equal(t, 5, len(res))
- assert.Equal(t, "id-4", res[0].ID)
- assert.Equal(t, "id-0", res[4].ID)
-
- // seek 255, ask 10 comments
- res, err = b.User("radio-t", "user1", 10, 255)
- assert.Nil(t, err)
- assert.Equal(t, 0, len(res))
-}
-
-func TestBoltDB_GetForUserCounter(t *testing.T) {
- var b, teardown = prep(t)
- defer teardown()
-
- count, err := b.UserCount("radio-t", "user1")
- assert.Nil(t, err)
- assert.Equal(t, 2, count)
-
- _, err = b.UserCount("bad", "user1")
- assert.EqualError(t, err, `site "bad" not found`)
-
- _, err = b.UserCount("radio-t", "userZ")
- assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_Ref(t *testing.T) {
- b := BoltDB{}
- comment := store.Comment{
- ID: "12345",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- res := b.makeRef(comment)
- assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
-
- url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
- assert.Nil(t, err)
- assert.Equal(t, "https://radio-t.com/2", url)
- assert.Equal(t, "12345", id)
-
- _, _, err = b.parseRef([]byte("https://radio-t.com/2"))
- assert.NotNil(t, err)
-}
-
-func TestBoltDB_New(t *testing.T) {
- _, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
- assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) (b *BoltDB, teardown func()) {
- os.Remove(testDb)
-
- boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
- assert.Nil(t, err)
- b = boltStore
-
- comment := store.Comment{
- ID: "id-1",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- _, err = b.Create(comment)
- assert.Nil(t, err)
-
- comment = store.Comment{
- ID: "id-2",
- Text: "some text2",
- Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- _, err = b.Create(comment)
- assert.Nil(t, err)
-
- teardown = func() {
- require.NoError(t, b.Close())
- os.Remove(testDb)
- }
- return b, teardown
-}
diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine/bolt_admin.go
deleted file mode 100644
index 429ac991bb..0000000000
--- a/backend/app/store/engine/bolt_admin.go
+++ /dev/null
@@ -1,339 +0,0 @@
-package engine
-
-import (
- "encoding/json"
- "time"
-
- bolt "github.com/coreos/bbolt"
- log "github.com/go-pkgz/lgr"
- "github.com/pkg/errors"
-
- "github.com/umputun/remark/backend/app/store"
-)
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-// From last bucket removed for real.
-func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return err
- }
-
- return bdb.Update(func(tx *bolt.Tx) error {
-
- postBkt, e := b.getPostBucket(tx, locator.URL)
- if e != nil {
- return e
- }
-
- comment := store.Comment{}
- if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
- return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
- }
- // set deleted status and clear fields
- comment.SetDeleted(mode)
-
- if err = b.save(postBkt, []byte(commentID), comment); err != nil {
- return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
- }
-
- // delete from "last" bucket
- lastBkt := tx.Bucket([]byte(lastBucketName))
- if err = lastBkt.Delete([]byte(commentID)); err != nil {
- return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
- }
-
- // decrement comments count for post url
- if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
- return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
- }
-
- return nil
- })
-}
-
-// DeleteAll removes all top-level buckets for given siteID
-func (b *BoltDB) DeleteAll(siteID string) error {
-
- bdb, err := b.db(siteID)
- if err != nil {
- return err
- }
-
- // delete all buckets except blocked users
- toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
-
- // delete top-level buckets
- err = bdb.Update(func(tx *bolt.Tx) error {
- for _, bktName := range toDelete {
-
- if e := tx.DeleteBucket([]byte(bktName)); e != nil {
- return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
- }
- if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
- return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
- }
- }
- return nil
- })
-
- return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
-func (b *BoltDB) DeleteUser(siteID string, userID string) error {
- bdb, err := b.db(siteID)
- if err != nil {
- return err
- }
-
- // get list of all comments outside of transaction loop
- posts, err := b.List(siteID, 0, 0)
- if err != nil {
- return err
- }
-
- type commentInfo struct {
- locator store.Locator
- commentID string
- }
-
- // get list of commentID for all user's comment
- comments := []commentInfo{}
- for _, postInfo := range posts {
- err = bdb.View(func(tx *bolt.Tx) error {
- postsBkt := tx.Bucket([]byte(postsBucketName))
- postBkt := postsBkt.Bucket([]byte(postInfo.URL))
- err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
- comment := store.Comment{}
- if err = json.Unmarshal(commentVal, &comment); err != nil {
- return errors.Wrap(err, "failed to unmarshal")
- }
- if comment.User.ID == userID {
- comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
- }
- return nil
- })
- return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
- })
- if err != nil {
- return err
- }
- }
-
- log.Printf("[DEBUG] comments for removal=%d", len(comments))
-
- // delete collected comments
- for _, ci := range comments {
- if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
- return errors.Wrapf(err, "failed to delete comment %+v", ci)
- }
- }
-
- // delete user bucket
- err = bdb.Update(func(tx *bolt.Tx) error {
- usersBkt := tx.Bucket([]byte(userBucketName))
- if usersBkt != nil {
- if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
- return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
- }
- }
- return nil
- })
-
- if err != nil {
- return errors.Wrap(err, "can't delete user meta")
- }
-
- if len(comments) == 0 {
- return errors.Errorf("unknown user %s", userID)
- }
-
- return err
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (b *BoltDB) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
- bdb, err := b.db(siteID)
- if err != nil {
- return err
- }
-
- return bdb.Update(func(tx *bolt.Tx) error {
- bucket := tx.Bucket([]byte(blocksBucketName))
- switch status {
- case true:
- val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
- if ttl > 0 {
- val = time.Now().Add(ttl).Format(tsNano)
- }
- if e := bucket.Put([]byte(userID), []byte(val)); e != nil {
- return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
- }
- case false:
- if e := bucket.Delete([]byte(userID)); e != nil {
- return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
- }
- }
- return nil
- })
-}
-
-// IsBlocked checks if user blocked
-func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
-
- bdb, err := b.db(siteID)
- if err != nil {
- return false
- }
-
- _ = bdb.View(func(tx *bolt.Tx) error {
- bucket := tx.Bucket([]byte(blocksBucketName))
- val := bucket.Get([]byte(userID))
- if val == nil {
- blocked = false
- return nil
- }
-
- until, e := time.Parse(tsNano, string(val))
- if e != nil {
- blocked = false
- return nil
- }
- blocked = time.Now().Before(until)
- return nil
- })
- return blocked
-}
-
-// Blocked get lists of blocked users for given site
-// bucket uses userID:
-func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
- users = []store.BlockedUser{}
- bdb, err := b.db(siteID)
- if err != nil {
- return nil, err
- }
-
- err = bdb.View(func(tx *bolt.Tx) error {
- bucket := tx.Bucket([]byte(blocksBucketName))
- return bucket.ForEach(func(k []byte, v []byte) error {
- ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
- if errParse != nil {
- return errors.Wrap(errParse, "can't parse block ts")
- }
- if time.Now().Before(ts) {
- // get user name from comment user section
- userName := ""
- userComments, errUser := b.User(siteID, string(k), 1, 0)
- if errUser == nil && len(userComments) > 0 {
- userName = userComments[0].User.Name
- }
- users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
- }
- return nil
- })
- })
-
- return users, err
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return err
- }
-
- return bdb.Update(func(tx *bolt.Tx) error {
- bucket := tx.Bucket([]byte(readonlyBucketName))
- switch status {
- case true:
- if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil {
- return errors.Wrapf(e, "failed to set ro for %s", locator.URL)
- }
- case false:
- if e := bucket.Delete([]byte(locator.URL)); e != nil {
- return errors.Wrapf(e, "failed to clean ro for %s", locator.URL)
- }
- }
- return nil
- })
-}
-
-// IsReadOnly checks if post in RO mode
-func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
-
- bdb, err := b.db(locator.SiteID)
- if err != nil {
- return false
- }
-
- _ = bdb.View(func(tx *bolt.Tx) error {
- bucket := tx.Bucket([]byte(readonlyBucketName))
- ro = bucket.Get([]byte(locator.URL)) != nil
- return nil
- })
- return ro
-}
-
-// SetVerified makes user verified or reset the flag
-func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error {
- bdb, err := b.db(siteID)
- if err != nil {
- return err
- }
-
- return bdb.Update(func(tx *bolt.Tx) error {
- bucket := tx.Bucket([]byte(verifiedBucketName))
- switch status {
- case true:
- if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
- return errors.Wrapf(e, "failed to set verified status for %s", userID)
- }
- case false:
- if e := bucket.Delete([]byte(userID)); e != nil {
- return errors.Wrapf(e, "failed to clean verified status for %s", userID)
- }
- }
- return nil
- })
-}
-
-// IsVerified checks if user verified
-func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
-
- bdb, err := b.db(siteID)
- if err != nil {
- return false
- }
-
- _ = bdb.View(func(tx *bolt.Tx) error {
- bucket := tx.Bucket([]byte(verifiedBucketName))
- verified = bucket.Get([]byte(userID)) != nil
- return nil
- })
- return verified
-}
-
-// Verified returns list of verified userIDs
-func (b *BoltDB) Verified(siteID string) (ids []string, err error) {
- bdb, err := b.db(siteID)
- if err != nil {
- return nil, err
- }
- err = bdb.View(func(tx *bolt.Tx) error {
- usersBkt := tx.Bucket([]byte(verifiedBucketName))
- _ = usersBkt.ForEach(func(k, _ []byte) error {
- ids = append(ids, string(k))
- return nil
- })
- return nil
- })
- return ids, err
-}
diff --git a/backend/app/store/engine/bolt_admin_test.go b/backend/app/store/engine/bolt_admin_test.go
deleted file mode 100644
index 06899b3b9b..0000000000
--- a/backend/app/store/engine/bolt_admin_test.go
+++ /dev/null
@@ -1,246 +0,0 @@
-package engine
-
-import (
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/umputun/remark/backend/app/store"
-)
-
-func TestBoltAdmin_Delete(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := b.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "initially 2 comments")
-
- count, err := b.Count(loc)
- require.NoError(t, err)
- assert.Equal(t, 2, count, "count=2 initially")
-
- err = b.Delete(loc, res[0].ID, store.SoftDelete)
- assert.Nil(t, err)
-
- res, err = b.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "", res[0].Text)
- assert.True(t, res[0].Deleted, "marked deleted")
- assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
- assert.Equal(t, "some text2", res[1].Text)
- assert.False(t, res[1].Deleted)
-
- comments, err := b.Last("radio-t", 10, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
- count, err = b.Count(loc)
- require.NoError(t, err)
- assert.Equal(t, 1, count)
-
- err = b.Delete(loc, "123456", store.SoftDelete)
- assert.NotNil(t, err)
-
- loc.SiteID = "bad"
- err = b.Delete(loc, res[0].ID, store.SoftDelete)
- assert.EqualError(t, err, `site "bad" not found`)
-
- loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
- err = b.Delete(loc, res[0].ID, store.SoftDelete)
- assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
-}
-
-func TestBoltAdmin_DeleteHard(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := b.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "initially 2 comments")
-
- err = b.Delete(loc, res[0].ID, store.HardDelete)
- assert.Nil(t, err)
-
- res, err = b.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "", res[0].Text)
- assert.True(t, res[0].Deleted, "marked deleted")
- assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestBoltAdmin_DeleteAll(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := b.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "initially 2 comments")
-
- err = b.DeleteAll("radio-t")
- assert.Nil(t, err)
-
- comments, err := b.Last("radio-t", 10, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 0, len(comments), "nothing left")
-
- c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 0, c, "0 count")
-
- err = b.DeleteAll("bad")
- assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_DeleteUser(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- err := b.DeleteUser("radio-t", "user1")
- require.NoError(t, err)
-
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := b.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "2 comments with deleted info")
- assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
- assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
- c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 0, c, "0 count")
-
- _, err = b.User("radio-t", "user1", 5, 0)
- assert.EqualError(t, err, "no comments for user user1 in store")
-
- comments, err := b.Last("radio-t", 10, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 0, len(comments), "nothing left")
-
- err = b.DeleteUser("radio-t-bad", "user1")
- assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltAdmin_BlockUser(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-
- assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
- assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-
- assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
- assert.NoError(t, b.SetBlock("radio-t", "user1", false, 0))
- assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
- assert.EqualError(t, b.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
- assert.NoError(t, b.SetBlock("radio-t", "userX", false, 0))
-
- assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
- assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
- assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
- time.Sleep(50 * time.Millisecond)
- assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
-}
-
-func TestBoltAdmin_BlockList(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
- assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
- assert.NoError(t, b.SetBlock("radio-t", "user3", false, 0))
-
- ids, err := b.Blocked("radio-t")
- assert.NoError(t, err)
-
- assert.Equal(t, 2, len(ids))
- assert.Equal(t, "user1", ids[0].ID)
- assert.Equal(t, "user2", ids[1].ID)
- t.Logf("%+v", ids)
-
- time.Sleep(50 * time.Millisecond)
- ids, err = b.Blocked("radio-t")
- assert.NoError(t, err)
- assert.Equal(t, 1, len(ids))
- assert.Equal(t, "user1", ids[0].ID)
-
- _, err = b.Blocked("bad")
- assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_ReadOnly(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
- assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
- assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
- assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
- assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
- assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
- assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`)
- assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
- assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_Verified(t *testing.T) {
-
- b, teardown := prep(t)
- defer teardown()
-
- assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
-
- assert.NoError(t, b.SetVerified("radio-t", "u1", true))
- assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified")
-
- assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified")
- assert.NoError(t, b.SetVerified("radio-t", "u1", false))
- assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
- assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`)
- assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
-
- assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-
- assert.NoError(t, b.SetVerified("radio-t", "u1", true))
- assert.NoError(t, b.SetVerified("radio-t", "u2", true))
- assert.NoError(t, b.SetVerified("radio-t", "u3", false))
-
- ids, err := b.Verified("radio-t")
- assert.NoError(t, err)
- assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
-
- _, err = b.Verified("radio-t-bad")
- assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
-}
diff --git a/backend/app/store/engine/bolt_test.go b/backend/app/store/engine/bolt_test.go
new file mode 100644
index 0000000000..5a284ef3e7
--- /dev/null
+++ b/backend/app/store/engine/bolt_test.go
@@ -0,0 +1,822 @@
+package engine
+
+import (
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ bolt "github.com/coreos/bbolt"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestBoltDB_CreateAndFind(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ var bb Interface = b
+ _ = bb
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, `some text, link`, res[0].Text)
+ assert.Equal(t, "user1", res[0].User.ID)
+ t.Log(res[0].ID)
+
+ _, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+ assert.NotNil(t, err)
+ assert.Equal(t, "key id-1 already in store", err.Error())
+
+ req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, Sort: "time"}
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `site "radio-t-bad" not found`)
+
+ assert.NoError(t, b.Close())
+}
+
+func TestBoltDB_CreateFailedReadOnly(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ comment := store.Comment{
+ ID: "id-ro",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+
+ flagReq := FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagTrue}
+ v, err := b.Flag(flagReq)
+ require.NoError(t, err)
+ assert.Equal(t, true, v)
+
+ _, err = b.Create(comment)
+ assert.NotNil(t, err)
+ assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+ flagReq = FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagFalse}
+ v, err = b.Flag(flagReq)
+ require.NoError(t, err)
+ assert.Equal(t, false, v)
+
+ _, err = b.Create(comment)
+ assert.NoError(t, err)
+}
+
+func TestBoltDB_Get(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "2 records initially")
+
+ comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+ assert.NoError(t, err)
+ assert.Equal(t, "some text2", comment.Text)
+
+ comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+ assert.NotNil(t, err)
+
+ _, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_Update(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "2 records initially")
+
+ comment := res[0]
+ comment.Text = "abc 123"
+ comment.Score = 100
+ err = b.Update(comment)
+ assert.NoError(t, err)
+
+ comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+ assert.NoError(t, err)
+ assert.Equal(t, "abc 123", comment.Text)
+ assert.Equal(t, res[0].ID, comment.ID)
+ assert.Equal(t, 100, comment.Score)
+
+ comment.Locator.SiteID = "bad"
+ err = b.Update(comment)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ comment.Locator.SiteID = "radio-t"
+ comment.Locator.URL = "https://radio-t.com-bad"
+ err = b.Update(comment)
+ assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
+}
+
+func TestBoltDB_FindLast(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Limit = 1
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Locator.SiteID = "bad"
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_FindLastSince(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindForUser(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res), "allow 1 comment")
+ assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(res), "allow 1 comment")
+ assert.Equal(t, `some text, link`, res[0].Text, "second comment")
+
+ req = FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+ _, err = b.Find(req)
+ assert.EqualError(t, err, `no comments for user userZ in store`)
+}
+
+func TestBoltDB_FindForUserPagination(t *testing.T) {
+ _ = os.Remove(testDb)
+ b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+ require.Nil(t, err)
+
+ defer func() {
+ require.NoError(t, b.Close())
+ _ = os.Remove(testDb)
+ }()
+
+ c := store.Comment{
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+
+ // write 200 comments
+ for i := 0; i < 200; i++ {
+ c.ID = fmt.Sprintf("id-%d", i)
+ c.Text = fmt.Sprintf("text #%d", i)
+ c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+ _, err = b.Create(c)
+ require.Nil(t, err)
+ }
+
+ // get all comments
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+ res, err := b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 200, len(res))
+ assert.Equal(t, "id-199", res[0].ID)
+
+ // seek 0, 5 comments
+ req.Limit = 5
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 5, len(res))
+ assert.Equal(t, "id-199", res[0].ID)
+ assert.Equal(t, "id-195", res[4].ID)
+
+ // seek 10, 3 comments
+ req.Skip, req.Limit = 10, 3
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 3, len(res))
+ assert.Equal(t, "id-189", res[0].ID)
+ assert.Equal(t, "id-187", res[2].ID)
+
+ // seek 195, ask 10 comments
+ req.Skip, req.Limit = 195, 10
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 5, len(res))
+ assert.Equal(t, "id-4", res[0].ID)
+ assert.Equal(t, "id-0", res[4].ID)
+
+ // seek 255, ask 10 comments
+ req.Skip, req.Limit = 255, 10
+ res, err = b.Find(req)
+ assert.NoError(t, err)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_CountPost(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+ c, err := b.Count(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, c)
+
+ req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+ c, err = b.Count(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 0, c)
+
+ req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+ _, err = b.Count(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_CountUser(t *testing.T) {
+ var b, teardown = prep(t)
+ defer teardown()
+
+ req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+ c, err := b.Count(req)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, c)
+
+ req = FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+ _, err = b.Count(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+ _, err = b.Count(req)
+ assert.EqualError(t, err, `no comments for user userZ in store for radio-t site`)
+}
+
+func TestBoltDB_InfoPost(t *testing.T) {
+ b, teardown := prep(t) // two comments for https://radio-t.com
+ defer teardown()
+
+ ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
+
+ // add one more for https://radio-t.com/2
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.NoError(t, err)
+
+ req := InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ r, err := b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+ r, err = b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+ ReadOnly: true}}, r)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ r, err = b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ _, err = b.Info(req)
+ require.NotNil(t, err)
+
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+ _, err = b.Info(req)
+ require.NotNil(t, err)
+
+ fr := FlagRequest{Flag: ReadOnly, Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: FlagTrue}
+ _, err = b.Flag(fr)
+ require.NoError(t, err)
+ req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+ r, err = b.Info(req)
+ require.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+ ReadOnly: true}}, r)
+}
+
+func TestBoltDB_InfoList(t *testing.T) {
+ b, teardown := prep(t) // two comments for https://radio-t.com
+ defer teardown()
+
+ // add one more for https://radio-t.com/2
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.Nil(t, err)
+
+ ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
+
+ req := InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+ res, err := b.Info(req)
+ assert.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+ {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+ res, err = b.Info(req)
+ assert.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+ {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+ res, err = b.Info(req)
+ assert.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+ res, err = b.Info(req)
+ assert.Nil(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ req = InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+ _, err = b.Info(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBolt_FlagBlockedUser(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ req := FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+ val, err := b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing blocked yet")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+ assert.NoError(t, err)
+ assert.True(t, val, "user1 blocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+ assert.NoError(t, err)
+ assert.True(t, val, "user1 still blocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagFalse}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+ assert.NoError(t, err)
+ assert.False(t, val, "user1 unblocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}, UserID: "user1", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "userX", Update: FlagTrue}
+ _, err = b.Flag(req)
+ assert.NoError(t, err, "non-existing user can't be blocked")
+
+ req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing blocked on wrong site")
+}
+
+func TestBolt_FlagReadOnlyPost(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ req := FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+ val, err := b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing ro")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagTrue}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.Equal(t, true, val)
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.True(t, val, "url-1 ro")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "url-2 still writable")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+ _, err = b.Flag(req)
+ assert.NoError(t, err)
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "url-1 writable")
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "bad", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+ _, err = b.Flag(req)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ req = FlagRequest{Locator: store.Locator{SiteID: "radio-t-bad", URL: "url-1"}, Flag: ReadOnly}
+ val, err = b.Flag(req)
+ assert.NoError(t, err)
+ assert.False(t, val, "nothing ro on wrong site")
+}
+
+func TestBolt_FlagVerified(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ isVerified := func(site, user string) bool {
+ req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+ v, err := b.Flag(req)
+ require.NoError(t, err)
+ return v
+ }
+
+ setVerified := func(site, user string, status FlagStatus) error {
+ req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+ _, err := b.Flag(req)
+ return err
+ }
+
+ assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+ assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+ assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+ assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+ assert.NoError(t, setVerified("radio-t", "u1", FlagFalse))
+ assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+ assert.EqualError(t, setVerified("bad", "u1", FlagTrue), `site "bad" not found`)
+ assert.NoError(t, setVerified("radio-t", "u1xyz", FlagFalse))
+
+ assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+ assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+ assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+ assert.NoError(t, setVerified("radio-t", "u3", FlagFalse))
+}
+
+func TestBolt_FlagListVerified(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ toIDs := func(inp []interface{}) (res []string) {
+ res = make([]string, len(inp))
+ for i, v := range inp {
+ vv, ok := v.(string)
+ require.True(t, ok)
+ res[i] = vv
+ }
+ return res
+ }
+
+ setVerified := func(site, user string, status FlagStatus) error {
+ req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+ _, err := b.Flag(req)
+ return err
+ }
+
+ ids, err := b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+ assert.NoError(t, err)
+ assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+ assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+ assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+ ids, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+ assert.NoError(t, err)
+ assert.Equal(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+ _, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+ assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
+}
+
+func TestBolt_FlagListBlocked(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ setBlocked := func(site, user string, status FlagStatus, ttl time.Duration) error {
+ req := FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status, TTL: ttl}
+ _, err := b.Flag(req)
+ return err
+ }
+
+ toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+ res = make([]store.BlockedUser, len(inp))
+ for i, v := range inp {
+ vv, ok := v.(store.BlockedUser)
+ require.True(t, ok)
+ res[i] = vv
+ }
+ return res
+ }
+ assert.NoError(t, setBlocked("radio-t", "user1", FlagTrue, 0))
+ assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 50*time.Millisecond))
+ assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0))
+
+ vv, err := b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+ assert.NoError(t, err)
+
+ blockedList := toBlocked(vv)
+ assert.Equal(t, 2, len(blockedList))
+ assert.Equal(t, "user1", blockedList[0].ID)
+ assert.Equal(t, "user2", blockedList[1].ID)
+ t.Logf("%+v", blockedList)
+
+ // check block expiration
+ time.Sleep(50 * time.Millisecond)
+ vv, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+ assert.NoError(t, err)
+ blockedList = toBlocked(vv)
+ assert.Equal(t, 1, len(blockedList))
+ assert.Equal(t, "user1", blockedList[0].ID)
+
+ _, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}})
+ assert.EqualError(t, err, `site "bad" not found`)
+
+}
+
+func TestBolt_DeleteComment(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "initially 2 comments")
+
+ count, err := b.Count(reqReq)
+ require.NoError(t, err)
+ assert.Equal(t, 2, count, "count=2 initially")
+
+ delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+ err = b.Delete(delReq)
+ assert.NoError(t, err)
+
+ res, err = b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "", res[0].Text)
+ assert.True(t, res[0].Deleted, "marked deleted")
+ assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+ assert.Equal(t, "some text2", res[1].Text)
+ assert.False(t, res[1].Deleted)
+
+ comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+ count, err = b.Count(reqReq)
+ require.NoError(t, err)
+ assert.Equal(t, 1, count)
+
+ delReq.CommentID = "123456"
+ err = b.Delete(delReq)
+ assert.NotNil(t, err)
+
+ delReq.Locator.SiteID = "bad"
+ delReq.CommentID = res[0].ID
+ err = b.Delete(delReq)
+ assert.EqualError(t, err, `site "bad" not found`)
+
+ delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+ err = b.Delete(delReq)
+ assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
+}
+
+func TestBolt_DeleteHard(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+ res, err := b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "initially 2 comments")
+
+ delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ CommentID: res[0].ID, DeleteMode: store.HardDelete}
+ err = b.Delete(delReq)
+ assert.NoError(t, err)
+
+ res, err = b.Find(reqReq)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "", res[0].Text)
+ assert.True(t, res[0].Deleted, "marked deleted")
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestBolt_DeleteAll(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ delReq := DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+ err := b.Delete(delReq)
+ assert.NoError(t, err)
+
+ comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, len(comments), "nothing left")
+
+ delReq = DeleteRequest{Locator: store.Locator{SiteID: "bad"}}
+ err = b.Delete(delReq)
+ assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUserHard(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
+ require.NoError(t, err)
+
+ comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+ c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, c, "0 count")
+
+ _, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+ assert.EqualError(t, err, "no comments for user user1 in store")
+
+ comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(comments), "nothing left")
+
+ err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+ assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
+
+ b, teardown := prep(t)
+ defer teardown()
+
+ err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
+ require.NoError(t, err)
+
+ comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+ assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+ assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+ c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, c, "0 count")
+
+ comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+ assert.NoError(t, err, "no comments for user user1 in store")
+ assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+ assert.True(t, comments[0].Deleted)
+ assert.True(t, comments[1].Deleted)
+ assert.Equal(t, "", comments[0].Text)
+ assert.Equal(t, "", comments[1].Text)
+
+ comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, len(comments), "nothing left")
+
+ err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+ assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltDB_ref(t *testing.T) {
+ b := BoltDB{}
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ res := b.makeRef(comment)
+ assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
+
+ url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
+ assert.NoError(t, err)
+ assert.Equal(t, "https://radio-t.com/2", url)
+ assert.Equal(t, "12345", id)
+
+ _, _, err = b.parseRef([]byte("https://radio-t.com/2"))
+ assert.NotNil(t, err)
+}
+
+func TestBoltDB_NewFailed(t *testing.T) {
+ _, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
+ assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
+ _ = os.Remove(testDb)
+
+ boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+ assert.Nil(t, err)
+ b = boltStore
+
+ comment := store.Comment{
+ ID: "id-1",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err = b.Create(comment)
+ assert.Nil(t, err)
+
+ comment = store.Comment{
+ ID: "id-2",
+ Text: "some text2",
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err = b.Create(comment)
+ assert.Nil(t, err)
+
+ teardown = func() {
+ require.NoError(t, b.Close())
+ _ = os.Remove(testDb)
+ }
+ return b, teardown
+}
+
+func getReq(locator store.Locator, commentID string) GetRequest {
+ return GetRequest{
+ Locator: locator,
+ CommentID: commentID,
+ }
+}
diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go
index fb6b25139e..5d30d7ee23 100644
--- a/backend/app/store/engine/engine.go
+++ b/backend/app/store/engine/engine.go
@@ -1,6 +1,7 @@
+package engine
+
// Package engine defines interfaces each supported storage should implement.
// Includes default implementation with boltdb
-package engine
import (
"sort"
@@ -13,38 +14,79 @@ import (
// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
-// UserRequest is the request send to get comments by user
-type UserRequest struct {
- SiteID string
- UserID string
- Limit int
- Skip int
-}
-
// Interface defines methods provided by low-level storage engine
type Interface interface {
- Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
- Get(locator store.Locator, commentID string) (store.Comment, error) // get comment by id
- Put(locator store.Locator, comment store.Comment) error // update comment, mutable parts only
- Find(locator store.Locator, sort string) ([]store.Comment, error) // find comments for locator
- Last(siteID string, limit int, since time.Time) ([]store.Comment, error) // last comments for given site, sorted by time
- User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
- UserCount(siteID, userID string) (int, error) // comments count by user
- Count(locator store.Locator) (int, error) // number of comments for the post
- List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts
- Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info
- Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
- DeleteAll(siteID string) error // delete all data from site
- DeleteUser(siteID string, userID string) error // remove all comments from user
- SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent)
- IsBlocked(siteID string, userID string) bool // check if user blocked
- Blocked(siteID string) ([]store.BlockedUser, error) // get list of blocked users
- SetReadOnly(locator store.Locator, status bool) error // set/reset read-only flag
- IsReadOnly(locator store.Locator) bool // check if post read-only
- SetVerified(siteID string, userID string, status bool) error // set/reset verified flag
- IsVerified(siteID string, userID string) bool // check verified status
- Verified(siteID string) ([]string, error) // list of verified user ids
- Close() error // close/stop engine
+ Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
+ Update(comment store.Comment) error // update comment, mutable parts only
+ Get(req GetRequest) (store.Comment, error) // get comment by id
+ Find(req FindRequest) ([]store.Comment, error) // find comments for locator or site
+ Info(req InfoRequest) ([]store.PostInfo, error) // get post(s) meta info
+ Count(req FindRequest) (int, error) // get count for post or user
+ Delete(req DeleteRequest) error // delete post(s) by id or by userID
+ Flag(req FlagRequest) (bool, error) // set and get flags
+ ListFlags(req FlagRequest) ([]interface{}, error) // get list of flagged keys, like blocked & verified user
+ Close() error // close storage engine
+}
+
+// GetRequest is the input for Get func
+type GetRequest struct {
+ Locator store.Locator `json:"locator"`
+ CommentID string `json:"comment_id"`
+}
+
+// FindRequest is the input for all find operations
+type FindRequest struct {
+ Locator store.Locator `json:"locator"` // lack of URL means site operation
+ UserID string `json:"user_id,omitempty"` // presence of UserID treated as user-related find
+ Sort string `json:"sort,omitempty"` // sort order with +/-field syntax
+ Since time.Time `json:"since,omitempty"` // time limit for found results
+ Limit int `json:"limit,omitempty"`
+ Skip int `json:"skip,omitempty"`
+}
+
+// InfoRequest is the input of Info operation used to get meta data about posts
+type InfoRequest struct {
+ Locator store.Locator `json:"locator"`
+ Limit int `json:"limit,omitempty"`
+ Skip int `json:"skip,omitempty"`
+ ReadOnlyAge int `json:"ro_age,omitempty"`
+}
+
+// DeleteRequest is the input for all delete operations (comments, sites, users)
+type DeleteRequest struct {
+ Locator store.Locator `json:"locator"` // lack of URL means site operation
+ CommentID string `json:"comment_id,omitempty"`
+ UserID string `json:"user_id,omitempty"`
+ DeleteMode store.DeleteMode `json:"del_mode"`
+}
+
+// Flag defines type of binary attribute
+type Flag string
+
+// FlagStatus represents values of the flag update
+type FlagStatus int
+
+// enum of update values
+const (
+ FlagNonSet FlagStatus = 0
+ FlagTrue FlagStatus = 1
+ FlagFalse FlagStatus = -1
+)
+
+// Enum of all flags
+const (
+ ReadOnly = Flag("readonly")
+ Verified = Flag("verified")
+ Blocked = Flag("blocked")
+)
+
+// FlagRequest is the input for both get/set for flags, like blocked, verified and so on
+type FlagRequest struct {
+ Flag Flag `json:"flag"` // flag type
+ Locator store.Locator `json:"locator"` // post locator
+ UserID string `json:"user_id,omitempty"` // for flags setting user status
+ Update FlagStatus `json:"update,omitempty"` // if FlagNonSet it will be get op, if set will set the value
+ TTL time.Duration `json:"ttl,omitempty"` // ttl for time-sensitive flags only, like blocked for some period
}
const (
diff --git a/backend/app/store/engine/engine_mock.go b/backend/app/store/engine/engine_mock.go
index e274d53858..7549c92d5d 100644
--- a/backend/app/store/engine/engine_mock.go
+++ b/backend/app/store/engine/engine_mock.go
@@ -3,36 +3,12 @@ package engine
import mock "github.com/stretchr/testify/mock"
import store "github.com/umputun/remark/backend/app/store"
-import time "time"
// MockInterface is an autogenerated mock type for the Interface type
type MockInterface struct {
mock.Mock
}
-// Blocked provides a mock function with given fields: siteID
-func (_m *MockInterface) Blocked(siteID string) ([]store.BlockedUser, error) {
- ret := _m.Called(siteID)
-
- var r0 []store.BlockedUser
- if rf, ok := ret.Get(0).(func(string) []store.BlockedUser); ok {
- r0 = rf(siteID)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).([]store.BlockedUser)
- }
- }
-
- var r1 error
- if rf, ok := ret.Get(1).(func(string) error); ok {
- r1 = rf(siteID)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
// Close provides a mock function with given fields:
func (_m *MockInterface) Close() error {
ret := _m.Called()
@@ -47,20 +23,20 @@ func (_m *MockInterface) Close() error {
return r0
}
-// Count provides a mock function with given fields: locator
-func (_m *MockInterface) Count(locator store.Locator) (int, error) {
- ret := _m.Called(locator)
+// Count provides a mock function with given fields: req
+func (_m *MockInterface) Count(req FindRequest) (int, error) {
+ ret := _m.Called(req)
var r0 int
- if rf, ok := ret.Get(0).(func(store.Locator) int); ok {
- r0 = rf(locator)
+ if rf, ok := ret.Get(0).(func(FindRequest) int); ok {
+ r0 = rf(req)
} else {
r0 = ret.Get(0).(int)
}
var r1 error
- if rf, ok := ret.Get(1).(func(store.Locator) error); ok {
- r1 = rf(locator)
+ if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+ r1 = rf(req)
} else {
r1 = ret.Error(1)
}
@@ -89,41 +65,13 @@ func (_m *MockInterface) Create(comment store.Comment) (string, error) {
return r0, r1
}
-// Delete provides a mock function with given fields: locator, commentID, mode
-func (_m *MockInterface) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
- ret := _m.Called(locator, commentID, mode)
-
- var r0 error
- if rf, ok := ret.Get(0).(func(store.Locator, string, store.DeleteMode) error); ok {
- r0 = rf(locator, commentID, mode)
- } else {
- r0 = ret.Error(0)
- }
-
- return r0
-}
-
-// DeleteAll provides a mock function with given fields: siteID
-func (_m *MockInterface) DeleteAll(siteID string) error {
- ret := _m.Called(siteID)
-
- var r0 error
- if rf, ok := ret.Get(0).(func(string) error); ok {
- r0 = rf(siteID)
- } else {
- r0 = ret.Error(0)
- }
-
- return r0
-}
-
-// DeleteUser provides a mock function with given fields: siteID, userID
-func (_m *MockInterface) DeleteUser(siteID string, userID string) error {
- ret := _m.Called(siteID, userID)
+// Delete provides a mock function with given fields: req
+func (_m *MockInterface) Delete(req DeleteRequest) error {
+ ret := _m.Called(req)
var r0 error
- if rf, ok := ret.Get(0).(func(string, string) error); ok {
- r0 = rf(siteID, userID)
+ if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok {
+ r0 = rf(req)
} else {
r0 = ret.Error(0)
}
@@ -131,13 +79,13 @@ func (_m *MockInterface) DeleteUser(siteID string, userID string) error {
return r0
}
-// Find provides a mock function with given fields: locator, sort
-func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comment, error) {
- ret := _m.Called(locator, sort)
+// Find provides a mock function with given fields: req
+func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) {
+ ret := _m.Called(req)
var r0 []store.Comment
- if rf, ok := ret.Get(0).(func(store.Locator, string) []store.Comment); ok {
- r0 = rf(locator, sort)
+ if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok {
+ r0 = rf(req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]store.Comment)
@@ -145,8 +93,8 @@ func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comme
}
var r1 error
- if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok {
- r1 = rf(locator, sort)
+ if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+ r1 = rf(req)
} else {
r1 = ret.Error(1)
}
@@ -154,20 +102,20 @@ func (_m *MockInterface) Find(locator store.Locator, sort string) ([]store.Comme
return r0, r1
}
-// Get provides a mock function with given fields: locator, commentID
-func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Comment, error) {
- ret := _m.Called(locator, commentID)
+// Flag provides a mock function with given fields: req
+func (_m *MockInterface) Flag(req FlagRequest) (bool, error) {
+ ret := _m.Called(req)
- var r0 store.Comment
- if rf, ok := ret.Get(0).(func(store.Locator, string) store.Comment); ok {
- r0 = rf(locator, commentID)
+ var r0 bool
+ if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok {
+ r0 = rf(req)
} else {
- r0 = ret.Get(0).(store.Comment)
+ r0 = ret.Get(0).(bool)
}
var r1 error
- if rf, ok := ret.Get(1).(func(store.Locator, string) error); ok {
- r1 = rf(locator, commentID)
+ if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+ r1 = rf(req)
} else {
r1 = ret.Error(1)
}
@@ -175,20 +123,20 @@ func (_m *MockInterface) Get(locator store.Locator, commentID string) (store.Com
return r0, r1
}
-// Info provides a mock function with given fields: locator, readonlyAge
-func (_m *MockInterface) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) {
- ret := _m.Called(locator, readonlyAge)
+// Get provides a mock function with given fields: req
+func (_m *MockInterface) Get(req GetRequest) (store.Comment, error) {
+ ret := _m.Called(req)
- var r0 store.PostInfo
- if rf, ok := ret.Get(0).(func(store.Locator, int) store.PostInfo); ok {
- r0 = rf(locator, readonlyAge)
+ var r0 store.Comment
+ if rf, ok := ret.Get(0).(func(GetRequest) store.Comment); ok {
+ r0 = rf(req)
} else {
- r0 = ret.Get(0).(store.PostInfo)
+ r0 = ret.Get(0).(store.Comment)
}
var r1 error
- if rf, ok := ret.Get(1).(func(store.Locator, int) error); ok {
- r1 = rf(locator, readonlyAge)
+ if rf, ok := ret.Get(1).(func(GetRequest) error); ok {
+ r1 = rf(req)
} else {
r1 = ret.Error(1)
}
@@ -196,64 +144,22 @@ func (_m *MockInterface) Info(locator store.Locator, readonlyAge int) (store.Pos
return r0, r1
}
-// IsBlocked provides a mock function with given fields: siteID, userID
-func (_m *MockInterface) IsBlocked(siteID string, userID string) bool {
- ret := _m.Called(siteID, userID)
-
- var r0 bool
- if rf, ok := ret.Get(0).(func(string, string) bool); ok {
- r0 = rf(siteID, userID)
- } else {
- r0 = ret.Get(0).(bool)
- }
-
- return r0
-}
-
-// IsReadOnly provides a mock function with given fields: locator
-func (_m *MockInterface) IsReadOnly(locator store.Locator) bool {
- ret := _m.Called(locator)
-
- var r0 bool
- if rf, ok := ret.Get(0).(func(store.Locator) bool); ok {
- r0 = rf(locator)
- } else {
- r0 = ret.Get(0).(bool)
- }
-
- return r0
-}
-
-// IsVerified provides a mock function with given fields: siteID, userID
-func (_m *MockInterface) IsVerified(siteID string, userID string) bool {
- ret := _m.Called(siteID, userID)
-
- var r0 bool
- if rf, ok := ret.Get(0).(func(string, string) bool); ok {
- r0 = rf(siteID, userID)
- } else {
- r0 = ret.Get(0).(bool)
- }
-
- return r0
-}
-
-// Last provides a mock function with given fields: siteID, limit, since
-func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]store.Comment, error) {
- ret := _m.Called(siteID, limit, since)
+// Info provides a mock function with given fields: req
+func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) {
+ ret := _m.Called(req)
- var r0 []store.Comment
- if rf, ok := ret.Get(0).(func(string, int, time.Time) []store.Comment); ok {
- r0 = rf(siteID, limit, since)
+ var r0 []store.PostInfo
+ if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok {
+ r0 = rf(req)
} else {
if ret.Get(0) != nil {
- r0 = ret.Get(0).([]store.Comment)
+ r0 = ret.Get(0).([]store.PostInfo)
}
}
var r1 error
- if rf, ok := ret.Get(1).(func(string, int, time.Time) error); ok {
- r1 = rf(siteID, limit, since)
+ if rf, ok := ret.Get(1).(func(InfoRequest) error); ok {
+ r1 = rf(req)
} else {
r1 = ret.Error(1)
}
@@ -261,22 +167,22 @@ func (_m *MockInterface) Last(siteID string, limit int, since time.Time) ([]stor
return r0, r1
}
-// List provides a mock function with given fields: siteID, limit, skip
-func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostInfo, error) {
- ret := _m.Called(siteID, limit, skip)
+// ListFlags provides a mock function with given fields: req
+func (_m *MockInterface) ListFlags(req FlagRequest) ([]interface{}, error) {
+ ret := _m.Called(req)
- var r0 []store.PostInfo
- if rf, ok := ret.Get(0).(func(string, int, int) []store.PostInfo); ok {
- r0 = rf(siteID, limit, skip)
+ var r0 []interface{}
+ if rf, ok := ret.Get(0).(func(FlagRequest) []interface{}); ok {
+ r0 = rf(req)
} else {
if ret.Get(0) != nil {
- r0 = ret.Get(0).([]store.PostInfo)
+ r0 = ret.Get(0).([]interface{})
}
}
var r1 error
- if rf, ok := ret.Get(1).(func(string, int, int) error); ok {
- r1 = rf(siteID, limit, skip)
+ if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+ r1 = rf(req)
} else {
r1 = ret.Error(1)
}
@@ -284,125 +190,16 @@ func (_m *MockInterface) List(siteID string, limit int, skip int) ([]store.PostI
return r0, r1
}
-// Put provides a mock function with given fields: locator, comment
-func (_m *MockInterface) Put(locator store.Locator, comment store.Comment) error {
- ret := _m.Called(locator, comment)
-
- var r0 error
- if rf, ok := ret.Get(0).(func(store.Locator, store.Comment) error); ok {
- r0 = rf(locator, comment)
- } else {
- r0 = ret.Error(0)
- }
-
- return r0
-}
-
-// SetBlock provides a mock function with given fields: siteID, userID, status, ttl
-func (_m *MockInterface) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
- ret := _m.Called(siteID, userID, status, ttl)
-
- var r0 error
- if rf, ok := ret.Get(0).(func(string, string, bool, time.Duration) error); ok {
- r0 = rf(siteID, userID, status, ttl)
- } else {
- r0 = ret.Error(0)
- }
-
- return r0
-}
-
-// SetReadOnly provides a mock function with given fields: locator, status
-func (_m *MockInterface) SetReadOnly(locator store.Locator, status bool) error {
- ret := _m.Called(locator, status)
-
- var r0 error
- if rf, ok := ret.Get(0).(func(store.Locator, bool) error); ok {
- r0 = rf(locator, status)
- } else {
- r0 = ret.Error(0)
- }
-
- return r0
-}
-
-// SetVerified provides a mock function with given fields: siteID, userID, status
-func (_m *MockInterface) SetVerified(siteID string, userID string, status bool) error {
- ret := _m.Called(siteID, userID, status)
+// Update provides a mock function with given fields: comment
+func (_m *MockInterface) Update(comment store.Comment) error {
+ ret := _m.Called(comment)
var r0 error
- if rf, ok := ret.Get(0).(func(string, string, bool) error); ok {
- r0 = rf(siteID, userID, status)
+ if rf, ok := ret.Get(0).(func(store.Comment) error); ok {
+ r0 = rf(comment)
} else {
r0 = ret.Error(0)
}
return r0
}
-
-// User provides a mock function with given fields: siteID, userID, limit, skip
-func (_m *MockInterface) User(siteID string, userID string, limit int, skip int) ([]store.Comment, error) {
- ret := _m.Called(siteID, userID, limit, skip)
-
- var r0 []store.Comment
- if rf, ok := ret.Get(0).(func(string, string, int, int) []store.Comment); ok {
- r0 = rf(siteID, userID, limit, skip)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).([]store.Comment)
- }
- }
-
- var r1 error
- if rf, ok := ret.Get(1).(func(string, string, int, int) error); ok {
- r1 = rf(siteID, userID, limit, skip)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// UserCount provides a mock function with given fields: siteID, userID
-func (_m *MockInterface) UserCount(siteID string, userID string) (int, error) {
- ret := _m.Called(siteID, userID)
-
- var r0 int
- if rf, ok := ret.Get(0).(func(string, string) int); ok {
- r0 = rf(siteID, userID)
- } else {
- r0 = ret.Get(0).(int)
- }
-
- var r1 error
- if rf, ok := ret.Get(1).(func(string, string) error); ok {
- r1 = rf(siteID, userID)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// Verified provides a mock function with given fields: siteID
-func (_m *MockInterface) Verified(siteID string) ([]string, error) {
- ret := _m.Called(siteID)
-
- var r0 []string
- if rf, ok := ret.Get(0).(func(string) []string); ok {
- r0 = rf(siteID)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).([]string)
- }
- }
-
- var r1 error
- if rf, ok := ret.Get(1).(func(string) error); ok {
- r1 = rf(siteID)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine/mongo.go
deleted file mode 100644
index 3520711d34..0000000000
--- a/backend/app/store/engine/mongo.go
+++ /dev/null
@@ -1,380 +0,0 @@
-package engine
-
-import (
- "time"
-
- "github.com/globalsign/mgo"
- "github.com/globalsign/mgo/bson"
- "github.com/go-pkgz/mongo"
- "github.com/hashicorp/go-multierror"
- "github.com/pkg/errors"
-
- "github.com/umputun/remark/backend/app/store"
-)
-
-// Mongo implements engine interface
-type Mongo struct {
- conn *mongo.Connection
- postWriter mongo.BufferedWriter
-}
-
-const (
- mongoPosts = "posts"
- mongoMetaPosts = "meta_posts"
- mongoMetaUsers = "meta_users"
-)
-
-type metaPost struct {
- ID string `bson:"_id"` // url
- SiteID string `bson:"site"`
- ReadOnly bool `bson:"read_only"`
-}
-
-type metaUser struct {
- ID string `bson:"_id"` // user_id
- SiteID string `bson:"site"`
- Verified bool `bson:"verified"`
- Blocked bool `bson:"blocked"`
- BlockedUntil time.Time `bson:"blocked_until"`
-}
-
-// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
-// flushDuration triggers automatic flush (write from buffer), 0 disables it and will flush as buffer size reached.
-// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
-func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
- writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
- result := Mongo{conn: conn, postWriter: writer}
- err := result.prepare()
- return &result, errors.Wrap(err, "failed to prepare mongo")
-}
-
-// Create new comment, write can be buffered and delayed.
-func (m *Mongo) Create(comment store.Comment) (commentID string, err error) {
- // err = m.postWriter.Write(comment)
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- return coll.Insert(&comment)
- })
- return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
- comments = []store.Comment{}
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}
- return coll.Find(query).Sort(sortFld).All(&comments)
- })
- return comments, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL}
- return coll.Find(query).One(&comment)
- })
- return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
- return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL},
- bson.M{"$set": bson.M{
- "text": comment.Text,
- "orig": comment.Orig,
- "score": comment.Score,
- "votes": comment.Votes,
- "pin": comment.Pin,
- "deleted": comment.Deleted,
- }})
- })
-}
-
-// Last returns up to max last comments for given siteID
-func (m *Mongo) Last(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
- comments = []store.Comment{}
- if max > lastLimit || max == 0 {
- max = lastLimit
- }
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- query := bson.M{"locator.site": siteID, "delete": false}
- if !since.IsZero() {
- query["time"] = bson.M{"$gt": since}
- }
- return coll.Find(query).Sort("-time").Limit(max).All(&comments)
- })
- return comments, err
-}
-
-// Count returns number of comments for locator
-func (m *Mongo) Count(locator store.Locator) (count int, err error) {
-
- e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false}
- count, err = coll.Find(query).Count()
- return err
- })
- return count, e
-}
-
-// List returns list of all commented posts with counters
-func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
- list = []store.PostInfo{}
-
- if limit <= 0 {
- limit = 1000
- }
- if skip < 0 {
- skip = 0
- }
-
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- pipeline := coll.Pipe([]bson.M{
- {"$match": bson.M{"locator.site": siteID}},
- {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
- {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
- "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
- {"$skip": skip},
- {"$limit": limit},
- })
- return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
- })
- return list, errors.Wrap(err, "can't get list")
-}
-
-// Info returns time range and count for locator
-func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) {
- list := []store.PostInfo{}
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- pipeline := coll.Pipe([]bson.M{
- {"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}},
- {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
- {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
- "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
- })
- return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
- })
- if err != nil {
- return info, err
- }
- if len(list) == 0 {
- return info, errors.Errorf("can't load info for %s", locator.URL)
- }
- info = list[0]
- // set read-only from age and manual bucket
- info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
- if m.IsReadOnly(locator) {
- info.ReadOnly = true
- }
- return info, nil
-}
-
-// User extracts all comments for given site and given userID
-func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
- comments = []store.Comment{}
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- query := bson.M{"locator.site": siteID, "user.id": userID}
- return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments)
- })
- return comments, errors.Wrapf(err, "can't get comments for user %s", userID)
-}
-
-// UserCount returns number of comments for user
-func (m *Mongo) UserCount(siteID, userID string) (count int, err error) {
- err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- var e error
- count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count()
- return e
- })
- return count, errors.Wrapf(err, "can't get comments count for user %s", userID)
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) {
- return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
- _, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}})
- return e
- })
-}
-
-// IsReadOnly checks if post in RO
-func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) {
- meta := metaPost{}
- err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
- return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta)
- })
- return err == nil && meta.ReadOnly
-}
-
-// SetVerified makes user verified or reset the flag
-func (m *Mongo) SetVerified(siteID string, userID string, status bool) error {
- return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
- _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}})
- return e
- })
-}
-
-// IsVerified checks if user verified
-func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
- meta := metaUser{}
- err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
- return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
- })
- return err == nil && meta.Verified
-}
-
-// Verified returns list of verified user IDs
-func (m *Mongo) Verified(siteID string) (ids []string, err error) {
- metas := []metaUser{}
- err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
- return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas)
- })
- if err != nil {
- return nil, err
- }
- for _, meta := range metas {
- ids = append(ids, meta.ID)
- }
- return ids, nil
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
- until := time.Time{}
- if status {
- until = time.Now().AddDate(100, 0, 0) // permanent is 50year
- if ttl > 0 {
- until = time.Now().Add(ttl)
- }
- }
- return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
- _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID},
- bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}})
- return errors.Wrapf(e, "failed to set block for %s", userID)
- })
-}
-
-// IsBlocked checks if user blocked
-func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) {
- meta := metaUser{}
- err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
- return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
- })
- return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now())
-}
-
-// Blocked get lists of blocked users for given site
-func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) {
- users = []store.BlockedUser{}
- metas := []metaUser{}
- err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
- return coll.Find(bson.M{"site": siteID,
- "blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas)
- })
- if err != nil {
- return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID)
- }
-
- for _, mu := range metas {
- blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil}
- if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 {
- blockedUser.Name = ucc[0].User.Name
- }
- users = append(users, blockedUser)
- }
- return users, nil
-}
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
- comment := store.Comment{}
- err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment)
- if e != nil {
- return e
- }
- comment.SetDeleted(mode)
- return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment)
- })
- return errors.Wrapf(err, "can't delete %s", commentID)
-}
-
-// DeleteAll removes all info about siteID
-func (m *Mongo) DeleteAll(siteID string) error {
- err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- _, e := coll.RemoveAll(bson.M{"locator.site": siteID})
- return e
- })
- return errors.Wrapf(err, "can't delete site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted".
-func (m *Mongo) DeleteUser(siteID string, userID string) error {
- comments := []store.Comment{}
- return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments)
- if e != nil {
- return e
- }
- for _, c := range comments {
- if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil {
- return e
- }
- }
- return nil
- })
-}
-
-// Close boltdb store
-func (m *Mongo) Close() error {
- if m.postWriter != nil {
- return m.postWriter.Close()
- }
- return nil
-}
-
-// prepare collections with all indexes
-func (m *Mongo) prepare() error {
- errs := new(multierror.Error)
- e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
- errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time"))
- errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time"))
- errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time"))
- errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score"))
- return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts)
- })
- if e != nil {
- return e
- }
-
- e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
- errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
- errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only"))
- return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts)
- })
- if e != nil {
- return e
- }
-
- return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
- errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
- errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked"))
- errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified"))
- return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers)
- })
-}
-
-func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query {
- if limit <= 0 {
- limit = 1000
- }
- if skip < 0 {
- skip = 0
- }
- return q.Skip(skip).Limit(limit)
-}
diff --git a/backend/app/store/engine/mongo_test.go b/backend/app/store/engine/mongo_test.go
deleted file mode 100644
index 5b3f4b27bd..0000000000
--- a/backend/app/store/engine/mongo_test.go
+++ /dev/null
@@ -1,604 +0,0 @@
-package engine
-
-import (
- "fmt"
- "math/rand"
- "testing"
- "time"
-
- "github.com/go-pkgz/mongo"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/umputun/remark/backend/app/store"
-)
-
-func TestMongo_CreateAndFind(t *testing.T) {
- var m Interface
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
- assert.Nil(t, err)
- require.Equal(t, 2, len(res))
- assert.Equal(t, `some text, link`, res[0].Text)
- assert.Equal(t, "user1", res[0].User.ID)
- t.Log(res[0].ID)
-
- _, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
- assert.NotNil(t, err, "reject dup")
-
- id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
- assert.Nil(t, err)
- assert.Equal(t, "id-3", id)
- res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time")
- assert.Nil(t, err)
- require.Equal(t, 1, len(res))
-
- assert.NoError(t, m.Close())
-}
-
-func TestMongo_Get(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
-
- comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
- assert.Nil(t, err)
- assert.Equal(t, "some text2", comment.Text)
-
- comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
- assert.NotNil(t, err, "not found")
-}
-
-func TestMongo_Put(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := m.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
-
- comment := res[0]
- comment.Text = "abc 123"
- comment.Score = 100
- err = m.Put(loc, comment)
- assert.Nil(t, err)
-
- comment, err = m.Get(loc, res[0].ID)
- assert.Nil(t, err)
- assert.Equal(t, "abc 123", comment.Text)
- assert.Equal(t, res[0].ID, comment.ID)
- assert.Equal(t, 100, comment.Score)
-
- err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
- assert.EqualError(t, err, `not found`)
-
- err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
- assert.EqualError(t, err, `not found`)
-}
-
-func TestMongo_Last(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- res, err := m.Last("radio-t", 0, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-
- res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local))
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-
- res, err = m.Last("radio-t", 0, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local))
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-
- res, err = m.Last("radio-t", 1, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res))
- assert.Equal(t, "some text2", res[0].Text)
-}
-
-func TestMongo_Count(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 2, c)
-
- c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 0, c)
-}
-
-func TestMongo_List(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- // add one more for https://radio-t.com/2
- comment := store.Comment{
- ID: "12345",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- _, err := m.Create(comment)
- assert.Nil(t, err)
-
- ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
-
- res, err := m.List("radio-t", 0, 0)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
- {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
- res)
-
- res, err = m.List("radio-t", -1, -1)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
- {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
- res, err = m.List("radio-t", 1, 0)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
- res, err = m.List("radio-t", 1, 1)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
- res, err = m.List("bad", 1, 1)
- assert.Nil(t, err)
- assert.Equal(t, []store.PostInfo{}, res)
-}
-
-func TestMongo_Info(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
-
- // add one more for https://radio-t.com/2
- comment := store.Comment{
- ID: "12345",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- _, err := m.Create(comment)
- assert.Nil(t, err)
-
- r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
- r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
- r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
- _, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
- require.NotNil(t, err)
-
- _, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
- require.NotNil(t, err)
-
- err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
- require.Nil(t, err)
- r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
- require.Nil(t, err)
- assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-}
-
-func TestMongo_ReadOnly(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
- assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
- assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
- assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
- assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
- assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
- assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"")
- assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
- assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestMongo_Verified(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified")
-
- assert.NoError(t, m.SetVerified("radio-t", "u1", true))
- assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified")
-
- assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified")
- assert.NoError(t, m.SetVerified("radio-t", "u1", false))
- assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
- assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`)
- assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false))
-
- assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-
- assert.NoError(t, m.SetVerified("radio-t", "u1", true))
- assert.NoError(t, m.SetVerified("radio-t", "u2", true))
- assert.NoError(t, m.SetVerified("radio-t", "u3", false))
-
- ids, err := m.Verified("radio-t")
- assert.NoError(t, err)
- assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
-
- ids, err = m.Verified("radio-t-bad")
- assert.NoError(t, err)
- assert.Equal(t, 0, len(ids))
-}
-
-func TestMongo_GetForUser(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- res, err := m.User("radio-t", "user1", 5, 0)
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
- res, err = m.User("radio-t", "user1", 1, 0)
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res), "allow 1 comment")
- assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
- res, err = m.User("radio-t", "user1", 1, 1)
- assert.Nil(t, err)
- assert.Equal(t, 1, len(res), "allow 1 comment")
- assert.Equal(t, `some text, link`, res[0].Text, "second comment")
-
- res, err = m.User("bad", "user1", 1, 0)
- assert.Nil(t, err)
- assert.Equal(t, 0, len(res))
-}
-
-func TestMongo_GetForUserPagination(t *testing.T) {
- m, skip := prepMongo(t, false)
- if skip {
- return
- }
- c := store.Comment{
- Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
-
- // write 200 comments
- for i := 0; i < 200; i++ {
- c.ID = fmt.Sprintf("id-%d", i)
- c.Text = fmt.Sprintf("text #%d", i)
- c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
- _, err := m.Create(c)
- require.Nil(t, err, c.ID)
- }
-
- // get all comments
- res, err := m.User("radio-t", "user1", 0, 0)
- assert.Nil(t, err)
- assert.Equal(t, 200, len(res))
- assert.Equal(t, "id-199", res[0].ID)
-
- // seek 0, 5 comments
- res, err = m.User("radio-t", "user1", 5, 0)
- assert.Nil(t, err)
- assert.Equal(t, 5, len(res))
- assert.Equal(t, "id-199", res[0].ID)
- assert.Equal(t, "id-195", res[4].ID)
-
- // seek 10, 3 comments
- res, err = m.User("radio-t", "user1", 3, 10)
- assert.Nil(t, err)
- assert.Equal(t, 3, len(res))
- assert.Equal(t, "id-189", res[0].ID)
- assert.Equal(t, "id-187", res[2].ID)
-
- // seek 195, ask 10 comments
- res, err = m.User("radio-t", "user1", 10, 195)
- assert.Nil(t, err)
- assert.Equal(t, 5, len(res))
- assert.Equal(t, "id-4", res[0].ID)
- assert.Equal(t, "id-0", res[4].ID)
-
- // seek 255, ask 10 comments
- res, err = m.User("radio-t", "user1", 10, 255)
- assert.Nil(t, err)
- assert.Equal(t, 0, len(res))
-}
-
-func TestMongo_BlockUser(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
-
- assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
- assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
-
- assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
- assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0))
- assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
- assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
- assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0))
-
- assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestMongo_BlockUserWithTTL(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
- assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond))
- assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
- time.Sleep(500 * time.Millisecond)
- assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
-}
-
-func TestMongo_GetForUserCounter(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- count, err := m.UserCount("radio-t", "user1")
- assert.Nil(t, err)
- assert.Equal(t, 2, count)
-
- count, err = m.UserCount("bad", "user1")
- assert.Nil(t, err)
- assert.Equal(t, 0, count)
-}
-
-func TestMongo_BlockList(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
- assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond))
- assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0))
-
- ids, err := m.Blocked("radio-t")
- assert.NoError(t, err)
-
- assert.Equal(t, 2, len(ids))
- assert.Equal(t, "user1", ids[0].ID)
- assert.Equal(t, "user2", ids[1].ID)
- t.Logf("%+v", ids)
-
- time.Sleep(500 * time.Millisecond)
- ids, err = m.Blocked("radio-t")
- assert.NoError(t, err)
- assert.Equal(t, 1, len(ids))
- assert.Equal(t, "user1", ids[0].ID)
-
- ids, err = m.Blocked("bad")
- assert.NoError(t, err)
- assert.Equal(t, 0, len(ids))
-}
-
-func TestMongo_Delete(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := m.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "initially 2 comments")
-
- err = m.Delete(loc, res[0].ID, store.SoftDelete)
- assert.Nil(t, err)
-
- res, err = m.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "", res[0].Text)
- assert.True(t, res[0].Deleted, "marked deleted")
- assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
- assert.Equal(t, "some text2", res[1].Text)
- assert.False(t, res[1].Deleted)
-
- comments, err := m.Last("radio-t", 10, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
- err = m.Delete(loc, "123456", store.SoftDelete)
- assert.NotNil(t, err)
-
- loc.SiteID = "bad"
- err = m.Delete(loc, res[0].ID, store.SoftDelete)
- assert.EqualError(t, err, `can't delete id-1: not found`)
-
- loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
- err = m.Delete(loc, res[0].ID, store.SoftDelete)
- assert.EqualError(t, err, `can't delete id-1: not found`)
-}
-
-func TestMongo_DeleteHard(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := m.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "initially 2 comments")
-
- err = m.Delete(loc, res[0].ID, store.HardDelete)
- assert.Nil(t, err)
-
- res, err = m.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res))
- assert.Equal(t, "", res[0].Text)
- assert.True(t, res[0].Deleted, "marked deleted")
- assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestMongo_DeleteAll(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := m.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "initially 2 comments")
-
- err = m.DeleteAll("radio-t")
- assert.Nil(t, err)
-
- comments, err := m.Last("radio-t", 10, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 0, len(comments), "nothing left")
-
- c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 0, c, "0 count")
-}
-
-func TestMongo_DeleteUser(t *testing.T) {
- m, skip := prepMongo(t, true) // adds two comments
- if skip {
- return
- }
- err := m.DeleteUser("radio-t", "user1")
- require.NoError(t, err)
-
- loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
- res, err := m.Find(loc, "time")
- assert.Nil(t, err)
- assert.Equal(t, 2, len(res), "2 comments with deleted info")
- assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
- assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
- c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
- assert.Nil(t, err)
- assert.Equal(t, 0, c, "0 count")
-
- cc, err := m.User("radio-t", "user1", 5, 0)
- assert.Nil(t, err, "no comments for user user1 in store")
- assert.Equal(t, 0, len(cc), "no comments for user user1 in store")
-
- comments, err := m.Last("radio-t", 10, time.Time{})
- assert.Nil(t, err)
- assert.Equal(t, 0, len(comments), "nothing left")
-}
-
-func TestMongo_Parallel(t *testing.T) {
- var m Interface
- var skip bool
- m, skip = prepMongoBuffered(t) // buffered engine, no comments
- if skip {
- return
- }
- go func() {
- for i := 0; i < 100; i++ {
- _, err := m.Create(store.Comment{
- ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
- require.Nil(t, err)
- time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)
- }
- }()
-
- for {
- time.Sleep(10 * time.Millisecond)
- res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
- assert.Nil(t, err)
- if len(res) == 100 {
- break
- }
- }
-}
-
-func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) {
- conn, err := mongo.MakeTestConnection(t)
- if err != nil {
- return nil, true
- }
- mongo.RemoveTestCollection(t, conn)
-
- m, err := NewMongo(conn, 1, 0*time.Microsecond)
- require.Nil(t, err)
-
- mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
- comment := store.Comment{
- ID: "id-1",
- Text: `some text, link`,
- Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- if writeRecs {
- _, err = m.Create(comment)
- assert.Nil(t, err)
- }
-
- comment = store.Comment{
- ID: "id-2",
- Text: "some text2",
- Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
- Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
- User: store.User{ID: "user1", Name: "user name"},
- }
- if writeRecs {
- _, err = m.Create(comment)
- assert.Nil(t, err)
- }
-
- return m, false
-}
-
-func prepMongoBuffered(t *testing.T) (*Mongo, bool) {
- conn, err := mongo.MakeTestConnection(t)
- if err != nil {
- return nil, true
- }
- mongo.RemoveTestCollection(t, conn)
-
- m, err := NewMongo(conn, 10, 10*time.Millisecond)
- mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
-
- require.Nil(t, err)
- return m, false
-}
diff --git a/backend/app/store/engine/remote.go b/backend/app/store/engine/remote.go
new file mode 100644
index 0000000000..f9b8b16b38
--- /dev/null
+++ b/backend/app/store/engine/remote.go
@@ -0,0 +1,104 @@
+package engine
+
+import (
+ "encoding/json"
+
+ "github.com/umputun/remark/backend/app/remote"
+ "github.com/umputun/remark/backend/app/store"
+)
+
+// Remote implements remote engine and delegates all Calls to remote http server
+type Remote struct {
+ remote.Client
+}
+
+// Create comment and return ID
+func (r *Remote) Create(comment store.Comment) (commentID string, err error) {
+
+ resp, err := r.Call("store.create", comment)
+ if err != nil {
+ return "", err
+ }
+
+ err = json.Unmarshal(*resp.Result, &commentID)
+ return commentID, err
+}
+
+// Get comment by ID
+func (r *Remote) Get(req GetRequest) (comment store.Comment, err error) {
+ resp, err := r.Call("store.get", req)
+ if err != nil {
+ return store.Comment{}, err
+ }
+
+ err = json.Unmarshal(*resp.Result, &comment)
+ return comment, err
+}
+
+// Update comment, mutable parts only
+func (r *Remote) Update(comment store.Comment) error {
+ _, err := r.Call("store.update", comment)
+ return err
+}
+
+// Find comments for locator
+func (r *Remote) Find(req FindRequest) (comments []store.Comment, err error) {
+ resp, err := r.Call("store.find", req)
+ if err != nil {
+ return nil, err
+ }
+ err = json.Unmarshal(*resp.Result, &comments)
+ return comments, err
+}
+
+// Info returns post(s) meta info
+func (r *Remote) Info(req InfoRequest) (info []store.PostInfo, err error) {
+ resp, err := r.Call("store.info", req)
+ if err != nil {
+ return nil, err
+ }
+ err = json.Unmarshal(*resp.Result, &info)
+ return info, err
+}
+
+// Flag sets and gets flags
+func (r *Remote) Flag(req FlagRequest) (status bool, err error) {
+ resp, err := r.Call("store.flag", req)
+ if err != nil {
+ return false, err
+ }
+ err = json.Unmarshal(*resp.Result, &status)
+ return status, err
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+func (r *Remote) ListFlags(req FlagRequest) (list []interface{}, err error) {
+ resp, err := r.Call("store.list_flags", req)
+ if err != nil {
+ return nil, err
+ }
+ err = json.Unmarshal(*resp.Result, &list)
+ return list, err
+}
+
+// Count gets comments count by user or site
+func (r *Remote) Count(req FindRequest) (count int, err error) {
+ resp, err := r.Call("store.count", req)
+ if err != nil {
+ return 0, err
+ }
+ err = json.Unmarshal(*resp.Result, &count)
+ return count, err
+}
+
+// Delete post(s) by id or by userID
+func (r *Remote) Delete(req DeleteRequest) error {
+ _, err := r.Call("store.delete", req)
+ return err
+}
+
+// Close storage engine
+func (r *Remote) Close() error {
+ _, err := r.Call("store.close")
+ return err
+}
diff --git a/backend/app/store/engine/remote_test.go b/backend/app/store/engine/remote_test.go
new file mode 100644
index 0000000000..459e370522
--- /dev/null
+++ b/backend/app/store/engine/remote_test.go
@@ -0,0 +1,178 @@
+package engine
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/remote"
+ "github.com/umputun/remark/backend/app/store"
+)
+
+func TestRemote_Create(t *testing.T) {
+ ts := testServer(t, `{"method":"store.create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`,
+ `{"result":"12345","id":1}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ var eng Interface = &c
+ _ = eng
+
+ res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"},
+ Text: "msg"})
+ assert.NoError(t, err)
+ assert.Equal(t, "12345", res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Get(t *testing.T) {
+ ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+ res, err := c.Get(req)
+ assert.NoError(t, err)
+ assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res)
+ t.Logf("%v %T", res, res)
+}
+
+func TestRemote_GetWithErrorResult(t *testing.T) {
+ ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"error":"failed"}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+ _, err := c.Get(req)
+ assert.EqualError(t, err, "failed")
+}
+
+func TestRemote_GetWithErrorDecode(t *testing.T) {
+ ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, ``)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+ _, err := c.Get(req)
+ assert.EqualError(t, err, "failed to decode response for store.get: EOF")
+}
+
+func TestRemote_GetWithErrorRemote(t *testing.T) {
+ c := Remote{Client: remote.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}}
+
+ req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+ _, err := c.Get(req)
+ assert.NotNil(t, err)
+ assert.True(t, strings.Contains(err.Error(), "remote call failed for store.get:"), err.Error())
+}
+
+func TestRemote_FailedStatus(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := ioutil.ReadAll(r.Body)
+ require.NoError(t, err)
+ t.Logf("req: %s", string(body))
+ w.WriteHeader(400)
+ }))
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+ _, err := c.Get(req)
+ assert.EqualError(t, err, "bad status 400 Bad Request for store.get")
+}
+
+func TestRemote_Update(t *testing.T) {
+ ts := testServer(t, `{"method":"store.update","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ err := c.Update(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"},
+ Text: "msg"})
+ assert.NoError(t, err)
+
+}
+
+func TestRemote_Find(t *testing.T) {
+ ts := testServer(t, `{"method":"store.find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ res, err := c.Find(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}, Sort: "-time", Limit: 10})
+ assert.NoError(t, err)
+ assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res)
+}
+
+func TestRemote_Info(t *testing.T) {
+ ts := testServer(t, `{"method":"store.info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ res, err := c.Info(InfoRequest{Locator: store.Locator{URL: "http://example.com/url"},
+ Limit: 10, Skip: 5, ReadOnlyAge: 10})
+ assert.NoError(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "u1", Count: 22}, {URL: "u2", Count: 33}}, res)
+}
+
+func TestRemote_Flag(t *testing.T) {
+ ts := testServer(t, `{"method":"store.flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, `{"result":false}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ res, err := c.Flag(FlagRequest{Locator: store.Locator{URL: "http://example.com/url"}, Flag: Verified})
+ assert.NoError(t, err)
+ assert.Equal(t, false, res)
+}
+
+func TestRemote_ListFlag(t *testing.T) {
+ ts := testServer(t, `{"method":"store.list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+ res, err := c.ListFlags(FlagRequest{Locator: store.Locator{SiteID: "site_id"}, Flag: Blocked})
+ assert.NoError(t, err)
+ assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res)
+}
+
+func TestRemote_Count(t *testing.T) {
+ ts := testServer(t, `{"method":"store.count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ res, err := c.Count(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}})
+ assert.NoError(t, err)
+ assert.Equal(t, 11, res)
+}
+
+func TestRemote_Delete(t *testing.T) {
+ ts := testServer(t, `{"method":"store.delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`,
+ `{}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+
+ err := c.Delete(DeleteRequest{Locator: store.Locator{URL: "http://example.com/url"}})
+ assert.NoError(t, err)
+}
+
+func TestRemote_Close(t *testing.T) {
+ ts := testServer(t, `{"method":"store.close","id":1}`, `{}`)
+ defer ts.Close()
+ c := Remote{Client: remote.Client{API: ts.URL, Client: http.Client{}}}
+ err := c.Close()
+ assert.NoError(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := ioutil.ReadAll(r.Body)
+ require.NoError(t, err)
+ assert.Equal(t, req, string(body))
+ t.Logf("req: %s", string(body))
+ fmt.Fprintf(w, resp)
+ }))
+}
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 6719e042e4..5a537bc661 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -24,7 +24,7 @@ import (
// DataStore wraps store.Interface with additional methods
type DataStore struct {
- engine.Interface
+ Engine engine.Interface
EditDuration time.Duration
AdminStore admin.Store
MaxCommentSize int
@@ -98,12 +98,14 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
}()
s.submitImages(comment)
- return s.Interface.Create(comment)
+ return s.Engine.Create(comment)
}
// Find wraps engine's Find call and alter results if needed
+// user used to filter results for self vs others
func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) {
- comments, err := s.Interface.Find(locator, sort)
+ req := engine.FindRequest{Locator: locator, Sort: sort}
+ comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
}
@@ -130,19 +132,26 @@ func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([
// Get comment by ID
func (s *DataStore) Get(locator store.Locator, commentID string, user store.User) (store.Comment, error) {
- c, err := s.Interface.Get(locator, commentID)
+ c, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
if err != nil {
return store.Comment{}, err
}
return s.alterComment(c, user), nil
}
+// Put updates comment, mutable parts only
+func (s *DataStore) Put(locator store.Locator, comment store.Comment) error {
+ comment.Locator = locator
+ return s.Engine.Update(comment)
+}
+
// submitImages initiated delayed commit of all images from the comment uploaded to remark42
func (s *DataStore) submitImages(comment store.Comment) {
s.ImageService.Submit(func() []string {
c := comment
- cc, err := s.Interface.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+ // this can be called after last edit, we have to retrieve fresh comment
+ cc, err := s.Engine.Get(engine.GetRequest{Locator: c.Locator, CommentID: c.ID})
if err != nil {
log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
return nil
@@ -182,14 +191,21 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err
return comment, nil
}
+// DeleteAll removes all data from site
+func (s *DataStore) DeleteAll(siteID string) error {
+ req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}}
+ return s.Engine.Delete(req)
+}
+
// SetPin pin/un-pin comment as special
func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error {
- comment, err := s.Interface.Get(locator, commentID)
+ comment, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
if err != nil {
return err
}
comment.Pin = status
- return s.Put(locator, comment)
+ comment.Locator = locator
+ return s.Engine.Update(comment)
}
// Vote for comment by id and locator
@@ -199,7 +215,7 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
cLock.Lock() // prevents race on voting
defer cLock.Unlock()
- comment, err = s.Interface.Get(locator, commentID)
+ comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
if err != nil {
return comment, err
}
@@ -257,8 +273,8 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
}
comment.Controversy = s.controversy(s.upsAndDowns(comment))
-
- return comment, s.Put(locator, comment)
+ comment.Locator = locator
+ return comment, s.Engine.Update(comment)
}
// controversy calculates controversial index of votes
@@ -287,7 +303,7 @@ type EditRequest struct {
// EditComment to edit text and update Edit info
func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) {
- comment, err = s.Interface.Get(locator, commentID)
+ comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
if err != nil {
return comment, err
}
@@ -303,7 +319,8 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
if req.Delete { // delete request
comment.Deleted = true
- return comment, s.Delete(locator, commentID, store.SoftDelete)
+ delReq := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: store.SoftDelete}
+ return comment, s.Engine.Delete(delReq)
}
if s.RestrictedWordsMatcher != nil && s.RestrictedWordsMatcher.Match(comment.Locator.SiteID, req.Text) {
@@ -316,9 +333,10 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
Timestamp: time.Now(),
Summary: req.Summary,
}
-
+ comment.Locator = locator
comment.Sanitize()
- err = s.Put(locator, comment)
+
+ err = s.Engine.Update(comment)
return comment, err
}
@@ -336,7 +354,8 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
return true
}
- comments, err := s.Interface.Last(comment.Locator.SiteID, maxLastCommentsReply, time.Time{})
+ req := engine.FindRequest{Locator: store.Locator{SiteID: comment.Locator.SiteID}, Limit: maxLastCommentsReply}
+ comments, err := s.Engine.Find(req)
if err != nil {
log.Printf("[WARN] can't get last comments for reply check, %v", err)
return false
@@ -395,7 +414,7 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
return comment, errors.New("no title extractor")
}
- comment, err = s.Interface.Get(locator, commentID)
+ comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
if err != nil {
return comment, err
}
@@ -406,7 +425,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
return comment, err
}
comment.PostTitle = title
- err = s.Put(locator, comment)
+ comment.Locator = locator
+ err = s.Engine.Update(comment)
return comment, err
}
@@ -414,7 +434,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
res := []store.PostInfo{}
for _, p := range postIDs {
- if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil {
+ req := engine.FindRequest{Locator: store.Locator{SiteID: siteID, URL: p}}
+ if c, err := s.Engine.Count(req); err == nil {
res = append(res, store.PostInfo{URL: p, Count: c})
}
}
@@ -441,7 +462,11 @@ func (s *DataStore) ValidateComment(c *store.Comment) error {
// IsAdmin checks if usesID in the list of admins
func (s *DataStore) IsAdmin(siteID string, userID string) bool {
- for _, a := range s.AdminStore.Admins(siteID) {
+ admins, err := s.AdminStore.Admins(siteID)
+ if err != nil {
+ return false
+ }
+ for _, a := range admins {
if a == userID {
return true
}
@@ -449,20 +474,126 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool {
return false
}
+// IsReadOnly checks if post read-only
+func (s *DataStore) IsReadOnly(locator store.Locator) bool {
+ req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly}
+ ro, err := s.Engine.Flag(req)
+ return err == nil && ro
+}
+
+// SetReadOnly set/reset read-only flag
+func (s *DataStore) SetReadOnly(locator store.Locator, status bool) error {
+ roStatus := engine.FlagFalse
+ if status {
+ roStatus = engine.FlagTrue
+
+ }
+ req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly, Update: roStatus}
+ _, err := s.Engine.Flag(req)
+ return err
+}
+
+// IsVerified checks if user verified
+func (s *DataStore) IsVerified(siteID string, userID string) bool {
+ req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified}
+ ro, err := s.Engine.Flag(req)
+ return err == nil && ro
+}
+
+// SetVerified set/reset verified status for user
+func (s *DataStore) SetVerified(siteID string, userID string, status bool) error {
+ roStatus := engine.FlagFalse
+ if status {
+ roStatus = engine.FlagTrue
+ }
+ req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified, Update: roStatus}
+ _, err := s.Engine.Flag(req)
+ return err
+}
+
+// IsBlocked checks if user blocked
+func (s *DataStore) IsBlocked(siteID string, userID string) bool {
+ req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Blocked}
+ ro, err := s.Engine.Flag(req)
+ return err == nil && ro
+}
+
+// SetBlock set/reset verified status for user
+func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
+ roStatus := engine.FlagFalse
+ if status {
+ roStatus = engine.FlagTrue
+ }
+ req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID,
+ Flag: engine.Blocked, Update: roStatus, TTL: ttl}
+ _, err := s.Engine.Flag(req)
+ return err
+}
+
+// Blocked returns list with all blocked users
+func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) {
+ blocked, e := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Blocked})
+ if e != nil {
+ return nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID)
+ }
+ for _, v := range blocked {
+ res = append(res, v.(store.BlockedUser))
+ }
+ return res, nil
+}
+
+// Info get post info
+func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) {
+ req := engine.InfoRequest{Locator: locator, ReadOnlyAge: readonlyAge}
+ res, err := s.Engine.Info(req)
+ if err != nil {
+ return store.PostInfo{}, err
+ }
+ if len(res) == 0 {
+ return store.PostInfo{}, errors.Errorf("post %+v not found", locator)
+ }
+ return res[0], nil
+}
+
+// Delete comment by id
+func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
+ req := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode}
+ return s.Engine.Delete(req)
+}
+
+// DeleteUser removes all comments from user
+func (s *DataStore) DeleteUser(siteID string, userID string, mode store.DeleteMode) error {
+ req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: mode}
+ return s.Engine.Delete(req)
+}
+
+// List of commented posts
+func (s *DataStore) List(siteID string, limit int, skip int) ([]store.PostInfo, error) {
+ req := engine.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip}
+ return s.Engine.Info(req)
+}
+
+// Count gets number of comments for the post
+func (s *DataStore) Count(locator store.Locator) (int, error) {
+ req := engine.FindRequest{Locator: locator}
+ return s.Engine.Count(req)
+}
+
// Metas returns metadata for users and posts
func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) {
umetas = []UserMetaData{}
pmetas = []PostMetaData{}
+
// set posts meta
- posts, err := s.List(siteID, 0, 0)
+ posts, err := s.Engine.Info(engine.InfoRequest{Locator: store.Locator{SiteID: siteID}})
if err != nil {
return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID)
}
+
for _, p := range posts {
if s.IsReadOnly(store.Locator{SiteID: siteID, URL: p.URL}) {
pmetas = append(pmetas, PostMetaData{URL: p.URL, ReadOnly: true})
}
-
}
// set users meta
@@ -484,11 +615,12 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
}
// process verified users
- verified, err := s.Verified(siteID)
+ verified, err := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Verified})
if err != nil {
return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID)
}
- for _, v := range verified {
+ for _, vi := range verified {
+ v := vi.(string)
val, ok := m[v]
if !ok {
val = UserMetaData{ID: v}
@@ -531,22 +663,35 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
// User gets comment for given userID on siteID
func (s *DataStore) User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error) {
- comments, err := s.Interface.User(siteID, userID, limit, skip)
+ req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Limit: limit, Skip: skip}
+ comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
}
return s.alterComments(comments, user), nil
}
+// UserCount is comments count by user
+func (s *DataStore) UserCount(siteID, userID string) (int, error) {
+ req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID}
+ return s.Engine.Count(req)
+}
+
// Last gets last comments for site, cross-post. Limited by count and optional since ts
func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error) {
- comments, err := s.Interface.Last(siteID, limit, since)
+ req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Since: since, Sort: "-time"}
+ comments, err := s.Engine.Find(req)
if err != nil {
return comments, err
}
return s.alterComments(comments, user), nil
}
+// Close store service
+func (s *DataStore) Close() error {
+ return s.Engine.Close()
+}
+
func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
for _, v := range c.Votes {
if v {
@@ -583,7 +728,9 @@ func (s *DataStore) alterComments(cc []store.Comment, user store.User) (res []st
func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Comment) {
- blocked := s.IsBlocked(c.Locator.SiteID, c.User.ID)
+ blocReq := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+ blocked, _ := s.Engine.Flag(blocReq)
+
// process blocked users
if blocked {
if !user.Admin { // reset comment to deleted for non-admins
@@ -595,7 +742,8 @@ func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Co
// set verified status retroactively
if !blocked {
- c.User.Verified = s.IsVerified(c.Locator.SiteID, c.User.ID)
+ verifReq := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+ c.User.Verified, _ = s.Engine.Flag(verifReq)
}
// hide info from non-admins
diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go
index d83d01b952..e19b659df5 100644
--- a/backend/app/store/service/service_test.go
+++ b/backend/app/store/service/service_test.go
@@ -30,7 +30,7 @@ var testDb = "/tmp/test-remark.db"
func TestService_CreateFromEmpty(t *testing.T) {
defer teardown(t)
ks := admin.NewStaticKeyStore("secret 123")
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks}
comment := store.Comment{
Text: "text",
User: store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
@@ -40,7 +40,7 @@ func TestService_CreateFromEmpty(t *testing.T) {
assert.NoError(t, err)
assert.True(t, id != "", id)
- res, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
+ res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id))
assert.NoError(t, err)
t.Logf("%+v", res)
assert.Equal(t, "text", res.Text)
@@ -54,7 +54,7 @@ func TestService_CreateFromEmpty(t *testing.T) {
func TestService_CreateFromPartial(t *testing.T) {
defer teardown(t)
ks := admin.NewStaticKeyStore("secret 123")
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks}
comment := store.Comment{
Text: "text",
Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
@@ -66,7 +66,7 @@ func TestService_CreateFromPartial(t *testing.T) {
assert.NoError(t, err)
assert.True(t, id != "", id)
- res, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
+ res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id))
assert.NoError(t, err)
t.Logf("%+v", res)
assert.Equal(t, "text", res.Text)
@@ -81,7 +81,7 @@ func TestService_CreateFromPartial(t *testing.T) {
func TestService_CreateFromPartialWithTitle(t *testing.T) {
defer teardown(t)
ks := admin.NewStaticKeyStore("secret 123")
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks,
TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
comment := store.Comment{
Text: "text",
@@ -94,7 +94,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
assert.NoError(t, err)
assert.True(t, id != "", id)
- res, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)
+ res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id))
assert.NoError(t, err)
t.Logf("%+v", res)
assert.Equal(t, "Радио-Т 630 — Радио-Т Подкаст", res.PostTitle)
@@ -102,7 +102,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
comment.PostTitle = "post blah"
id, err = b.Create(comment)
assert.NoError(t, err)
- res, err = b.Interface.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)
+ res, err = b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id))
assert.NoError(t, err)
t.Logf("%+v", res)
assert.Equal(t, "post blah", res.PostTitle, "keep comment title")
@@ -131,7 +131,7 @@ func TestService_SetTitle(t *testing.T) {
defer tss.Close()
ks := admin.NewStaticKeyStore("secret 123")
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks,
TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
comment := store.Comment{
Text: "text",
@@ -145,7 +145,7 @@ func TestService_SetTitle(t *testing.T) {
assert.NoError(t, err)
assert.True(t, id != "", id)
- res, err := b.Interface.Get(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+ res, err := b.Engine.Get(getReq(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id))
assert.NoError(t, err)
t.Logf("%+v", res)
assert.Equal(t, "", res.PostTitle)
@@ -157,14 +157,14 @@ func TestService_SetTitle(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, "post1 blah 123", c.PostTitle)
- b = DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+ b = DataStore{Engine: prepStoreEngine(t), AdminStore: ks}
_, err = b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
require.EqualError(t, err, "no title extractor")
}
func TestService_Vote(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
comment := store.Comment{
Text: "text",
@@ -174,19 +174,32 @@ func TestService_Vote(t *testing.T) {
_, err := b.Create(comment)
assert.NoError(t, err)
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[0])
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, 0, res[0].Score)
assert.Equal(t, 0, res[0].Vote)
assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
+ // vote +1 as user1
c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, 1, c.Score)
assert.Equal(t, 1, c.Vote)
assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +")
+ // check result as user1
+ c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.User{ID: "user1"})
+ assert.NoError(t, err)
+ assert.Equal(t, 1, c.Score)
+ assert.Equal(t, 1, c.Vote, "can see own vote result")
+ assert.Nil(t, c.Votes)
+ // check result as user2
+ c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.User{ID: "user2"})
+ assert.NoError(t, err)
+ assert.Equal(t, 1, c.Score)
+ assert.Equal(t, 0, c.Vote, "can't see other user vote result")
+ assert.Nil(t, c.Votes)
c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true)
assert.NotNil(t, err, "self-voting not allowed")
@@ -195,17 +208,28 @@ func TestService_Vote(t *testing.T) {
assert.NotNil(t, err, "double-voting rejected")
assert.True(t, strings.HasPrefix(err.Error(), "user user1 already voted"))
- res, err = b.Interface.Last("radio-t", 0, time.Time{})
- assert.Nil(t, err)
+ // check in last as user1
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user1"})
+ assert.NoError(t, err)
+ t.Logf("%+v", res[0])
assert.Equal(t, 3, len(res))
assert.Equal(t, 1, res[0].Score)
assert.Equal(t, 1, res[0].Vote)
assert.Equal(t, 0.0, res[0].Controversy)
+ // check in last as user2
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user2"})
+ assert.NoError(t, err)
+ t.Logf("%+v", res[0])
+ assert.Equal(t, 3, len(res))
+ assert.Equal(t, 1, res[0].Score)
+ assert.Equal(t, 0, res[0].Vote)
+ assert.Equal(t, 0.0, res[0].Controversy)
+
_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
- assert.Nil(t, err, "vote reset")
- res, err = b.Interface.Last("radio-t", 0, time.Time{})
- assert.Nil(t, err)
+ assert.NoError(t, err, "vote reset")
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
+ assert.NoError(t, err)
assert.Equal(t, 3, len(res))
assert.Equal(t, 0, res[0].Score)
assert.Equal(t, 0, res[0].Vote)
@@ -214,25 +238,25 @@ func TestService_Vote(t *testing.T) {
func TestService_VoteLimit(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
- assert.Nil(t, err)
+ assert.NoError(t, err)
_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true)
- assert.Nil(t, err)
+ assert.NoError(t, err)
_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user4", true)
assert.NotNil(t, err, "vote limit reached")
assert.True(t, strings.HasPrefix(err.Error(), "maximum number of votes exceeded for comment id-1"))
_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user4", true)
- assert.Nil(t, err)
+ assert.NoError(t, err)
}
func TestService_VotesDisabled(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
assert.EqualError(t, err, "maximum number of votes exceeded for comment id-1")
@@ -240,7 +264,7 @@ func TestService_VotesDisabled(t *testing.T) {
func TestService_VoteAggressive(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
comment := store.Comment{
Text: "text",
@@ -250,8 +274,8 @@ func TestService_VoteAggressive(t *testing.T) {
_, err := b.Create(comment)
assert.NoError(t, err)
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
- require.Nil(t, err)
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
+ require.NoError(t, err)
t.Logf("%+v", res[0])
assert.Equal(t, 3, len(res))
assert.Equal(t, 0, res[0].Score)
@@ -259,7 +283,7 @@ func TestService_VoteAggressive(t *testing.T) {
// add a vote as user2
_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true)
- require.Nil(t, err)
+ require.NoError(t, err)
// crazy vote +1 as user1
var wg sync.WaitGroup
@@ -271,13 +295,14 @@ func TestService_VoteAggressive(t *testing.T) {
}()
}
wg.Wait()
- res, err = b.Interface.Last("radio-t", 0, time.Time{})
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user1"})
require.NoError(t, err)
t.Logf("%+v", res[0])
assert.Equal(t, 3, len(res))
assert.Equal(t, 2, res[0].Score, "add single +1")
- assert.Equal(t, 2, len(res[0].Votes), "made a single vote")
+ assert.Equal(t, 1, res[0].Vote, "user1 voted +1")
+ assert.Equal(t, 0, len(res[0].Votes), "votes hidden")
// random +1/-1 result should be [0..2]
rand.Seed(time.Now().UnixNano())
@@ -290,7 +315,7 @@ func TestService_VoteAggressive(t *testing.T) {
}()
}
wg.Wait()
- res, err = b.Interface.Last("radio-t", 0, time.Time{})
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
require.NoError(t, err)
assert.Equal(t, 3, len(res))
t.Logf("%+v %d", res[0], res[0].Score)
@@ -300,7 +325,7 @@ func TestService_VoteAggressive(t *testing.T) {
func TestService_VoteConcurrent(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
comment := store.Comment{
Text: "text",
@@ -309,7 +334,7 @@ func TestService_VoteConcurrent(t *testing.T) {
}
_, err := b.Create(comment)
assert.NoError(t, err)
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
require.Nil(t, err)
// concurrent vote +1 as multiple users for the same comment
@@ -324,35 +349,35 @@ func TestService_VoteConcurrent(t *testing.T) {
}()
}
wg.Wait()
- res, err = b.Interface.Last("radio-t", 0, time.Time{})
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
require.NoError(t, err)
assert.Equal(t, 100, res[0].Score, "should have 100 score")
- assert.Equal(t, 100, len(res[0].Votes), "should have 100 votes")
+ assert.Equal(t, 0, len(res[0].Votes), "should hide votes")
assert.Equal(t, 0.0, res[0].Controversy, "should have 0 controversy")
}
func TestService_VotePositive(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
MaxVotes: -1, PositiveScore: true}
_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
assert.EqualError(t, err, "minimal score reached for comment id-1")
_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true)
- assert.Nil(t, err, "minimal score doesn't affect positive vote")
+ assert.NoError(t, err, "minimal score doesn't affect positive vote")
- b = DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+ b = DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
MaxVotes: -1, PositiveScore: false}
c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
- assert.Nil(t, err, "minimal score ignored")
+ assert.NoError(t, err, "minimal score ignored")
assert.Equal(t, -1, c.Score)
assert.Equal(t, 0.0, c.Controversy)
}
func TestService_VoteControversy(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user2", false)
assert.NoError(t, err)
@@ -370,7 +395,7 @@ func TestService_VoteControversy(t *testing.T) {
assert.InDelta(t, 1.73, c.Controversy, 0.01)
// check if stored
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
require.NoError(t, err)
assert.Equal(t, 1, res[0].Score, "should have 1 score")
assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
@@ -402,81 +427,82 @@ func TestService_Controversy(t *testing.T) {
func TestService_Pin(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[0])
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, 2, len(res))
assert.Equal(t, false, res[0].Pin)
err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true)
- assert.Nil(t, err)
+ assert.NoError(t, err)
- c, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
- assert.Nil(t, err)
+ c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+ assert.NoError(t, err)
assert.Equal(t, true, c.Pin)
err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false)
- assert.Nil(t, err)
- c, err = b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
- assert.Nil(t, err)
+ assert.NoError(t, err)
+ c, err = b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+ assert.NoError(t, err)
assert.Equal(t, false, c.Pin)
}
func TestService_EditComment(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[0])
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, 2, len(res))
assert.Nil(t, res[0].Edit)
comment, err := b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, "my edit", comment.Edit.Summary)
assert.Equal(t, "xxx", comment.Text)
assert.Equal(t, "yyy", comment.Orig)
- c, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
- assert.Nil(t, err)
+ c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+ assert.NoError(t, err)
assert.Equal(t, "my edit", c.Edit.Summary)
assert.Equal(t, "xxx", c.Text)
_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
- assert.Nil(t, err, "allow second edit")
+ assert.NoError(t, err, "allow second edit")
}
func TestService_DeleteComment(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[0])
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, 2, len(res))
assert.Nil(t, res[0].Edit)
_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Delete: true})
- assert.Nil(t, err)
+ assert.NoError(t, err)
- c, err := b.Interface.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
- assert.Nil(t, err)
+ c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+ assert.NoError(t, err)
assert.True(t, c.Deleted)
t.Logf("%+v", c)
}
func TestService_EditCommentDurationFailed(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[0])
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, 2, len(res))
assert.Nil(t, res[0].Edit)
@@ -489,11 +515,11 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
func TestService_EditCommentReplyFailed(t *testing.T) {
defer teardown(t)
- b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+ b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
- res, err := b.Interface.Last("radio-t", 0, time.Time{})
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
t.Logf("%+v", res[1])
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, 2, len(res))
assert.Nil(t, res[1].Edit)
@@ -531,7 +557,7 @@ func TestService_ValidateComment(t *testing.T) {
for n, tt := range tbl {
e := b.ValidateComment(&tt.inp)
if tt.err == nil {
- assert.Nil(t, e, "check #%d", n)
+ assert.NoError(t, e, "check #%d", n)
continue
}
require.NotNil(t, e)
@@ -552,15 +578,15 @@ func TestService_Counts(t *testing.T) {
User: store.User{ID: "user1", Name: "user name"},
}
_, err := b.Create(comment)
- assert.Nil(t, err)
+ assert.NoError(t, err)
- svc := DataStore{Interface: b}
+ svc := DataStore{Engine: b}
res, err := svc.Counts("radio-t", []string{"https://radio-t.com/2"})
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res)
res, err = svc.Counts("radio-t", []string{"https://radio-t.com", "https://radio-t.com/2", "blah"})
- assert.Nil(t, err)
+ assert.NoError(t, err)
assert.Equal(t, []store.PostInfo{
{URL: "https://radio-t.com", Count: 2},
{URL: "https://radio-t.com/2", Count: 1},
@@ -571,7 +597,7 @@ func TestService_Counts(t *testing.T) {
func TestService_GetMetas(t *testing.T) {
defer teardown(t)
// two comments for https://radio-t.com
- b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticKeyStore("secret 123")}
um, pm, err := b.Metas("radio-t")
@@ -602,7 +628,7 @@ func TestService_GetMetas(t *testing.T) {
func TestService_SetMetas(t *testing.T) {
defer teardown(t)
// two comments for https://radio-t.com
- b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticKeyStore("secret 123")}
umetas := []UserMetaData{}
pmetas := []PostMetaData{}
@@ -626,7 +652,7 @@ func TestService_SetMetas(t *testing.T) {
func TestService_IsAdmin(t *testing.T) {
defer teardown(t)
// two comments for https://radio-t.com
- b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
assert.False(t, b.IsAdmin("radio-t", "user1"))
@@ -637,7 +663,7 @@ func TestService_HasReplies(t *testing.T) {
defer teardown(t)
// two comments for https://radio-t.com, no reply
- b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
comment := store.Comment{
@@ -668,7 +694,7 @@ func TestService_UserReplies(t *testing.T) {
defer teardown(t)
// two comments for https://radio-t.com, no reply
- b := DataStore{Interface: prepStoreEngine(t),
+ b := DataStore{Engine: prepStoreEngine(t),
AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
c1 := store.Comment{
@@ -738,13 +764,15 @@ func TestService_UserReplies(t *testing.T) {
cc, u, err = b.UserReplies("radio-t", "uxxx", 10, time.Hour)
assert.NoError(t, err)
assert.Equal(t, 0, len(cc), "0 replies to uxxx")
+ assert.Equal(t, "", u)
+
}
func TestService_Find(t *testing.T) {
defer teardown(t)
// two comments for https://radio-t.com, no reply
- b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time", store.User{})
@@ -761,8 +789,8 @@ func TestService_Find(t *testing.T) {
Score: 1,
Votes: map[string]bool{"id-1": true, "id-2": true, "123456": false},
}
- _, err = b.Interface.Create(comment) // create directly with engine, doesn't set Controversy
- assert.Nil(t, err)
+ _, err = b.Engine.Create(comment) // create directly with engine, doesn't set Controversy
+ assert.NoError(t, err)
// make sure Controversy altered
res, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "-controversy", store.User{})
@@ -774,6 +802,192 @@ func TestService_Find(t *testing.T) {
assert.InDelta(t, 0, res[1].Controversy, 0.01)
}
+func TestService_Info(t *testing.T) {
+ defer teardown(t)
+
+ // two comments for https://radio-t.com, no reply
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+ info, err := b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
+ require.NoError(t, err)
+ assert.Equal(t, "https://radio-t.com", info.URL)
+ assert.Equal(t, 2, info.Count)
+ assert.False(t, info.ReadOnly)
+ assert.True(t, info.LastTS.After(info.FirstTS))
+
+ time.Sleep(1 * time.Second) // make post RO in 1sec
+ info, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 1)
+ require.NoError(t, err)
+ assert.Equal(t, "https://radio-t.com", info.URL)
+ assert.True(t, info.ReadOnly)
+}
+
+func TestService_Delete(t *testing.T) {
+ defer teardown(t)
+
+ // two comments for https://radio-t.com, no reply
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
+ assert.Equal(t, 2, len(res))
+ assert.NoError(t, err)
+
+ err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.SoftDelete)
+ assert.NoError(t, err)
+
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
+ assert.Equal(t, 1, len(res), "one left")
+ assert.NoError(t, err)
+}
+
+// DeleteUser removes all comments from user
+func TestService_DeleteUser(t *testing.T) {
+ defer teardown(t)
+ // two comments for https://radio-t.com, no reply
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+ // add one more for user2
+ comment := store.Comment{
+ ID: "123456xyz",
+ Text: `some text, link`,
+ Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user2", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.NoError(t, err)
+
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
+ assert.Equal(t, 3, len(res), "3 comments initially, for 2 diff users and 2 posts")
+ assert.NoError(t, err)
+
+ err = b.DeleteUser("radio-t", "user1", store.HardDelete)
+ assert.NoError(t, err)
+
+ res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
+ assert.Equal(t, 1, len(res), "only one comment left for user2")
+ assert.NoError(t, err)
+ assert.Equal(t, "user2", res[0].User.ID)
+}
+
+func TestService_List(t *testing.T) {
+ defer teardown(t)
+ // two comments for https://radio-t.com, no reply
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+ // add one more for user2
+ comment := store.Comment{
+ ID: "id-3",
+ Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+ Text: `some text, link`,
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user2", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.NoError(t, err)
+
+ res, err := b.List("radio-t", 0, 0)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, len(res), "2 posts")
+ assert.Equal(t, "https://radio-t.com/2", res[0].URL)
+ assert.Equal(t, 1, res[0].Count)
+ assert.Equal(t, time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), res[0].FirstTS)
+ assert.Equal(t, time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), res[0].LastTS)
+
+ assert.Equal(t, "https://radio-t.com", res[1].URL)
+ assert.Equal(t, 2, res[1].Count)
+ assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), res[1].FirstTS)
+ assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), res[1].LastTS)
+}
+
+func TestService_Count(t *testing.T) {
+ defer teardown(t)
+ // two comments for https://radio-t.com, no reply
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+ // add one more for user2
+ comment := store.Comment{
+ ID: "id-3",
+ Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+ Text: `some text, link`,
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user2", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.NoError(t, err)
+
+ c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
+ assert.NoError(t, err)
+ assert.Equal(t, 2, c)
+
+ c, err = b.Count(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"})
+ assert.NoError(t, err)
+ assert.Equal(t, 1, c)
+
+ c, err = b.Count(store.Locator{URL: "https://radio-t.com/3", SiteID: "radio-t"})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, c)
+}
+
+func TestService_UserCount(t *testing.T) {
+ defer teardown(t)
+ // two comments for https://radio-t.com, no reply
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+ // add one more for user2
+ comment := store.Comment{
+ ID: "id-3",
+ Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+ Text: `some text, link`,
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user2", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.NoError(t, err)
+
+ c, err := b.UserCount("radio-t", "user1")
+ assert.NoError(t, err)
+ assert.Equal(t, 2, c)
+
+ c, err = b.UserCount("radio-t", "user2")
+ assert.NoError(t, err)
+ assert.Equal(t, 1, c)
+
+ _, err = b.UserCount("radio-t", "userBad")
+ assert.EqualError(t, err, "no comments for user userBad in store for radio-t site")
+}
+
+func TestService_DeleteAll(t *testing.T) {
+ defer teardown(t)
+ // two comments for https://radio-t.com, no reply
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+ AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+ // add one more for user2
+ comment := store.Comment{
+ ID: "id-3",
+ Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+ Text: `some text, link`,
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user2", Name: "user name"},
+ }
+ _, err := b.Create(comment)
+ assert.NoError(t, err)
+
+ err = b.DeleteAll("radio-t")
+ assert.NoError(t, err)
+
+ res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
+ assert.NoError(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
func TestService_submitImages(t *testing.T) {
defer teardown(t)
lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
@@ -783,7 +997,7 @@ func TestService_submitImages(t *testing.T) {
imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50}
// two comments for https://radio-t.com
- b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
+ b := DataStore{Engine: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
c := store.Comment{
@@ -793,7 +1007,7 @@ func TestService_submitImages(t *testing.T) {
Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
User: store.User{ID: "user1", Name: "user name"},
}
- _, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages
+ _, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages
assert.NoError(t, err)
b.submitImages(c)
@@ -804,31 +1018,33 @@ func TestService_alterComment(t *testing.T) {
defer teardown(t)
engineMock := engine.MockInterface{}
- engineMock.On("IsBlocked", mock.Anything, mock.Anything).Return(false)
- engineMock.On("IsVerified", mock.Anything, mock.Anything).Return(false)
- svc := DataStore{Interface: &engineMock}
+ engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(false, nil)
+ engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil)
+ svc := DataStore{Engine: &engineMock}
- r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1"}}, store.User{Name: "dev", Admin: false})
- assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: ""}}, r, "ip cleaned")
- r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1"}}, store.User{Name: "dev", Admin: true})
- assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "127.0.0.1"}}, r, "ip not cleaned")
+ r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}},
+ store.User{Name: "dev", ID: "devid", Admin: false})
+ assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid"}}, r, "ip cleaned")
+ r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}},
+ store.User{Name: "dev", ID: "devid", Admin: true})
+ assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, r, "ip not cleaned")
engineMock = engine.MockInterface{}
- engineMock.On("IsBlocked", mock.Anything, mock.Anything).Return(false)
- engineMock.On("IsVerified", mock.Anything, mock.Anything).Return(true)
- svc = DataStore{Interface: &engineMock}
- r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", Verified: true}},
- store.User{Name: "dev", Admin: false})
- assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true}}, r, "verified set")
+ engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(false, nil)
+ engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(true, nil)
+ svc = DataStore{Engine: &engineMock}
+ r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}},
+ store.User{Name: "dev", ID: "devid", Admin: false})
+ assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid", Verified: true}}, r, "verified set")
engineMock = engine.MockInterface{}
- engineMock.On("IsBlocked", mock.Anything, mock.Anything).Return(true)
- engineMock.On("IsVerified", mock.Anything, mock.Anything).Return(false)
- svc = DataStore{Interface: &engineMock}
- r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", Verified: true}},
- store.User{Name: "dev", Admin: false})
- assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true, Blocked: true}, Deleted: true}, r,
- "blocked")
+ engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(true, nil)
+ engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil)
+ svc = DataStore{Engine: &engineMock}
+ r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}},
+ store.User{Name: "dev", ID: "devid", Admin: false})
+ assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true, Blocked: true, ID: "devid"},
+ Deleted: true}, r, "blocked")
}
// makes new boltdb, put two records
@@ -836,7 +1052,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
_ = os.Remove(testDb)
boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
- assert.Nil(t, err)
+ assert.NoError(t, err)
b := boltStore
comment := store.Comment{
@@ -847,7 +1063,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
User: store.User{ID: "user1", Name: "user name"},
}
_, err = b.Create(comment)
- assert.Nil(t, err)
+ assert.NoError(t, err)
comment = store.Comment{
ID: "id-2",
@@ -857,7 +1073,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
User: store.User{ID: "user1", Name: "user name"},
}
_, err = b.Create(comment)
- assert.Nil(t, err)
+ assert.NoError(t, err)
return b
}
@@ -865,3 +1081,10 @@ func prepStoreEngine(t *testing.T) engine.Interface {
func teardown(_ *testing.T) {
_ = os.Remove(testDb)
}
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+ return engine.GetRequest{
+ Locator: locator,
+ CommentID: commentID,
+ }
+}
diff --git a/backend/go.mod b/backend/go.mod
index d00e249f7f..8fccd3d50e 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -11,14 +11,13 @@ require (
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/didip/tollbooth v4.0.0+incompatible
github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
- github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
github.com/go-chi/chi v4.0.2+incompatible
github.com/go-chi/cors v1.0.0
github.com/go-chi/render v1.0.1
github.com/go-pkgz/auth v0.5.2
github.com/go-pkgz/lcw v0.3.1
github.com/go-pkgz/lgr v0.6.2
- github.com/go-pkgz/mongo v1.1.2
+ github.com/go-pkgz/mongo v1.1.2 // indirect
github.com/go-pkgz/repeater v1.1.2
github.com/go-pkgz/rest v1.4.1
github.com/go-pkgz/syncs v1.1.1