Merge pull request #165 from Ptt-Alertor/clean_category

Clean Board Category, Close #92

jobs/categorycleaner.go

@@ -0,0 +1,80 @@
+package jobs
+
+import (
+	"time"
+
+	log "github.com/Ptt-Alertor/logrus"
+	"github.com/Ptt-Alertor/ptt-alertor/models"
+	"github.com/Ptt-Alertor/ptt-alertor/models/author"
+	"github.com/Ptt-Alertor/ptt-alertor/models/keyword"
+	"github.com/Ptt-Alertor/ptt-alertor/models/pushsum"
+	"github.com/Ptt-Alertor/ptt-alertor/models/subscription"
+	"github.com/Ptt-Alertor/ptt-alertor/myutil"
+	"github.com/Ptt-Alertor/ptt-alertor/ptt/rss"
+)
+
+type categoryCleaner struct {
+}
+
+func NewCategoryCleaner() *categoryCleaner {
+	return &categoryCleaner{}
+}
+
+func (cc categoryCleaner) Run() {
+	boardNames := myutil.StringSlice(models.Board().List())
+	boardNames.AppendNonRepeat(myutil.StringSlice(pushsum.List()), false)
+
+	for _, boardName := range boardNames {
+		time.Sleep(100 * time.Millisecond)
+		if !rss.CheckBoardExist(boardName) {
+			log.WithField("category", boardName).Info("Delete Category")
+			cc.CleanAccountSetting(boardName)
+			cc.CleanKeywordAuthorBoard(boardName)
+			cc.CleanPushsumBoard(boardName)
+		}
+	}
+
+}
+
+func (cc categoryCleaner) CleanAccountSetting(boardName string) {
+	subs := myutil.StringSlice(keyword.Subscribers(boardName))
+	subs.AppendNonRepeat(author.Subscribers(boardName), false)
+	subs.AppendNonRepeat(pushsum.ListSubscribers(boardName), false)
+
+	for _, sub := range subs {
+		u := models.User().Find(sub)
+		u.Subscribes.Delete(subscription.Subscription{Board: boardName})
+		if err := u.Update(); err != nil {
+			log.WithFields(log.Fields{
+				"category": boardName,
+				"account":  u.Account,
+			}).WithError(err).Error("Remove Category in User Failed")
+		}
+	}
+}
+
+func (cc categoryCleaner) CleanKeywordAuthorBoard(boardName string) {
+	board := models.Board()
+	board.Name = boardName
+
+	if err := board.Delete(); err != nil {
+		log.WithField("category", boardName).WithError(err).Error("Delete Board Category Failed")
+	}
+
+	if err := keyword.Destroy(boardName); err != nil {
+		log.WithField("category", boardName).WithError(err).Error("Delete Keyword Category Failed")
+	}
+
+	if err := author.Destroy(boardName); err != nil {
+		log.WithField("category", boardName).WithError(err).Error("Delete Author Category Failed")
+	}
+}
+
+func (cc categoryCleaner) CleanPushsumBoard(boardName string) {
+	if err := pushsum.Remove(boardName); err != nil {
+		log.WithField("category", boardName).WithError(err).Error("Remove Pushsum Category Failed")
+	}
+	if err := pushsum.Destroy(boardName); err != nil {
+		log.WithField("category", boardName).WithError(err).Error("Delete Pushsum Category Failed")
+	}
+}

main.go

@@ -162,4 +162,5 @@ func init() {
 	// jobs.NewGenerator().Run()
 	// jobs.NewFetcher().Run()
 	// jobs.NewMigrateDB().Run()
+	jobs.NewCategoryCleaner().Run()
 }

models/board/board.go

@@ -168,7 +168,7 @@ func CheckBoardExist(boardName string) (bool, string) {
 	if bd.Exist() {
 		return true, ""
 	}
-	if web.CheckBoardExist(boardName) {
+	if rss.CheckBoardExist(boardName) {
 		bd.Create()
 		return true, ""
 	}

ptt/rss/rss.go

@@ -11,6 +11,19 @@ import (
 
 var ErrTooManyRequests = errors.New("Too Many Requests")
 
+// CheckBoardExist use for checking board exist or not
+func CheckBoardExist(board string) bool {
+	feed, err := parseURL("https://www.ptt.cc/atom/" + board + ".xml")
+	if err != nil {
+		return false
+	}
+	// category didn't have any items but has rss link as well
+	if len(feed.Items) == 0 {
+		return false
+	}
+	return true
+}
+
 func BuildArticles(board string) (articles article.Articles, err error) {
 	feed, err := parseURL("https://www.ptt.cc/atom/" + board + ".xml")
 	if err != nil {

ptt/rss/rss_test.go

@@ -0,0 +1,24 @@
+package rss
+
+import "testing"
+
+func TestCheckBoardExist(t *testing.T) {
+	type args struct {
+		board string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{"exist", args{"movie"}, true},
+		{"exist", args{"movies"}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := CheckBoardExist(tt.args.board); got != tt.want {
+				t.Errorf("CheckBoardExist() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}