diff --git a/pkg/api/external_references.go b/pkg/api/external_references.go index 71ad0bd14..b231c8e10 100644 --- a/pkg/api/external_references.go +++ b/pkg/api/external_references.go @@ -49,6 +49,16 @@ func (i ExternalReference) WebService() *restful.WebService { ws.Route(ws.GET("/generic/scrape_all").To(i.genericActorScraper). Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/stashdb/link2scene/{scene-id}/{stashdb-id}").To(i.linkScene2Stashdb). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes(models.Scene{})) + ws.Route(ws.GET("/stashdb/search/{scene-id}").To(i.searchForStashdbScene). + Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/stashdb/link2actor/{actor-id}/{stashdb-id}").To(i.linkActor2Stashdb). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes(models.Scene{})) + ws.Route(ws.GET("/stashdb/searchactor/{actor-id}").To(i.searchForStashdbActor). + Metadata(restfulspec.KeyOpenAPITags, tags)) ws.Route(ws.POST("/generic/scrape_single").To(i.genericSingleActorScraper). Metadata(restfulspec.KeyOpenAPITags, tags)) diff --git a/pkg/api/stashdb.go b/pkg/api/stashdb.go index b4e02b590..d34c047b1 100644 --- a/pkg/api/stashdb.go +++ b/pkg/api/stashdb.go @@ -1,7 +1,14 @@ package api import ( + "encoding/json" + "fmt" + "math" "net/http" + "regexp" + "sort" + "strconv" + "strings" "time" "github.com/emicklei/go-restful/v3" @@ -31,6 +38,638 @@ func (i ExternalReference) stashDbUpdateData(req *restful.Request, resp *restful func (i ExternalReference) stashRunAll(req *restful.Request, resp *restful.Response) { StashdbRunAll() } +func (i ExternalReference) linkScene2Stashdb(req *restful.Request, resp *restful.Response) { + sceneId := req.PathParameter("scene-id") + stashdbId := req.PathParameter("stashdb-id") + stashdbId = strings.TrimPrefix(stashdbId, "https://stashdb.org/scenes/") + var scene models.Scene + + db, _ := models.GetDB() + defer db.Close() + + if strings.Contains(sceneId, "-") { + scene.GetIfExist(sceneId) + } else { + id, _ := strconv.Atoi(req.PathParameter("scene-id")) + scene.GetIfExistByPK(uint(id)) + } + if scene.ID == 0 { + return + } + stashScene := scrape.GetStashDbScene(stashdbId) + + var existingRef models.ExternalReference + existingRef.FindExternalId("stashdb scene", stashdbId) + + jsonData, _ := json.MarshalIndent(stashScene.Data.Scene, "", " ") + + // chek if we have the performers, may not in the case of loading scenes from the parent studio + for _, performer := range stashScene.Data.Scene.Performers { + scrape.UpdatePerformer(performer.Performer) + } + + var xbrLink []models.ExternalReferenceLink + xbrLink = append(xbrLink, models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: scene.ID, InternalNameId: scene.SceneID, ExternalSource: "stashdb scene", ExternalId: stashdbId, MatchType: 5}) + ext := models.ExternalReference{ExternalSource: "stashdb scene", ExternalURL: "https://stashdb.org/scenes/" + stashdbId, ExternalId: stashdbId, ExternalDate: time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC), ExternalData: string(jsonData), + XbvrLinks: xbrLink} + ext.AddUpdateWithId() + + // check for actor not yet linked + for _, actor := range scene.Cast { + var extreflinks []models.ExternalReferenceLink + db.Preload("ExternalReference").Where(&models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, ExternalSource: "stashdb performer"}).Find(&extreflinks) + if len(extreflinks) == 0 { + stashPerformerId := "" + for _, stashPerf := range stashScene.Data.Scene.Performers { + if strings.EqualFold(stashPerf.Performer.Name, actor.Name) || strings.EqualFold(stashPerf.As, actor.Name) { + stashPerformerId = stashPerf.Performer.ID + continue + } + for _, alias := range stashPerf.Performer.Aliases { + if strings.EqualFold(alias, actor.Name) { + stashPerformerId = stashPerf.Performer.ID + } + } + } + if stashPerformerId != "" { + scrape.RefreshPerformer(stashPerformerId) + var actorRef models.ExternalReference + actorRef.FindExternalId("stashdb performer", stashPerformerId) + var performer models.StashPerformer + json.Unmarshal([]byte(actorRef.ExternalData), &performer) + + xbvrLink := models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, InternalNameId: actor.Name, MatchType: 90, + ExternalReferenceID: actorRef.ID, ExternalSource: actorRef.ExternalSource, ExternalId: actorRef.ExternalId} + actorRef.XbvrLinks = append(actorRef.XbvrLinks, xbvrLink) + actorRef.AddUpdateWithId() + + externalreference.UpdateXbvrActor(performer, actor.ID) + } + } + } + + // reread the scene to return updated data + scene.GetIfExistByPK(scene.ID) + resp.WriteHeaderAndEntity(http.StatusOK, scene) +} + +func (i ExternalReference) searchForStashdbScene(req *restful.Request, resp *restful.Response) { + query := req.QueryParameter("q") + + var warnings []string + type StashSearchScenePerformerResult struct { + Name string + Url string + } + type StashSearchSceneResult struct { + Url string + ImageUrl string + Performers []StashSearchScenePerformerResult + Title string + Studio string + Duration string + Description string + Weight int + Date string + Id string + } + type StashSearchSceneResponse struct { + Status string + Results []StashSearchSceneResult + } + results := make(map[string]StashSearchSceneResult) + + sceneId := req.PathParameter("scene-id") + var scene models.Scene + + db, _ := models.GetDB() + defer db.Close() + + if strings.Contains(sceneId, "-") { + scene.GetIfExist(sceneId) + } else { + id, _ := strconv.Atoi(req.PathParameter("scene-id")) + scene.GetIfExistByPK(uint(id)) + } + if scene.ID == 0 { + var response StashSearchSceneResponse + response.Results = []StashSearchSceneResult{} + response.Status = "XBVR Scene not found" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + + setupStashSearchResult := func(stashScene models.StashScene, weight int) StashSearchSceneResult { + //common function to call to setup stash response details + result := StashSearchSceneResult{Id: stashScene.ID, Url: "https://stashdb.org/scenes/" + stashScene.ID, Weight: weight, Title: stashScene.Title, Description: stashScene.Details, Date: stashScene.Date, Studio: stashScene.Studio.Name} + if len(stashScene.Images) > 0 { + result.ImageUrl = stashScene.Images[0].URL + } + for _, perf := range stashScene.Performers { + result.Performers = append(result.Performers, StashSearchScenePerformerResult{Name: perf.Performer.Name, Url: `https://stashdb.org/performers/` + perf.Performer.ID}) + } + if stashScene.Duration > 0 { + hours := stashScene.Duration / 3600 // calculate hours + stashScene.Duration %= 3600 // remaining seconds after hours + minutes := stashScene.Duration / 60 // calculate minutes + stashScene.Duration %= 60 // remaining seconds after minutes + + // Format the time string + result.Duration = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, stashScene.Duration) + } + return result + } + + var guidRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + idTest := strings.TrimPrefix(strings.TrimSpace(query), "https://stashdb.org/scenes/") + + if guidRegex.MatchString(idTest) { + stashScene := scrape.GetStashDbScene(idTest) + if stashScene.Data.Scene.ID != "" { + results[stashScene.Data.Scene.ID] = setupStashSearchResult(stashScene.Data.Scene, 10000) + var response StashSearchSceneResponse + response.Results = []StashSearchSceneResult{results[stashScene.Data.Scene.ID]} + response.Status = "" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + } + + stashStudioIds := findStashStudioIds(scene.ScraperId) + if len(stashStudioIds) == 0 { + var response StashSearchSceneResponse + response.Results = []StashSearchSceneResult{} + response.Status = "Cannot find Stashdb Studio" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + + var xbvrperformers []string + for _, actor := range scene.Cast { + var stashlinks []models.ExternalReferenceLink + db.Preload("ExternalReference").Where(&models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, ExternalSource: "stashdb performer"}).Find(&stashlinks) + if len(stashlinks) == 0 { + warnings = append(warnings, actor.Name+" is not linked to Stashdb") + } else { + for _, stashPerformer := range stashlinks { + xbvrperformers = append(xbvrperformers, `"`+stashPerformer.ExternalId+`"`) + } + } + } + + // define a function to update the results found + scoreResults := func(stashScenes scrape.QueryScenesResult, weightIncrement int, performers []string, studios []string) { + for _, stashscene := range stashScenes.Data.QueryScenes.Scenes { + // consider adding weight bump for duration and date + scoreBump := 0 + if stashscene.Date == scene.ReleaseDateText { + scoreBump += 15 + } + if stashscene.Title == scene.Title { + scoreBump += 25 + } + if stashscene.Duration > 0 && scene.Duration > 0 { + stashDur := float64((stashscene.Duration / 60) - scene.Duration) + if math.Abs(stashDur) <= 2 { + scoreBump += 5 * int(3-math.Abs(stashDur)) + } + } + // check duration from video files + for _, file := range scene.Files { + if file.Type == "video" { + diff := file.VideoDuration - float64(stashscene.Duration) + if math.Abs(diff) <= 2 { + scoreBump += 5 * int(3-math.Abs(diff)) + } + } + } + + // check it is from a studio we expect + for _, studio := range studios { + if strings.ReplaceAll(studio, `"`, ``) == stashscene.Studio.ID { + scoreBump += 20 + } + } + + foundActorBump := -5 + for _, sp := range stashscene.Performers { + for _, xp := range performers { + if strings.Contains(xp, sp.Performer.ID) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 10 + } else { + foundActorBump += 5 + } + } + } + } + // we have checked if stash performers match xbvr, now check for xbvr performers not matched in stash + for _, xp := range xbvrperformers { + for _, sp := range stashscene.Performers { + if strings.Contains(xp, sp.Performer.ID) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 15 + } else { + foundActorBump += 5 + } + } + } + } + // check actor matches using names and aliases + for _, actor := range scene.Cast { + for _, sp := range stashscene.Performers { + if strings.EqualFold(actor.Name, sp.Performer.Name) || strings.EqualFold(actor.Name, sp.As) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 15 + } else { + foundActorBump += 5 + } + continue + } + // try aliases + for _, alias := range sp.Performer.Aliases { + if strings.EqualFold(alias, actor.Name) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 15 + } else { + foundActorBump += 5 + } + continue + } + } + } + } + if mapEntry, exists := results[stashscene.ID]; exists { + mapEntry.Weight += weightIncrement + scoreBump + foundActorBump + results[stashscene.ID] = mapEntry + } else { + results[stashscene.ID] = setupStashSearchResult(stashscene, weightIncrement+scoreBump+foundActorBump) + } + } + } + + var fingerprints []string + for _, file := range scene.Files { + if file.Type == "video" { + file.OsHash = "00000000000000000" + file.OsHash + fingerprints = append(fingerprints, `"`+file.OsHash[len(file.OsHash)-16:]+`"`) + } + } + if len(fingerprints) > 0 { + fingerprintList := strings.Join(fingerprints, ",") + fingerprintQuery := ` + {"input":{ + "page": 1, + "per_page": 150, + "sort": "UPDATED_AT", + "fingerprints": {"value": [` + + fingerprintList + + `], "modifier":"EQUALS"} + } + }` + stashScenes := scrape.GetScenePage(fingerprintQuery) + scoreResults(stashScenes, 400, xbvrperformers, stashStudioIds) + } + + stashScenes := scrape.QueryScenesResult{} + for _, studio := range stashStudioIds { + // Exact Title submatch + titleQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": 1, + "per_page": 150, + "sort": "UPDATED_AT", + "title": "\"` + + scene.Title + `\"" + } + }` + stashScenes = scrape.GetScenePage(titleQuery) + scoreResults(stashScenes, 150, xbvrperformers, stashStudioIds) + } + + if len(xbvrperformers) > 0 { + performerList := strings.Join(xbvrperformers, ",") + for _, studio := range stashStudioIds { + performerQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": 1, + "per_page": 150, + "sort": "UPDATED_AT", + "performers": {"value": [` + + performerList + + `], "modifier":"INCLUDES_ALL"} + } + }` + stashScenes = scrape.GetScenePage(performerQuery) + scoreResults(stashScenes, 200, xbvrperformers, stashStudioIds) + if len(stashScenes.Data.QueryScenes.Scenes) == 0 { + performerQuery = strings.ReplaceAll(performerQuery, "INCLUDES_ALL", "INCLUDES") + stashScenes := scrape.GetScenePage(performerQuery) + scoreResults(stashScenes, 100, xbvrperformers, stashStudioIds) + } + } + } + + if len(results) == 0 { + for _, studio := range stashStudioIds { + // No match yet, try match any words from the title, not likely to find, as this returns too many results + titleQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": 1, + "per_page": 100, + "sort": "UPDATED_AT", + "title": "` + + scene.Title + `" + } + }` + stashScenes = scrape.GetScenePage(titleQuery) + scoreResults(stashScenes, 150, xbvrperformers, stashStudioIds) + page := 2 + for i := 101; i < stashScenes.Data.QueryScenes.Count && page <= 5; { + titleQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": ` + strconv.Itoa(page) + `, + "per_page": 100, + "sort": "UPDATED_AT", + "title": "` + + scene.Title + `" + } + }` + stashScenes = scrape.GetScenePage(titleQuery) + scoreResults(stashScenes, 150, xbvrperformers, stashStudioIds) + i = i + 100 + page += 1 + } + } + } + + if len(results) == 0 { + warnings = append(warnings, "No Stashdb Scenes Found") + } + // sort and limit the number of results + // Convert map to a slice of key-value pairs + pairs := make([]StashSearchSceneResult, 0, len(results)) + for _, v := range results { + pairs = append(pairs, v) + } + // Sort the slice by weight in descending order + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Weight > pairs[j].Weight + }) + // Take the first 100 entries (or less if there are fewer than 100 entries) + top100 := pairs[:min(len(pairs), 100)] + var response StashSearchSceneResponse + response.Results = top100 + response.Status = strings.Join(warnings, ", ") + resp.WriteHeaderAndEntity(http.StatusOK, response) +} + +func (i ExternalReference) linkActor2Stashdb(req *restful.Request, resp *restful.Response) { + actorId := req.PathParameter("actor-id") + stashPerformerId := req.PathParameter("stashdb-id") + stashPerformerId = strings.TrimPrefix(stashPerformerId, "https://stashdb.org/performers/") + var actor models.Actor + + db, _ := models.GetDB() + defer db.Close() + + id, _ := strconv.Atoi(actorId) + if id == 0 { + actor.GetIfExist(actorId) + } else { + actor.GetIfExistByPK(uint(id)) + } + if actor.ID == 0 { + return + } + + scrape.RefreshPerformer(stashPerformerId) + var actorRef models.ExternalReference + actorRef.FindExternalId("stashdb performer", stashPerformerId) + // change the External Date, this is used to find the most recent change and query + // stash for changes since then. If wew manually load an actor, we may miss other updates + actorRef.ExternalDate = time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC) + actorRef.Save() + + var performer models.StashPerformer + json.Unmarshal([]byte(actorRef.ExternalData), &performer) + + xbvrLink := models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, InternalNameId: actor.Name, MatchType: 90, + ExternalReferenceID: actorRef.ID, ExternalSource: actorRef.ExternalSource, ExternalId: actorRef.ExternalId} + actorRef.XbvrLinks = append(actorRef.XbvrLinks, xbvrLink) + actorRef.AddUpdateWithId() + + externalreference.UpdateXbvrActor(performer, actor.ID) + + // reread the actor to return updated data + actor.GetIfExistByPK(actor.ID) + resp.WriteHeaderAndEntity(http.StatusOK, actor) +} + +func (i ExternalReference) searchForStashdbActor(req *restful.Request, resp *restful.Response) { + query := req.QueryParameter("q") + query = strings.TrimSpace(strings.TrimPrefix(query, "aka:")) + + var warnings []string + type StashSearchPerformerScenesResult struct { + Title string + Id string + Url string + Duration string + ImageUrl string + Studio string + } + type StashSearchPerformerStudioResult struct { + Name string + Id string + Url string + SceneCount int + Matched bool + } + type StashSearchPerformerAliasResult struct { + Alias string + Matched bool + } + type StashSearchPerformerResult struct { + Url string + Name string + Disambiguation string + Aliases []StashSearchPerformerAliasResult + Id string + ImageUrl []string + DOB string + Weight int + Studios []StashSearchPerformerStudioResult + } + type StashSearchPerformersResponse struct { + Status string + Results []StashSearchPerformerResult + } + results := make(map[string]StashSearchPerformerResult) + + actorId := req.PathParameter("actor-id") + var actor models.Actor + + db, _ := models.GetDB() + defer db.Close() + + id, _ := strconv.Atoi(req.PathParameter("actor-id")) + if id == 0 { + actor.GetIfExist(actorId) + } else { + actor.GetIfExistByPK(uint(id)) + } + if actor.ID == 0 { + var response StashSearchPerformersResponse + response.Results = []StashSearchPerformerResult{} + response.Status = "XBVR Actor not found" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + + matchedStudios := map[string]struct{}{} + matchedAlias := map[string]struct{}{} + + setupStashSearchResult := func(stashPerformer models.StashPerformer, weight int) StashSearchPerformerResult { + //common function to call to setup stash response details + result := StashSearchPerformerResult{Id: stashPerformer.ID, Url: "https://stashdb.org/performers/" + stashPerformer.ID, Weight: weight, Name: stashPerformer.Name, DOB: stashPerformer.BirthDate, Disambiguation: stashPerformer.Disambiguation} + for _, image := range stashPerformer.Images { + result.ImageUrl = append(result.ImageUrl, image.URL) + } + for _, studio := range stashPerformer.Studios { + _, matched := matchedStudios[studio.Studio.ID] + if matched { + matched = true + } + result.Studios = append(result.Studios, StashSearchPerformerStudioResult{Name: studio.Studio.Name, Id: studio.Studio.ID, Url: `https://stashdb.org/performers/` + stashPerformer.ID + `?studios=` + studio.Studio.ID, SceneCount: studio.SceneCount, Matched: matched}) + } + for _, alias := range stashPerformer.Aliases { + _, matched := matchedAlias[strings.ToLower(alias)] + if matched { + matched = true + } + result.Aliases = append(result.Aliases, StashSearchPerformerAliasResult{Alias: alias, Matched: matched}) + } + + sort.Slice(result.Studios, func(i, j int) bool { + return result.Studios[i].Name < result.Studios[j].Name + }) + return result + } + + var guidRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + idTest := strings.TrimPrefix(strings.TrimSpace(query), "https://stashdb.org/performers/") + + if guidRegex.MatchString(idTest) { + stashPerformer := scrape.GetStashPerformerFull(idTest) + if stashPerformer.Data.Performer.ID != "" { + results[stashPerformer.Data.Performer.ID] = setupStashSearchResult(stashPerformer.Data.Performer, 10000) + // need to get studios + var response StashSearchPerformersResponse + response.Results = []StashSearchPerformerResult{results[stashPerformer.Data.Performer.ID]} + response.Status = "" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + } + + if strings.TrimSpace(query) == "" { + query = actor.Name + } + // define a function to update the results found + scoreResults := func(stashPerformers []models.StashPerformer, weightIncrement int) { + for _, stashPerformer := range stashPerformers { + // consider adding weight bump for duration and date + scoreBump := 0 + lcaseActorName := strings.ToLower(actor.Name) + if stashPerformer.Name == actor.Name { + scoreBump += 200 + } + if strings.Contains(strings.ToLower(stashPerformer.Name), lcaseActorName) { + scoreBump += 30 + } + for _, alias := range stashPerformer.Aliases { + lcaseAlias := strings.ToLower(alias) + if strings.EqualFold(lcaseAlias, lcaseActorName) || strings.EqualFold(lcaseAlias, strings.ToLower(query)) { + scoreBump += 50 + matchedAlias[strings.ToLower(alias)] = struct{}{} + } else { + if strings.Contains(lcaseAlias, lcaseActorName) || strings.Contains(lcaseActorName, lcaseAlias) || strings.Contains(lcaseAlias, strings.ToLower(query)) { + scoreBump += 20 + matchedAlias[strings.ToLower(alias)] = struct{}{} + } + } + } + // dob checks + + if actor.Gender != "" && strings.EqualFold(actor.Gender, stashPerformer.Gender) { + scoreBump += 10 + } + + var stashExtLink models.ExternalReferenceLink + commonDb, _ := models.GetCommonDB() + for _, xbvrScene := range actor.Scenes { + for _, stashStudio := range stashPerformer.Studios { + xbvrsite := xbvrScene.Site + var siteRef models.ExternalReferenceLink + commonDb.Where(&models.ExternalReferenceLink{InternalTable: "sites", InternalNameId: xbvrScene.ScraperId, ExternalSource: "stashdb studio"}).First(&siteRef) + if strings.Index(xbvrsite, " (") != -1 { + xbvrsite = xbvrsite[:strings.Index(xbvrsite, " (")] + } + if strings.EqualFold(stashStudio.Studio.Name, xbvrsite) || siteRef.ExternalId == stashStudio.Studio.ID { + scoreBump += 5 + matchedStudios[stashStudio.Studio.ID] = struct{}{} + } + } + // check if we have a linked scene with this performer + links := stashExtLink.FindByExternalSource("scenes", xbvrScene.ID, "stashdb scene") + if len(links) == 0 { + continue + } + for _, link := range links { + if strings.Contains(link.ExternalReference.ExternalData, stashPerformer.ID) { + scoreBump += 50 + } + } + } + + if mapEntry, exists := results[stashPerformer.ID]; exists { + mapEntry.Weight += weightIncrement + scoreBump + results[stashPerformer.ID] = mapEntry + } else { + results[stashPerformer.ID] = setupStashSearchResult(stashPerformer, weightIncrement+scoreBump) + } + } + } + + stashPerformers := scrape.SearchPerformerResult{} + stashPerformers = scrape.SearchStashPerformer(query) + scoreResults(stashPerformers.Data.Performers, 150) + + if len(results) == 0 { + warnings = append(warnings, "No Stashdb Performers Found") + } + // Sort the results by the weight score and limit to 100 + // Convert map to a slice of key-value pairs + pairs := make([]StashSearchPerformerResult, 0, len(results)) + for _, v := range results { + pairs = append(pairs, v) + } + // Sort the slice by weight in descending order + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Weight > pairs[j].Weight + }) + // Take the first 100 entries (or less if there are fewer than 100 entries) + top100 := pairs[:min(len(pairs), 100)] + var response StashSearchPerformersResponse + response.Results = top100 + response.Status = strings.Join(warnings, ", ") + resp.WriteHeaderAndEntity(http.StatusOK, response) +} func StashdbRunAll() { go func() { @@ -52,3 +691,37 @@ func StashdbRunAll() { } }() } +func findStashStudioIds(scraper string) []string { + stashIds := map[string]struct{}{} + var site models.Site + site.GetIfExist(scraper) + + db, _ := models.GetCommonDB() + var refs []models.ExternalReferenceLink + db.Preload("ExternalReference").Where(&models.ExternalReferenceLink{InternalTable: "sites", InternalNameId: scraper, ExternalSource: "stashdb studio"}).Find(&refs) + + for _, site := range refs { + stashIds[site.ExternalId] = struct{}{} + } + + config := models.BuildActorScraperRules() + s := config.StashSceneMatching[scraper] + for _, value := range s { + stashIds[value.StashId] = struct{}{} + } + + if len(stashIds) == 0 { + // if we don't have any lookup stashdb using the sitename + sitename := site.Name + if i := strings.Index(sitename, " ("); i != -1 { + sitename = sitename[:i] + } + studio := scrape.FindStashdbStudio(sitename, "name") + stashIds[studio.Data.Studio.ID] = struct{}{} + } + var results []string + for key, _ := range stashIds { + results = append(results, `"`+key+`"`) + } + return results +} diff --git a/pkg/externalreference/stashdb.go b/pkg/externalreference/stashdb.go index b49d365e9..fe73ed9f6 100644 --- a/pkg/externalreference/stashdb.go +++ b/pkg/externalreference/stashdb.go @@ -253,7 +253,6 @@ func matchSceneOnRules(sitename string, config models.StashSiteConfig) { func simplystring(str string) string { str = strings.TrimSpace(str) - str = strings.ReplaceAll(str, " and ", "&") str = strings.ReplaceAll(str, " ", "") str = strings.ReplaceAll(str, ".", "") str = strings.ReplaceAll(str, ":", "") @@ -262,6 +261,7 @@ func simplystring(str string) string { str = strings.ReplaceAll(str, ";", "") str = strings.ReplaceAll(str, ",", "") str = strings.ReplaceAll(str, "#", "") + str = strings.ReplaceAll(str, " and ", "&") str = strings.ReplaceAll(str, "@", "") str = strings.ReplaceAll(str, "$", "") str = strings.ReplaceAll(str, "%", "") diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go index 825a0b5d7..51abf89b8 100644 --- a/pkg/models/model_external_reference.go +++ b/pkg/models/model_external_reference.go @@ -89,6 +89,7 @@ type SceneMatchRule struct { XbvrField string XbvrMatch string XbvrMatchResultPosition int + StashField string StashRule string StashMatchResultPosition int } @@ -123,6 +124,12 @@ func (o *ExternalReferenceLink) FindByInternalName(internalTable string, interna commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalNameId: internalName}).Find(&refs) return refs } +func (o *ExternalReferenceLink) FindByExternalSource(internalTable string, internalId uint, externalSource string) []ExternalReferenceLink { + commonDb, _ := GetCommonDB() + var refs []ExternalReferenceLink + commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalDbId: internalId, ExternalSource: externalSource}).Find(&refs) + return refs +} func (o *ExternalReferenceLink) FindByExternaID(externalSource string, externalId string) { commonDb, _ := GetCommonDB() commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{ExternalSource: externalSource, ExternalId: externalId}).Find(&o) @@ -1054,6 +1061,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() { XbvrField: "Enter xbvr field you are matching to, scene_url or scene_id", XbvrMatch: "Enter regex express to extract value from field to match on", XbvrMatchResultPosition: 0, + StashField: "Enter the stash field to cmpare, default Url", StashRule: "Enter rule name, ie title, title/date, studio_code or regex expression to extract value to match from the stash url", StashMatchResultPosition: 0, }} diff --git a/pkg/models/model_stashdb.go b/pkg/models/model_stashdb.go index 3777c791c..b4052a018 100644 --- a/pkg/models/model_stashdb.go +++ b/pkg/models/model_stashdb.go @@ -15,6 +15,10 @@ type StashStudio struct { Parent IdName `json:"parent"` Updated time.Time `json:"updated"` } +type StashPerformerStudio struct { + SceneCount int `json:"scene_count"` + Studio StashStudio `json:"studio"` +} type StashPerformer struct { ID string `json:"id"` @@ -44,6 +48,7 @@ type StashPerformer struct { MergedIds []string `json:"merged_ids"` Created string `json:"created"` Updated time.Time `json:"updated"` + Studios []StashPerformerStudio `json:"studios"` } type StashBodyModification struct { @@ -68,6 +73,7 @@ type StashScene struct { Studio StashStudio `json:"studio"` Duration int `json:"duration"` Code string `json:"code"` + Images []Image `json:"images"` } type StashPerformerAs struct { @@ -81,3 +87,8 @@ type DELETEStashPerformerMin struct { Gender string `json:"gender"` Name string `json:"name"` } +type StashImage struct { + Url StashPerformer `json:"url"` + Width int `json:"width"` + Height int `json:"height"` +} diff --git a/pkg/scrape/stashdb.go b/pkg/scrape/stashdb.go index ab1b2a89f..8b59a3615 100644 --- a/pkg/scrape/stashdb.go +++ b/pkg/scrape/stashdb.go @@ -35,6 +35,13 @@ type QueryScenesData struct { type QueryScenesResult struct { Data QueryScenesData `json:"data"` } +type FindScenesData struct { + Scene models.StashScene `json:"findScene"` +} +type FindScenesResult struct { + Data FindScenesData `json:"data"` +} + type FindStudioResponse struct { Studio models.StashStudio `json:"studio"` } @@ -52,6 +59,12 @@ type FindPerformerResult struct { type FindPerformerData struct { Performer models.StashPerformer `json:"findPerformer"` } +type SearchPerformerResult struct { + Data SearchPerformerData `json:"data"` +} +type SearchPerformerData struct { + Performers []models.StashPerformer `json:"searchPerformer"` +} type QueryPerformerResult struct { Data QueryPerformerResultTypeData `json:"data"` @@ -63,6 +76,12 @@ type QueryPerformerResultType struct { Count int `json:"count"` Performers []models.StashPerformer `json:"performers"` } +type FindPerformerScenesData struct { + Performer models.StashPerformer `json:"findPerformer"` +} +type FindPerformerScenesResult struct { + Data FindPerformerScenesData `json:"data"` +} type Image struct { ID string `json:"id"` Url string `json:"url"` @@ -70,6 +89,56 @@ type Image struct { Height int `json:"height"` } +const sceneFieldList = ` +id +title +details +release_date +date +updated +urls{ + url + type + site { + id + name + description + url + regex + valid_types + } +} +studio{ + id + name + updated + parent { id } +} +images{ + url + width + height +} +performers{ + performer{ + id + updated + gender + name + aliases + } + as +} +fingerprints{ + hash + duration + submissions +} +duration +code +deleted +` + var Config models.ActorScraperConfig func StashDb() { @@ -94,7 +163,7 @@ func StashDb() { if i := strings.Index(sitename, " ("); i != -1 { sitename = sitename[:i] } - studio := findStudio(sitename, "name") + studio := FindStashdbStudio(sitename, "name") sitecfg, cfgExists := Config.StashSceneMatching[site.ID] if !cfgExists && studio.Data.Studio.ID != "" { @@ -103,7 +172,7 @@ func StashDb() { // check for a config entry if site not found for _, cfgEntry := range sitecfg { - studio = findStudio(cfgEntry.StashId, "id") + studio = FindStashdbStudio(cfgEntry.StashId, "id") siteConfig := Config.StashSceneMatching[site.ID] var ext models.ExternalReference ext.FindExternalId("stashdb studio", studio.Data.Studio.ID) @@ -132,7 +201,29 @@ func StashDb() { } } -func findStudio(studio string, field string) FindStudioResult { +func GetStashDbScene(stashId string) FindScenesResult { + var result FindScenesResult + if config.Config.Advanced.StashApiKey == "" { + return result + } + tlog := log.WithField("task", "scrape") + tlog.Infof("Scraping stash studio %s", stashId) + query := ` + query findScene($id: ID!) { + findScene(id: $id) { + ` + sceneFieldList + ` + } + } + ` + variables := `{"id": "` + stashId + `"}` + resp := CallStashDb(query, variables) + json.Unmarshal(resp, &result) + + tlog.Info("Scrape of Stashdb completed") + return result +} + +func FindStashdbStudio(studio string, field string) FindStudioResult { fieldType := "String" if field == "id" { fieldType = "ID" @@ -175,7 +266,7 @@ func processStudioPerformers(studioId string) { } for _, performer := range performerList.Data.QueryPerformers.Performers { - updatePerformer(performer) + UpdatePerformer(performer) } } func getPerformersPage(studioId string, page int) QueryPerformerResult { @@ -260,12 +351,17 @@ func getStudioSceneQueryVariable(studioId string, page int, count int) string { // Builds a query variable to get scenes from the Parent Studio // Uses the tagId to filter just scenes tag as Virtual Reality func getParentSceneQueryVariable(parentId string, tagId string, page int, count int) string { - return ` - {"input":{ + tag := "" + if tagId != "" { + tag = ` "tags": { "value": "` + tagId + `", "modifier": "INCLUDES" }, + ` + } + return ` + {"input":{` + tag + ` "parentStudio": "` + parentId + `", "page": ` + strconv.Itoa(page) + `, @@ -283,52 +379,7 @@ func GetScenePage(variables string) QueryScenesResult { queryScenes(input: $input) { count scenes{ - id - title - details - release_date - date - updated - urls{ - url - type - site { - id - name - description - url - regex - valid_types - } - } - studio{ - id - name - updated - } - images{ - url - width - height - } - performers{ - performer{ - id - updated - gender - name - aliases - } - as - } - fingerprints{ - hash - duration - submissions - } - duration - code - deleted +` + sceneFieldList + ` } } } @@ -373,7 +424,7 @@ func saveScenesToExternalReferences(scenes QueryScenesResult, studioId string) { // chek if we have the performers, may not in the case of loading scenes from the parent studio for _, performer := range scene.Performers { - updatePerformer(performer.Performer) + UpdatePerformer(performer.Performer) } // see if we can link to an xbvr scene based on the urls @@ -399,13 +450,13 @@ func saveScenesToExternalReferences(scenes QueryScenesResult, studioId string) { } } -func updatePerformer(newPerformer models.StashPerformer) { +func UpdatePerformer(newPerformer models.StashPerformer) { var ext models.ExternalReference ext.FindExternalId("stashdb performer", newPerformer.ID) var oldPerformer models.StashPerformer json.Unmarshal([]byte(ext.ExternalData), &oldPerformer) if ext.ID == 0 || newPerformer.Updated.UTC().Sub(oldPerformer.Updated.UTC()).Seconds() > 1 { - fullDetails := getStashPerformer(newPerformer.ID).Data.Performer + fullDetails := GetStashPerformer(newPerformer.ID).Data.Performer jsonData, _ := json.MarshalIndent(fullDetails, "", " ") newext := models.ExternalReference{ExternalSource: "stashdb performer", ExternalURL: "https://stashdb.org/performers/" + fullDetails.ID, ExternalId: fullDetails.ID, ExternalDate: fullDetails.Updated, ExternalData: string(jsonData)} if ext.ID != 0 { @@ -425,7 +476,7 @@ func RefreshPerformer(performerId string) { } var ext models.ExternalReference ext.FindExternalId("stashdb performer", performerId) - fullDetails := getStashPerformer(performerId).Data.Performer + fullDetails := GetStashPerformer(performerId).Data.Performer if fullDetails.ID == "" { return } @@ -440,7 +491,7 @@ func RefreshPerformer(performerId string) { } } -func getStashPerformer(performer string) FindPerformerResult { +func GetStashPerformer(performer string) FindPerformerResult { query := ` query findPerformer($id: ID!) { @@ -509,6 +560,127 @@ func getStashPerformer(performer string) FindPerformerResult { } return data } +func SearchStashPerformer(performer string) SearchPerformerResult { + + query := ` + query SearchAll($term: String!, $limit: Int = 100) + { searchPerformer(term: $term, limit: $limit) { + id + name + disambiguation + aliases + gender + birth_date + age + ethnicity + country + eye_color + hair_color + height + cup_size + band_size + waist_size + hip_size + breast_type + career_start_year + career_end_year + studios { + scene_count + studio { + name + id + } +} + images{ + id + url + width + height + } + + deleted + merged_ids + created + updated + + } + } + ` + + // Define the variables needed for your query as a Go map + var data SearchPerformerResult + variables := `{"term": "` + performer + `"}` + resp := CallStashDb(query, variables) + err := json.Unmarshal(resp, &data) + if err != nil { + log.Errorf("Eror extracting actor json") + } + return data +} + +func GetStashPerformerFull(performer string) FindPerformerScenesResult { + + query := ` + query findPerformer($id: ID!) { + findPerformer(id: $id) { + id + name + disambiguation + aliases + gender + birth_date + images{ + id + url + width + height + } + studios { + scene_count + studio { + name + id + } + } + deleted + created + updated + scenes { + id + title + details + release_date + date + studio{ + id + name + } + studio { + name + id + } + images{ + url + width + height + } + duration + deleted + } +} + } +` + + // Define the variables needed for your query as a Go map + var data FindPerformerScenesResult + variables := `{"id": "` + performer + `"}` + resp := CallStashDb(query, variables) + err := json.Unmarshal(resp, &data) + if err != nil { + log.Errorf("Eror extracting actor json") + } + return data +} func CallStashDb(query string, rawVariables string) []byte { var variables map[string]interface{} diff --git a/ui/src/App.vue b/ui/src/App.vue index a4061ec67..a5063ef6c 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -13,6 +13,8 @@ + + @@ -30,9 +32,11 @@ import Details from './views/scenes/Details' import EditScene from './views/scenes/EditScene' import ActorDetails from './views/actors/ActorDetails' import EditActor from './views/actors/EditActor' +import SearchStashdbScenes from './views/scenes/SearchStashdbScenes' +import SearchStashdbActors from './views/actors/SearchStashdbActors' export default { - components: { Navbar, Socket, QuickFind, GlobalEvents, Details, EditScene, ActorDetails, EditActor }, + components: { Navbar, Socket, QuickFind, GlobalEvents, Details, EditScene, ActorDetails, EditActor, SearchStashdbScenes,SearchStashdbActors }, computed: { showOverlay () { return this.$store.state.overlay.details.show @@ -46,6 +50,12 @@ export default { showActorEdit() { return this.$store.state.overlay.actoredit.show }, + showSearchStashdbScenes() { + return this.$store.state.overlay.searchStashDbScenes.show + }, + showSearchStashdbActors() { + return this.$store.state.overlay.searchStashDbActors.show + }, } } diff --git a/ui/src/components/LinkStashdbButton.vue b/ui/src/components/LinkStashdbButton.vue new file mode 100644 index 000000000..3391ec6a8 --- /dev/null +++ b/ui/src/components/LinkStashdbButton.vue @@ -0,0 +1,50 @@ + + + + \ No newline at end of file diff --git a/ui/src/store/overlay.js b/ui/src/store/overlay.js index a36a808cf..c225209fb 100644 --- a/ui/src/store/overlay.js +++ b/ui/src/store/overlay.js @@ -40,6 +40,15 @@ const state = { show:false, site: '', }, + searchStashDbScenes: { + show: false, + scene: null + }, + searchStashDbActors: { + show: false, + actor: null + }, + changeDetailsTab: -1, } const mutations = { @@ -135,6 +144,25 @@ const mutations = { hideSceneMatchParams (state, payload) { state.sceneMatchParams.show = false }, + showSearchStashdbScenes (state, payload) { + state.searchStashDbScenes.scene = payload.item + state.searchStashDbScenes.show = true + }, + hideSearchStashdbScenes (state) { + state.searchStashDbScenes.scene = null + state.searchStashDbScenes.show = false + }, + showSearchStashdbActors (state, payload) { + state.searchStashDbActors.actor = payload.item + state.searchStashDbActors.show = true + }, + hideSearchStashdbActors (state) { + state.searchStashDbActors.actor = null + state.searchStashDbActors.show = false + }, + changeDetailsTab (state, payload) { + state.changeDetailsTab = payload.tab + }, } export default { diff --git a/ui/src/views/actors/ActorCard.vue b/ui/src/views/actors/ActorCard.vue index fbc716875..99e6084d5 100644 --- a/ui/src/views/actors/ActorCard.vue +++ b/ui/src/views/actors/ActorCard.vue @@ -21,6 +21,7 @@   + @@ -54,13 +55,14 @@ import { format, parseISO } from 'date-fns' import ActorFavouriteButton from '../../components/ActorFavouriteButton' import ActorWatchlistButton from '../../components/ActorWatchlistButton' import ActorEditButton from '../../components/ActorEditButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import VueLoadImage from 'vue-load-image' import { tr } from 'date-fns/locale' export default { name: 'ActorCard', props: { actor: Object, colleague: String }, - components: {ActorFavouriteButton, ActorWatchlistButton, VueLoadImage, ActorEditButton}, + components: {ActorFavouriteButton, ActorWatchlistButton, VueLoadImage, ActorEditButton, LinkStashdbButton}, data () { return { preview: false, diff --git a/ui/src/views/actors/ActorDetails.vue b/ui/src/views/actors/ActorDetails.vue index 7ec127b28..88c4bad7e 100644 --- a/ui/src/views/actors/ActorDetails.vue +++ b/ui/src/views/actors/ActorDetails.vue @@ -10,6 +10,7 @@ @keydown.f="$store.commit('actorList/toggleActorList', {actor_id: actor.id, list: 'favourite'})" @keydown.exact.w="$store.commit('actorList/toggleActorList', {actor_id: actor.id, list: 'watchlist'})" @keydown.e="$store.commit('overlay/editActorDetails', {actor: actor})" + @keydown.s="$store.commit('overlay/showSearchStashdbActors', {actor: item})" @keydown.48="setRating(0)" /> @@ -91,6 +92,7 @@       + @@ -266,12 +268,13 @@ import StarRating from 'vue-star-rating' import ActorFavouriteButton from '../../components/ActorFavouriteButton' import ActorWatchlistButton from '../../components/ActorWatchlistButton' import ActorEditButton from '../../components/ActorEditButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import SceneCard from '../scenes/SceneCard' import ActorCard from './ActorCard' export default { name: 'ActorDetails', - components: { VueLoadImage, GlobalEvents, StarRating, ActorWatchlistButton, ActorFavouriteButton, SceneCard, ActorEditButton, ActorCard }, + components: { VueLoadImage, GlobalEvents, StarRating, ActorWatchlistButton, ActorFavouriteButton, SceneCard, ActorEditButton, ActorCard, LinkStashdbButton }, data () { return { index: 1, diff --git a/ui/src/views/actors/SearchStashdbActors.vue b/ui/src/views/actors/SearchStashdbActors.vue new file mode 100644 index 000000000..70ced48bb --- /dev/null +++ b/ui/src/views/actors/SearchStashdbActors.vue @@ -0,0 +1,250 @@ + + + + + diff --git a/ui/src/views/scenes/Details.vue b/ui/src/views/scenes/Details.vue index e80dff764..19fd0c284 100644 --- a/ui/src/views/scenes/Details.vue +++ b/ui/src/views/scenes/Details.vue @@ -12,6 +12,7 @@ @keydown.shift.w="$store.commit('sceneList/toggleSceneList', {scene_id: item.scene_id, list: 'watched'})" @keydown.t="$store.commit('sceneList/toggleSceneList', {scene_id: item.scene_id, list: 'trailerlist'})" @keydown.e="$store.commit('overlay/editDetails', {scene: item})" + @keydown.s="$store.commit('overlay/showSearchStashdbScenes', {scene: item})" @keydown.g="toggleGallery" @keydown.48="setRating(0)" /> @@ -126,6 +127,7 @@ + @@ -406,6 +408,7 @@ import VueLoadImage from 'vue-load-image' import GlobalEvents from 'vue-global-events' import StarRating from 'vue-star-rating' import FavouriteButton from '../../components/FavouriteButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import WatchlistButton from '../../components/WatchlistButton' import WishlistButton from '../../components/WishlistButton' import WatchedButton from '../../components/WatchedButton' @@ -417,7 +420,7 @@ import HiddenButton from '../../components/HiddenButton' export default { name: 'Details', - components: { VueLoadImage, GlobalEvents, StarRating, WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, RefreshButton, RescrapeButton, TrailerlistButton, HiddenButton }, + components: { VueLoadImage, GlobalEvents, StarRating, WatchlistButton, FavouriteButton, LinkStashdbButton, WishlistButton, WatchedButton, EditButton, RefreshButton, RescrapeButton, TrailerlistButton, HiddenButton }, data () { return { index: 1, @@ -584,6 +587,9 @@ export default { return 0; // Return 0 or handle error as needed } }, + changeDetailsTab() { + return this.$store.state.overlay.changeDetailsTab + }, quickFindOverlayState() { return this.$store.state.overlay.quickFind.show }, @@ -632,6 +638,13 @@ watch:{ } } }, + changeDetailsTab(newVal, oldVal){ + if (newVal == -1 ) { + return + } + this.activeTab = newVal + this.$store.commit('overlay/changeDetailsTab', { tab: -1 }) + } }, methods: { setupPlayer () { diff --git a/ui/src/views/scenes/SceneCard.vue b/ui/src/views/scenes/SceneCard.vue index df6a0fc79..0d3fdf769 100644 --- a/ui/src/views/scenes/SceneCard.vue +++ b/ui/src/views/scenes/SceneCard.vue @@ -60,6 +60,7 @@ + @@ -91,6 +92,7 @@ import FavouriteButton from '../../components/FavouriteButton' import WishlistButton from '../../components/WishlistButton' import WatchedButton from '../../components/WatchedButton' import EditButton from '../../components/EditButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import TrailerlistButton from '../../components/TrailerlistButton' import HiddenButton from '../../components/HiddenButton' import ky from 'ky' @@ -99,13 +101,14 @@ import VueLoadImage from 'vue-load-image' export default { name: 'SceneCard', props: { item: Object, reRead: Boolean }, - components: { WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, TrailerlistButton, HiddenButton, VueLoadImage }, + components: { WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, LinkStashdbButton, TrailerlistButton, HiddenButton, VueLoadImage }, data () { return { preview: false, format, parseISO, alternateSources: [], + stashLinkExists: false, } }, computed: { @@ -156,6 +159,7 @@ export default { return this.$store.state.optionsWeb.web.isAvailOpacity / 100 }, async getAlternateSceneSourcesWithTitles() { + this.stashLinkExists = false try { const response = await ky.get('/api/scene/alternate_source/' + this.item.id).json(); this.alternateSources = []; @@ -173,6 +177,9 @@ export default { } else if (altsrc.external_source == "stashdb scene") { title = extdata.title || 'No Title'; } + if (altsrc.external_source.includes('stashdb')) { + this.stashLinkExists = true + } return { ...altsrc, title: title diff --git a/ui/src/views/scenes/SearchStashdbScenes.vue b/ui/src/views/scenes/SearchStashdbScenes.vue new file mode 100644 index 000000000..68d61d214 --- /dev/null +++ b/ui/src/views/scenes/SearchStashdbScenes.vue @@ -0,0 +1,197 @@ + + + + +