diff --git a/pkg/api/external_references.go b/pkg/api/external_references.go index 71ad0bd14..b231c8e10 100644 --- a/pkg/api/external_references.go +++ b/pkg/api/external_references.go @@ -49,6 +49,16 @@ func (i ExternalReference) WebService() *restful.WebService { ws.Route(ws.GET("/generic/scrape_all").To(i.genericActorScraper). Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/stashdb/link2scene/{scene-id}/{stashdb-id}").To(i.linkScene2Stashdb). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes(models.Scene{})) + ws.Route(ws.GET("/stashdb/search/{scene-id}").To(i.searchForStashdbScene). + Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/stashdb/link2actor/{actor-id}/{stashdb-id}").To(i.linkActor2Stashdb). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes(models.Scene{})) + ws.Route(ws.GET("/stashdb/searchactor/{actor-id}").To(i.searchForStashdbActor). + Metadata(restfulspec.KeyOpenAPITags, tags)) ws.Route(ws.POST("/generic/scrape_single").To(i.genericSingleActorScraper). Metadata(restfulspec.KeyOpenAPITags, tags)) diff --git a/pkg/api/options.go b/pkg/api/options.go index 8d1fdf51d..50f0ed464 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -1017,6 +1017,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res scrapers := make(map[string][]config.ScraperConfig) scrapers["povr"] = scraperConfig.CustomScrapers.PovrScrapers scrapers["slr"] = scraperConfig.CustomScrapers.SlrScrapers + scrapers["stashdb"] = scraperConfig.CustomScrapers.StashDbScrapers scrapers["vrphub"] = scraperConfig.CustomScrapers.VrphubScrapers scrapers["vrporn"] = scraperConfig.CustomScrapers.VrpornScrapers scrapers["realvr"] = scraperConfig.CustomScrapers.RealVRScrapers @@ -1041,6 +1042,8 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res scrapers["povr"] = append(scrapers["povr"], scraper) case "sexlikereal": scrapers["slr"] = append(scrapers["slr"], scraper) + case "stashdb": + scrapers["stashdb"] = append(scrapers["stashdb"], scraper) case "vrphub": scrapers["vrphub"] = append(scrapers["vrphub"], scraper) case "vrporn": @@ -1051,6 +1054,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res } scraperConfig.CustomScrapers.PovrScrapers = scrapers["povr"] scraperConfig.CustomScrapers.SlrScrapers = scrapers["slr"] + scraperConfig.CustomScrapers.StashDbScrapers = scrapers["stashdb"] scraperConfig.CustomScrapers.VrphubScrapers = scrapers["vrphub"] scraperConfig.CustomScrapers.VrpornScrapers = scrapers["vrporn"] scraperConfig.CustomScrapers.RealVRScrapers = scrapers["realvr"] diff --git a/pkg/api/scenes.go b/pkg/api/scenes.go index 389128d57..bd86370fe 100644 --- a/pkg/api/scenes.go +++ b/pkg/api/scenes.go @@ -1029,7 +1029,7 @@ func (i SceneResource) getSceneAlternateSources(req *restful.Request, resp *rest var site models.Site if ref.ExternalSource == "stashdb scene" { - ressults = append(ressults, ResponseGetAlternateSources{Url: ref.ExternalReference.ExternalURL, Icon: "https://docs.stashapp.cc/favicon.ico", ExternalSource: ref.ExternalReference.ExternalSource, ExternalId: ref.ExternalReference.ExternalId, ExternalData: ref.ExternalReference.ExternalData}) + ressults = append(ressults, ResponseGetAlternateSources{Url: ref.ExternalReference.ExternalURL, Icon: "https://guidelines.stashdb.org/favicon.ico", ExternalSource: ref.ExternalReference.ExternalSource, ExternalId: ref.ExternalReference.ExternalId, ExternalData: ref.ExternalReference.ExternalData}) } else { json.Unmarshal([]byte(ref.ExternalReference.ExternalData), &altscene) site.GetIfExist(altscene.Scene.ScraperId) diff --git a/pkg/api/stashdb.go b/pkg/api/stashdb.go index b4e02b590..d34c047b1 100644 --- a/pkg/api/stashdb.go +++ b/pkg/api/stashdb.go @@ -1,7 +1,14 @@ package api import ( + "encoding/json" + "fmt" + "math" "net/http" + "regexp" + "sort" + "strconv" + "strings" "time" "github.com/emicklei/go-restful/v3" @@ -31,6 +38,638 @@ func (i ExternalReference) stashDbUpdateData(req *restful.Request, resp *restful func (i ExternalReference) stashRunAll(req *restful.Request, resp *restful.Response) { StashdbRunAll() } +func (i ExternalReference) linkScene2Stashdb(req *restful.Request, resp *restful.Response) { + sceneId := req.PathParameter("scene-id") + stashdbId := req.PathParameter("stashdb-id") + stashdbId = strings.TrimPrefix(stashdbId, "https://stashdb.org/scenes/") + var scene models.Scene + + db, _ := models.GetDB() + defer db.Close() + + if strings.Contains(sceneId, "-") { + scene.GetIfExist(sceneId) + } else { + id, _ := strconv.Atoi(req.PathParameter("scene-id")) + scene.GetIfExistByPK(uint(id)) + } + if scene.ID == 0 { + return + } + stashScene := scrape.GetStashDbScene(stashdbId) + + var existingRef models.ExternalReference + existingRef.FindExternalId("stashdb scene", stashdbId) + + jsonData, _ := json.MarshalIndent(stashScene.Data.Scene, "", " ") + + // chek if we have the performers, may not in the case of loading scenes from the parent studio + for _, performer := range stashScene.Data.Scene.Performers { + scrape.UpdatePerformer(performer.Performer) + } + + var xbrLink []models.ExternalReferenceLink + xbrLink = append(xbrLink, models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: scene.ID, InternalNameId: scene.SceneID, ExternalSource: "stashdb scene", ExternalId: stashdbId, MatchType: 5}) + ext := models.ExternalReference{ExternalSource: "stashdb scene", ExternalURL: "https://stashdb.org/scenes/" + stashdbId, ExternalId: stashdbId, ExternalDate: time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC), ExternalData: string(jsonData), + XbvrLinks: xbrLink} + ext.AddUpdateWithId() + + // check for actor not yet linked + for _, actor := range scene.Cast { + var extreflinks []models.ExternalReferenceLink + db.Preload("ExternalReference").Where(&models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, ExternalSource: "stashdb performer"}).Find(&extreflinks) + if len(extreflinks) == 0 { + stashPerformerId := "" + for _, stashPerf := range stashScene.Data.Scene.Performers { + if strings.EqualFold(stashPerf.Performer.Name, actor.Name) || strings.EqualFold(stashPerf.As, actor.Name) { + stashPerformerId = stashPerf.Performer.ID + continue + } + for _, alias := range stashPerf.Performer.Aliases { + if strings.EqualFold(alias, actor.Name) { + stashPerformerId = stashPerf.Performer.ID + } + } + } + if stashPerformerId != "" { + scrape.RefreshPerformer(stashPerformerId) + var actorRef models.ExternalReference + actorRef.FindExternalId("stashdb performer", stashPerformerId) + var performer models.StashPerformer + json.Unmarshal([]byte(actorRef.ExternalData), &performer) + + xbvrLink := models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, InternalNameId: actor.Name, MatchType: 90, + ExternalReferenceID: actorRef.ID, ExternalSource: actorRef.ExternalSource, ExternalId: actorRef.ExternalId} + actorRef.XbvrLinks = append(actorRef.XbvrLinks, xbvrLink) + actorRef.AddUpdateWithId() + + externalreference.UpdateXbvrActor(performer, actor.ID) + } + } + } + + // reread the scene to return updated data + scene.GetIfExistByPK(scene.ID) + resp.WriteHeaderAndEntity(http.StatusOK, scene) +} + +func (i ExternalReference) searchForStashdbScene(req *restful.Request, resp *restful.Response) { + query := req.QueryParameter("q") + + var warnings []string + type StashSearchScenePerformerResult struct { + Name string + Url string + } + type StashSearchSceneResult struct { + Url string + ImageUrl string + Performers []StashSearchScenePerformerResult + Title string + Studio string + Duration string + Description string + Weight int + Date string + Id string + } + type StashSearchSceneResponse struct { + Status string + Results []StashSearchSceneResult + } + results := make(map[string]StashSearchSceneResult) + + sceneId := req.PathParameter("scene-id") + var scene models.Scene + + db, _ := models.GetDB() + defer db.Close() + + if strings.Contains(sceneId, "-") { + scene.GetIfExist(sceneId) + } else { + id, _ := strconv.Atoi(req.PathParameter("scene-id")) + scene.GetIfExistByPK(uint(id)) + } + if scene.ID == 0 { + var response StashSearchSceneResponse + response.Results = []StashSearchSceneResult{} + response.Status = "XBVR Scene not found" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + + setupStashSearchResult := func(stashScene models.StashScene, weight int) StashSearchSceneResult { + //common function to call to setup stash response details + result := StashSearchSceneResult{Id: stashScene.ID, Url: "https://stashdb.org/scenes/" + stashScene.ID, Weight: weight, Title: stashScene.Title, Description: stashScene.Details, Date: stashScene.Date, Studio: stashScene.Studio.Name} + if len(stashScene.Images) > 0 { + result.ImageUrl = stashScene.Images[0].URL + } + for _, perf := range stashScene.Performers { + result.Performers = append(result.Performers, StashSearchScenePerformerResult{Name: perf.Performer.Name, Url: `https://stashdb.org/performers/` + perf.Performer.ID}) + } + if stashScene.Duration > 0 { + hours := stashScene.Duration / 3600 // calculate hours + stashScene.Duration %= 3600 // remaining seconds after hours + minutes := stashScene.Duration / 60 // calculate minutes + stashScene.Duration %= 60 // remaining seconds after minutes + + // Format the time string + result.Duration = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, stashScene.Duration) + } + return result + } + + var guidRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + idTest := strings.TrimPrefix(strings.TrimSpace(query), "https://stashdb.org/scenes/") + + if guidRegex.MatchString(idTest) { + stashScene := scrape.GetStashDbScene(idTest) + if stashScene.Data.Scene.ID != "" { + results[stashScene.Data.Scene.ID] = setupStashSearchResult(stashScene.Data.Scene, 10000) + var response StashSearchSceneResponse + response.Results = []StashSearchSceneResult{results[stashScene.Data.Scene.ID]} + response.Status = "" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + } + + stashStudioIds := findStashStudioIds(scene.ScraperId) + if len(stashStudioIds) == 0 { + var response StashSearchSceneResponse + response.Results = []StashSearchSceneResult{} + response.Status = "Cannot find Stashdb Studio" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + + var xbvrperformers []string + for _, actor := range scene.Cast { + var stashlinks []models.ExternalReferenceLink + db.Preload("ExternalReference").Where(&models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, ExternalSource: "stashdb performer"}).Find(&stashlinks) + if len(stashlinks) == 0 { + warnings = append(warnings, actor.Name+" is not linked to Stashdb") + } else { + for _, stashPerformer := range stashlinks { + xbvrperformers = append(xbvrperformers, `"`+stashPerformer.ExternalId+`"`) + } + } + } + + // define a function to update the results found + scoreResults := func(stashScenes scrape.QueryScenesResult, weightIncrement int, performers []string, studios []string) { + for _, stashscene := range stashScenes.Data.QueryScenes.Scenes { + // consider adding weight bump for duration and date + scoreBump := 0 + if stashscene.Date == scene.ReleaseDateText { + scoreBump += 15 + } + if stashscene.Title == scene.Title { + scoreBump += 25 + } + if stashscene.Duration > 0 && scene.Duration > 0 { + stashDur := float64((stashscene.Duration / 60) - scene.Duration) + if math.Abs(stashDur) <= 2 { + scoreBump += 5 * int(3-math.Abs(stashDur)) + } + } + // check duration from video files + for _, file := range scene.Files { + if file.Type == "video" { + diff := file.VideoDuration - float64(stashscene.Duration) + if math.Abs(diff) <= 2 { + scoreBump += 5 * int(3-math.Abs(diff)) + } + } + } + + // check it is from a studio we expect + for _, studio := range studios { + if strings.ReplaceAll(studio, `"`, ``) == stashscene.Studio.ID { + scoreBump += 20 + } + } + + foundActorBump := -5 + for _, sp := range stashscene.Performers { + for _, xp := range performers { + if strings.Contains(xp, sp.Performer.ID) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 10 + } else { + foundActorBump += 5 + } + } + } + } + // we have checked if stash performers match xbvr, now check for xbvr performers not matched in stash + for _, xp := range xbvrperformers { + for _, sp := range stashscene.Performers { + if strings.Contains(xp, sp.Performer.ID) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 15 + } else { + foundActorBump += 5 + } + } + } + } + // check actor matches using names and aliases + for _, actor := range scene.Cast { + for _, sp := range stashscene.Performers { + if strings.EqualFold(actor.Name, sp.Performer.Name) || strings.EqualFold(actor.Name, sp.As) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 15 + } else { + foundActorBump += 5 + } + continue + } + // try aliases + for _, alias := range sp.Performer.Aliases { + if strings.EqualFold(alias, actor.Name) { + if sp.Performer.Gender == "FEMALE" { + foundActorBump += 15 + } else { + foundActorBump += 5 + } + continue + } + } + } + } + if mapEntry, exists := results[stashscene.ID]; exists { + mapEntry.Weight += weightIncrement + scoreBump + foundActorBump + results[stashscene.ID] = mapEntry + } else { + results[stashscene.ID] = setupStashSearchResult(stashscene, weightIncrement+scoreBump+foundActorBump) + } + } + } + + var fingerprints []string + for _, file := range scene.Files { + if file.Type == "video" { + file.OsHash = "00000000000000000" + file.OsHash + fingerprints = append(fingerprints, `"`+file.OsHash[len(file.OsHash)-16:]+`"`) + } + } + if len(fingerprints) > 0 { + fingerprintList := strings.Join(fingerprints, ",") + fingerprintQuery := ` + {"input":{ + "page": 1, + "per_page": 150, + "sort": "UPDATED_AT", + "fingerprints": {"value": [` + + fingerprintList + + `], "modifier":"EQUALS"} + } + }` + stashScenes := scrape.GetScenePage(fingerprintQuery) + scoreResults(stashScenes, 400, xbvrperformers, stashStudioIds) + } + + stashScenes := scrape.QueryScenesResult{} + for _, studio := range stashStudioIds { + // Exact Title submatch + titleQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": 1, + "per_page": 150, + "sort": "UPDATED_AT", + "title": "\"` + + scene.Title + `\"" + } + }` + stashScenes = scrape.GetScenePage(titleQuery) + scoreResults(stashScenes, 150, xbvrperformers, stashStudioIds) + } + + if len(xbvrperformers) > 0 { + performerList := strings.Join(xbvrperformers, ",") + for _, studio := range stashStudioIds { + performerQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": 1, + "per_page": 150, + "sort": "UPDATED_AT", + "performers": {"value": [` + + performerList + + `], "modifier":"INCLUDES_ALL"} + } + }` + stashScenes = scrape.GetScenePage(performerQuery) + scoreResults(stashScenes, 200, xbvrperformers, stashStudioIds) + if len(stashScenes.Data.QueryScenes.Scenes) == 0 { + performerQuery = strings.ReplaceAll(performerQuery, "INCLUDES_ALL", "INCLUDES") + stashScenes := scrape.GetScenePage(performerQuery) + scoreResults(stashScenes, 100, xbvrperformers, stashStudioIds) + } + } + } + + if len(results) == 0 { + for _, studio := range stashStudioIds { + // No match yet, try match any words from the title, not likely to find, as this returns too many results + titleQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": 1, + "per_page": 100, + "sort": "UPDATED_AT", + "title": "` + + scene.Title + `" + } + }` + stashScenes = scrape.GetScenePage(titleQuery) + scoreResults(stashScenes, 150, xbvrperformers, stashStudioIds) + page := 2 + for i := 101; i < stashScenes.Data.QueryScenes.Count && page <= 5; { + titleQuery := ` + {"input":{ + "parentStudio": ` + studio + `, + "page": ` + strconv.Itoa(page) + `, + "per_page": 100, + "sort": "UPDATED_AT", + "title": "` + + scene.Title + `" + } + }` + stashScenes = scrape.GetScenePage(titleQuery) + scoreResults(stashScenes, 150, xbvrperformers, stashStudioIds) + i = i + 100 + page += 1 + } + } + } + + if len(results) == 0 { + warnings = append(warnings, "No Stashdb Scenes Found") + } + // sort and limit the number of results + // Convert map to a slice of key-value pairs + pairs := make([]StashSearchSceneResult, 0, len(results)) + for _, v := range results { + pairs = append(pairs, v) + } + // Sort the slice by weight in descending order + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Weight > pairs[j].Weight + }) + // Take the first 100 entries (or less if there are fewer than 100 entries) + top100 := pairs[:min(len(pairs), 100)] + var response StashSearchSceneResponse + response.Results = top100 + response.Status = strings.Join(warnings, ", ") + resp.WriteHeaderAndEntity(http.StatusOK, response) +} + +func (i ExternalReference) linkActor2Stashdb(req *restful.Request, resp *restful.Response) { + actorId := req.PathParameter("actor-id") + stashPerformerId := req.PathParameter("stashdb-id") + stashPerformerId = strings.TrimPrefix(stashPerformerId, "https://stashdb.org/performers/") + var actor models.Actor + + db, _ := models.GetDB() + defer db.Close() + + id, _ := strconv.Atoi(actorId) + if id == 0 { + actor.GetIfExist(actorId) + } else { + actor.GetIfExistByPK(uint(id)) + } + if actor.ID == 0 { + return + } + + scrape.RefreshPerformer(stashPerformerId) + var actorRef models.ExternalReference + actorRef.FindExternalId("stashdb performer", stashPerformerId) + // change the External Date, this is used to find the most recent change and query + // stash for changes since then. If wew manually load an actor, we may miss other updates + actorRef.ExternalDate = time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC) + actorRef.Save() + + var performer models.StashPerformer + json.Unmarshal([]byte(actorRef.ExternalData), &performer) + + xbvrLink := models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, InternalNameId: actor.Name, MatchType: 90, + ExternalReferenceID: actorRef.ID, ExternalSource: actorRef.ExternalSource, ExternalId: actorRef.ExternalId} + actorRef.XbvrLinks = append(actorRef.XbvrLinks, xbvrLink) + actorRef.AddUpdateWithId() + + externalreference.UpdateXbvrActor(performer, actor.ID) + + // reread the actor to return updated data + actor.GetIfExistByPK(actor.ID) + resp.WriteHeaderAndEntity(http.StatusOK, actor) +} + +func (i ExternalReference) searchForStashdbActor(req *restful.Request, resp *restful.Response) { + query := req.QueryParameter("q") + query = strings.TrimSpace(strings.TrimPrefix(query, "aka:")) + + var warnings []string + type StashSearchPerformerScenesResult struct { + Title string + Id string + Url string + Duration string + ImageUrl string + Studio string + } + type StashSearchPerformerStudioResult struct { + Name string + Id string + Url string + SceneCount int + Matched bool + } + type StashSearchPerformerAliasResult struct { + Alias string + Matched bool + } + type StashSearchPerformerResult struct { + Url string + Name string + Disambiguation string + Aliases []StashSearchPerformerAliasResult + Id string + ImageUrl []string + DOB string + Weight int + Studios []StashSearchPerformerStudioResult + } + type StashSearchPerformersResponse struct { + Status string + Results []StashSearchPerformerResult + } + results := make(map[string]StashSearchPerformerResult) + + actorId := req.PathParameter("actor-id") + var actor models.Actor + + db, _ := models.GetDB() + defer db.Close() + + id, _ := strconv.Atoi(req.PathParameter("actor-id")) + if id == 0 { + actor.GetIfExist(actorId) + } else { + actor.GetIfExistByPK(uint(id)) + } + if actor.ID == 0 { + var response StashSearchPerformersResponse + response.Results = []StashSearchPerformerResult{} + response.Status = "XBVR Actor not found" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + + matchedStudios := map[string]struct{}{} + matchedAlias := map[string]struct{}{} + + setupStashSearchResult := func(stashPerformer models.StashPerformer, weight int) StashSearchPerformerResult { + //common function to call to setup stash response details + result := StashSearchPerformerResult{Id: stashPerformer.ID, Url: "https://stashdb.org/performers/" + stashPerformer.ID, Weight: weight, Name: stashPerformer.Name, DOB: stashPerformer.BirthDate, Disambiguation: stashPerformer.Disambiguation} + for _, image := range stashPerformer.Images { + result.ImageUrl = append(result.ImageUrl, image.URL) + } + for _, studio := range stashPerformer.Studios { + _, matched := matchedStudios[studio.Studio.ID] + if matched { + matched = true + } + result.Studios = append(result.Studios, StashSearchPerformerStudioResult{Name: studio.Studio.Name, Id: studio.Studio.ID, Url: `https://stashdb.org/performers/` + stashPerformer.ID + `?studios=` + studio.Studio.ID, SceneCount: studio.SceneCount, Matched: matched}) + } + for _, alias := range stashPerformer.Aliases { + _, matched := matchedAlias[strings.ToLower(alias)] + if matched { + matched = true + } + result.Aliases = append(result.Aliases, StashSearchPerformerAliasResult{Alias: alias, Matched: matched}) + } + + sort.Slice(result.Studios, func(i, j int) bool { + return result.Studios[i].Name < result.Studios[j].Name + }) + return result + } + + var guidRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + idTest := strings.TrimPrefix(strings.TrimSpace(query), "https://stashdb.org/performers/") + + if guidRegex.MatchString(idTest) { + stashPerformer := scrape.GetStashPerformerFull(idTest) + if stashPerformer.Data.Performer.ID != "" { + results[stashPerformer.Data.Performer.ID] = setupStashSearchResult(stashPerformer.Data.Performer, 10000) + // need to get studios + var response StashSearchPerformersResponse + response.Results = []StashSearchPerformerResult{results[stashPerformer.Data.Performer.ID]} + response.Status = "" + resp.WriteHeaderAndEntity(http.StatusOK, response) + return + } + } + + if strings.TrimSpace(query) == "" { + query = actor.Name + } + // define a function to update the results found + scoreResults := func(stashPerformers []models.StashPerformer, weightIncrement int) { + for _, stashPerformer := range stashPerformers { + // consider adding weight bump for duration and date + scoreBump := 0 + lcaseActorName := strings.ToLower(actor.Name) + if stashPerformer.Name == actor.Name { + scoreBump += 200 + } + if strings.Contains(strings.ToLower(stashPerformer.Name), lcaseActorName) { + scoreBump += 30 + } + for _, alias := range stashPerformer.Aliases { + lcaseAlias := strings.ToLower(alias) + if strings.EqualFold(lcaseAlias, lcaseActorName) || strings.EqualFold(lcaseAlias, strings.ToLower(query)) { + scoreBump += 50 + matchedAlias[strings.ToLower(alias)] = struct{}{} + } else { + if strings.Contains(lcaseAlias, lcaseActorName) || strings.Contains(lcaseActorName, lcaseAlias) || strings.Contains(lcaseAlias, strings.ToLower(query)) { + scoreBump += 20 + matchedAlias[strings.ToLower(alias)] = struct{}{} + } + } + } + // dob checks + + if actor.Gender != "" && strings.EqualFold(actor.Gender, stashPerformer.Gender) { + scoreBump += 10 + } + + var stashExtLink models.ExternalReferenceLink + commonDb, _ := models.GetCommonDB() + for _, xbvrScene := range actor.Scenes { + for _, stashStudio := range stashPerformer.Studios { + xbvrsite := xbvrScene.Site + var siteRef models.ExternalReferenceLink + commonDb.Where(&models.ExternalReferenceLink{InternalTable: "sites", InternalNameId: xbvrScene.ScraperId, ExternalSource: "stashdb studio"}).First(&siteRef) + if strings.Index(xbvrsite, " (") != -1 { + xbvrsite = xbvrsite[:strings.Index(xbvrsite, " (")] + } + if strings.EqualFold(stashStudio.Studio.Name, xbvrsite) || siteRef.ExternalId == stashStudio.Studio.ID { + scoreBump += 5 + matchedStudios[stashStudio.Studio.ID] = struct{}{} + } + } + // check if we have a linked scene with this performer + links := stashExtLink.FindByExternalSource("scenes", xbvrScene.ID, "stashdb scene") + if len(links) == 0 { + continue + } + for _, link := range links { + if strings.Contains(link.ExternalReference.ExternalData, stashPerformer.ID) { + scoreBump += 50 + } + } + } + + if mapEntry, exists := results[stashPerformer.ID]; exists { + mapEntry.Weight += weightIncrement + scoreBump + results[stashPerformer.ID] = mapEntry + } else { + results[stashPerformer.ID] = setupStashSearchResult(stashPerformer, weightIncrement+scoreBump) + } + } + } + + stashPerformers := scrape.SearchPerformerResult{} + stashPerformers = scrape.SearchStashPerformer(query) + scoreResults(stashPerformers.Data.Performers, 150) + + if len(results) == 0 { + warnings = append(warnings, "No Stashdb Performers Found") + } + // Sort the results by the weight score and limit to 100 + // Convert map to a slice of key-value pairs + pairs := make([]StashSearchPerformerResult, 0, len(results)) + for _, v := range results { + pairs = append(pairs, v) + } + // Sort the slice by weight in descending order + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Weight > pairs[j].Weight + }) + // Take the first 100 entries (or less if there are fewer than 100 entries) + top100 := pairs[:min(len(pairs), 100)] + var response StashSearchPerformersResponse + response.Results = top100 + response.Status = strings.Join(warnings, ", ") + resp.WriteHeaderAndEntity(http.StatusOK, response) +} func StashdbRunAll() { go func() { @@ -52,3 +691,37 @@ func StashdbRunAll() { } }() } +func findStashStudioIds(scraper string) []string { + stashIds := map[string]struct{}{} + var site models.Site + site.GetIfExist(scraper) + + db, _ := models.GetCommonDB() + var refs []models.ExternalReferenceLink + db.Preload("ExternalReference").Where(&models.ExternalReferenceLink{InternalTable: "sites", InternalNameId: scraper, ExternalSource: "stashdb studio"}).Find(&refs) + + for _, site := range refs { + stashIds[site.ExternalId] = struct{}{} + } + + config := models.BuildActorScraperRules() + s := config.StashSceneMatching[scraper] + for _, value := range s { + stashIds[value.StashId] = struct{}{} + } + + if len(stashIds) == 0 { + // if we don't have any lookup stashdb using the sitename + sitename := site.Name + if i := strings.Index(sitename, " ("); i != -1 { + sitename = sitename[:i] + } + studio := scrape.FindStashdbStudio(sitename, "name") + stashIds[studio.Data.Studio.ID] = struct{}{} + } + var results []string + for key, _ := range stashIds { + results = append(results, `"`+key+`"`) + } + return results +} diff --git a/pkg/config/scraper_list.go b/pkg/config/scraper_list.go index 2be1c4e83..bdfb3a2cc 100644 --- a/pkg/config/scraper_list.go +++ b/pkg/config/scraper_list.go @@ -21,18 +21,20 @@ type ScraperList struct { XbvrScrapers XbvrScrapers `json:"xbvr"` } type XbvrScrapers struct { - PovrScrapers []ScraperConfig `json:"povr"` - SlrScrapers []ScraperConfig `json:"slr"` - VrpornScrapers []ScraperConfig `json:"vrporn"` - VrphubScrapers []ScraperConfig `json:"vrphub"` - RealVRScrapers []ScraperConfig `json:"realvr"` + PovrScrapers []ScraperConfig `json:"povr"` + SlrScrapers []ScraperConfig `json:"slr"` + StashDbScrapers []ScraperConfig `json:"stashdb"` + RealVRScrapers []ScraperConfig `json:"realvr"` + VrpornScrapers []ScraperConfig `json:"vrporn"` + VrphubScrapers []ScraperConfig `json:"vrphub"` } type CustomScrapers struct { - PovrScrapers []ScraperConfig `json:"povr"` - SlrScrapers []ScraperConfig `json:"slr"` - VrpornScrapers []ScraperConfig `json:"vrporn"` - VrphubScrapers []ScraperConfig `json:"vrphub"` - RealVRScrapers []ScraperConfig `json:"realvr"` + PovrScrapers []ScraperConfig `json:"povr"` + SlrScrapers []ScraperConfig `json:"slr"` + StashDbScrapers []ScraperConfig `json:"stashdb"` + RealVRScrapers []ScraperConfig `json:"realvr"` + VrpornScrapers []ScraperConfig `json:"vrporn"` + VrphubScrapers []ScraperConfig `json:"vrphub"` } type ScraperConfig struct { ID string `json:"-"` @@ -73,11 +75,13 @@ func (o *ScraperList) Load() error { SetSiteId(&o.XbvrScrapers.PovrScrapers, "") SetSiteId(&o.XbvrScrapers.SlrScrapers, "") + SetSiteId(&o.XbvrScrapers.StashDbScrapers, "") SetSiteId(&o.XbvrScrapers.VrphubScrapers, "") SetSiteId(&o.XbvrScrapers.VrpornScrapers, "") SetSiteId(&o.XbvrScrapers.RealVRScrapers, "") SetSiteId(&o.CustomScrapers.PovrScrapers, "povr") SetSiteId(&o.CustomScrapers.SlrScrapers, "slr") + SetSiteId(&o.CustomScrapers.StashDbScrapers, "stashdb") SetSiteId(&o.CustomScrapers.VrphubScrapers, "vrphub") SetSiteId(&o.CustomScrapers.VrpornScrapers, "vrporn") SetSiteId(&o.CustomScrapers.RealVRScrapers, "realvr") @@ -85,6 +89,7 @@ func (o *ScraperList) Load() error { // remove custom sites that are now offical for the same aggregation site o.CustomScrapers.PovrScrapers = RemoveCustomListNowOffical(o.CustomScrapers.PovrScrapers, o.XbvrScrapers.PovrScrapers) o.CustomScrapers.SlrScrapers = RemoveCustomListNowOffical(o.CustomScrapers.SlrScrapers, o.XbvrScrapers.SlrScrapers) + o.CustomScrapers.StashDbScrapers = RemoveCustomListNowOffical(o.CustomScrapers.StashDbScrapers, o.XbvrScrapers.StashDbScrapers) o.CustomScrapers.VrphubScrapers = RemoveCustomListNowOffical(o.CustomScrapers.VrphubScrapers, o.XbvrScrapers.VrphubScrapers) o.CustomScrapers.VrpornScrapers = RemoveCustomListNowOffical(o.CustomScrapers.VrpornScrapers, o.XbvrScrapers.VrpornScrapers) o.CustomScrapers.RealVRScrapers = RemoveCustomListNowOffical(o.CustomScrapers.RealVRScrapers, o.XbvrScrapers.RealVRScrapers) diff --git a/pkg/externalreference/stashdb.go b/pkg/externalreference/stashdb.go index b49d365e9..5adf3f279 100644 --- a/pkg/externalreference/stashdb.go +++ b/pkg/externalreference/stashdb.go @@ -91,22 +91,39 @@ func matchOnSceneUrl() { var xbvrId uint var xbvrSceneId string + for _, unmatchedXbvrScene := range unmatchedXbvrScenes { + stashId := strings.TrimPrefix(unmatchedXbvrScene.SceneID, "stash-") + if stashId == stashScene.ExternalId { + // xbvrLink := models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: scene.ID, InternalNameId: scene.SceneID, + // ExternalReferenceID: stashScene.ID, ExternalSource: stashScene.ExternalSource, ExternalId: stashScene.ExternalId, MatchType: 20} + //stashScene.XbvrLinks = append(stashScene.XbvrLinks, xbvrLink) + //stashScene.Save() + //var externalData models.StashScene + //json.Unmarshal([]byte(stashScene.ExternalData), &externalData) + //matchPerformerName(externalData, scene, 20) + xbvrId = unmatchedXbvrScene.ID + xbvrSceneId = unmatchedXbvrScene.SceneID + } + } + // see if we can link to an xbvr scene based on the urls - for _, url := range scene.URLs { - if url.Type == "STUDIO" { - var xbvrScene models.Scene - for _, scene := range unmatchedXbvrScenes { - sceneurl := removeQueryFromURL(scene.SceneURL) - tmpurl := removeQueryFromURL(url.URL) - sceneurl = simplifyUrl(sceneurl) - tmpurl = simplifyUrl(tmpurl) - if strings.EqualFold(sceneurl, tmpurl) { - xbvrScene = scene + if xbvrId == 0 { + for _, url := range scene.URLs { + if url.Type == "STUDIO" { + var xbvrScene models.Scene + for _, scene := range unmatchedXbvrScenes { + sceneurl := removeQueryFromURL(scene.SceneURL) + tmpurl := removeQueryFromURL(url.URL) + sceneurl = simplifyUrl(sceneurl) + tmpurl = simplifyUrl(tmpurl) + if strings.EqualFold(sceneurl, tmpurl) { + xbvrScene = scene + } + } + if xbvrScene.ID != 0 { + xbvrId = xbvrScene.ID + xbvrSceneId = xbvrScene.SceneID } - } - if xbvrScene.ID != 0 { - xbvrId = xbvrScene.ID - xbvrSceneId = xbvrScene.SceneID } } } @@ -253,7 +270,6 @@ func matchSceneOnRules(sitename string, config models.StashSiteConfig) { func simplystring(str string) string { str = strings.TrimSpace(str) - str = strings.ReplaceAll(str, " and ", "&") str = strings.ReplaceAll(str, " ", "") str = strings.ReplaceAll(str, ".", "") str = strings.ReplaceAll(str, ":", "") @@ -262,6 +278,7 @@ func simplystring(str string) string { str = strings.ReplaceAll(str, ";", "") str = strings.ReplaceAll(str, ",", "") str = strings.ReplaceAll(str, "#", "") + str = strings.ReplaceAll(str, " and ", "&") str = strings.ReplaceAll(str, "@", "") str = strings.ReplaceAll(str, "$", "") str = strings.ReplaceAll(str, "%", "") diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go index 19c378eb6..edbbac21b 100644 --- a/pkg/models/model_external_reference.go +++ b/pkg/models/model_external_reference.go @@ -89,6 +89,7 @@ type SceneMatchRule struct { XbvrField string XbvrMatch string XbvrMatchResultPosition int + StashField string StashRule string StashMatchResultPosition int } @@ -123,6 +124,12 @@ func (o *ExternalReferenceLink) FindByInternalName(internalTable string, interna commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalNameId: internalName}).Find(&refs) return refs } +func (o *ExternalReferenceLink) FindByExternalSource(internalTable string, internalId uint, externalSource string) []ExternalReferenceLink { + commonDb, _ := GetCommonDB() + var refs []ExternalReferenceLink + commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalDbId: internalId, ExternalSource: externalSource}).Find(&refs) + return refs +} func (o *ExternalReferenceLink) FindByExternaID(externalSource string, externalId string) { commonDb, _ := GetCommonDB() commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{ExternalSource: externalSource, ExternalId: externalId}).Find(&o) @@ -1059,6 +1066,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() { XbvrField: "Enter xbvr field you are matching to, scene_url or scene_id", XbvrMatch: "Enter regex express to extract value from field to match on", XbvrMatchResultPosition: 0, + StashField: "Enter the stash field to cmpare, default Url", StashRule: "Enter rule name, ie title, title/date, studio_code or regex expression to extract value to match from the stash url", StashMatchResultPosition: 0, }} diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index b022eb20b..03050ba56 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -462,8 +462,12 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error { } } if ext.ActorDetails[name].ProfileUrl != "" { - if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) { - saveActor = true + if strings.HasPrefix(ext.ActorDetails[name].ProfileUrl, "https://stashdb.org/performers/") { + + } else { + if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) { + saveActor = true + } } } if saveActor { diff --git a/pkg/models/model_scraper.go b/pkg/models/model_scraper.go index 317e747c0..2af2b0292 100644 --- a/pkg/models/model_scraper.go +++ b/pkg/models/model_scraper.go @@ -54,6 +54,7 @@ type ActorDetails struct { ImageUrl string ProfileUrl string Source string + StashData string } type TrailerScrape struct { SceneUrl string `json:"scene_url"` // url of the page to be scrapped diff --git a/pkg/models/model_stashdb.go b/pkg/models/model_stashdb.go index 3777c791c..349cf977d 100644 --- a/pkg/models/model_stashdb.go +++ b/pkg/models/model_stashdb.go @@ -15,6 +15,10 @@ type StashStudio struct { Parent IdName `json:"parent"` Updated time.Time `json:"updated"` } +type StashPerformerStudio struct { + SceneCount int `json:"scene_count"` + Studio StashStudio `json:"studio"` +} type StashPerformer struct { ID string `json:"id"` @@ -44,6 +48,7 @@ type StashPerformer struct { MergedIds []string `json:"merged_ids"` Created string `json:"created"` Updated time.Time `json:"updated"` + Studios []StashPerformerStudio `json:"studios"` } type StashBodyModification struct { @@ -68,6 +73,8 @@ type StashScene struct { Studio StashStudio `json:"studio"` Duration int `json:"duration"` Code string `json:"code"` + Images []StashImages `json:"images"` + Tags []StashTags `json:"tags"` } type StashPerformerAs struct { @@ -75,9 +82,24 @@ type StashPerformerAs struct { As string `` } +type StashImages struct { + URL string `json:"url"` + Width string `json:"width"` + Height Site `json:"height"` +} +type StashTags struct { + ID string `json:"id"` + Name string `json:"name"` +} + type DELETEStashPerformerMin struct { ID string `json:"id"` Updated string `json:"updated"` Gender string `json:"gender"` Name string `json:"name"` } +type StashImage struct { + Url StashPerformer `json:"url"` + Width int `json:"width"` + Height int `json:"height"` +} diff --git a/pkg/scrape/stashdb.go b/pkg/scrape/stashdb.go index ab1b2a89f..f2aa65553 100644 --- a/pkg/scrape/stashdb.go +++ b/pkg/scrape/stashdb.go @@ -35,6 +35,13 @@ type QueryScenesData struct { type QueryScenesResult struct { Data QueryScenesData `json:"data"` } +type FindScenesData struct { + Scene models.StashScene `json:"findScene"` +} +type FindScenesResult struct { + Data FindScenesData `json:"data"` +} + type FindStudioResponse struct { Studio models.StashStudio `json:"studio"` } @@ -52,6 +59,12 @@ type FindPerformerResult struct { type FindPerformerData struct { Performer models.StashPerformer `json:"findPerformer"` } +type SearchPerformerResult struct { + Data SearchPerformerData `json:"data"` +} +type SearchPerformerData struct { + Performers []models.StashPerformer `json:"searchPerformer"` +} type QueryPerformerResult struct { Data QueryPerformerResultTypeData `json:"data"` @@ -63,6 +76,18 @@ type QueryPerformerResultType struct { Count int `json:"count"` Performers []models.StashPerformer `json:"performers"` } +type FindSceneResult struct { + Data FindSceneData `json:"data"` +} +type FindSceneData struct { + Scene models.StashScene `json:"findScene"` +} +type FindPerformerScenesData struct { + Performer models.StashPerformer `json:"findPerformer"` +} +type FindPerformerScenesResult struct { + Data FindPerformerScenesData `json:"data"` +} type Image struct { ID string `json:"id"` Url string `json:"url"` @@ -70,6 +95,61 @@ type Image struct { Height int `json:"height"` } +const sceneFieldList = ` +id +title +details +release_date +date +updated +urls{ + url + type + site { + id + name + description + url + regex + valid_types + } +} +studio{ + id + name + updated + parent { id } +} +images{ + url + width + height +} +performers{ + performer{ + id + updated + gender + name + aliases + } + as +} +fingerprints{ + hash + duration + submissions +} +duration + tags { + id + name +} +======= +code +deleted +` + var Config models.ActorScraperConfig func StashDb() { @@ -94,7 +174,7 @@ func StashDb() { if i := strings.Index(sitename, " ("); i != -1 { sitename = sitename[:i] } - studio := findStudio(sitename, "name") + studio := FindStashdbStudio(sitename, "name") sitecfg, cfgExists := Config.StashSceneMatching[site.ID] if !cfgExists && studio.Data.Studio.ID != "" { @@ -103,7 +183,7 @@ func StashDb() { // check for a config entry if site not found for _, cfgEntry := range sitecfg { - studio = findStudio(cfgEntry.StashId, "id") + studio = FindStashdbStudio(cfgEntry.StashId, "id") siteConfig := Config.StashSceneMatching[site.ID] var ext models.ExternalReference ext.FindExternalId("stashdb studio", studio.Data.Studio.ID) @@ -132,7 +212,29 @@ func StashDb() { } } -func findStudio(studio string, field string) FindStudioResult { +func GetStashDbScene(stashId string) FindScenesResult { + var result FindScenesResult + if config.Config.Advanced.StashApiKey == "" { + return result + } + tlog := log.WithField("task", "scrape") + tlog.Infof("Scraping stash studio %s", stashId) + query := ` + query findScene($id: ID!) { + findScene(id: $id) { + ` + sceneFieldList + ` + } + } + ` + variables := `{"id": "` + stashId + `"}` + resp := CallStashDb(query, variables) + json.Unmarshal(resp, &result) + + tlog.Info("Scrape of Stashdb completed") + return result +} + +func FindStashdbStudio(studio string, field string) FindStudioResult { fieldType := "String" if field == "id" { fieldType = "ID" @@ -175,7 +277,7 @@ func processStudioPerformers(studioId string) { } for _, performer := range performerList.Data.QueryPerformers.Performers { - updatePerformer(performer) + UpdatePerformer(performer) } } func getPerformersPage(studioId string, page int) QueryPerformerResult { @@ -260,12 +362,17 @@ func getStudioSceneQueryVariable(studioId string, page int, count int) string { // Builds a query variable to get scenes from the Parent Studio // Uses the tagId to filter just scenes tag as Virtual Reality func getParentSceneQueryVariable(parentId string, tagId string, page int, count int) string { - return ` - {"input":{ + tag := "" + if tagId != "" { + tag = ` "tags": { "value": "` + tagId + `", "modifier": "INCLUDES" }, + ` + } + return ` + {"input":{` + tag + ` "parentStudio": "` + parentId + `", "page": ` + strconv.Itoa(page) + `, @@ -305,6 +412,10 @@ func GetScenePage(variables string) QueryScenesResult { id name updated + parent { + id + name + } } images{ url @@ -318,6 +429,11 @@ func GetScenePage(variables string) QueryScenesResult { gender name aliases + images{ + url + width + height + } } as } @@ -326,6 +442,10 @@ func GetScenePage(variables string) QueryScenesResult { duration submissions } + tags { + id + name + } duration code deleted @@ -341,6 +461,81 @@ func GetScenePage(variables string) QueryScenesResult { return data } +func GetSceneFromStash(sceneId string) models.StashScene { + variables := `{"id": "` + sceneId + `"} ` + query := ` + query findScene($id: ID!) { + findScene(id: $id) { + id + title + details + release_date + date + updated + urls{ + url + type + site { + id + name + description + url + regex + valid_types + } + } + studio{ + id + name + updated + parent { + id + name + } + } + images{ + url + width + height + } + performers{ + performer{ + id + updated + gender + name + aliases + images{ + url + width + height + } + } + as + } + fingerprints{ + hash + duration + submissions + } + tags { + id + name + } + duration + code + deleted + } + } + ` + + // Define the variables needed for your query as a Go map + resp := CallStashDb(query, variables) + var data FindSceneResult + json.Unmarshal(resp, &data) + return data.Data.Scene +} + func saveScenesToExternalReferences(scenes QueryScenesResult, studioId string) { tlog := log.WithField("task", "scrape") startTime := time.Now() @@ -373,7 +568,7 @@ func saveScenesToExternalReferences(scenes QueryScenesResult, studioId string) { // chek if we have the performers, may not in the case of loading scenes from the parent studio for _, performer := range scene.Performers { - updatePerformer(performer.Performer) + UpdatePerformer(performer.Performer) } // see if we can link to an xbvr scene based on the urls @@ -399,13 +594,13 @@ func saveScenesToExternalReferences(scenes QueryScenesResult, studioId string) { } } -func updatePerformer(newPerformer models.StashPerformer) { +func UpdatePerformer(newPerformer models.StashPerformer) { var ext models.ExternalReference ext.FindExternalId("stashdb performer", newPerformer.ID) var oldPerformer models.StashPerformer json.Unmarshal([]byte(ext.ExternalData), &oldPerformer) if ext.ID == 0 || newPerformer.Updated.UTC().Sub(oldPerformer.Updated.UTC()).Seconds() > 1 { - fullDetails := getStashPerformer(newPerformer.ID).Data.Performer + fullDetails := GetStashPerformer(newPerformer.ID).Data.Performer jsonData, _ := json.MarshalIndent(fullDetails, "", " ") newext := models.ExternalReference{ExternalSource: "stashdb performer", ExternalURL: "https://stashdb.org/performers/" + fullDetails.ID, ExternalId: fullDetails.ID, ExternalDate: fullDetails.Updated, ExternalData: string(jsonData)} if ext.ID != 0 { @@ -425,7 +620,7 @@ func RefreshPerformer(performerId string) { } var ext models.ExternalReference ext.FindExternalId("stashdb performer", performerId) - fullDetails := getStashPerformer(performerId).Data.Performer + fullDetails := GetStashPerformer(performerId).Data.Performer if fullDetails.ID == "" { return } @@ -440,7 +635,7 @@ func RefreshPerformer(performerId string) { } } -func getStashPerformer(performer string) FindPerformerResult { +func GetStashPerformer(performer string) FindPerformerResult { query := ` query findPerformer($id: ID!) { @@ -509,6 +704,127 @@ func getStashPerformer(performer string) FindPerformerResult { } return data } +func SearchStashPerformer(performer string) SearchPerformerResult { + + query := ` + query SearchAll($term: String!, $limit: Int = 100) + { searchPerformer(term: $term, limit: $limit) { + id + name + disambiguation + aliases + gender + birth_date + age + ethnicity + country + eye_color + hair_color + height + cup_size + band_size + waist_size + hip_size + breast_type + career_start_year + career_end_year + studios { + scene_count + studio { + name + id + } +} + images{ + id + url + width + height + } + + deleted + merged_ids + created + updated + + } + } + ` + + // Define the variables needed for your query as a Go map + var data SearchPerformerResult + variables := `{"term": "` + performer + `"}` + resp := CallStashDb(query, variables) + err := json.Unmarshal(resp, &data) + if err != nil { + log.Errorf("Eror extracting actor json") + } + return data +} + +func GetStashPerformerFull(performer string) FindPerformerScenesResult { + + query := ` + query findPerformer($id: ID!) { + findPerformer(id: $id) { + id + name + disambiguation + aliases + gender + birth_date + images{ + id + url + width + height + } + studios { + scene_count + studio { + name + id + } + } + deleted + created + updated + scenes { + id + title + details + release_date + date + studio{ + id + name + } + studio { + name + id + } + images{ + url + width + height + } + duration + deleted + } +} + } +` + + // Define the variables needed for your query as a Go map + var data FindPerformerScenesResult + variables := `{"id": "` + performer + `"}` + resp := CallStashDb(query, variables) + err := json.Unmarshal(resp, &data) + if err != nil { + log.Errorf("Eror extracting actor json") + } + return data +} func CallStashDb(query string, rawVariables string) []byte { var variables map[string]interface{} diff --git a/pkg/scrape/stashdb_studios.go b/pkg/scrape/stashdb_studios.go new file mode 100644 index 000000000..3120ce2b5 --- /dev/null +++ b/pkg/scrape/stashdb_studios.go @@ -0,0 +1,153 @@ +package scrape + +import ( + "strings" + + "github.com/jinzhu/gorm" + "github.com/mozillazg/go-slugify" + "github.com/thoas/go-funk" + "github.com/xbapps/xbvr/pkg/config" + "github.com/xbapps/xbvr/pkg/externalreference" + "github.com/xbapps/xbvr/pkg/models" +) + +func StashStudio(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, scraper string, name string, limitScraping bool, stashGuid string, masterSiteId string) error { + defer wg.Done() + commonDb, _ := models.GetCommonDB() + stashGuid = strings.TrimPrefix(stashGuid, "https://stashdb.org/studios/") + + scraperID := scraper + siteID := name + logScrapeStart(scraperID, siteID) + + if singleSceneURL != "" { + stashGuid := strings.TrimPrefix(strings.ToLower(singleSceneURL), "https://stashdb.org/scenes/") + stashScene := GetSceneFromStash(stashGuid) + if stashScene.ID != "" { + sc := processScrapedScene(stashScene, "", commonDb) + out <- sc + } + } else { + scenes := getStashdbScenes(stashGuid, "", "", limitScraping) + for _, stashScene := range scenes.Data.QueryScenes.Scenes { + if !funk.ContainsString(knownScenes, "https://stashdb.org/scenes/"+stashScene.ID) { + sc := processScrapedScene(stashScene, masterSiteId, commonDb) + out <- sc + } + } + } + + if updateSite { + updateSiteLastUpdate(scraperID) + } + logScrapeFinished(scraperID, siteID) + return nil +} +func processScrapedScene(stashScene models.StashScene, masterSiteId string, commonDb *gorm.DB) models.ScrapedScene { + var scene models.Scene + scene.GetIfExist("stash-" + stashScene.ID) + + sc := models.ScrapedScene{} + sc.MasterSiteId = masterSiteId + sc.ScraperID = slugify.Slugify(stashScene.Studio.Name) + "-stashdb" + sc.SceneType = "2D" + sc.Studio = stashScene.Studio.Name + if stashScene.Studio.Parent.Name != "" { + sc.Studio = stashScene.Studio.Parent.Name + } + sc.Site = stashScene.Studio.Name + " (Stash)" + sc.HomepageURL = "https://stashdb.org/scenes/" + stashScene.ID + sc.SceneID = "stash-" + stashScene.ID + + for _, urldata := range stashScene.URLs { + if urldata.Type == "STUDIO" { + sc.MembersUrl = urldata.URL + continue + } + } + sc.Released = stashScene.Date + sc.Title = stashScene.Title + sc.Synopsis = stashScene.Details + if len(stashScene.Images) > 0 { + sc.Covers = append(sc.Covers, stashScene.Images[0].URL) + } + + for _, tag := range stashScene.Tags { + sc.Tags = append(sc.Tags, tag.Name) + if tag.Name == "Virtual Reailty" || tag.Name == "VR" { + sc.SceneType = "VR" + } + } + + // Cast + sc.ActorDetails = make(map[string]models.ActorDetails) + for _, model := range stashScene.Performers { + modelName := model.Performer.Name + if model.Performer.Disambiguation != "" { + modelName = model.Performer.Name + "(" + model.Performer.Disambiguation + ")" + } + sc.Cast = append(sc.Cast, modelName) + + tmpActor := models.Actor{} + commonDb.Where(&models.Actor{Name: strings.Replace(modelName, ".", "", -1)}).First(&tmpActor) + if tmpActor.ID == 0 { + commonDb.Where(&models.Actor{Name: strings.Replace(modelName, ".", "", -1)}).FirstOrCreate(&tmpActor) + stashPerformer := GetStashPerformer(model.Performer.ID) + externalreference.UpdateXbvrActor(stashPerformer.Data.Performer, tmpActor.ID) + } + } + + sc.Duration = stashScene.Duration / 60 + return sc +} + +func init() { + addStashScraper("single_scene", "Stashdb - Other", "", "", "") + var scrapers config.ScraperList + scrapers.Load() + for _, scraper := range scrapers.XbvrScrapers.StashDbScrapers { + addStashScraper(slugify.Slugify(scraper.Name), scraper.Name, scraper.AvatarUrl, scraper.URL, scraper.MasterSiteId) + } + for _, scraper := range scrapers.CustomScrapers.StashDbScrapers { + addStashScraper(slugify.Slugify(scraper.Name), scraper.Name, scraper.AvatarUrl, scraper.URL, scraper.MasterSiteId) + } +} +func addStashScraper(id string, name string, avatarURL string, stashGuid string, masterSiteId string) { + if masterSiteId == "" { + registerScraper(id+"-stashdb", name+" (Stashdb)", avatarURL, "stashdb.org", func(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return StashStudio(wg, updateSite, knownScenes, out, singleSceneURL, singeScrapeAdditionalInfo, id, name, limitScraping, stashGuid, masterSiteId) + }) + } else { + registerAlternateScraper(id+"-stashdb", name+" (Stashdb)", avatarURL, "stashdb.org", masterSiteId, func(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return StashStudio(wg, updateSite, knownScenes, out, singleSceneURL, singeScrapeAdditionalInfo, id, name, limitScraping, stashGuid, masterSiteId) + }) + } +} + +func getStashdbScenes(studioId string, parentId string, tagId string, limitScraping bool) QueryScenesResult { + const count = 25 + page := 1 + var sceneList QueryScenesResult + var nextList QueryScenesResult + var variables string + if parentId != "" { + variables = getParentSceneQueryVariable(parentId, tagId, page, count) + } else { + variables = getStudioSceneQueryVariable(studioId, page, count) + } + sceneList = GetScenePage(variables) + nextList = sceneList + for limitScraping == false && + len(nextList.Data.QueryScenes.Scenes) > 0 && + len(sceneList.Data.QueryScenes.Scenes) < sceneList.Data.QueryScenes.Count { + page += 1 + if parentId != "" { + variables = getParentSceneQueryVariable(parentId, tagId, page, count) + } else { + variables = getStudioSceneQueryVariable(studioId, page, count) + } + nextList = GetScenePage(variables) + sceneList.Data.QueryScenes.Scenes = append(sceneList.Data.QueryScenes.Scenes, nextList.Data.QueryScenes.Scenes...) + } + return sceneList +} diff --git a/ui/src/App.vue b/ui/src/App.vue index a4061ec67..a5063ef6c 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -13,6 +13,8 @@ + + @@ -30,9 +32,11 @@ import Details from './views/scenes/Details' import EditScene from './views/scenes/EditScene' import ActorDetails from './views/actors/ActorDetails' import EditActor from './views/actors/EditActor' +import SearchStashdbScenes from './views/scenes/SearchStashdbScenes' +import SearchStashdbActors from './views/actors/SearchStashdbActors' export default { - components: { Navbar, Socket, QuickFind, GlobalEvents, Details, EditScene, ActorDetails, EditActor }, + components: { Navbar, Socket, QuickFind, GlobalEvents, Details, EditScene, ActorDetails, EditActor, SearchStashdbScenes,SearchStashdbActors }, computed: { showOverlay () { return this.$store.state.overlay.details.show @@ -46,6 +50,12 @@ export default { showActorEdit() { return this.$store.state.overlay.actoredit.show }, + showSearchStashdbScenes() { + return this.$store.state.overlay.searchStashDbScenes.show + }, + showSearchStashdbActors() { + return this.$store.state.overlay.searchStashDbActors.show + }, } } diff --git a/ui/src/components/LinkStashdbButton.vue b/ui/src/components/LinkStashdbButton.vue new file mode 100644 index 000000000..3391ec6a8 --- /dev/null +++ b/ui/src/components/LinkStashdbButton.vue @@ -0,0 +1,50 @@ + + + + \ No newline at end of file diff --git a/ui/src/store/overlay.js b/ui/src/store/overlay.js index a36a808cf..c225209fb 100644 --- a/ui/src/store/overlay.js +++ b/ui/src/store/overlay.js @@ -40,6 +40,15 @@ const state = { show:false, site: '', }, + searchStashDbScenes: { + show: false, + scene: null + }, + searchStashDbActors: { + show: false, + actor: null + }, + changeDetailsTab: -1, } const mutations = { @@ -135,6 +144,25 @@ const mutations = { hideSceneMatchParams (state, payload) { state.sceneMatchParams.show = false }, + showSearchStashdbScenes (state, payload) { + state.searchStashDbScenes.scene = payload.item + state.searchStashDbScenes.show = true + }, + hideSearchStashdbScenes (state) { + state.searchStashDbScenes.scene = null + state.searchStashDbScenes.show = false + }, + showSearchStashdbActors (state, payload) { + state.searchStashDbActors.actor = payload.item + state.searchStashDbActors.show = true + }, + hideSearchStashdbActors (state) { + state.searchStashDbActors.actor = null + state.searchStashDbActors.show = false + }, + changeDetailsTab (state, payload) { + state.changeDetailsTab = payload.tab + }, } export default { diff --git a/ui/src/views/actors/ActorCard.vue b/ui/src/views/actors/ActorCard.vue index fbc716875..99e6084d5 100644 --- a/ui/src/views/actors/ActorCard.vue +++ b/ui/src/views/actors/ActorCard.vue @@ -21,6 +21,7 @@   + @@ -54,13 +55,14 @@ import { format, parseISO } from 'date-fns' import ActorFavouriteButton from '../../components/ActorFavouriteButton' import ActorWatchlistButton from '../../components/ActorWatchlistButton' import ActorEditButton from '../../components/ActorEditButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import VueLoadImage from 'vue-load-image' import { tr } from 'date-fns/locale' export default { name: 'ActorCard', props: { actor: Object, colleague: String }, - components: {ActorFavouriteButton, ActorWatchlistButton, VueLoadImage, ActorEditButton}, + components: {ActorFavouriteButton, ActorWatchlistButton, VueLoadImage, ActorEditButton, LinkStashdbButton}, data () { return { preview: false, diff --git a/ui/src/views/actors/ActorDetails.vue b/ui/src/views/actors/ActorDetails.vue index 7ec127b28..88c4bad7e 100644 --- a/ui/src/views/actors/ActorDetails.vue +++ b/ui/src/views/actors/ActorDetails.vue @@ -10,6 +10,7 @@ @keydown.f="$store.commit('actorList/toggleActorList', {actor_id: actor.id, list: 'favourite'})" @keydown.exact.w="$store.commit('actorList/toggleActorList', {actor_id: actor.id, list: 'watchlist'})" @keydown.e="$store.commit('overlay/editActorDetails', {actor: actor})" + @keydown.s="$store.commit('overlay/showSearchStashdbActors', {actor: item})" @keydown.48="setRating(0)" /> @@ -91,6 +92,7 @@       + @@ -266,12 +268,13 @@ import StarRating from 'vue-star-rating' import ActorFavouriteButton from '../../components/ActorFavouriteButton' import ActorWatchlistButton from '../../components/ActorWatchlistButton' import ActorEditButton from '../../components/ActorEditButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import SceneCard from '../scenes/SceneCard' import ActorCard from './ActorCard' export default { name: 'ActorDetails', - components: { VueLoadImage, GlobalEvents, StarRating, ActorWatchlistButton, ActorFavouriteButton, SceneCard, ActorEditButton, ActorCard }, + components: { VueLoadImage, GlobalEvents, StarRating, ActorWatchlistButton, ActorFavouriteButton, SceneCard, ActorEditButton, ActorCard, LinkStashdbButton }, data () { return { index: 1, diff --git a/ui/src/views/actors/SearchStashdbActors.vue b/ui/src/views/actors/SearchStashdbActors.vue new file mode 100644 index 000000000..70ced48bb --- /dev/null +++ b/ui/src/views/actors/SearchStashdbActors.vue @@ -0,0 +1,250 @@ + + + + + diff --git a/ui/src/views/options/sections/InterfaceAdvanced.vue b/ui/src/views/options/sections/InterfaceAdvanced.vue index 346a739c0..9f5ab7c48 100644 --- a/ui/src/views/options/sections/InterfaceAdvanced.vue +++ b/ui/src/views/options/sections/InterfaceAdvanced.vue @@ -195,7 +195,7 @@ export default { this.scraperFieldsValid=false if (this.scraperName != "") { if (this.scraperUrl.startsWith("https://") || this.scraperUrl.startsWith("http://") ) { - if (this.scraperUrl.includes("povr.com") || this.scraperUrl.includes("sexlikereal.com") || this.scraperUrl.includes("vrphub.com") || this.scraperUrl.includes("vrporn.com") || this.scraperUrl.includes("realvr.com")) { + if (this.scraperUrl.includes("povr.com") || this.scraperUrl.includes("sexlikereal.com") || this.scraperUrl.includes("vrphub.com") || this.scraperUrl.includes("vrporn.com") || this.scraperUrl.includes("stashdb.org")) || this.scraperUrl.includes("realvr.com")) { this.scraperFieldsValid=true } } diff --git a/ui/src/views/options/sections/OptionsSceneCreate.vue b/ui/src/views/options/sections/OptionsSceneCreate.vue index 5ffdc3dcb..e7e561a28 100644 --- a/ui/src/views/options/sections/OptionsSceneCreate.vue +++ b/ui/src/views/options/sections/OptionsSceneCreate.vue @@ -186,6 +186,8 @@ export default { } if (this.scrapeUrl.toLowerCase().includes("realvr.com")) { site = "realvr-single_scene" + if (this.scrapeUrl.toLowerCase().includes("stashdb.org")) { + site = "single_scene-stashdb" } if (site == "") { this.$buefy.toast.open({message: `No scrapers exist for this domain`, type: 'is-danger', duration: 5000}) diff --git a/ui/src/views/scenes/Details.vue b/ui/src/views/scenes/Details.vue index e80dff764..19fd0c284 100644 --- a/ui/src/views/scenes/Details.vue +++ b/ui/src/views/scenes/Details.vue @@ -12,6 +12,7 @@ @keydown.shift.w="$store.commit('sceneList/toggleSceneList', {scene_id: item.scene_id, list: 'watched'})" @keydown.t="$store.commit('sceneList/toggleSceneList', {scene_id: item.scene_id, list: 'trailerlist'})" @keydown.e="$store.commit('overlay/editDetails', {scene: item})" + @keydown.s="$store.commit('overlay/showSearchStashdbScenes', {scene: item})" @keydown.g="toggleGallery" @keydown.48="setRating(0)" /> @@ -126,6 +127,7 @@ + @@ -406,6 +408,7 @@ import VueLoadImage from 'vue-load-image' import GlobalEvents from 'vue-global-events' import StarRating from 'vue-star-rating' import FavouriteButton from '../../components/FavouriteButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import WatchlistButton from '../../components/WatchlistButton' import WishlistButton from '../../components/WishlistButton' import WatchedButton from '../../components/WatchedButton' @@ -417,7 +420,7 @@ import HiddenButton from '../../components/HiddenButton' export default { name: 'Details', - components: { VueLoadImage, GlobalEvents, StarRating, WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, RefreshButton, RescrapeButton, TrailerlistButton, HiddenButton }, + components: { VueLoadImage, GlobalEvents, StarRating, WatchlistButton, FavouriteButton, LinkStashdbButton, WishlistButton, WatchedButton, EditButton, RefreshButton, RescrapeButton, TrailerlistButton, HiddenButton }, data () { return { index: 1, @@ -584,6 +587,9 @@ export default { return 0; // Return 0 or handle error as needed } }, + changeDetailsTab() { + return this.$store.state.overlay.changeDetailsTab + }, quickFindOverlayState() { return this.$store.state.overlay.quickFind.show }, @@ -632,6 +638,13 @@ watch:{ } } }, + changeDetailsTab(newVal, oldVal){ + if (newVal == -1 ) { + return + } + this.activeTab = newVal + this.$store.commit('overlay/changeDetailsTab', { tab: -1 }) + } }, methods: { setupPlayer () { diff --git a/ui/src/views/scenes/SceneCard.vue b/ui/src/views/scenes/SceneCard.vue index df6a0fc79..0d3fdf769 100644 --- a/ui/src/views/scenes/SceneCard.vue +++ b/ui/src/views/scenes/SceneCard.vue @@ -60,6 +60,7 @@ + @@ -91,6 +92,7 @@ import FavouriteButton from '../../components/FavouriteButton' import WishlistButton from '../../components/WishlistButton' import WatchedButton from '../../components/WatchedButton' import EditButton from '../../components/EditButton' +import LinkStashdbButton from '../../components/LinkStashdbButton' import TrailerlistButton from '../../components/TrailerlistButton' import HiddenButton from '../../components/HiddenButton' import ky from 'ky' @@ -99,13 +101,14 @@ import VueLoadImage from 'vue-load-image' export default { name: 'SceneCard', props: { item: Object, reRead: Boolean }, - components: { WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, TrailerlistButton, HiddenButton, VueLoadImage }, + components: { WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, LinkStashdbButton, TrailerlistButton, HiddenButton, VueLoadImage }, data () { return { preview: false, format, parseISO, alternateSources: [], + stashLinkExists: false, } }, computed: { @@ -156,6 +159,7 @@ export default { return this.$store.state.optionsWeb.web.isAvailOpacity / 100 }, async getAlternateSceneSourcesWithTitles() { + this.stashLinkExists = false try { const response = await ky.get('/api/scene/alternate_source/' + this.item.id).json(); this.alternateSources = []; @@ -173,6 +177,9 @@ export default { } else if (altsrc.external_source == "stashdb scene") { title = extdata.title || 'No Title'; } + if (altsrc.external_source.includes('stashdb')) { + this.stashLinkExists = true + } return { ...altsrc, title: title diff --git a/ui/src/views/scenes/SearchStashdbScenes.vue b/ui/src/views/scenes/SearchStashdbScenes.vue new file mode 100644 index 000000000..68d61d214 --- /dev/null +++ b/ui/src/views/scenes/SearchStashdbScenes.vue @@ -0,0 +1,197 @@ + + + + +