add relational fetchers and use max page size for variant resolver

non optional variants add fetchers to resolvers add credset by study fetcher use fetcher relations for credsets relation fetcher for l2g, pagingation max size for variant formatting revert DeferredValues
opentargets · Oct 29, 2024 · 06c8c16 · 06c8c16
1 parent 5c9db80
commit 06c8c16
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 44 deletions.
diff --git a/Makefile b/Makefile
@@ -28,7 +28,7 @@ ch_tunnel: ## Create tunnel connection to Clickhouse eg. make ch_tunnel zone eur
 	@gcloud compute ssh ${instance} --zone="${zone}" --tunnel-through-iap -- -L 8123:localhost:8123
 
 run_with_standalone: ## Runs API with standalone platform
-	@sbt "run 8090" -DELASTICSEARCH_HOST=elasticsearch -DSLICK_CLICKHOUSE_URL=jdbc:clickhouse://clickhouse:8123 -DPLATFORM_API_IGNORE_CACHE=true
+	@sbt "run 8090" -J-Xms2g -J-Xmx7g -DELASTICSEARCH_HOST=elasticsearch -DSLICK_CLICKHOUSE_URL=jdbc:clickhouse://clickhouse:8123 -DPLATFORM_API_IGNORE_CACHE=true
 
 debug_with_standalone: ## Runs API with standalone platform
 	@sbt -jvm-debug 9999 run 8090 -DSLICK_CLICKHOUSE_URL=jdbc:clickhouse://clickhouse:8123 -DPLATFORM_API_IGNORE_CACHE=true

diff --git a/app/models/Backend.scala b/app/models/Backend.scala
@@ -137,49 +137,42 @@ class Backend @Inject() (implicit
     esRetriever.getByIds(targetIndexName, ids, fromJsValue[GeneOntologyTerm])
   }
 
-  def getL2GPredictions(id: String,
-                        pagination: Option[Pagination]
-  ): Future[IndexedSeq[L2GPredictions]] = {
+  def getL2GPredictions(ids: Seq[String]): Future[IndexedSeq[L2GPredictions]] = {
     val indexName = getIndexOrDefault("l2g_predictions")
-    val pag = pagination.getOrElse(Pagination.mkDefault)
-
     esRetriever
-      .getByIndexedTermsMust(indexName,
-                             Map("studyLocusId.keyword" -> Seq(id)),
-                             pag,
-                             fromJsValue[L2GPredictions],
-                             sortByField = ElasticRetriever.sortBy("score", SortOrder.Desc)
+      .getByIndexedTermsMust(
+        indexName,
+        Map("studyLocusId.keyword" -> ids),
+        Pagination(Pagination.indexDefault, Pagination.sizeMax),
+        fromJsValue[L2GPredictions],
+        sortByField = ElasticRetriever.sortBy("score", SortOrder.Desc)
       )
       .map(_._1)
   }
 
   def getVariants(ids: Seq[String]): Future[IndexedSeq[VariantIndex]] = {
     val indexName = getIndexOrDefault("variant_index")
-
-    //esRetriever.getByIds(indexName, ids, fromJsValue[VariantIndex])
-    esRetriever
+    val pag = Pagination(Pagination.indexDefault, Pagination.sizeMax)
+    val r = esRetriever
       .getByIndexedTermsMust(indexName,
                              Map("variantId.keyword" -> ids),
-                             Pagination.mkDefault,
+                             pag,
                              fromJsValue[VariantIndex]
       )
       .map(_._1)
+    r
   }
 
-  def getBiosample(id: String): Future[Option[Biosample]] = {
+  def getBiosamples(ids: Seq[String]): Future[IndexedSeq[Biosample]] = {
     val indexName = getIndexOrDefault("biosample", Some("biosample"))
     esRetriever
       .getByIndexedTermsMust(
         indexName,
-        Map("biosampleId.keyword" -> Seq(id)),
+        Map("biosampleId.keyword" -> ids),
         Pagination.mkDefault,
         fromJsValue[Biosample]
       )
-      .map {
-        case (Seq(), _) => None
-        case (hits, aggs) =>
-          Some(hits.head)
-      }
+      .map(_._1)
   }
 
   def getStudies(queryArgs: StudyQueryArgs,
@@ -243,7 +236,7 @@ class Backend @Inject() (implicit
       queryArgs: CredibleSetQueryArgs,
       pagination: Option[Pagination]
   ): Future[IndexedSeq[JsValue]] = {
-    val pag = pagination.getOrElse(Pagination.mkDefault)
+    val pag = pagination.getOrElse(Pagination(Pagination.indexDefault, Pagination.sizeMax))
     val indexName = getIndexOrDefault("credible_set")
     val termsQuery = Map(
       "studyLocusId.keyword" -> queryArgs.ids,

diff --git a/app/models/GQLSchema.scala b/app/models/GQLSchema.scala
@@ -19,6 +19,9 @@ trait GQLEntities extends Logging {}
 
 object GQLSchema {
   val resolvers: DeferredResolver[Backend] = DeferredResolver.fetchers(
+    biosamplesFetcher,
+    credibleSetFetcher,
+    l2gFetcher,
     targetsFetcher,
     drugsFetcher,
     diseasesFetcher,

diff --git a/app/models/entities/Configuration.scala b/app/models/entities/Configuration.scala
@@ -5,7 +5,7 @@ import play.api.libs.json.Reads._
 import play.api.libs.functional.syntax._
 
 object Configuration {
-  val batchSize = 100
+  val batchSize = 5000
 
   case class Logging(otHeader: String, ignoredQueries: Seq[String])
 

diff --git a/app/models/entities/CredibleSet.scala b/app/models/entities/CredibleSet.scala
@@ -4,7 +4,13 @@ import models.Backend
 import models.gql.StudyTypeEnum
 import models.gql.Arguments.StudyType
 import models.entities.GwasIndex.{gwasImp, gwasWithoutCredSetsImp}
-import models.gql.Fetchers.{gwasFetcher, targetsFetcher, variantFetcher}
+import models.gql.Fetchers.{
+  gwasFetcher,
+  l2gFetcher,
+  l2gByStudyLocusIdRel,
+  targetsFetcher,
+  variantFetcher
+}
 import models.gql.Objects.{logger, targetImp, variantIndexImp, colocalisationImp, l2gPredictionsImp}
 import play.api.Logging
 import play.api.libs.json.{JsValue, Json, OFormat, OWrites}
@@ -103,7 +109,7 @@ object CredibleSet extends Logging {
         OptionType(variantIndexImp),
         description = None,
         resolve = r => {
-          val variantId = (r.value.variantId)
+          val variantId = r.value.variantId.getOrElse("")
           logger.debug(s"Finding variant index: $variantId")
           variantFetcher.deferOpt(variantId)
         }
@@ -127,7 +133,7 @@ object CredibleSet extends Logging {
       OptionType(variantIndexImp),
       description = None,
       resolve = js => {
-        val id = (js.value \ "variantId").asOpt[String]
+        val id = (js.value \ "variantId").as[String]
         logger.debug(s"Finding variant for id: $id")
         variantFetcher.deferOpt(id)
       }
@@ -136,10 +142,9 @@ object CredibleSet extends Logging {
       "l2Gpredictions",
       OptionType(ListType(l2gPredictionsImp)),
       description = None,
-      arguments = pageArg :: Nil,
       resolve = js => {
         val id = (js.value \ "studyLocusId").as[String]
-        js.ctx.getL2GPredictions(id, js.arg(pageArg))
+        l2gFetcher.deferRelSeq(l2gByStudyLocusIdRel, id)
       }
     ),
     Field(

diff --git a/app/models/entities/GwasIndex.scala b/app/models/entities/GwasIndex.scala
@@ -1,10 +1,16 @@
 package models.entities
 
 import models.Backend
-import models.gql.Fetchers.{diseasesFetcher, targetsFetcher}
+import models.gql.Fetchers.{
+  biosamplesFetcher,
+  credibleSetFetcher,
+  credibleSetByStudyRel,
+  diseasesFetcher,
+  targetsFetcher
+}
 import play.api.Logging
 import play.api.libs.json.{JsValue, Json, OFormat}
-import models.entities.CredibleSet.credibleSetImp
+import models.entities.CredibleSet.{credibleSetImp, credibleSetWithoutStudyImp}
 import models.gql.Objects.{diseaseImp, targetImp, biosampleImp}
 import sangria.schema.{
   BooleanType,
@@ -16,7 +22,6 @@ import sangria.schema.{
   StringType,
   fields
 }
-import models.entities.CredibleSet.credibleSetWithoutStudyImp
 import models.gql.StudyTypeEnum
 import models.gql.Arguments.{pageArg, StudyType}
 
@@ -82,14 +87,10 @@ object GwasIndex extends Logging {
     Field(
       "biosample",
       OptionType(biosampleImp),
-      Some("Biosample"),
+      Some("biosample"),
       resolve = js => {
-        val biosampleId = (js.value \ "biosampleFromSourceId").asOpt[String].getOrElse("")
-        if (biosampleId.isEmpty) {
-          None
-        } else {
-          js.ctx.getBiosample(biosampleId)
-        }
+        val biosampleId = (js.value \ "biosampleFromSourceId").asOpt[String]
+        biosamplesFetcher.deferOpt(biosampleId)
       }
     ),
     Field(
@@ -232,13 +233,12 @@ object GwasIndex extends Logging {
   lazy val credibleSetField: Field[Backend, JsValue] =
     Field(
       "credibleSets",
-      OptionType(ListType(credibleSetWithoutStudyImp)),
+      OptionType(ListType(credibleSetImp)),
       arguments = pageArg :: Nil,
       description = Some("Credible sets"),
       resolve = js => {
-        val studyIdSeq = Seq((js.value \ "studyId").as[String])
-        val credSetQueryArgs = CredibleSetQueryArgs(studyIds = studyIdSeq)
-        js.ctx.getCredibleSets(credSetQueryArgs, js.arg(pageArg))
+        val studyId = (js.value \ "studyId").as[String]
+        credibleSetFetcher.deferRelSeq(credibleSetByStudyRel, studyId)
       }
     )
   lazy val gwasImp: ObjectType[Backend, JsValue] = ObjectType(

diff --git a/app/models/gql/Fetchers.scala b/app/models/gql/Fetchers.scala
@@ -2,13 +2,16 @@ package models.gql
 
 import models.Helpers.fromJsValue
 import models.entities.{
+  Biosample,
+  CredibleSet,
   Disease,
   Drug,
   Expressions,
   GeneOntologyTerm,
   GwasIndex,
   HPO,
   Indications,
+  L2GPredictions,
   OtarProjects,
   Reactome,
   Target,
@@ -17,7 +20,16 @@ import models.entities.{
 import models.{Backend, entities}
 import play.api.Logging
 import play.api.libs.json.{JsValue, __}
-import sangria.execution.deferred.{Fetcher, FetcherCache, FetcherConfig, HasId, SimpleFetcherCache}
+import sangria.execution.deferred.{
+  Relation,
+  RelationIds,
+  Fetcher,
+  FetcherCache,
+  FetcherConfig,
+  HasId,
+  SimpleFetcherCache
+}
+import models.gql.Arguments.studyId
 
 object Fetchers extends Logging {
   val soTermsFetcherCache = FetcherCache.simple
@@ -77,6 +89,16 @@ object Fetchers extends Logging {
     }
   )
 
+  implicit val biosampleHasId: HasId[Biosample, String] = HasId[Biosample, String](_.biosampleId)
+  val biosamplesFetcherCache = FetcherCache.simple
+  val biosamplesFetcher: Fetcher[Backend, Biosample, Biosample, String] = Fetcher(
+    config =
+      FetcherConfig.maxBatchSize(entities.Configuration.batchSize).caching(biosamplesFetcherCache),
+    fetch = (ctx: Backend, ids: Seq[String]) => {
+      ctx.getBiosamples(ids)
+    }
+  )
+
   //hpo fetcher
   implicit val hpoHasId: HasId[HPO, String] = HasId[HPO, String](_.id)
 
@@ -128,6 +150,42 @@ object Fetchers extends Logging {
     }
   )
 
+  val credibleSetFetcherCache = FetcherCache.simple
+  val credibleSetByStudyRel =
+    Relation[JsValue, String]("byStudy", js => Seq((js \ "studyId").as[String]))
+  val credibleSetFetcher: Fetcher[Backend, JsValue, JsValue, String] = {
+    implicit val credibleSetFetcherId: HasId[JsValue, String] =
+      HasId[JsValue, String](js => (js \ "studyLocusId").as[String])
+    Fetcher.rel(
+      config = FetcherConfig
+        .maxBatchSize(entities.Configuration.batchSize)
+        .caching(credibleSetFetcherCache),
+      fetch = (ctx: Backend, ids: Seq[String]) => {
+        ctx.getCredibleSets(entities.CredibleSetQueryArgs(ids = ids), None)
+      },
+      fetchRel = (ctx: Backend, ids: RelationIds[JsValue]) => {
+        ctx.getCredibleSets(entities.CredibleSetQueryArgs(studyIds = ids(credibleSetByStudyRel)),
+                            None
+        )
+      }
+    )
+  }
+
+  val l2gFetcherCache = FetcherCache.simple
+  implicit val l2gHasId: HasId[L2GPredictions, String] =
+    HasId[L2GPredictions, String](_.studyLocusId)
+  val l2gByStudyLocusIdRel =
+    Relation[L2GPredictions, String]("byStudyLocus", l2g => Seq(l2g.studyLocusId))
+  val l2gFetcher: Fetcher[Backend, L2GPredictions, L2GPredictions, String] = Fetcher.rel(
+    config = FetcherConfig.maxBatchSize(entities.Configuration.batchSize).caching(l2gFetcherCache),
+    fetch = (ctx: Backend, ids: Seq[String]) => {
+      ctx.getL2GPredictions(ids)
+    },
+    fetchRel = (ctx: Backend, ids: RelationIds[L2GPredictions]) => {
+      ctx.getL2GPredictions(ids(l2gByStudyLocusIdRel))
+    }
+  )
+
   val gwasFetcherCache = FetcherCache.simple
   val gwasFetcher: Fetcher[Backend, JsValue, JsValue, String] = {
     implicit val gwasFetcherId: HasId[JsValue, String] =
@@ -160,6 +218,13 @@ object Fetchers extends Logging {
   def resetCache(): Unit = {
     logger.info("Clearing all GraphQL caches.")
     val fetchers: List[SimpleFetcherCache] = List(
+      biosamplesFetcherCache,
+      credibleSetFetcherCache,
+      gwasFetcherCache,
+      hpoFetcherCache,
+      goFetcherCache,
+      variantFetcherCache,
+      l2gFetcherCache,
       targetsFetcherCache,
       drugsFetcherCache,
       diseasesFetcherCache,