[#129] search improvements with threshold relevance score

dspback/config/__init__.py

@@ -74,6 +74,7 @@ class Settings(BaseSettings):
     jwt_algorithm: str = "HS256"
     access_token_expire_minutes: int = 12 * 60
     access_token_expiration_buffer_seconds: int = 30 * 60
+    search_relevance_score_threshold: float = 1.0
 
     session_secret_key: str
 

dspback/routers/discovery.py

@@ -35,7 +35,7 @@ def is_one_char_off(str1, str2):
 @router.get("/search")
 async def search(
     request: Request,
-    term: str,
+    term: str = None,
     sortBy: str = None,
     contentType: str = None,
     providerName: str = None,
@@ -63,18 +63,28 @@ async def search(
         term,
     )
 
-    should = [{'autocomplete': {'query': term, 'path': key}} for key in search_paths]
+    compound = {'filter': filters, 'must': must}
+    if term:
+        should = [{'autocomplete': {'query': term, 'path': key, 'fuzzy': {'maxEdits': 1}}} for key in search_paths]
+        compound['should'] = should
 
     stages.insert(
         0,
         {
             '$search': {
                 'index': 'fuzzy_search',
-                'compound': {'filter': filters, 'should': should, 'must': must, 'minimumShouldMatch': 1},
-                'highlight': {'path': search_paths},
+                'compound': compound,
             }
         },
     )
+    stages.append(
+        {'$set': {'score': {'$meta': 'searchScore'}, 'highlights': {'$meta': 'searchHighlights'}}},
+    )
+    if term:
+        stages[0]['$search']['highlight'] = {'path': search_paths}
+        # get only results which meet minimum relevance score threshold
+        score_threshold = get_settings().search_relevance_score_threshold
+        stages.append({'$match': {'score': {'$gt': score_threshold}}})
 
     results = await request.app.db[get_settings().mongo_database]["discovery"].aggregate(stages).to_list(pageSize)
 
@@ -206,6 +216,7 @@ async def base_search(
     must = []
     stages = []
     filters = []
+    must.append({'term': {'path': '@type', 'query': "Dataset"}})
     if publishedStart:
         filters.append(
             {