Minor optimisations

elastic · Jan 17, 2025 · df1783c · df1783c
1 parent 1ed28e5
commit df1783c
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 17 deletions.
diff --git a/connectors/es/index.py b/connectors/es/index.py
@@ -219,11 +219,6 @@ async def fetch_by_id(self, doc_id):
 
     async def fetch_response_by_id(self, doc_id):
         try:
-            if not self.serverless:
-                await self._retrier.execute_with_retry(
-                    partial(self.client.indices.refresh, index=self.index_name)
-                )
-
             resp = await self._retrier.execute_with_retry(
                 partial(self.client.get, index=self.index_name, id=doc_id)
             )
@@ -286,11 +281,6 @@ async def get_all_docs(self, query=None, sort=None, page_size=DEFAULT_PAGE_SIZE)
         Returns:
             Iterator
         """
-        if not self.serverless:
-            await self._retrier.execute_with_retry(
-                partial(self.client.indices.refresh, index=self.index_name)
-            )
-
         if query is None:
             query = {"match_all": {}}
 

diff --git a/connectors/sync_job_runner.py b/connectors/sync_job_runner.py
@@ -35,8 +35,8 @@
 
 UTF_8 = "utf-8"
 
-JOB_REPORTING_INTERVAL = 10
-JOB_CHECK_INTERVAL = 1
+JOB_REPORTING_INTERVAL = 15
+JOB_CHECK_INTERVAL = 5
 ES_ID_SIZE_LIMIT = 512
 
 
@@ -324,11 +324,6 @@ async def _sync_done(self, sync_status, sync_error=None):
         }
 
         if await self.reload_sync_job():
-            if await self.reload_connector():
-                persisted_stats[
-                    "total_document_count"
-                ] = await self.connector.document_count()
-
             if sync_status == JobStatus.ERROR:
                 await self.sync_job.fail(sync_error, ingestion_stats=persisted_stats)
             elif sync_status == JobStatus.SUSPENDED:
@@ -339,6 +334,9 @@ async def _sync_done(self, sync_status, sync_error=None):
                 await self.sync_job.done(ingestion_stats=persisted_stats)
 
         if await self.reload_connector():
+            persisted_stats[
+                "total_document_count"
+            ] = await self.connector.document_count()
             sync_cursor = (
                 None
                 if not self.data_provider  # If we failed before initializing the data provider, we don't need to change the cursor