From df1783c841b89a07db22af302ec6d801db9cb4b4 Mon Sep 17 00:00:00 2001 From: Artem Shelkovnikov Date: Fri, 17 Jan 2025 17:57:31 +0400 Subject: [PATCH] Minor optimisations --- connectors/es/index.py | 10 ---------- connectors/sync_job_runner.py | 12 +++++------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/connectors/es/index.py b/connectors/es/index.py index 4e6b753a2..3435ed388 100644 --- a/connectors/es/index.py +++ b/connectors/es/index.py @@ -219,11 +219,6 @@ async def fetch_by_id(self, doc_id): async def fetch_response_by_id(self, doc_id): try: - if not self.serverless: - await self._retrier.execute_with_retry( - partial(self.client.indices.refresh, index=self.index_name) - ) - resp = await self._retrier.execute_with_retry( partial(self.client.get, index=self.index_name, id=doc_id) ) @@ -286,11 +281,6 @@ async def get_all_docs(self, query=None, sort=None, page_size=DEFAULT_PAGE_SIZE) Returns: Iterator """ - if not self.serverless: - await self._retrier.execute_with_retry( - partial(self.client.indices.refresh, index=self.index_name) - ) - if query is None: query = {"match_all": {}} diff --git a/connectors/sync_job_runner.py b/connectors/sync_job_runner.py index 9e7015e5d..5351bae37 100644 --- a/connectors/sync_job_runner.py +++ b/connectors/sync_job_runner.py @@ -35,8 +35,8 @@ UTF_8 = "utf-8" -JOB_REPORTING_INTERVAL = 10 -JOB_CHECK_INTERVAL = 1 +JOB_REPORTING_INTERVAL = 15 +JOB_CHECK_INTERVAL = 5 ES_ID_SIZE_LIMIT = 512 @@ -324,11 +324,6 @@ async def _sync_done(self, sync_status, sync_error=None): } if await self.reload_sync_job(): - if await self.reload_connector(): - persisted_stats[ - "total_document_count" - ] = await self.connector.document_count() - if sync_status == JobStatus.ERROR: await self.sync_job.fail(sync_error, ingestion_stats=persisted_stats) elif sync_status == JobStatus.SUSPENDED: @@ -339,6 +334,9 @@ async def _sync_done(self, sync_status, sync_error=None): await self.sync_job.done(ingestion_stats=persisted_stats) if await self.reload_connector(): + persisted_stats[ + "total_document_count" + ] = await self.connector.document_count() sync_cursor = ( None if not self.data_provider # If we failed before initializing the data provider, we don't need to change the cursor