Export View (#417)

* Export View

* Fix Bugs

* Mark skipped expenses in fyle (#418)

* Mark skipped expenses in fyle

* Fix bug

* Update In Progress Expenses in Fyle (#419)

* Update In Progress Expenses in Fyle

* Comments Resolved

* Update Failed expenses in Fyle (#420)

* Update Failed expenses in Fyle

* update failed status in general exception

* Post Exported expenses in Fyle (#421)

* Post Exported expenses in Fyle

* Fix tests

* Add missing test case

* Comments resolved

* Add missing script for direct export (#422)

* Add missing script for direct export

* Fix comments

* add workspace id while creating expense

apps/fyle/actions.py

@@ -0,0 +1,235 @@
+from datetime import datetime, timezone
+from typing import List
+import logging
+
+from django.conf import settings
+from django.db.models import Q
+
+from fyle_integrations_platform_connector import PlatformConnector
+from fyle.platform.internals.decorators import retry
+from fyle.platform.exceptions import InternalServerError, RetryException
+from fyle_accounting_mappings.models import ExpenseAttribute
+
+from apps.fyle.constants import DEFAULT_FYLE_CONDITIONS
+from apps.fyle.models import ExpenseGroup, Expense
+from apps.workspaces.models import FyleCredential, Workspace
+
+from .helpers import get_updated_accounting_export_summary, get_batched_expenses
+
+
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
+
+
+def __bulk_update_expenses(expense_to_be_updated: List[Expense]) -> None:
+    """
+    Bulk update expenses
+    :param expense_to_be_updated: expenses to be updated
+    :return: None
+    """
+    if expense_to_be_updated:
+        Expense.objects.bulk_update(expense_to_be_updated, ['is_skipped', 'accounting_export_summary'], batch_size=50)
+
+
+def update_expenses_in_progress(in_progress_expenses: List[Expense]) -> None:
+    """
+    Update expenses in progress in bulk
+    :param in_progress_expenses: in progress expenses
+    :return: None
+    """
+    expense_to_be_updated = []
+    for expense in in_progress_expenses:
+        expense_to_be_updated.append(
+            Expense(
+                id=expense.id,
+                accounting_export_summary=get_updated_accounting_export_summary(
+                    expense.expense_id,
+                    'IN_PROGRESS',
+                    None,
+                    '{}/workspaces/main/dashboard'.format(settings.INTACCT_INTEGRATION_APP_URL),
+                    False
+                )
+            )
+        )
+
+    __bulk_update_expenses(expense_to_be_updated)
+
+
+def mark_expenses_as_skipped(final_query: Q, expenses_object_ids: List, workspace: Workspace) -> None:
+    """
+    Mark expenses as skipped in bulk
+    :param final_query: final query
+    :param expenses_object_ids: expenses object ids
+    :param workspace: workspace object
+    :return: None
+    """
+    # We'll iterate through the list of expenses to be skipped, construct accounting export summary and update expenses
+    expense_to_be_updated = []
+    expenses_to_be_skipped = Expense.objects.filter(
+        final_query,
+        id__in=expenses_object_ids,
+        expensegroup__isnull=True,
+        org_id=workspace.fyle_org_id
+    )
+
+    for expense in expenses_to_be_skipped:
+        expense_to_be_updated.append(
+            Expense(
+                id=expense.id,
+                is_skipped=True,
+                accounting_export_summary=get_updated_accounting_export_summary(
+                    expense.expense_id,
+                    'SKIPPED',
+                    None,
+                    '{}/workspaces/main/export_log'.format(settings.INTACCT_INTEGRATION_APP_URL),
+                    False
+                )
+            )
+        )
+
+    __bulk_update_expenses(expense_to_be_updated)
+
+
+def mark_accounting_export_summary_as_synced(expenses: List[Expense]) -> None:
+    """
+    Mark accounting export summary as synced in bulk
+    :param expenses: List of expenses
+    :return: None
+    """
+    # Mark all expenses as synced
+    expense_to_be_updated = []
+    for expense in expenses:
+        expense.accounting_export_summary['synced'] = True
+        updated_accounting_export_summary = expense.accounting_export_summary
+        expense_to_be_updated.append(
+            Expense(
+                id=expense.id,
+                accounting_export_summary=updated_accounting_export_summary,
+                previous_export_state=updated_accounting_export_summary['state']
+            )
+        )
+
+    Expense.objects.bulk_update(expense_to_be_updated, ['accounting_export_summary', 'previous_export_state'], batch_size=50)
+
+
+def update_failed_expenses(failed_expenses: List[Expense], is_mapping_error: bool) -> None:
+    """
+    Update failed expenses
+    :param failed_expenses: Failed expenses
+    """
+    expense_to_be_updated = []
+    for expense in failed_expenses:
+        error_type = 'MAPPING' if is_mapping_error else 'ACCOUNTING_INTEGRATION_ERROR'
+
+        # Skip dummy updates (if it is already in error state with the same error type)
+        if not (expense.accounting_export_summary.get('state') == 'ERROR' and \
+            expense.accounting_export_summary.get('error_type') == error_type):
+            expense_to_be_updated.append(
+                Expense(
+                    id=expense.id,
+                    accounting_export_summary=get_updated_accounting_export_summary(
+                        expense.expense_id,
+                        'ERROR',
+                        error_type,
+                        '{}/workspaces/main/dashboard'.format(settings.INTACCT_INTEGRATION_APP_URL),
+                        False
+                    )
+                )
+            )
+
+    __bulk_update_expenses(expense_to_be_updated)
+
+
+def update_complete_expenses(exported_expenses: List[Expense], url: str) -> None:
+    """
+    Update complete expenses
+    :param exported_expenses: Exported expenses
+    :param url: Export url
+    :return: None
+    """
+    expense_to_be_updated = []
+    for expense in exported_expenses:
+        expense_to_be_updated.append(
+            Expense(
+                id=expense.id,
+                accounting_export_summary=get_updated_accounting_export_summary(
+                    expense.expense_id,
+                    'COMPLETE',
+                    None,
+                    url,
+                    False
+                )
+            )
+        )
+
+    __bulk_update_expenses(expense_to_be_updated)
+
+
+def __handle_post_accounting_export_summary_exception(exception: Exception, workspace_id: int) -> None:
+    """
+    Handle post accounting export summary exception
+    :param exception: Exception
+    :param workspace_id: Workspace id
+    :return: None
+    """
+    error_response = exception.__dict__
+    expense_to_be_updated = []
+    if (
+        'message' in error_response and error_response['message'] == 'Some of the parameters are wrong'
+        and 'response' in error_response and 'data' in error_response['response'] and error_response['response']['data']
+    ):
+        logger.info('Error while syncing workspace %s %s',workspace_id, error_response)
+        for expense in error_response['response']['data']:
+            if expense['message'] == 'Permission denied to perform this action.':
+                expense_instance = Expense.objects.get(expense_id=expense['key'], workspace_id=workspace_id)
+                expense_to_be_updated.append(
+                    Expense(
+                        id=expense_instance.id,
+                        accounting_export_summary=get_updated_accounting_export_summary(
+                            expense_instance.expense_id,
+                            'DELETED',
+                            None,
+                            '{}/workspaces/main/dashboard'.format(settings.INTACCT_INTEGRATION_APP_URL),
+                            True
+                        )
+                    )
+                )
+        if expense_to_be_updated:
+            Expense.objects.bulk_update(expense_to_be_updated, ['accounting_export_summary'], batch_size=50)
+    else:
+        logger.error('Error while syncing accounting export summary, workspace_id: %s %s', workspace_id, str(error_response))
+
+
+@retry(n=3, backoff=1, exceptions=InternalServerError)
+def bulk_post_accounting_export_summary(platform: PlatformConnector, payload: List[dict]):
+    """
+    Bulk post accounting export summary with retry of 3 times and backoff of 1 second which handles InternalServerError
+    :param platform: Platform connector object
+    :param payload: Payload
+    :return: None
+    """
+    platform.expenses.post_bulk_accounting_export_summary(payload)
+
+
+def create_generator_and_post_in_batches(accounting_export_summary_batches: List[dict], platform: PlatformConnector, workspace_id: int) -> None:
+    """
+    Create generator and post in batches
+    :param accounting_export_summary_batches: Accounting export summary batches
+    :param platform: Platform connector object
+    :param workspace_id: Workspace id
+    :return: None
+    """
+    for batched_payload in accounting_export_summary_batches:
+        try:
+            if batched_payload:
+                bulk_post_accounting_export_summary(platform, batched_payload)
+
+                batched_expenses = get_batched_expenses(batched_payload, workspace_id)
+                mark_accounting_export_summary_as_synced(batched_expenses)
+        except RetryException:
+            logger.error(
+                'Internal server error while posting accounting export summary to Fyle workspace_id: %s',
+                workspace_id
+            )
+        except Exception as exception:
+            __handle_post_accounting_export_summary_exception(exception, workspace_id)

apps/fyle/helpers.py

@@ -1,20 +1,25 @@
 import json
+import traceback
 import requests
 from datetime import datetime, timezone
 from fyle_integrations_platform_connector import PlatformConnector
 import logging
+from typing import List, Union
 
 from django.utils.module_loading import import_string
 from django.conf import settings
 from django.db.models import Q
 
-from apps.fyle.models import ExpenseFilter, ExpenseGroupSettings
-from apps.workspaces.models import FyleCredential, Workspace
+from apps.fyle.models import ExpenseFilter, ExpenseGroupSettings, Expense
+from apps.tasks.models import TaskLog
+from apps.workspaces.models import FyleCredential, Workspace, Configuration
 
 from typing import List
 
 logger = logging.getLogger(__name__)
 
+SOURCE_ACCOUNT_MAP = {'PERSONAL': 'PERSONAL_CASH_ACCOUNT', 'CCC': 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT'}
+
 
 def post_request(url, body, refresh_token=None):
     """
@@ -231,3 +236,75 @@ def connect_to_platform(workspace_id: int) -> PlatformConnector:
     fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id)
 
     return PlatformConnector(fyle_credentials=fyle_credentials)
+
+def get_updated_accounting_export_summary(
+        expense_id: str, state: str, error_type: Union[str, None], url: Union[str, None], is_synced: bool) -> dict:
+    """
+    Get updated accounting export summary
+    :param expense_id: expense id
+    :param state: state
+    :param error_type: error type
+    :param url: url
+    :param is_synced: is synced
+    :return: updated accounting export summary
+    """
+    return {
+        'id': expense_id,
+        'state': state,
+        'error_type': error_type,
+        'url': url,
+        'synced': is_synced
+    }
+
+def get_batched_expenses(batched_payload: List[dict], workspace_id: int) -> List[Expense]:
+    """
+    Get batched expenses
+    :param batched_payload: batched payload
+    :param workspace_id: workspace id
+    :return: batched expenses
+    """
+    expense_ids = [expense['id'] for expense in batched_payload]
+    return Expense.objects.filter(expense_id__in=expense_ids, workspace_id=workspace_id)
+
+
+def get_source_account_type(fund_source: List[str]) -> List[str]:
+    """
+    Get source account type
+    :param fund_source: fund source
+    :return: source account type
+    """
+    source_account_type = []
+    for source in fund_source:
+        source_account_type.append(SOURCE_ACCOUNT_MAP[source])
+
+    return source_account_type
+
+
+
+def get_fund_source(workspace_id: int) -> List[str]:
+    """
+    Get fund source
+    :param workspace_id: workspace id
+    :return: fund source
+    """
+    general_settings = Configuration.objects.get(workspace_id=workspace_id)
+    fund_source = []
+    if general_settings.reimbursable_expenses_object:
+        fund_source.append('PERSONAL')
+    if general_settings.corporate_credit_card_expenses_object:
+        fund_source.append('CCC')
+
+    return fund_source
+
+
+def handle_import_exception(task_log: TaskLog) -> None:
+    """
+    Handle import exception
+    :param task_log: task log
+    :return: None
+    """
+    error = traceback.format_exc()
+    task_log.detail = {'error': error}
+    task_log.status = 'FATAL'
+    task_log.save()
+    logger.error('Something unexpected happened workspace_id: %s %s', task_log.workspace_id, task_log.detail)

apps/fyle/models.py

@@ -190,6 +190,7 @@ def create_expense_objects(expenses: List[Dict], workspace_id: int):
                     'payment_number': expense['payment_number'],
                     'file_ids': expense['file_ids'],
                     'corporate_card_id': expense['corporate_card_id'],
+                    'workspace_id': workspace_id
                 }
             )
 

apps/fyle/queue.py

@@ -0,0 +1,24 @@
+from django_q.tasks import async_task
+
+
+def async_post_accounting_export_summary(org_id: str, workspace_id: int) -> None:
+    """
+    Async'ly post accounting export summary to Fyle
+    :param org_id: org id
+    :param workspace_id: workspace id
+    :return: None
+    """
+    # This function calls post_accounting_export_summary asynchrously
+    async_task('apps.fyle.tasks.post_accounting_export_summary', org_id, workspace_id)
+
+
+def async_import_and_export_expenses(body: dict) -> None:
+    """
+    Async'ly import and export expenses
+    :param body: body
+    :return: None
+    """
+    if body.get('action') == 'ACCOUNTING_EXPORT_INITIATED' and body.get('data'):
+        report_id = body['data']['id']
+        org_id = body['data']['org_id']
+        async_task('apps.fyle.tasks.import_and_export_expenses', report_id, org_id)