Merge branch 'master' into add-ormq-check

.gitignore

@@ -160,3 +160,5 @@ test_scripts/
 fyle_accounting_mappings/
 fyle_integrations_platform_connector/
 ./fyle
+
+commits.csv

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "fyle_integrations_imports"]
 	path = fyle_integrations_imports
 	url = https://github.com/fylein/fyle_integrations_imports
+[submodule "fyle-integrations-db-migrations"]
+	path = fyle-integrations-db-migrations
+	url = https://github.com/fylein/fyle-integrations-db-migrations

README.md

@@ -13,6 +13,41 @@ Django Rest Framework API for Fyle Netsuite Integration.
     $ git submodule update
     ```
 
+* ### Adding a New View, Function, or Trigger:
+    Follow these steps to ensure your changes are applied correctly:
+
+    1. **Make changes** in the [`fyle-integrations-db-migrations`](https://github.com/fylein/fyle-integrations-db-migrations) repository.
+    2. **Update the submodule** in the Netsuite API:
+        ```bash
+        git submodule init
+        git submodule update
+        ```
+    3. **Enter the Netsuite API container**:
+        ```bash
+        enter netsuite-api
+        ```
+    4. **Generate a migration file** using the provided convenient command:
+        ```bash
+        python3 manage.py create_sql_migration <file-path1>
+        ```
+        - Replace `<file-path1>` with the relative path to your SQL file from the fyle-integrations-db-migrations folder.
+        - The migration will always be created in the `internal` app.
+
+        **Example:**
+        ```bash
+        python3 manage.py create_sql_migration fyle-integrations-db-migrations/netsuite/functions/re_export_expenses_netsuite.sql
+        ```
+
+    5. **Review the newly generated migration file**:
+        Navigate to the `apps/internal/migrations/` directory and ensure the migration file content is as expected.
+
+    6. **Restart the Netsuite API service and apply the migration**:
+        ```bash
+        restart netsuite-api
+        logs netsuite-api
+        ```
+        Confirm in the logs that the migration has been applied successfully.
+
 * Download and install Docker desktop for Mac from [here.](https://www.docker.com/products/docker-desktop)
 
 * If you're using a linux machine, please download docker according to the distrubution you're on.

apps/fyle/models.py

@@ -10,7 +10,7 @@
 
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.fields import ArrayField
-from django.contrib.postgres.fields.jsonb import KeyTextTransform
+from django.db.models.fields.json import KeyTextTransform
 from django.db import models
 from django.db.models import Count, Q, JSONField
 

apps/internal/helpers.py

@@ -0,0 +1,25 @@
+import os
+from django.db import migrations
+from django.db.utils import ProgrammingError
+def safe_run_sql(sql_files):
+    """
+    Safely create migrations.RunSQL operations from a list of SQL file paths.
+    Handles FileNotFoundError for missing files and lets ProgrammingError surface naturally.
+    """
+    operations = []
+    for file_path in sql_files:
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"SQL file not found: {file_path}")
+        try:
+            with open(file_path, 'r', encoding='utf-8') as sql_file:
+                sql = sql_file.read()
+            operations.append(migrations.RunSQL(sql=sql, reverse_sql=None))
+        except ProgrammingError as pe:
+            raise ProgrammingError(
+                f"SQL syntax error in file {file_path}: {pe}"
+            ) from pe
+        except Exception as e:
+            raise RuntimeError(
+                f"Unexpected error in file {file_path}: {e}"
+            ) from e
+    return operations

apps/internal/management/commands/create_sql_migration.py

@@ -0,0 +1,82 @@
+import os
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.apps import apps
+from django.db.migrations import loader
+class Command(BaseCommand):
+    help = 'Generate a migration file for running SQL files'
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'sql_files',
+            nargs='+',
+            help='Paths to the SQL files to include in the migration',
+        )
+        parser.add_argument(
+            '--app',
+            default='internal',
+            help='The Django app to create the migration for',
+        )
+    def get_latest_migration(self, app_name):
+        """Get the latest migration name using Django's migration loader."""
+        migration_loader = loader.MigrationLoader(None)
+        app_migrations = [
+            name for app, name in migration_loader.disk_migrations.keys()
+            if app == app_name
+        ]
+        if app_migrations:
+            return sorted(app_migrations)[-1]
+        return None
+    def handle(self, *args, **options):
+        app_name = options['app']
+        sql_files = options['sql_files']
+        validated_sql_files = []
+        for file_path in sql_files:
+            absolute_path = os.path.join(settings.BASE_DIR, file_path)
+            if not os.path.exists(absolute_path):
+                self.stderr.write(
+                    self.style.ERROR(f"File not found: {file_path} "
+                                   f"(looked in {absolute_path})")
+                )
+                return
+            validated_sql_files.append(file_path)
+        try:
+            app_config = apps.get_app_config(app_name)
+            app_path = app_config.path
+            migrations_dir = os.path.join(app_path, 'migrations')
+        except LookupError:
+            self.stderr.write(self.style.ERROR(f"App '{app_name}' not found. Make sure it's in INSTALLED_APPS"))
+            return
+        if not os.path.exists(migrations_dir):
+            os.makedirs(migrations_dir)
+        latest_migration = self.get_latest_migration(app_name)
+        if latest_migration:
+            dependencies = f"[('{app_name}', '{latest_migration}')]"
+        else:
+            dependencies = "[]  # This is the first migration"
+        existing_migrations = [f for f in os.listdir(migrations_dir) if f.endswith('.py')]
+        migration_number = len([f for f in existing_migrations if f != '__init__.py']) + 1
+        migration_name = f"{migration_number:04d}_auto_generated_sql.py"
+        migration_file_path = os.path.join(migrations_dir, migration_name)
+        formatted_sql_files = []
+        for file_path in validated_sql_files:
+            normalized_path = file_path.replace('\\', '/')
+            formatted_sql_files.append(f"'{normalized_path}'")
+        sql_files_str = ',\n    '.join(formatted_sql_files)
+        migration_content = f"""# Generated by Django
+from django.db import migrations
+from apps.internal.helpers import safe_run_sql
+sql_files = [
+    {sql_files_str}
+]
+class Migration(migrations.Migration):
+    dependencies = {dependencies}
+    operations = safe_run_sql(sql_files)
+"""
+        with open(migration_file_path, 'w') as migration_file:
+            migration_file.write(migration_content)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Migration file created at: {migration_file_path}\n"
+                f"Dependencies set to: {dependencies}"
+            )
+        )

apps/internal/migrations/0001_auto_generated_sql.py

@@ -0,0 +1,64 @@
+# Generated by Django
+from django.db import migrations
+from apps.internal.helpers import safe_run_sql
+sql_files = [
+    # Netsuite Specific Functions
+    'fyle-integrations-db-migrations/netsuite/functions/delete_workspace.sql',
+    'fyle-integrations-db-migrations/netsuite/functions/re_export_expenses_netsuite.sql',
+    'fyle-integrations-db-migrations/netsuite/functions/trigger_auto_import.sql',
+
+    # Global Shared Functions
+    'fyle-integrations-db-migrations/common/global_shared/functions/json_diff.sql',
+    'fyle-integrations-db-migrations/common/global_shared/functions/log_delete_event.sql',
+
+    # Grouped Functions
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/functions/delete_failed_expenses_qbo_intacct_xero_netsuite.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/functions/delete_test_orgs_schedule.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/functions/log_update_event.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/functions/trigger_auto_import_export.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/functions/update_in_progress_tasks_to_failed.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/functions/ws_org_id.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/functions/ws_search.sql',
+
+    # Global Views
+    'fyle-integrations-db-migrations/common/global_shared/views/extended_category_mappings_view.sql',
+    'fyle-integrations-db-migrations/common/global_shared/views/extended_employee_mappings_view.sql',
+    'fyle-integrations-db-migrations/common/global_shared/views/extended_mappings_view.sql',
+    'fyle-integrations-db-migrations/common/global_shared/views/prod_workspaces_view.sql',
+
+    # Global Alert Views
+    'fyle-integrations-db-migrations/common/global_shared/views/alerts/_import_logs_fatal_failed_in_progress_tasks_view.sql',
+    'fyle-integrations-db-migrations/common/global_shared/views/alerts/import_logs_fatal_failed_in_progress_tasks_view.sql',
+    'fyle-integrations-db-migrations/common/global_shared/views/alerts/ormq_count_view.sql',
+    'fyle-integrations-db-migrations/common/global_shared/views/alerts/repetition_error_count_view.sql',
+
+    # Grouped Views
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/prod_active_workspaces_view.sql',
+    # Grouped Alert Views
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/_django_queue_fatal_tasks_view.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/_django_queue_in_progress_tasks_view.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/direct_export_errored_expenses_view.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/_direct_export_errored_expenses_view.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/django_queue_fatal_tasks_view.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/django_queue_in_progress_tasks_view.sql',
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/inactive_workspaces_view.sql',
+
+    # Netsuite Specific Views
+    'fyle-integrations-db-migrations/netsuite/views/extended_expenses_view.sql',
+    'fyle-integrations-db-migrations/netsuite/views/extended_settings_view.sql',
+    'fyle-integrations-db-migrations/netsuite/views/product_advance_settings_view.sql',
+    'fyle-integrations-db-migrations/netsuite/views/product_export_settings_view.sql',
+    'fyle-integrations-db-migrations/netsuite/views/product_import_settings_view.sql',
+
+    # Netsuite Specific Trigger
+    'fyle-integrations-db-migrations/netsuite/triggers/update_logs.sql'
+]
+class Migration(migrations.Migration):
+    dependencies = [
+        ('fyle', '0037_auto_20241226_0929'),
+        ('users', '0002_auto_20201228_0754'),
+        ('netsuite', '0027_auto_20240924_0820'),
+        ('tasks', '0012_alter_tasklog_expense_group'),
+        ('mappings', '0017_alter_generalmapping_is_tax_balancing_enabled')
+    ]
+    operations = safe_run_sql(sql_files)

apps/internal/migrations/0002_auto_generated_sql.py

@@ -0,0 +1,9 @@
+# Generated by Django
+from django.db import migrations
+from apps.internal.helpers import safe_run_sql
+sql_files = [
+    'fyle-integrations-db-migrations/netsuite/functions/delete_workspace.sql'
+]
+class Migration(migrations.Migration):
+    dependencies = [('internal', '0001_auto_generated_sql')]
+    operations = safe_run_sql(sql_files)

apps/internal/migrations/0003_auto_generated_sql.py

@@ -0,0 +1,12 @@
+# Generated by Django
+from django.db import migrations
+from apps.internal.helpers import safe_run_sql
+
+sql_files = [
+    'fyle-integrations-db-migrations/common/qbo_intacct_netsuite_xero/views/alerts/inactive_workspaces_view.sql'
+]
+
+
+class Migration(migrations.Migration):
+    dependencies = [('internal', '0002_auto_generated_sql')]
+    operations = safe_run_sql(sql_files)

apps/tasks/migrations/0001_initial.py

@@ -13,6 +13,7 @@ class Migration(migrations.Migration):
     dependencies = [
         ('fyle', '0001_initial'),
         ('workspaces', '0001_initial'),
+        ('netsuite', '0001_initial')
     ]
 
     operations = [

deploy_dump.sh

@@ -0,0 +1,11 @@
+if [ -z "$1" ]; then
+  echo "Usage: sh $0 '2024-12-09'"
+  exit 1
+fi
+
+base_url="https://github.com/fylein/fyle-netsuite-api/commit"
+branch_name=$(git rev-parse --abbrev-ref HEAD)
+
+git log --since="$1" --pretty=format:"$base_url/%H,%an,%ad,%s,$branch_name,netsuite-api" > commits.csv
+
+open commits.csv

fyle_netsuite_api/sentry.py

@@ -16,7 +16,7 @@ def init():
             environment=os.environ.get('SENTRY_ENV'),
             attach_stacktrace=True,
             before_send=Sentry.before_send,
-            request_bodies='small',
+            max_request_body_size='small',
             in_app_include=['apps.users',
             'apps.workspaces',
             'apps.mappings',

fyle_netsuite_api/settings.py

@@ -71,7 +71,6 @@
     'fyle_netsuite_api.logging_middleware.ErrorHandlerMiddleware',
     'corsheaders.middleware.CorsMiddleware',
     'django.middleware.common.CommonMiddleware',
-    'corsheaders.middleware.CorsPostCsrfMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',

fyle_netsuite_api/tests/settings.py

@@ -48,7 +48,6 @@
     'fyle_netsuite_api.logging_middleware.ErrorHandlerMiddleware',
     'corsheaders.middleware.CorsMiddleware',
     'django.middleware.common.CommonMiddleware',
-    'corsheaders.middleware.CorsPostCsrfMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',

requirements.txt

@@ -1,66 +1,39 @@
-arrow==0.15.4
-asgiref==3.4.1
-astroid==2.6.6
-attrs==21.2.0
-blessed==1.16.1
-cached-property==1.5.2
-certifi==2022.12.7
-chardet==3.0.4
-coverage==6.2
-cryptography==3.3.2
-dj-database-url==0.5.0
-Django==3.2.14
-django-cors-headers==3.2.0
-django-picklefield==3.1
-fyle-djangoq2==1.0.0
+cryptography==44.0.0
+dj-database-url==2.3.0
+Django==4.2.18
+django-cors-headers==4.6.0
+django-picklefield==3.2
+fyle-djangoq2==2.0.0
 django-request-logging==0.7.5
-django-rest-framework==0.1.0
 django-db-geventpool==4.0.1
-djangorestframework==3.11.2
+djangorestframework==3.15.2
 django-sendgrid-v5==1.2.0
 enum34==1.1.10
-future==0.18.2
-fyle==0.37.0
+future==1.0.0
+fyle==0.37.2
 fyle-accounting-mappings==1.36.3
 fyle-integrations-platform-connector==1.39.3
 fyle-rest-auth==1.7.2
-gunicorn==20.1.0
-gevent==23.9.1
-idna==2.8
-iniconfig==1.1.1
+gunicorn==23.0.0
+gevent==24.11.1
 isodate==0.6.0
 isort==5.10.1
-lazy-object-proxy==1.6.0
 lxml==4.6.5
-mccabe==0.6.1
 netsuitesdk==3.0.1
-oauthlib==3.2.1
-packaging==21.3
-platformdirs==2.4.0
-pluggy==1.0.0
-psycopg2-binary==2.8.4
-py==1.11.0
+psycopg2-binary==2.9.10
 pylint==2.7.4
-pyparsing==3.0.6
-pytest==6.2.5
-pytest-cov==2.12.1
-pytest-django==4.4.0
-pytest-mock==3.6.1
-python-dateutil==2.8.1
-pytz==2019.3
-requests==2.25.0
-requests-file==1.5.1
-requests-oauthlib==1.3.0
-requests-toolbelt==0.9.1
+pytest==8.3.4
+pytest-cov==6.0.0
+pytest-django==4.9.0
+pytest-mock==3.14.0
+python-dateutil==2.8.2
+pytz==2024.2
+requests==2.32.3
+requests-oauthlib==2.0.0
 psutil==5.9.5
-sentry-sdk==1.19.1
-six==1.13.0
-sqlparse==0.3.0
-toml==0.10.2
+sentry-sdk==2.19.2
+six==1.17.0
 Unidecode==1.1.2
-urllib3==1.26.11
-wcwidth==0.1.8
-wrapt==1.12.1
-zeep==4.1.0
-django-filter
-sendgrid==6.9.7
+zeep==4.3.1
+django-filter==24.3
+sendgrid==6.11.0