feat: Complete invoice generation (#7300)

.travis.yml

@@ -4,6 +4,7 @@ dist: bionic
 
 services:
   - docker
+  - redis-server
 
 addons:
   apt:

app/api/helpers/payment.py

@@ -215,7 +215,7 @@ def create_payment(order, return_url, cancel_url, payee_email=None):
                 "transactions": [
                     {
                         "amount": {
-                            "total": round_money(order.amount),
+                            "total": float(round_money(order.amount)),
                             "currency": order.event.payment_currency,
                         },
                         "payee": {"email": payee_email},

app/api/helpers/scheduled_jobs.py

@@ -3,12 +3,14 @@
 
 import pytz
 from flask_celeryext import RequestContextTask
+from redis.exceptions import LockError
+from sqlalchemy import distinct, or_
 
 from app.api.helpers.db import save_to_db
 from app.api.helpers.mail import send_email_after_event
 from app.api.helpers.notification import send_notif_after_event
 from app.api.helpers.query import get_user_event_roles_by_role_name
-from app.api.helpers.utilities import make_dict
+from app.api.helpers.utilities import make_dict, monthdelta
 from app.instance import celery
 from app.models import db
 from app.models.event import Event
@@ -18,6 +20,7 @@
 from app.models.speaker import Speaker
 from app.models.ticket_holder import TicketHolder
 from app.settings import get_settings
+from app.views.redis_store import redis_store
 
 logger = logging.getLogger(__name__)
 
@@ -133,18 +136,83 @@ def delete_ticket_holders_no_order_id():
     db.session.commit()
 
 
+def this_month_date() -> datetime.datetime:
+    return datetime.datetime.combine(
+        datetime.date.today().replace(day=1), datetime.time()
+    )
+
+
 @celery.task(base=RequestContextTask, name='send.monthly.event.invoice')
-def send_monthly_event_invoice():
-    events = Event.query.filter_by(deleted_at=None).filter(Event.owner != None).all()
+def send_monthly_event_invoice(send_notification: bool = True):
+    this_month = this_month_date()
+    last_month = monthdelta(this_month, -1)
+    # Find all event IDs which had a completed order last month
+    last_order_event_ids = Order.query.filter(
+        Order.completed_at.between(last_month, this_month)
+    ).with_entities(distinct(Order.event_id))
+    # SQLAlchemy returns touples instead of list of IDs
+    last_order_event_ids = [r[0] for r in last_order_event_ids]
+    events = (
+        Event.query.filter(Event.owner != None)
+        .filter(
+            or_(
+                Event.starts_at.between(last_month, this_month),
+                Event.ends_at.between(last_month, this_month),
+                Event.id.in_(last_order_event_ids),
+            )
+        )
+        .all()
+    )
 
     for event in events:
-        event_invoice = EventInvoice(event=event)
-        pdf_url = event_invoice.populate()
-        if pdf_url:
-            save_to_db(event_invoice)
+        send_event_invoice.delay(event.id, send_notification=send_notification)
+
+
+@celery.task(base=RequestContextTask, bind=True, max_retries=5)
+def send_event_invoice(
+    self, event_id: int, send_notification: bool = True, force: bool = False
+):
+    this_month = this_month_date()
+    # Check if this month's invoice has been generated
+    event_invoice = (
+        EventInvoice.query.filter_by(event_id=event_id)
+        .filter(EventInvoice.issued_at >= this_month)
+        .first()
+    )
+    if not force and event_invoice:
+        logger.warn(
+            'Event Invoice of this month for this event has already been created: %s',
+            event_id,
+        )
+        return
+
+    event = Event.query.get(event_id)
+    try:
+        # For keeping invoice numbers gapless and non-repeating, we need to generate invoices
+        # one at a time. Hence, we try acquiring an expiring lock for 20 seconds, and then retry.
+        # To avoid the condition of a deadlock, lock automatically expires after 5 seconds
+        saved = False
+        pdf_url = None
+        with redis_store.lock('event_invoice_generate', timeout=5, blocking_timeout=20):
+            event_invoice = EventInvoice(event=event, issued_at=this_month)
+            pdf_url = event_invoice.populate()
+            if pdf_url:
+                try:
+                    save_to_db(event_invoice)
+                    saved = True
+                except Exception as e:
+                    # For some reason, like duplicate identifier, the record might not be saved, so we
+                    # retry generating the invoice and hope the error doesn't happen again
+                    logger.exception('Error while saving invoice. Retrying')
+                    raise self.retry(exc=e)
+            else:
+                logger.error('Error in generating event invoice for event %s', event)
+        if saved and send_notification:
             event_invoice.send_notification()
-        else:
-            logger.error('Error in generating event invoice for event %s', event)
+        return pdf_url
+    except LockError as e:
+        logger.exception('Error while acquiring lock. Retrying')
+        self.retry(exc=e)
 
 
 @celery.on_after_configure.connect

app/api/schema/event_invoices.py

@@ -24,6 +24,7 @@ class Meta:
     identifier = fields.Str(allow_none=True)
     amount = fields.Float(validate=lambda n: n >= 0, allow_none=True)
     created_at = fields.DateTime(allow_none=True)
+    issued_at = fields.DateTime(allow_none=True)
     completed_at = fields.DateTime(default=None)
     transaction_id = fields.Str(allow_none=True)
     paid_via = fields.Str(

app/models/event_invoice.py

@@ -4,7 +4,6 @@
 from flask.templating import render_template
 from sqlalchemy.sql import func
 
-from app.api.helpers.db import get_new_identifier
 from app.api.helpers.files import create_save_pdf
 from app.api.helpers.mail import send_email_for_monthly_fee_payment
 from app.api.helpers.notification import send_notif_monthly_fee_payment
@@ -20,23 +19,20 @@
 logger = logging.getLogger(__name__)
 
 
-def get_new_id():
-    return get_new_identifier(EventInvoice, length=8)
-
-
 class EventInvoice(SoftDeletionModel):
     DUE_DATE_DAYS = 30
 
     __tablename__ = 'event_invoices'
 
     id = db.Column(db.Integer, primary_key=True)
-    identifier = db.Column(db.String, unique=True, default=get_new_id)
+    identifier = db.Column(db.String, unique=True, nullable=False)
     amount = db.Column(db.Float)
 
     user_id = db.Column(db.Integer, db.ForeignKey('users.id', ondelete='SET NULL'))
     event_id = db.Column(db.Integer, db.ForeignKey('events.id', ondelete='SET NULL'))
 
     created_at = db.Column(db.DateTime(timezone=True), default=func.now())
+    issued_at = db.Column(db.DateTime(timezone=True), nullable=False)
 
     # Payment Fields
     completed_at = db.Column(db.DateTime(timezone=True), nullable=True, default=None)
@@ -59,18 +55,32 @@ class EventInvoice(SoftDeletionModel):
     def __init__(self, **kwargs):
         super(EventInvoice, self).__init__(**kwargs)
 
-        if not self.created_at:
-            self.created_at = datetime.utcnow()
+        if not self.issued_at:
+            self.issued_at = datetime.now()
 
         if not self.identifier:
-            self.identifier = self.created_at.strftime('%Y%mU-') + get_new_id()
+            self.identifier = self.get_new_id()
 
     def __repr__(self):
         return '<EventInvoice %r>' % self.invoice_pdf_url
 
+    def get_new_id(self) -> str:
+        with db.session.no_autoflush:
+            identifier = self.issued_at.strftime('%Y%mU-') + str(
+                EventInvoice.query.count() + 1
+            )
+            count = EventInvoice.query.filter_by(identifier=identifier).count()
+            if count == 0:
+                return identifier
+            return self.get_new_id()
+
+    @property
+    def previous_month_date(self):
+        return monthdelta(self.issued_at, -1)
+
     @property
     def due_at(self):
-        return self.created_at + timedelta(days=EventInvoice.DUE_DATE_DAYS)
+        return self.issued_at + timedelta(days=EventInvoice.DUE_DATE_DAYS)
 
     def populate(self):
         assert self.event is not None
@@ -83,8 +93,8 @@ def generate_pdf(self):
         with db.session.no_autoflush:
             latest_invoice_date = (
                 EventInvoice.query.filter_by(event=self.event)
-                .filter(EventInvoice.created_at < self.created_at)
-                .with_entities(func.max(EventInvoice.created_at))
+                .filter(EventInvoice.issued_at < self.issued_at)
+                .with_entities(func.max(EventInvoice.issued_at))
                 .scalar()
             )
 
@@ -103,15 +113,15 @@ def generate_pdf(self):
             ticket_fee_percentage = ticket_fee_object.service_fee
             ticket_fee_maximum = ticket_fee_object.maximum_fee
             gross_revenue = self.event.calc_revenue(
-                start=latest_invoice_date, end=self.created_at
+                start=latest_invoice_date, end=self.issued_at
             )
             invoice_amount = gross_revenue * (ticket_fee_percentage / 100)
             if invoice_amount > ticket_fee_maximum:
                 invoice_amount = ticket_fee_maximum
             self.amount = round_money(invoice_amount)
             net_revenue = round_money(gross_revenue - invoice_amount)
             orders_query = self.event.get_orders_query(
-                start=latest_invoice_date, end=self.created_at
+                start=latest_invoice_date, end=self.issued_at
             )
             first_order_date = orders_query.with_entities(
                 func.min(Order.completed_at)
@@ -123,8 +133,8 @@ def generate_pdf(self):
                 'tickets_sold': self.event.tickets_sold,
                 'gross_revenue': round_money(gross_revenue),
                 'net_revenue': round_money(net_revenue),
-                'first_date': first_order_date,
-                'last_date': last_order_date,
+                'first_date': first_order_date or self.previous_month_date,
+                'last_date': last_order_date or self.issued_at,
             }
             self.invoice_pdf_url = create_save_pdf(
                 render_template(
@@ -133,7 +143,7 @@ def generate_pdf(self):
                     admin_info=admin_info,
                     currency=currency,
                     event=self.event,
-                    ticket_fee_object=ticket_fee_object,
+                    ticket_fee=ticket_fee_object,
                     payment_details=payment_details,
                     net_revenue=net_revenue,
                     invoice=self,
@@ -148,9 +158,7 @@ def generate_pdf(self):
         return self.invoice_pdf_url
 
     def send_notification(self, follow_up=False):
-        prev_month = monthdelta(self.created_at, 1).strftime(
-            "%b %Y"
-        )  # Displayed as Aug 2016
+        prev_month = self.previous_month_date.strftime("%b %Y")  # Displayed as Aug 2016
         app_name = get_settings()['app_name']
         frontend_url = get_settings()['frontend_url']
         link = '{}/event-invoice/{}/review'.format(frontend_url, self.identifier)

app/templates/pdf/event_invoice.html

@@ -105,6 +105,10 @@
           text-align: left;
         }
 
+        table .fees {
+          text-align: right;
+        }
+
         table td {
           padding: 20px;
           text-align: right;
@@ -142,8 +146,12 @@ <h1>INVOICE {{ invoice.identifier }}</h1>
         <div>{{ admin_info.admin_company or '' }}</div>
         <div>{{ admin_info.full_billing_address or '' }}</div>
         <div>{{ admin_info.admin_billing_phone or '' }}</div>
-        <div>{{ 'Tax ID: ' ~ admin_info.admin_billing_tax_info or '' }}</div>
-        <div><a href="{{ admin_info.admin_billing_email }}">{{ admin_info.admin_billing_email }}</a></div>
+        {% if admin_info.admin_billing_tax_info %}
+          <div>{{ 'Tax ID: ' ~ admin_info.admin_billing_tax_info or '' }}</div>
+        {% endif %}
+        {% if admin_info.admin_billing_tax_info %}
+          <div><a href="{{ admin_info.admin_billing_email }}">{{ admin_info.admin_billing_email }}</a></div>
+        {% endif %}
       </div>
       <div id="project">
         <div><span>CLIENT</span> {{ user.billing_contact_name or user.fullname }}</div>
@@ -152,7 +160,7 @@ <h1>INVOICE {{ invoice.identifier }}</h1>
         {% if user.billing_tax_info %}
           <div><span>TAX ID</span> {{user.billing_tax_info or ''}}</div>
         {% endif %}
-        <div><span>DATE</span> {{ invoice.created_at | date }}</div>
+        <div><span>DATE</span> {{ invoice.issued_at | date }}</div>
         <div><span>DUE DATE</span> {{ invoice.due_at | date }}</div>
       </div>
     </header>
@@ -162,7 +170,7 @@ <h1>INVOICE {{ invoice.identifier }}</h1>
           <tr>
             <th class="service">PRODUCT</th>
             <th class="desc">DESCRIPTION</th>
-            <th>FEES</th>
+            <th class="fees">FEES ({{ ticket_fee.service_fee }}%)</th>
           </tr>
         </thead>
         <tbody>
@@ -180,7 +188,7 @@ <h1>INVOICE {{ invoice.identifier }}</h1>
             <td class="total">{{ currency | currency_symbol }} {{ invoice.amount }}</td>
           </tr>
           <tr>
-            <td colspan="2">TAX N/A</td>
+            <td colspan="2">TAX (N/A)</td>
             <td class="total">{{ currency | currency_symbol }} 0.00</td>
           </tr>
           <tr>

migrations/versions/rev-2020-09-30-12:50:33-08e4b2df44a3_.py

@@ -0,0 +1,31 @@
+"""empty message
+
+Revision ID: 08e4b2df44a3
+Revises: 8e86e0ddc9a4
+Create Date: 2020-09-30 12:50:33.433044
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '08e4b2df44a3'
+down_revision = '8e86e0ddc9a4'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('event_invoices', 'identifier',
+               existing_type=sa.VARCHAR(),
+               nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('event_invoices', 'identifier',
+               existing_type=sa.VARCHAR(),
+               nullable=True)
+    # ### end Alembic commands ###

migrations/versions/rev-2020-09-30-13:00:05-99217a8bc9b2_.py

@@ -0,0 +1,27 @@
+"""empty message
+
+Revision ID: 99217a8bc9b2
+Revises: 08e4b2df44a3
+Create Date: 2020-09-30 13:00:05.338258
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '99217a8bc9b2'
+down_revision = '08e4b2df44a3'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('event_invoices', sa.Column('issued_at', sa.DateTime(timezone=True), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('event_invoices', 'issued_at')
+    # ### end Alembic commands ###

tests/all/unit/api/helpers/test_utilities.py

@@ -95,6 +95,14 @@ def test_monthdelta():
     test_future_date = monthdelta(test_date, 3)
     assert test_future_date == datetime.datetime(2000, 9, 18)
 
+    test_date = datetime.datetime(2000, 1, 1)
+    test_past_date = monthdelta(test_date, -1)
+    assert test_past_date == datetime.datetime(1999, 12, 1)
+
+    test_date = datetime.datetime(2000, 3, 1)
+    test_past_date = monthdelta(test_date, -1)
+    assert test_past_date == datetime.datetime(2000, 2, 1)
+
 
 def test_dict_to_snake_case():
     assert dict_to_snake_case(None) is None