From 304986e4f3462a7187eed2670266d9ea0f928e17 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Thu, 29 Dec 2022 21:48:15 -0500 Subject: [PATCH 01/12] relay dataset and data tables are created --- dags/bqetl_subplat.py | 70 +++++++ .../relay/active_subscription_ids/view.sql | 23 ++ .../relay/active_subscriptions/view.sql | 22 ++ .../relay/dataset_metadata.yaml | 10 + .../relay/subscription_events/view.sql | 22 ++ .../relay/subscriptions/view.sql | 7 + .../active_subscription_ids_live/view.sql | 23 ++ .../active_subscription_ids_v1/metadata.yaml | 16 ++ .../active_subscription_ids_v1/query.sql | 6 + .../active_subscriptions_live/view.sql | 47 +++++ .../active_subscriptions_v1/metadata.yaml | 13 ++ .../active_subscriptions_v1/query.sql | 6 + .../active_subscriptions_v1/schema.yaml | 46 ++++ .../relay_derived/dataset_metadata.yaml | 10 + .../subscription_events_live/view.sql | 152 ++++++++++++++ .../subscription_events_v1/metadata.yaml | 14 ++ .../subscription_events_v1/query.sql | 6 + .../subscription_events_v1/schema.yaml | 49 +++++ .../subscriptions_v1/metadata.yaml | 15 ++ .../relay_derived/subscriptions_v1/query.sql | 197 ++++++++++++++++++ .../subscriptions_v1/schema.yaml | 112 ++++++++++ 21 files changed, 866 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/relay/active_subscription_ids/view.sql create mode 100644 sql/moz-fx-data-shared-prod/relay/active_subscriptions/view.sql create mode 100644 sql/moz-fx-data-shared-prod/relay/dataset_metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay/subscription_events/view.sql create mode 100644 sql/moz-fx-data-shared-prod/relay/subscriptions/view.sql create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql create mode 100755 sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql create mode 100755 sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/dataset_metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql create mode 100755 sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml diff --git a/dags/bqetl_subplat.py b/dags/bqetl_subplat.py index 10800675971..4edc3034671 100644 --- a/dags/bqetl_subplat.py +++ b/dags/bqetl_subplat.py @@ -555,6 +555,54 @@ ], ) + relay_derived__active_subscription_ids__v1 = bigquery_etl_query( + task_id="relay_derived__active_subscription_ids__v1", + destination_table='active_subscription_ids_v1${{ macros.ds_format(macros.ds_add(ds, -7), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="relay_derived", + project_id="moz-fx-data-shared-prod", + owner="srose@mozilla.com", + email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=True, + parameters=["date:DATE:{{macros.ds_add(ds, -7)}}"], + ) + + relay_derived__active_subscriptions__v1 = bigquery_etl_query( + task_id="relay_derived__active_subscriptions__v1", + destination_table='active_subscriptions_v1${{ macros.ds_format(macros.ds_add(ds, -7), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="relay_derived", + project_id="moz-fx-data-shared-prod", + owner="srose@mozilla.com", + email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["date:DATE:{{macros.ds_add(ds, -7)}}"], + ) + + relay_derived__subscription_events__v1 = bigquery_etl_query( + task_id="relay_derived__subscription_events__v1", + destination_table='subscription_events_v1${{ macros.ds_format(macros.ds_add(ds, -8), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="relay_derived", + project_id="moz-fx-data-shared-prod", + owner="srose@mozilla.com", + email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["date:DATE:{{macros.ds_add(ds, -8)}}"], + ) + + relay_derived__subscriptions__v1 = bigquery_etl_query( + task_id="relay_derived__subscriptions__v1", + destination_table="subscriptions_v1", + dataset_id="relay_derived", + project_id="moz-fx-data-shared-prod", + owner="srose@mozilla.com", + email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + task_concurrency=1, + ) + stripe_external__itemized_payout_reconciliation__v5 = gke_command( task_id="stripe_external__itemized_payout_reconciliation__v5", command=[ @@ -739,3 +787,25 @@ ) mozilla_vpn_derived__users__v1.set_upstream(mozilla_vpn_external__users__v1) + + relay_derived__active_subscription_ids__v1.set_upstream( + relay_derived__subscriptions__v1 + ) + + relay_derived__active_subscriptions__v1.set_upstream( + relay_derived__active_subscription_ids__v1 + ) + + relay_derived__active_subscriptions__v1.set_upstream( + relay_derived__subscriptions__v1 + ) + + relay_derived__subscription_events__v1.set_upstream( + relay_derived__active_subscription_ids__v1 + ) + + relay_derived__subscription_events__v1.set_upstream( + relay_derived__subscriptions__v1 + ) + + relay_derived__subscriptions__v1.set_upstream(fivetran_stripe_sync_wait) diff --git a/sql/moz-fx-data-shared-prod/relay/active_subscription_ids/view.sql b/sql/moz-fx-data-shared-prod/relay/active_subscription_ids/view.sql new file mode 100644 index 00000000000..1fb90ceffb0 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay/active_subscription_ids/view.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.relay.active_subscription_ids` +AS +WITH max_active_date AS ( + SELECT AS VALUE + MAX(active_date) + FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_v1 +) +SELECT + active_subscription_ids_live.* +FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_live +CROSS JOIN + max_active_date +WHERE + -- static partition filter not needed because live view doesn't use date partitioned tables + active_date > max_active_date +UNION ALL +SELECT + * +FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_v1 diff --git a/sql/moz-fx-data-shared-prod/relay/active_subscriptions/view.sql b/sql/moz-fx-data-shared-prod/relay/active_subscriptions/view.sql new file mode 100644 index 00000000000..f880022e89e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay/active_subscriptions/view.sql @@ -0,0 +1,22 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.relay.active_subscriptions` +AS +WITH max_agg_date AS ( + SELECT AS VALUE + MAX(active_date) + FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_v1 +) +SELECT + active_subscriptions_live.* +FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_live +CROSS JOIN + max_agg_date +WHERE + active_date > max_agg_date +UNION ALL +SELECT + * +FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_v1 diff --git a/sql/moz-fx-data-shared-prod/relay/dataset_metadata.yaml b/sql/moz-fx-data-shared-prod/relay/dataset_metadata.yaml new file mode 100644 index 00000000000..f38c4ed887f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay/dataset_metadata.yaml @@ -0,0 +1,10 @@ +friendly_name: Firefox Relay +description: |- + Data related to the Firefox Relay service +dataset_base_acl: view +user_facing: true +labels: {} +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential diff --git a/sql/moz-fx-data-shared-prod/relay/subscription_events/view.sql b/sql/moz-fx-data-shared-prod/relay/subscription_events/view.sql new file mode 100644 index 00000000000..9fbd21bceaf --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay/subscription_events/view.sql @@ -0,0 +1,22 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.relay.subscription_events` +AS +WITH max_agg_date AS ( + SELECT AS VALUE + MAX(event_date) + FROM + `moz-fx-data-shared-prod`.relay_derived.subscription_events_v1 +) +SELECT + subscription_events_live.* +FROM + `moz-fx-data-shared-prod`.relay_derived.subscription_events_live +CROSS JOIN + max_agg_date +WHERE + event_date > max_agg_date +UNION ALL +SELECT + * +FROM + `moz-fx-data-shared-prod`.relay_derived.subscription_events_v1 diff --git a/sql/moz-fx-data-shared-prod/relay/subscriptions/view.sql b/sql/moz-fx-data-shared-prod/relay/subscriptions/view.sql new file mode 100644 index 00000000000..72d90ed0640 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay/subscriptions/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.relay.subscriptions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod`.relay_derived.subscriptions_v1 diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql new file mode 100644 index 00000000000..08ccd43aba0 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.relay_derived.active_subscription_ids_live` +AS +SELECT + active_date, + subscription_id, +FROM + mozdata.relay.subscriptions +CROSS JOIN + UNNEST( + GENERATE_DATE_ARRAY( + DATE(subscription_start_date), + GREATEST(DATE(subscription_start_date), DATE(end_date) - 1) + ) + ) AS active_date +WHERE + subscription_start_date IS NOT NULL + AND DATE(subscription_start_date) < ( + SELECT + DATE(MAX(end_date)) + FROM + mozdata.relay.subscriptions + ) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/metadata.yaml new file mode 100755 index 00000000000..87d71659a55 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/metadata.yaml @@ -0,0 +1,16 @@ +friendly_name: Active Firefox Relay Subscription IDs +description: > + IDs of active Firefox Relay subscriptions by date. +owners: + - srose@mozilla.com +labels: + application: relay + schedule: daily +scheduling: + dag_name: bqetl_subplat + # While this ETL doesn't depend on its own previous runs, other ETLs are built + # on the assumption that this is built sequentially day-by-day with no gaps. + depends_on_past: true + # delay aggregates by 7 days, to ensure data is complete + date_partition_offset: -7 + date_partition_parameter: date diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/query.sql new file mode 100644 index 00000000000..898d172fea2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/query.sql @@ -0,0 +1,6 @@ +SELECT + * +FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_live +WHERE + IF(CAST(@date AS DATE) IS NULL, active_date < CURRENT_DATE - 7, active_date = @date) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql new file mode 100644 index 00000000000..cef62377311 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql @@ -0,0 +1,47 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.relay_derived.active_subscriptions_live` +AS +WITH + subscriptions AS ( + SELECT + *, + TO_JSON_STRING(promotion_codes) AS json_promotion_codes + FROM + mozdata.relay.subscriptions) +SELECT + active_subscription_ids.active_date, + subscriptions.plan_id, + subscriptions.status, + subscriptions.country, + subscriptions.provider, + subscriptions.plan_amount, + subscriptions.plan_currency, + subscriptions.plan_interval, + subscriptions.plan_interval_count, + subscriptions.product_id, + subscriptions.product_name, + subscriptions.pricing_plan, + JSON_VALUE_ARRAY(subscriptions.json_promotion_codes) AS promotion_codes, + subscriptions.promotion_discounts_amount, + COUNT(*) AS `count`, +FROM + subscriptions +JOIN + mozdata.relay.active_subscription_ids +USING + (subscription_id) +GROUP BY + active_date, + plan_id, + status, + country, + provider, + plan_amount, + plan_currency, + plan_interval, + plan_interval_count, + product_id, + product_name, + pricing_plan, + json_promotion_codes, + promotion_discounts_amount diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/metadata.yaml new file mode 100755 index 00000000000..54d20318222 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/metadata.yaml @@ -0,0 +1,13 @@ +friendly_name: Active Firefox Relay Subscriptions +description: > + Aggregated count of active Firefox Relay subscriptions. +owners: + - srose@mozilla.com +labels: + application: relay + schedule: daily +scheduling: + dag_name: bqetl_subplat + # delay aggregates by 7 days, to ensure data is complete + date_partition_offset: -7 + date_partition_parameter: date diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/query.sql new file mode 100644 index 00000000000..adba04f3b2b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/query.sql @@ -0,0 +1,6 @@ +SELECT + * +FROM + `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_live +WHERE + IF(CAST(@date AS DATE) IS NULL, active_date < CURRENT_DATE - 7, active_date = @date) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml new file mode 100644 index 00000000000..3eee9ceb2e9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml @@ -0,0 +1,46 @@ +fields: +- name: active_date + type: DATE + mode: NULLABLE +- name: plan_id + type: STRING + mode: NULLABLE +- name: status + type: STRING + mode: NULLABLE +- name: country + type: STRING + mode: NULLABLE +- name: provider + type: STRING + mode: NULLABLE +- name: plan_amount + type: INTEGER + mode: NULLABLE +- name: plan_currency + type: STRING + mode: NULLABLE +- name: plan_interval + type: STRING + mode: NULLABLE +- name: plan_interval_count + type: INTEGER + mode: NULLABLE +- name: product_id + type: STRING + mode: NULLABLE +- name: product_name + type: STRING + mode: NULLABLE +- name: pricing_plan + type: STRING + mode: NULLABLE +- name: promotion_codes + type: STRING + mode: REPEATED +- name: promotion_discounts_amount + type: INTEGER + mode: NULLABLE +- name: count + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/relay_derived/dataset_metadata.yaml b/sql/moz-fx-data-shared-prod/relay_derived/dataset_metadata.yaml new file mode 100644 index 00000000000..07b23dd720f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/dataset_metadata.yaml @@ -0,0 +1,10 @@ +friendly_name: Firefox Relay Derived +description: |- + Derived data related to the Firefox Relay service +dataset_base_acl: derived +user_facing: false +labels: {} +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql new file mode 100644 index 00000000000..355f1b6b7c3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql @@ -0,0 +1,152 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.relay_derived.subscription_events_live` +AS +WITH subscriptions AS ( + SELECT + *, + TO_JSON_STRING(promotion_codes) AS json_promotion_codes + FROM + mozdata.relay.subscriptions +), +max_active_date AS ( + SELECT AS VALUE + MAX(active_date) + FROM + mozdata.relay.active_subscription_ids +), +trials AS ( + SELECT + * + FROM + subscriptions + WHERE + trial_start IS NOT NULL + AND DATE(trial_start) <= (SELECT max_active_date FROM max_active_date) +), +new_trial_events AS ( + SELECT + DATE(trial_start) AS event_date, + subscription_id, + "New Trial" AS event_type, + FROM + trials +), +cancelled_trial_events AS ( + SELECT + DATE(ended_at) AS event_date, + subscription_id, + "Cancelled Trial" AS event_type, + FROM + trials + WHERE + subscription_start_date IS NULL + AND ended_at IS NOT NULL +), +new_events AS ( + SELECT + active_date AS event_date, + subscription_id, + "New" AS event_type, + FROM + mozdata.relay.active_subscription_ids + WHERE + TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING + QUALIFY + LAG(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM ( + active_date - 1 + ) +), +cancelled_events AS ( + SELECT + active_date + 1 AS event_date, + subscription_id, + "Cancelled" AS event_type, + FROM + mozdata.relay.active_subscription_ids + CROSS JOIN + max_active_date + WHERE + TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING + QUALIFY + LEAD(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM ( + active_date + 1 + ) + AND active_date < max_active_date +), +events AS ( + SELECT + * + FROM + new_trial_events + UNION ALL + SELECT + * + FROM + cancelled_trial_events + UNION ALL + SELECT + * + FROM + new_events + UNION ALL + SELECT + * + FROM + cancelled_events +) +SELECT + events.event_date, + events.event_type, + CASE + WHEN + events.event_type IN ("New Trial", "Cancelled Trial") + THEN + events.event_type + WHEN + events.event_type = "New" + THEN + subscriptions.subscription_start_reason + WHEN + events.event_type = "Cancelled" + THEN + COALESCE( + subscriptions.ended_reason, + IF(subscriptions.provider = "Apple Store", "Cancelled by IAP", "Payment Failed") + ) + END + AS granular_event_type, + subscriptions.plan_id, + subscriptions.status, + subscriptions.country, + subscriptions.provider, + subscriptions.plan_amount, + subscriptions.plan_currency, + subscriptions.plan_interval, + subscriptions.plan_interval_count, + subscriptions.product_id, + subscriptions.product_name, + subscriptions.pricing_plan, + JSON_VALUE_ARRAY(subscriptions.json_promotion_codes) AS promotion_codes, + COUNT(*) AS `count`, +FROM + subscriptions +JOIN + events +USING + (subscription_id) +GROUP BY + event_date, + event_type, + granular_event_type, + plan_id, + status, + country, + provider, + plan_amount, + plan_currency, + plan_interval, + plan_interval_count, + product_id, + product_name, + pricing_plan, + json_promotion_codes diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/metadata.yaml new file mode 100755 index 00000000000..a32294cb51d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/metadata.yaml @@ -0,0 +1,14 @@ +friendly_name: Firefox Relay Subscription Events +description: > + Aggregated count of Firefox Relay subscription start/end events. +owners: + - srose@mozilla.com +labels: + application: relay + schedule: daily +scheduling: + dag_name: bqetl_subplat + # Delay aggregates by 8 days, to ensure data is complete. Upstream tables are + # delayed 7 days, and this needs an additional day of delay for cancel events. + date_partition_offset: -8 + date_partition_parameter: date diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql new file mode 100644 index 00000000000..7188cce025e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql @@ -0,0 +1,6 @@ +SELECT + * +FROM + `moz-fx-data-shared-prod`.relay_derived.subscription_events_live +WHERE + IF(CAST(@date AS DATE) IS NULL, event_date < CURRENT_DATE - 7, event_date = @date) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml new file mode 100644 index 00000000000..a24ea50e8c2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml @@ -0,0 +1,49 @@ +fields: +- name: event_date + type: DATE + mode: NULLABLE +- name: event_type + type: STRING + mode: NULLABLE +- name: granular_event_type + type: STRING + mode: NULLABLE +- name: plan_id + type: STRING + mode: NULLABLE +- name: status + type: STRING + mode: NULLABLE +- name: country + type: STRING + mode: NULLABLE +- name: provider + type: STRING + mode: NULLABLE +- name: plan_amount + type: INTEGER + mode: NULLABLE +- name: plan_currency + type: STRING + mode: NULLABLE +- name: plan_interval + type: STRING + mode: NULLABLE +- name: plan_interval_count + type: INTEGER + mode: NULLABLE +- name: product_id + type: STRING + mode: NULLABLE +- name: product_name + type: STRING + mode: NULLABLE +- name: pricing_plan + type: STRING + mode: NULLABLE +- name: promotion_codes + type: STRING + mode: REPEATED +- name: count + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml new file mode 100644 index 00000000000..1e2f60fd8b3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml @@ -0,0 +1,15 @@ +friendly_name: Relay Subscriptions +description: |- + Firefox Relay subscriptions +owners: + - srose@mozilla.com +labels: + application: relay + schedule: daily +scheduling: + dag_name: bqetl_subplat + # destination is the whole table, not a single partition, + # so don't use date_partition_parameter + date_partition_parameter: null + depends_on_fivetran: + - task_id: fivetran_stripe diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql new file mode 100644 index 00000000000..56d7d0f019c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql @@ -0,0 +1,197 @@ +WITH standardized_country AS ( -- CTE for all_subscriptions + SELECT + raw_country AS country, + standardized_country AS country_name, + FROM + mozdata.static.third_party_standardized_country_names +), +stripe_subscriptions_history AS (-- CTE for all_subscriptions + SELECT + *, + CONCAT( + subscription_id, + COALESCE( + CONCAT( + "-", + NULLIF(ROW_NUMBER() OVER (PARTITION BY subscription_id ORDER BY valid_from), 1) + ), + "" + ) + ) AS subscription_sequence_id + FROM + `mozdata.subscription_platform.stripe_subscriptions_history` + WHERE + -- Only include the current history records and the last history records for previous plans. + (valid_to IS NULL OR plan_ended_at IS NOT NULL) + AND status NOT IN ("incomplete", "incomplete_expired") +), +stripe_subscriptions AS (-- CTE for all_subscriptions + SELECT + -- user_id, + customer_id, + subscription_sequence_id AS subscription_id, + IF( + subscription_sequence_id != subscription_id, + subscription_id, + NULL + ) AS original_subscription_id, + plan_id, + status, + synced_at AS event_timestamp, + IF( + (trial_end > TIMESTAMP(CURRENT_DATE) OR ended_at <= trial_end), + NULL, + COALESCE(plan_started_at, subscription_start_date) + ) AS subscription_start_date, + --first subscription start date associated with the subscription id + IF( + (trial_end > TIMESTAMP(CURRENT_DATE) OR ended_at <= trial_end), + NULL, + subscription_start_date + ) AS original_subscription_start_date, + IF(plan_started_at IS NOT NULL, "Plan Change", NULL) AS subscription_start_reason, + -- created, + trial_start, + trial_end, + canceled_at, + canceled_for_customer_at, + cancel_at, + cancel_at_period_end, + COALESCE(plan_ended_at, IF(ended_at < TIMESTAMP(CURRENT_DATE), ended_at, NULL)) AS ended_at, + IF(plan_ended_at IS NOT NULL, "Plan Change", NULL) AS ended_reason, + -- fxa_uid, + country, + country_name, + -- user_registration_date, + -- entrypoint_experiment, + -- entrypoint_variation, + -- utm_campaign, + -- utm_content, + -- utm_medium, + -- utm_source, + -- utm_term, + provider, + plan_amount, + -- billing_scheme, + plan_currency, + plan_interval, + plan_interval_count, + plan_interval_timezone, + product_id, + product_name, + CONCAT( + plan_interval_count, + "-", + plan_interval, + "-", + plan_currency, + "-", + (plan_amount / 100) + ) AS pricing_plan, + -- Stripe billing grace period is 7 day and Paypal is billed by Stripe + INTERVAL 7 DAY AS billing_grace_period, + promotion_codes, + promotion_discounts_amount, + FROM + stripe_subscriptions_history + LEFT JOIN + standardized_country + USING + (country) + -- LEFT JOIN + -- users + -- USING + -- (fxa_uid) + -- LEFT JOIN + -- attribution + -- USING + -- (fxa_uid) + WHERE + (product_name = "Relay Premium" OR product_name = "Mozilla VPN & Firefox Relay") +), +relay_subscriptions_with_end_date AS (-- CTE for all_subscriptions + SELECT + *, + IF( + customer_id IS NOT NULL, + MIN(subscription_start_date) OVER (PARTITION BY customer_id), + subscription_start_date + ) AS customer_start_date, + COALESCE(ended_at, TIMESTAMP(CURRENT_DATE)) AS end_date, + FROM + stripe_subscriptions +) +SELECT + * REPLACE ( + CASE + WHEN + subscription_start_date IS NULL + THEN + NULL + WHEN + subscription_start_reason IS NOT NULL + THEN + subscription_start_reason + WHEN + trial_start IS NOT NULL + THEN + "Converted Trial" + WHEN + DATE(subscription_start_date) = DATE(customer_start_date) + THEN + "New" + ELSE + "Resurrected" + END + AS subscription_start_reason, + CASE + WHEN + ended_at IS NULL + THEN + NULL + WHEN + ended_reason IS NOT NULL + THEN + ended_reason + WHEN + canceled_for_customer_at IS NOT NULL + OR cancel_at_period_end + THEN + "Cancelled by Customer" + ELSE + "Payment Failed" + END + AS ended_reason + ), + mozfun.norm.diff_months( + start => DATETIME(subscription_start_date, plan_interval_timezone), + `end` => DATETIME(end_date, plan_interval_timezone), + grace_period => billing_grace_period, + inclusive => FALSE + ) AS months_retained, + mozfun.norm.diff_months( + start => DATETIME( + COALESCE(original_subscription_start_date, subscription_start_date), + plan_interval_timezone + ), + `end` => DATETIME(end_date, plan_interval_timezone), + grace_period => billing_grace_period, + inclusive => FALSE + ) AS original_subscription_months_retained, + mozfun.norm.diff_months( + start => DATETIME(subscription_start_date, plan_interval_timezone), + `end` => DATETIME(TIMESTAMP(CURRENT_DATE), plan_interval_timezone), + grace_period => billing_grace_period, + inclusive => FALSE + ) AS current_months_since_subscription_start, + mozfun.norm.diff_months( + start => DATETIME( + COALESCE(original_subscription_start_date, subscription_start_date), + plan_interval_timezone + ), + `end` => DATETIME(TIMESTAMP(CURRENT_DATE), plan_interval_timezone), + grace_period => billing_grace_period, + inclusive => FALSE + ) AS current_months_since_original_subscription_start, +FROM + relay_subscriptions_with_end_date diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml new file mode 100644 index 00000000000..c3be1eafb58 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml @@ -0,0 +1,112 @@ +fields: +- mode: NULLABLE + name: customer_id + type: STRING +- mode: NULLABLE + name: subscription_id + type: STRING +- mode: NULLABLE + name: original_subscription_id + type: STRING +- mode: NULLABLE + name: plan_id + type: STRING +- mode: NULLABLE + name: status + type: STRING +- mode: NULLABLE + name: event_timestamp + type: TIMESTAMP +- mode: NULLABLE + name: customer_start_date + type: TIMESTAMP +- mode: NULLABLE + name: subscription_start_date + type: TIMESTAMP +- mode: NULLABLE + name: original_subscription_start_date + type: TIMESTAMP +- mode: NULLABLE + name: subscription_start_reason + type: STRING +- mode: NULLABLE + name: trial_start + type: TIMESTAMP +- mode: NULLABLE + name: trial_end + type: TIMESTAMP +- mode: NULLABLE + name: canceled_at + type: TIMESTAMP +- mode: NULLABLE + name: canceled_for_customer_at + type: STRING +- mode: NULLABLE + name: cancel_at + type: TIMESTAMP +- mode: NULLABLE + name: cancel_at_period_end + type: BOOLEAN +- mode: NULLABLE + name: ended_at + type: TIMESTAMP +- mode: NULLABLE + name: ended_reason + type: STRING +- mode: NULLABLE + name: end_date + type: TIMESTAMP +- mode: NULLABLE + name: country + type: STRING +- mode: NULLABLE + name: country_name + type: STRING +- mode: NULLABLE + name: provider + type: STRING +- mode: NULLABLE + name: plan_amount + type: INTEGER +- mode: NULLABLE + name: plan_currency + type: STRING +- mode: NULLABLE + name: plan_interval + type: STRING +- mode: NULLABLE + name: plan_interval_count + type: INTEGER +- mode: NULLABLE + name: plan_interval_timezone + type: STRING +- mode: NULLABLE + name: product_id + type: STRING +- mode: NULLABLE + name: product_name + type: STRING +- mode: NULLABLE + name: pricing_plan + type: STRING +- mode: NULLABLE + name: billing_grace_period + type: INTERVAL +- mode: REPEATED + name: promotion_codes + type: STRING +- mode: NULLABLE + name: promotion_discounts_amount + type: INTEGER +- mode: NULLABLE + name: months_retained + type: INTEGER +- mode: NULLABLE + name: original_subscription_months_retained + type: INTEGER +- mode: NULLABLE + name: current_months_since_subscription_start + type: INTEGER +- mode: NULLABLE + name: current_months_since_original_subscription_start + type: INTEGER From 1e4c057535d07bcd277c53f27b0cfa44a715f694 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Thu, 29 Dec 2022 22:22:37 -0500 Subject: [PATCH 02/12] added country_name columns to all tables --- .../relay_derived/active_subscriptions_live/view.sql | 2 ++ .../relay_derived/active_subscriptions_v1/schema.yaml | 3 +++ .../relay_derived/subscription_events_live/view.sql | 1 + .../relay_derived/subscription_events_v1/schema.yaml | 3 +++ 4 files changed, 9 insertions(+) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql index cef62377311..d31a559e3e4 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql @@ -13,6 +13,7 @@ SELECT subscriptions.plan_id, subscriptions.status, subscriptions.country, + subscriptions.country_name, subscriptions.provider, subscriptions.plan_amount, subscriptions.plan_currency, @@ -35,6 +36,7 @@ GROUP BY plan_id, status, country, + country_name, provider, plan_amount, plan_currency, diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml index 3eee9ceb2e9..b758bc01be4 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml @@ -11,6 +11,9 @@ fields: - name: country type: STRING mode: NULLABLE +- name: country_name + type: STRING + mode: NULLABLE - name: provider type: STRING mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql index 355f1b6b7c3..cf2aa35c9ee 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql @@ -118,6 +118,7 @@ SELECT subscriptions.plan_id, subscriptions.status, subscriptions.country, + subscriptions.country_name, subscriptions.provider, subscriptions.plan_amount, subscriptions.plan_currency, diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml index a24ea50e8c2..4f34fcd81fe 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml @@ -17,6 +17,9 @@ fields: - name: country type: STRING mode: NULLABLE +- name: country_name + type: STRING + mode: NULLABLE - name: provider type: STRING mode: NULLABLE From 2390dcf3bd9fc6833c985e3b85406ca281e1d10d Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 30 Dec 2022 08:44:10 -0500 Subject: [PATCH 03/12] reformat query files for circle ci error fixing --- .../relay_derived/active_subscription_ids_live/view.sql | 7 +------ .../relay_derived/active_subscriptions_live/view.sql | 6 +++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql index 08ccd43aba0..96ca8569e9f 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql @@ -15,9 +15,4 @@ CROSS JOIN ) AS active_date WHERE subscription_start_date IS NOT NULL - AND DATE(subscription_start_date) < ( - SELECT - DATE(MAX(end_date)) - FROM - mozdata.relay.subscriptions - ) + AND DATE(subscription_start_date) < (SELECT DATE(MAX(end_date)) FROM mozdata.relay.subscriptions) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql index d31a559e3e4..35ce960b90f 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql @@ -1,13 +1,13 @@ CREATE OR REPLACE VIEW `moz-fx-data-shared-prod.relay_derived.active_subscriptions_live` AS -WITH - subscriptions AS ( +WITH subscriptions AS ( SELECT *, TO_JSON_STRING(promotion_codes) AS json_promotion_codes FROM - mozdata.relay.subscriptions) + mozdata.relay.subscriptions +) SELECT active_subscription_ids.active_date, subscriptions.plan_id, From 79fc3defc490cad053c131665cc67a383864a1bf Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 30 Dec 2022 21:13:56 -0500 Subject: [PATCH 04/12] Schema edited and dry run list updated --- bigquery_etl/dryrun.py | 1 + .../active_subscription_ids_v1/schema.yaml | 7 + .../subscriptions_v1/schema.yaml | 148 +++++++++--------- 3 files changed, 82 insertions(+), 74 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/schema.yaml diff --git a/bigquery_etl/dryrun.py b/bigquery_etl/dryrun.py index e978ac69322..ed699f754eb 100644 --- a/bigquery_etl/dryrun.py +++ b/bigquery_etl/dryrun.py @@ -129,6 +129,7 @@ "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/init.sql", "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql", "sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql", + "sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql", *glob.glob("sql/moz-fx-data-shared-prod/search_terms*/**/*.sql", recursive=True), "sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql", "sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql", diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/schema.yaml new file mode 100644 index 00000000000..72db0f81403 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: active_date + type: DATE + mode: NULLABLE +- name: subscription_id + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml index c3be1eafb58..219a65afeac 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml @@ -1,112 +1,112 @@ fields: -- mode: NULLABLE - name: customer_id +- name: customer_id type: STRING -- mode: NULLABLE - name: subscription_id + mode: NULLABLE +- name: subscription_id type: STRING -- mode: NULLABLE - name: original_subscription_id + mode: NULLABLE +- name: original_subscription_id type: STRING -- mode: NULLABLE - name: plan_id + mode: NULLABLE +- name: plan_id type: STRING -- mode: NULLABLE - name: status + mode: NULLABLE +- name: status type: STRING -- mode: NULLABLE - name: event_timestamp + mode: NULLABLE +- name: event_timestamp type: TIMESTAMP -- mode: NULLABLE - name: customer_start_date + mode: NULLABLE +- name: customer_start_date type: TIMESTAMP -- mode: NULLABLE - name: subscription_start_date + mode: NULLABLE +- name: subscription_start_date type: TIMESTAMP -- mode: NULLABLE - name: original_subscription_start_date + mode: NULLABLE +- name: original_subscription_start_date type: TIMESTAMP -- mode: NULLABLE - name: subscription_start_reason + mode: NULLABLE +- name: subscription_start_reason type: STRING -- mode: NULLABLE - name: trial_start + mode: NULLABLE +- name: trial_start type: TIMESTAMP -- mode: NULLABLE - name: trial_end + mode: NULLABLE +- name: trial_end type: TIMESTAMP -- mode: NULLABLE - name: canceled_at + mode: NULLABLE +- name: canceled_at type: TIMESTAMP -- mode: NULLABLE - name: canceled_for_customer_at + mode: NULLABLE +- name: canceled_for_customer_at type: STRING -- mode: NULLABLE - name: cancel_at + mode: NULLABLE +- name: cancel_at type: TIMESTAMP -- mode: NULLABLE - name: cancel_at_period_end + mode: NULLABLE +- name: cancel_at_period_end type: BOOLEAN -- mode: NULLABLE - name: ended_at + mode: NULLABLE +- name: ended_at type: TIMESTAMP -- mode: NULLABLE - name: ended_reason + mode: NULLABLE +- name: ended_reason type: STRING -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: TIMESTAMP -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: country_name + mode: NULLABLE +- name: country_name type: STRING -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: plan_amount + mode: NULLABLE +- name: plan_amount type: INTEGER -- mode: NULLABLE - name: plan_currency + mode: NULLABLE +- name: plan_currency type: STRING -- mode: NULLABLE - name: plan_interval + mode: NULLABLE +- name: plan_interval type: STRING -- mode: NULLABLE - name: plan_interval_count + mode: NULLABLE +- name: plan_interval_count type: INTEGER -- mode: NULLABLE - name: plan_interval_timezone + mode: NULLABLE +- name: plan_interval_timezone type: STRING -- mode: NULLABLE - name: product_id + mode: NULLABLE +- name: product_id type: STRING -- mode: NULLABLE - name: product_name + mode: NULLABLE +- name: product_name type: STRING -- mode: NULLABLE - name: pricing_plan + mode: NULLABLE +- name: pricing_plan type: STRING -- mode: NULLABLE - name: billing_grace_period + mode: NULLABLE +- name: billing_grace_period type: INTERVAL -- mode: REPEATED - name: promotion_codes + mode: REPEATED +- name: promotion_codes type: STRING -- mode: NULLABLE - name: promotion_discounts_amount + mode: NULLABLE +- name: promotion_discounts_amount type: INTEGER -- mode: NULLABLE - name: months_retained + mode: NULLABLE +- name: months_retained type: INTEGER -- mode: NULLABLE - name: original_subscription_months_retained + mode: NULLABLE +- name: original_subscription_months_retained type: INTEGER -- mode: NULLABLE - name: current_months_since_subscription_start + mode: NULLABLE +- name: current_months_since_subscription_start type: INTEGER -- mode: NULLABLE - name: current_months_since_original_subscription_start + mode: NULLABLE +- name: current_months_since_original_subscription_start type: INTEGER + mode: NULLABLE From 45f2e082e0aff2d22b34c020f18f2734ca337257 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 30 Dec 2022 21:31:48 -0500 Subject: [PATCH 05/12] removing status column in multiple places --- .../relay_derived/active_subscriptions_live/view.sql | 2 -- .../relay_derived/active_subscriptions_v1/schema.yaml | 3 --- .../relay_derived/subscription_events_live/view.sql | 2 -- .../relay_derived/subscription_events_v1/schema.yaml | 3 --- 4 files changed, 10 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql index 35ce960b90f..6cab56592c1 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql @@ -11,7 +11,6 @@ WITH subscriptions AS ( SELECT active_subscription_ids.active_date, subscriptions.plan_id, - subscriptions.status, subscriptions.country, subscriptions.country_name, subscriptions.provider, @@ -34,7 +33,6 @@ USING GROUP BY active_date, plan_id, - status, country, country_name, provider, diff --git a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml index b758bc01be4..f0ecc8c1fed 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml @@ -5,9 +5,6 @@ fields: - name: plan_id type: STRING mode: NULLABLE -- name: status - type: STRING - mode: NULLABLE - name: country type: STRING mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql index cf2aa35c9ee..06858fb835a 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_live/view.sql @@ -116,7 +116,6 @@ SELECT END AS granular_event_type, subscriptions.plan_id, - subscriptions.status, subscriptions.country, subscriptions.country_name, subscriptions.provider, @@ -140,7 +139,6 @@ GROUP BY event_type, granular_event_type, plan_id, - status, country, provider, plan_amount, diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml index 4f34fcd81fe..1593340f032 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/schema.yaml @@ -11,9 +11,6 @@ fields: - name: plan_id type: STRING mode: NULLABLE -- name: status - type: STRING - mode: NULLABLE - name: country type: STRING mode: NULLABLE From c6b1de73e019c0335cec2657d05849cb8b0bd2b0 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 30 Dec 2022 21:38:28 -0500 Subject: [PATCH 06/12] matching date_partition_offset setting with the table Co-authored-by: Sean Rose <1994030+sean-rose@users.noreply.github.com> --- .../relay_derived/subscription_events_v1/query.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql index 7188cce025e..6fbbd5858da 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscription_events_v1/query.sql @@ -3,4 +3,4 @@ SELECT FROM `moz-fx-data-shared-prod`.relay_derived.subscription_events_live WHERE - IF(CAST(@date AS DATE) IS NULL, event_date < CURRENT_DATE - 7, event_date = @date) + IF(CAST(@date AS DATE) IS NULL, event_date < CURRENT_DATE - 8, event_date = @date) From de35a8d020a73a8167b07193659f52ac84a36d99 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 30 Dec 2022 21:40:33 -0500 Subject: [PATCH 07/12] keep the consistency of friendly_name Co-authored-by: Sean Rose <1994030+sean-rose@users.noreply.github.com> --- .../relay_derived/subscriptions_v1/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml index 1e2f60fd8b3..49dcc7d567c 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/metadata.yaml @@ -1,4 +1,4 @@ -friendly_name: Relay Subscriptions +friendly_name: Firefox Relay Subscriptions description: |- Firefox Relay subscriptions owners: From 23591c79cf043f336abb20f1fba9949063b0f561 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 30 Dec 2022 21:58:01 -0500 Subject: [PATCH 08/12] removing unnecessary comment --- .../relay_derived/subscriptions_v1/query.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql index 56d7d0f019c..b19b8cd06f4 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql @@ -1,11 +1,11 @@ -WITH standardized_country AS ( -- CTE for all_subscriptions +WITH standardized_country AS ( SELECT raw_country AS country, standardized_country AS country_name, FROM mozdata.static.third_party_standardized_country_names ), -stripe_subscriptions_history AS (-- CTE for all_subscriptions +stripe_subscriptions_history AS ( SELECT *, CONCAT( @@ -25,7 +25,7 @@ stripe_subscriptions_history AS (-- CTE for all_subscriptions (valid_to IS NULL OR plan_ended_at IS NOT NULL) AND status NOT IN ("incomplete", "incomplete_expired") ), -stripe_subscriptions AS (-- CTE for all_subscriptions +stripe_subscriptions AS ( SELECT -- user_id, customer_id, @@ -109,7 +109,7 @@ stripe_subscriptions AS (-- CTE for all_subscriptions WHERE (product_name = "Relay Premium" OR product_name = "Mozilla VPN & Firefox Relay") ), -relay_subscriptions_with_end_date AS (-- CTE for all_subscriptions +relay_subscriptions_with_end_date AS ( SELECT *, IF( From c44c7e9a0f8b5891c9fe95f10e2033d4f0a33d40 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 30 Dec 2022 22:30:26 -0500 Subject: [PATCH 09/12] subscriptions query updated --- .../relay_derived/subscriptions_v1/query.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql index b19b8cd06f4..ed46ddbeb11 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql @@ -25,7 +25,7 @@ stripe_subscriptions_history AS ( (valid_to IS NULL OR plan_ended_at IS NOT NULL) AND status NOT IN ("incomplete", "incomplete_expired") ), -stripe_subscriptions AS ( +relay_subscriptions AS ( SELECT -- user_id, customer_id, @@ -59,7 +59,7 @@ stripe_subscriptions AS ( cancel_at_period_end, COALESCE(plan_ended_at, IF(ended_at < TIMESTAMP(CURRENT_DATE), ended_at, NULL)) AS ended_at, IF(plan_ended_at IS NOT NULL, "Plan Change", NULL) AS ended_reason, - -- fxa_uid, + fxa_uid, country, country_name, -- user_registration_date, @@ -107,7 +107,10 @@ stripe_subscriptions AS ( -- USING -- (fxa_uid) WHERE - (product_name = "Relay Premium" OR product_name = "Mozilla VPN & Firefox Relay") + ( + "premium-relay" IN UNNEST(stripe_subscriptions_history.product_capabilities) + OR "premium-relay" IN UNNEST(stripe_subscriptions_history.plan_capabilities) + ) ), relay_subscriptions_with_end_date AS ( SELECT From e874d33bc208eebc79c663906c4c0f01c6c9f360 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 6 Jan 2023 10:46:27 -0500 Subject: [PATCH 10/12] renaming a CTE Co-authored-by: Sean Rose <1994030+sean-rose@users.noreply.github.com> --- .../relay_derived/subscriptions_v1/query.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql index ed46ddbeb11..cd989117bb3 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql @@ -122,7 +122,7 @@ relay_subscriptions_with_end_date AS ( ) AS customer_start_date, COALESCE(ended_at, TIMESTAMP(CURRENT_DATE)) AS end_date, FROM - stripe_subscriptions + relay_subscriptions ) SELECT * REPLACE ( From 8c38cb0602c9674821bbabdc3c71345512fed322 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Fri, 6 Jan 2023 14:55:25 -0500 Subject: [PATCH 11/12] subscriptions_v1 schema editted --- .../relay_derived/subscriptions_v1/schema.yaml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml index 219a65afeac..cfc6495c939 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/schema.yaml @@ -17,9 +17,6 @@ fields: - name: event_timestamp type: TIMESTAMP mode: NULLABLE -- name: customer_start_date - type: TIMESTAMP - mode: NULLABLE - name: subscription_start_date type: TIMESTAMP mode: NULLABLE @@ -53,8 +50,8 @@ fields: - name: ended_reason type: STRING mode: NULLABLE -- name: end_date - type: TIMESTAMP +- name: fxa_uid + type: STRING mode: NULLABLE - name: country type: STRING @@ -91,13 +88,19 @@ fields: mode: NULLABLE - name: billing_grace_period type: INTERVAL - mode: REPEATED + mode: NULLABLE - name: promotion_codes type: STRING - mode: NULLABLE + mode: REPEATED - name: promotion_discounts_amount type: INTEGER mode: NULLABLE +- name: customer_start_date + type: TIMESTAMP + mode: NULLABLE +- name: end_date + type: TIMESTAMP + mode: NULLABLE - name: months_retained type: INTEGER mode: NULLABLE From 2804833d0619f2fb5aa957f2c5c1d8243dc9aba6 Mon Sep 17 00:00:00 2001 From: Yeonjoo Yoo Date: Mon, 9 Jan 2023 16:01:49 -0500 Subject: [PATCH 12/12] add "relay-phones" to be included for filtering --- .../relay_derived/subscriptions_v1/query.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql index cd989117bb3..715ba8601b4 100644 --- a/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql @@ -109,7 +109,9 @@ relay_subscriptions AS ( WHERE ( "premium-relay" IN UNNEST(stripe_subscriptions_history.product_capabilities) + OR "relay-phones" IN UNNEST(stripe_subscriptions_history.product_capabilities) OR "premium-relay" IN UNNEST(stripe_subscriptions_history.plan_capabilities) + OR "relay-phones" IN UNNEST(stripe_subscriptions_history.plan_capabilities) ) ), relay_subscriptions_with_end_date AS (