(DS-2284, DS-2520)Creating Relay derived tables and views for Relay SaaSboard

bigquery_etl/dryrun.py

@@ -130,6 +130,7 @@
     "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/init.sql",
     "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql",
     "sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql",
+    "sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql",
     *glob.glob("sql/moz-fx-data-shared-prod/search_terms*/**/*.sql", recursive=True),
     "sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql",
     "sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql",

dags/bqetl_subplat.py

@@ -568,6 +568,54 @@
         ],
     )
 
+    relay_derived__active_subscription_ids__v1 = bigquery_etl_query(
+        task_id="relay_derived__active_subscription_ids__v1",
+        destination_table='active_subscription_ids_v1${{ macros.ds_format(macros.ds_add(ds, -7), "%Y-%m-%d", "%Y%m%d") }}',
+        dataset_id="relay_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="[email protected]",
+        email=["[email protected]", "[email protected]"],
+        date_partition_parameter=None,
+        depends_on_past=True,
+        parameters=["date:DATE:{{macros.ds_add(ds, -7)}}"],
+    )
+
+    relay_derived__active_subscriptions__v1 = bigquery_etl_query(
+        task_id="relay_derived__active_subscriptions__v1",
+        destination_table='active_subscriptions_v1${{ macros.ds_format(macros.ds_add(ds, -7), "%Y-%m-%d", "%Y%m%d") }}',
+        dataset_id="relay_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="[email protected]",
+        email=["[email protected]", "[email protected]"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        parameters=["date:DATE:{{macros.ds_add(ds, -7)}}"],
+    )
+
+    relay_derived__subscription_events__v1 = bigquery_etl_query(
+        task_id="relay_derived__subscription_events__v1",
+        destination_table='subscription_events_v1${{ macros.ds_format(macros.ds_add(ds, -8), "%Y-%m-%d", "%Y%m%d") }}',
+        dataset_id="relay_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="[email protected]",
+        email=["[email protected]", "[email protected]"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        parameters=["date:DATE:{{macros.ds_add(ds, -8)}}"],
+    )
+
+    relay_derived__subscriptions__v1 = bigquery_etl_query(
+        task_id="relay_derived__subscriptions__v1",
+        destination_table="subscriptions_v1",
+        dataset_id="relay_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="[email protected]",
+        email=["[email protected]", "[email protected]"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
     stripe_external__itemized_payout_reconciliation__v5 = gke_command(
         task_id="stripe_external__itemized_payout_reconciliation__v5",
         command=[
@@ -766,3 +814,25 @@
     )
 
     mozilla_vpn_derived__users__v1.set_upstream(mozilla_vpn_external__users__v1)
+
+    relay_derived__active_subscription_ids__v1.set_upstream(
+        relay_derived__subscriptions__v1
+    )
+
+    relay_derived__active_subscriptions__v1.set_upstream(
+        relay_derived__active_subscription_ids__v1
+    )
+
+    relay_derived__active_subscriptions__v1.set_upstream(
+        relay_derived__subscriptions__v1
+    )
+
+    relay_derived__subscription_events__v1.set_upstream(
+        relay_derived__active_subscription_ids__v1
+    )
+
+    relay_derived__subscription_events__v1.set_upstream(
+        relay_derived__subscriptions__v1
+    )
+
+    relay_derived__subscriptions__v1.set_upstream(fivetran_stripe_sync_wait)

sql/moz-fx-data-shared-prod/relay/active_subscription_ids/view.sql

@@ -0,0 +1,23 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.relay.active_subscription_ids`
+AS
+WITH max_active_date AS (
+  SELECT AS VALUE
+    MAX(active_date)
+  FROM
+    `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_v1
+)
+SELECT
+  active_subscription_ids_live.*
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_live
+CROSS JOIN
+  max_active_date
+WHERE
+  -- static partition filter not needed because live view doesn't use date partitioned tables
+  active_date > max_active_date
+UNION ALL
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_v1

sql/moz-fx-data-shared-prod/relay/active_subscriptions/view.sql

@@ -0,0 +1,22 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.relay.active_subscriptions`
+AS
+WITH max_agg_date AS (
+  SELECT AS VALUE
+    MAX(active_date)
+  FROM
+    `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_v1
+)
+SELECT
+  active_subscriptions_live.*
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_live
+CROSS JOIN
+  max_agg_date
+WHERE
+  active_date > max_agg_date
+UNION ALL
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_v1

sql/moz-fx-data-shared-prod/relay/dataset_metadata.yaml

@@ -0,0 +1,10 @@
+friendly_name: Firefox Relay
+description: |-
+  Data related to the Firefox Relay service
+dataset_base_acl: view
+user_facing: true
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential

sql/moz-fx-data-shared-prod/relay/subscription_events/view.sql

@@ -0,0 +1,22 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.relay.subscription_events`
+AS
+WITH max_agg_date AS (
+  SELECT AS VALUE
+    MAX(event_date)
+  FROM
+    `moz-fx-data-shared-prod`.relay_derived.subscription_events_v1
+)
+SELECT
+  subscription_events_live.*
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.subscription_events_live
+CROSS JOIN
+  max_agg_date
+WHERE
+  event_date > max_agg_date
+UNION ALL
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.subscription_events_v1

sql/moz-fx-data-shared-prod/relay/subscriptions/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.relay.subscriptions`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.subscriptions_v1

sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_live/view.sql

@@ -0,0 +1,18 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.relay_derived.active_subscription_ids_live`
+AS
+SELECT
+  active_date,
+  subscription_id,
+FROM
+  mozdata.relay.subscriptions
+CROSS JOIN
+  UNNEST(
+    GENERATE_DATE_ARRAY(
+      DATE(subscription_start_date),
+      GREATEST(DATE(subscription_start_date), DATE(end_date) - 1)
+    )
+  ) AS active_date
+WHERE
+  subscription_start_date IS NOT NULL
+  AND DATE(subscription_start_date) < (SELECT DATE(MAX(end_date)) FROM mozdata.relay.subscriptions)

sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/metadata.yaml

@@ -0,0 +1,16 @@
+friendly_name: Active Firefox Relay Subscription IDs
+description: >
+  IDs of active Firefox Relay subscriptions by date.
+owners:
+  - [email protected]
+labels:
+  application: relay
+  schedule: daily
+scheduling:
+  dag_name: bqetl_subplat
+  # While this ETL doesn't depend on its own previous runs, other ETLs are built
+  # on the assumption that this is built sequentially day-by-day with no gaps.
+  depends_on_past: true
+  # delay aggregates by 7 days, to ensure data is complete
+  date_partition_offset: -7
+  date_partition_parameter: date

sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/query.sql

@@ -0,0 +1,6 @@
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.active_subscription_ids_live
+WHERE
+  IF(CAST(@date AS DATE) IS NULL, active_date < CURRENT_DATE - 7, active_date = @date)

sql/moz-fx-data-shared-prod/relay_derived/active_subscription_ids_v1/schema.yaml

@@ -0,0 +1,7 @@
+fields:
+- name: active_date
+  type: DATE
+  mode: NULLABLE
+- name: subscription_id
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_live/view.sql

@@ -0,0 +1,47 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.relay_derived.active_subscriptions_live`
+AS
+WITH subscriptions AS (
+  SELECT
+    *,
+    TO_JSON_STRING(promotion_codes) AS json_promotion_codes
+  FROM
+    mozdata.relay.subscriptions
+)
+SELECT
+  active_subscription_ids.active_date,
+  subscriptions.plan_id,
+  subscriptions.country,
+  subscriptions.country_name,
+  subscriptions.provider,
+  subscriptions.plan_amount,
+  subscriptions.plan_currency,
+  subscriptions.plan_interval,
+  subscriptions.plan_interval_count,
+  subscriptions.product_id,
+  subscriptions.product_name,
+  subscriptions.pricing_plan,
+  JSON_VALUE_ARRAY(subscriptions.json_promotion_codes) AS promotion_codes,
+  subscriptions.promotion_discounts_amount,
+  COUNT(*) AS `count`,
+FROM
+  subscriptions
+JOIN
+  mozdata.relay.active_subscription_ids
+USING
+  (subscription_id)
+GROUP BY
+  active_date,
+  plan_id,
+  country,
+  country_name,
+  provider,
+  plan_amount,
+  plan_currency,
+  plan_interval,
+  plan_interval_count,
+  product_id,
+  product_name,
+  pricing_plan,
+  json_promotion_codes,
+  promotion_discounts_amount

sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/metadata.yaml

@@ -0,0 +1,13 @@
+friendly_name: Active Firefox Relay Subscriptions
+description: >
+  Aggregated count of active Firefox Relay subscriptions.
+owners:
+  - [email protected]
+labels:
+  application: relay
+  schedule: daily
+scheduling:
+  dag_name: bqetl_subplat
+  # delay aggregates by 7 days, to ensure data is complete
+  date_partition_offset: -7
+  date_partition_parameter: date

sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/query.sql

@@ -0,0 +1,6 @@
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.relay_derived.active_subscriptions_live
+WHERE
+  IF(CAST(@date AS DATE) IS NULL, active_date < CURRENT_DATE - 7, active_date = @date)

sql/moz-fx-data-shared-prod/relay_derived/active_subscriptions_v1/schema.yaml

@@ -0,0 +1,46 @@
+fields:
+- name: active_date
+  type: DATE
+  mode: NULLABLE
+- name: plan_id
+  type: STRING
+  mode: NULLABLE
+- name: country
+  type: STRING
+  mode: NULLABLE
+- name: country_name
+  type: STRING
+  mode: NULLABLE
+- name: provider
+  type: STRING
+  mode: NULLABLE
+- name: plan_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: plan_currency
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval_count
+  type: INTEGER
+  mode: NULLABLE
+- name: product_id
+  type: STRING
+  mode: NULLABLE
+- name: product_name
+  type: STRING
+  mode: NULLABLE
+- name: pricing_plan
+  type: STRING
+  mode: NULLABLE
+- name: promotion_codes
+  type: STRING
+  mode: REPEATED
+- name: promotion_discounts_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: count
+  type: INTEGER
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/relay_derived/dataset_metadata.yaml

@@ -0,0 +1,10 @@
+friendly_name: Firefox Relay Derived
+description: |-
+  Derived data related to the Firefox Relay service
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential