migrate DM NPS survey (singer-io#85)

* migrate DM NPS survey

* comment question exclusion

data/data_typeform_organization_mapping.csv

@@ -0,0 +1,51 @@
+response_id,organization_id
+f4ee66dccede10144292e1be2b05fc46,527
+1721eded9214cc0e559e62adcb6864fc,497
+301a21540c3f2f6a3fe07af7c7faa7cb,459
+c00ded171ade6dc4fdbd77165c1a8809,396
+902c1e1a5dbf3d391ed473c7f71f3263,388
+5ba2e14bd167fcc7c59fc55f79ecd761,317
+d83d7b5dc1c47559f7039646092cb051,318
+dbf8c977210621d90ec3c29ddd8bee3c,318
+ba02ed4dbcf845962821569d8213808c,294
+22ecc6735ae5eb24ff9261064ddf251c,292
+f155f9867251a260760da40be557af35,596
+589a9c2158bff2792df028b5b8876192,591
+b40056d2e7c640b41de6753488ed8016,581
+38073779e4e76934a63cf509f11db9d9,561
+7aaaffbd066f786e6c117df97aa4e362,567
+1773ebedf4a4f232090463357db141f1,556
+73858506fa256b88f88af183ed823131,544
+c86c6a1e1afe8ec75de6c6cba66932c3,533
+b3ed48d71893090406abe3a2a405f843,523
+d75059512436a8c268ddc90cb9be76ca,515
+318049bfbfa3064ce0af611889cd98b5,507
+ec0552f60bf8e97237ac03b1e55b2648,491
+f6251cab157543ca6a5be9cc3f59fa1a,506
+458db242405310f6a1820e288a50110f,499
+c92d3f1f3c79298ad77df8cb19808ac0,500
+f88760e3768888979607b3518d8881af,490
+3f540d362c8f561d1bad813530fe08b3,486
+984dab7a1a1a63dcab4ac14e06e668f5,453
+99439d3bb33331c6517795603bc22970,456
+b2d6fbd07372bade5959f9a1a3008b2c,462
+d56d7b2100ef645296595cbcb6a1b275,447
+8a66a3fff1acbaf0439f8dbb51f1fed2,445
+c88b2436b7900d95a7cbf3222eb19045,409
+eb223afa7672890d7797b967fa03ba5b,413
+c6cecb985f6c9ad06fff5b37391895e3,414
+926ae314f4f812d0e25f80d42d5f93b0,411
+231fe1260bc9cfa6423af8d75fe81d20,395
+07a3e9a63f672d59a7a676fadc045899,399
+ae652be0691d9005261f7c54e07de4be,398
+c26997b0dd17f6babfa66b64f9e97a23,327
+2e893866ac6bca2b544579c3ab5c5f64,380
+02c21271332f97ff2664ea5f49c45801,370
+1fbb8134d4a455eb50c3f11fd3f4a6a7,273
+cb4864571b9de1ff2f0b5d809e69744a,320
+000c4c8492d3a1455839c3405825a3f8,320
+3549f0d04ddb2506e31949fa6e44af07,276
+4ef99f28e9da193944d56ee6f59c9b48,283
+d2df6cf8783acaa7bad805407110a53b,289
+16c11fe7bf85a7a20e7dfb0b7163cc8e,295
+28433d95162ee476fec42732f7649b21,276

models/delighted/base/delighted_nps_dm_survey.sql

@@ -5,4 +5,8 @@ select email
 	, tags
 	, comment
 	, created_at as timestamp
+	, updated_at
+	, contact_type
+	, month_since_billing_start_date
+	, delighted_workspace
 from delighted.nps_survey_dm

models/delighted/base/delighted_nps_patient_survey.sql

@@ -6,6 +6,7 @@ select email
 	, tags
 	, comment
 	, created_at as timestamp
+	, updated_at
 	, date_trunc('month', created_at) as date_month
 	, date_trunc('week', created_at) as date_week
 from delighted.nps_survey

models/meta/nps/nps_patient_survey_with_user_id.sql

@@ -13,6 +13,7 @@ select nps_survey.episode_id
 	, nps_survey.tags
 	, nps_survey.comment
 	, nps_survey.timestamp
+	, nps_survey.updated_at
 	, users.user_id
 from nps_survey
 inner join users

models/meta/survey/survey_onboarding.sql

@@ -0,0 +1,28 @@
+with
+	responses as (
+		select * from {{ ref('typeform_responses_detailed') }}
+	)
+
+	, organizations as (
+		select * from {{ ref('organizations') }}
+	)
+
+select responses.*
+	, organizations.organization_name
+	, organizations.account_id
+	, organizations.account_name
+from responses
+left join organizations
+	using (organization_id)
+where
+-- exclude identifying questions
+-- These questions don't include any useful information
+	responses.question_id not in (
+		'ZLkkdqEuAthg'
+		, 'K2E9T8kHiuJh'
+		, '67870899'
+		, '67870887'
+	) and responses.form_title in (
+		'Sondage de satisfaction suite au lancement'
+        , 'Post-Onboarding survey'
+    )

models/meta/table/nps_dm_survey.sql

@@ -0,0 +1,46 @@
+with
+	typeform_nps as (
+		select * from {{ ref('typeform_nps_dm_survey') }}
+	)
+
+	, delighted_nps as (
+		select * from {{ ref('delighted_nps_dm_survey') }}
+	)
+
+	, organizations as (
+		select * from {{ ref('organizations') }}
+	)
+
+	, nps_all as (
+		select email
+			, organization_id
+			, score
+			, category
+			, tags
+			, comment
+			, timestamp
+			, updated_at
+			, contact_type
+			, month_since_billing_start_date
+		from typeform_nps
+		union all
+		select email
+			, organization_id
+			, score
+			, category
+			, tags
+			, comment
+			, timestamp
+			, updated_at
+			, contact_type
+			, month_since_billing_start_date
+		from delighted_nps
+	)
+
+select nps_all.*
+	, date_trunc('month', timestamp) as date_month
+	, coalesce(organizations.organization_name, 'N/A') as organization_name
+	, coalesce(organizations.account_name, 'N/A') as account_name
+from nps_all
+left join organizations
+	using (organization_id)

models/typeform/base/schema.yml

@@ -0,0 +1,45 @@
+
+version: 2
+
+models:
+  - name: typeform_responses
+    columns:
+      - name: response_id
+        tests:
+          - not_null
+
+      - name: question_id
+        tests:
+          - not_null
+          - relationships:
+              to: ref('typeform_responses')
+              field: question_id
+
+    tests:
+      - unique:
+          column_name: "concat(response_id, question_id)"
+
+  - name: typeform_questions
+    columns:
+      - name: question_id
+        tests:
+          - not_null
+          - unique
+
+  - name: typeform_organization_mapping
+    description: |
+      Mapping between responses and organizations to back-fill data
+      when Typeform was not set up to capture organization IDs
+    columns:
+      - name: response_id
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('typeform_responses')
+              field: response_id
+
+      - name: organization_id
+        tests:
+          - not_null
+

models/typeform/base/typeform_organization_mapping.sql

@@ -0,0 +1 @@
+select * from {{ ref('data_typeform_organization_mapping') }}

models/typeform/base/typeform_questions.sql

@@ -0,0 +1,4 @@
+select *
+	, id as question_id
+	, title as question_title
+from typeform.questions

models/typeform/base/typeform_responses.sql

@@ -0,0 +1,31 @@
+with
+	responses as (
+		select *
+			, row_number() over(partition by response_id, question_id) as rank
+		from typeform.responses
+	)
+
+select response_id
+	, form_score
+	, form_submitted_at
+	, form_landed_at
+	, question_id
+	, question_type
+	, answer_type
+	, choices_answer
+	, free_text_answer
+	, number_answer
+	, boolean_answer
+	, hidden
+	, hidden::json->>'user_id' as user_id
+	, case
+		when (hidden::json->>'org_id') similar to '[0-9]+'
+		then (hidden::json->>'org_id')::int
+	end as organization_id
+	, coalesce(choice_answer
+		, free_text_answer
+		, boolean_answer::text
+		, number_answer::text
+	) as answer
+from responses
+where rank = 1

models/typeform/table/schema.yml

@@ -0,0 +1,20 @@
+
+version: 2
+
+models:
+  - name: typeform_responses_detailed
+    columns:
+      - name: response_id
+        tests:
+          - not_null
+
+      - name: question_id
+        tests:
+          - not_null
+          - relationships:
+              to: ref('typeform_responses')
+              field: question_id
+
+    tests:
+      - unique:
+          column_name: "concat(response_id, question_id)"

models/typeform/table/typeform_responses_detailed.sql

@@ -0,0 +1,37 @@
+with
+	responses as (
+		select * from {{ ref('typeform_responses') }}
+	)
+
+	, questions as (
+		select * from {{ ref('typeform_questions') }}
+	)
+
+	, organizations_mapping as (
+		select * from {{ ref('typeform_organization_mapping') }}
+	)
+
+select responses.response_id
+	, responses.form_score
+	, responses.form_submitted_at
+	, responses.form_landed_at
+	, responses.question_id
+	, responses.question_type
+	, responses.answer_type
+	, responses.choices_answer
+	, responses.free_text_answer
+	, responses.number_answer
+	, responses.boolean_answer
+	, responses.hidden
+	, responses.user_id
+	, responses.answer
+	, coalesce(organizations_mapping.organization_id
+		, responses.organization_id
+		) as organization_id
+	, questions.form_title
+	, questions.question_title
+from responses
+inner join questions
+	using (question_id)
+left join organizations_mapping
+	using (response_id)

models/typeform/transform/typeform_nps_dm_survey.sql

@@ -0,0 +1,28 @@
+with
+	responses as (
+		select * from {{ ref('typeform_responses_detailed') }}
+	)
+
+	, questions as (
+		select * from {{ ref('typeform_questions') }}
+	)
+
+select ''::text as email
+	, number_answer as score
+	, case
+		when number_answer >= 9 then 'promoter'
+		when number_answer <= 6 then 'detractor'
+		else 'passive'
+	end as category
+	, ''::text as comment
+	, form_submitted_at as timestamp
+	, form_submitted_at as updated_at
+	, ''::text as contact_type
+	, organization_id
+	, null::text as delighted_workspace
+	, null::text[] as tags
+	, 0 as month_since_billing_start_date
+from questions
+inner join responses
+	using (question_id)
+where questions.question_id in ('Zj5euVUa95Gn', '67871435')