From 3e3a162eb5684ad825d7f02b3b119496c0472cc4 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 12 Nov 2024 16:22:40 -0800 Subject: [PATCH 1/4] add personal loans --- .../models/L1_staging/loans/_loans_noel.yml | 8 +++ .../L1_staging/loans/stg_noel_noel_source.sql | 71 +++++++++++++++++++ .../L1_staging/loans/stg_noel_noel_source.yml | 61 ++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 transform/models/L1_staging/loans/_loans_noel.yml create mode 100644 transform/models/L1_staging/loans/stg_noel_noel_source.sql create mode 100644 transform/models/L1_staging/loans/stg_noel_noel_source.yml diff --git a/transform/models/L1_staging/loans/_loans_noel.yml b/transform/models/L1_staging/loans/_loans_noel.yml new file mode 100644 index 0000000..a7e4726 --- /dev/null +++ b/transform/models/L1_staging/loans/_loans_noel.yml @@ -0,0 +1,8 @@ +version: 2 + +sources: + - name: NOEL + database: RAW + tables: + - name: NOEL_SOURCE + description: '' diff --git a/transform/models/L1_staging/loans/stg_noel_noel_source.sql b/transform/models/L1_staging/loans/stg_noel_noel_source.sql new file mode 100644 index 0000000..6cf5b66 --- /dev/null +++ b/transform/models/L1_staging/loans/stg_noel_noel_source.sql @@ -0,0 +1,71 @@ +with raw_source as ( + + select * + from {{ source('NOEL', 'NOEL_SOURCE') }} + +), + +final as ( + + select + "_AIRBYTE_RAW_ID"::varchar as airbyte_raw_id, + "_AIRBYTE_EXTRACTED_AT"::timestamp_tz as airbyte_extracted_at, + "_AIRBYTE_META"::variant as airbyte_meta, + "TOTAL_ACC"::float as total_acc, + "ANNUAL_INC"::float as annual_inc, + "EMP_LENGTH"::varchar as emp_length, + "DESC"::varchar as desc, + "TOTAL_PYMNT"::float as total_pymnt, + "LAST_PYMNT_D"::varchar as last_pymnt_d, + "ADDR_STATE"::varchar as addr_state, + "NEXT_PYMNT_D"::varchar as next_pymnt_d, + "EMP_TITLE"::varchar as emp_title, + "COLLECTION_RECOVERY_FEE"::float as collection_recovery_fee, + "MTHS_SINCE_LAST_MAJOR_DEROG"::float as mths_since_last_major_derog, + "INQ_LAST_6MTHS"::float as inq_last_6mths, + "SUB_GRADE"::varchar as sub_grade, + "FUNDED_AMNT_INV"::float as funded_amnt_inv, + "DELINQ_2YRS"::float as delinq_2yrs, + "LOAN_ID"::varchar as loan_id, + "FUNDED_AMNT"::float as funded_amnt, + "VERIFICATION_STATUS"::varchar as verification_status, + "DTI"::float as dti, + "TOTAL_REC_PRNCP"::float as total_rec_prncp, + "GRADE"::varchar as grade, + "HOME_OWNERSHIP"::varchar as home_ownership, + "ISSUE_D"::varchar as issue_d, + "MTHS_SINCE_LAST_DELINQ"::float as mths_since_last_delinq, + "OUT_PRNCP"::float as out_prncp, + "PUB_REC"::float as pub_rec, + "INT_RATE"::float as int_rate, + "ZIP_CODE"::varchar as zip_code, + "OPEN_ACC"::float as open_acc, + "TERM"::varchar as term, + "PYMNT_PLAN"::varchar as pymnt_plan, + "URL"::varchar as url, + "REVOL_BAL"::float as revol_bal, + "RECOVERIES"::float as recoveries, + "LAST_PYMNT_AMNT"::float as last_pymnt_amnt, + "LOAN_AMNT"::float as loan_amnt, + "PURPOSE"::varchar as purpose, + "INITIAL_LIST_STATUS"::varchar as initial_list_status, + "TOTAL_REC_INT"::float as total_rec_int, + "TOTAL_PYMNT_INV"::float as total_pymnt_inv, + "MTHS_SINCE_LAST_RECORD"::float as mths_since_last_record, + "LAST_CREDIT_PULL_D"::varchar as last_credit_pull_d, + "TOTAL_REC_LATE_FEE"::float as total_rec_late_fee, + "MEMBER_ID"::float as member_id, + "POLICY_CODE"::float as policy_code, + "TITLE"::varchar as title, + "LOAN_STATUS"::varchar as loan_status, + "INSTALLMENT"::float as installment, + "EARLIEST_CR_LINE"::varchar as earliest_cr_line, + "REVOL_UTIL"::varchar as revol_util, + "OUT_PRNCP_INV"::float as out_prncp_inv, + "COLLECTIONS_12_MTHS_EX_MED"::float as collections_12_mths_ex_med + + from raw_source + +) + +select * from final diff --git a/transform/models/L1_staging/loans/stg_noel_noel_source.yml b/transform/models/L1_staging/loans/stg_noel_noel_source.yml new file mode 100644 index 0000000..88c1c84 --- /dev/null +++ b/transform/models/L1_staging/loans/stg_noel_noel_source.yml @@ -0,0 +1,61 @@ +version: 2 + +models: + - name: NOEL_SOURCE + description: '' + columns: + - name: airbyte_raw_id + - name: airbyte_extracted_at + - name: airbyte_meta + - name: total_acc + - name: annual_inc + - name: emp_length + - name: desc + - name: total_pymnt + - name: last_pymnt_d + - name: addr_state + - name: next_pymnt_d + - name: emp_title + - name: collection_recovery_fee + - name: mths_since_last_major_derog + - name: inq_last_6mths + - name: sub_grade + - name: funded_amnt_inv + - name: delinq_2yrs + - name: loan_id + - name: funded_amnt + - name: verification_status + - name: dti + - name: total_rec_prncp + - name: grade + - name: home_ownership + - name: issue_d + - name: mths_since_last_delinq + - name: out_prncp + - name: pub_rec + - name: int_rate + - name: zip_code + - name: open_acc + - name: term + - name: pymnt_plan + - name: url + - name: revol_bal + - name: recoveries + - name: last_pymnt_amnt + - name: loan_amnt + - name: purpose + - name: initial_list_status + - name: total_rec_int + - name: total_pymnt_inv + - name: mths_since_last_record + - name: last_credit_pull_d + - name: total_rec_late_fee + - name: member_id + - name: policy_code + - name: title + - name: loan_status + - name: installment + - name: earliest_cr_line + - name: revol_util + - name: out_prncp_inv + - name: collections_12_mths_ex_med From 545c6c6886acd167dc1bd7a3facb21a29b9df771 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Wed, 13 Nov 2024 15:29:45 -0800 Subject: [PATCH 2/4] add avg model --- .../L1_staging/loans/stg_noel_noel_source.yml | 2 +- .../models/L2_core/noel_avg_by_grade.sql | 20 +++++++++++++++++++ .../models/L2_core/noel_avg_by_grade.yml | 9 +++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 transform/models/L2_core/noel_avg_by_grade.sql create mode 100644 transform/models/L2_core/noel_avg_by_grade.yml diff --git a/transform/models/L1_staging/loans/stg_noel_noel_source.yml b/transform/models/L1_staging/loans/stg_noel_noel_source.yml index 88c1c84..b2a71df 100644 --- a/transform/models/L1_staging/loans/stg_noel_noel_source.yml +++ b/transform/models/L1_staging/loans/stg_noel_noel_source.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: NOEL_SOURCE + - name: stg_noel_noel_source description: '' columns: - name: airbyte_raw_id diff --git a/transform/models/L2_core/noel_avg_by_grade.sql b/transform/models/L2_core/noel_avg_by_grade.sql new file mode 100644 index 0000000..78c9273 --- /dev/null +++ b/transform/models/L2_core/noel_avg_by_grade.sql @@ -0,0 +1,20 @@ +with raw_source as ( + + select * from {{ ref('stg_noel_noel_source') }} + +), + +final as ( + + select + grade, + avg(loan_amnt) as avg_loan_amount, + count(*) as total_loans + from raw_source + where loan_status = 'Fully Paid' + group by grade + order by grade + +) + +select * from final diff --git a/transform/models/L2_core/noel_avg_by_grade.yml b/transform/models/L2_core/noel_avg_by_grade.yml new file mode 100644 index 0000000..4a6cdb3 --- /dev/null +++ b/transform/models/L2_core/noel_avg_by_grade.yml @@ -0,0 +1,9 @@ +version: 2 + +models: + - name: noel_avg_by_grade + description: 'my model' + columns: + - name: grade + - name: avg_loan_amount + - name: total_loans From 855d07f3cdb0b91563d8aba412ebd612dab72309 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Wed, 13 Nov 2024 15:38:35 -0800 Subject: [PATCH 3/4] update description --- transform/models/L1_staging/country_data/_country_data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_staging/country_data/_country_data.yml b/transform/models/L1_staging/country_data/_country_data.yml index e0f3613..9fae7f1 100644 --- a/transform/models/L1_staging/country_data/_country_data.yml +++ b/transform/models/L1_staging/country_data/_country_data.yml @@ -7,4 +7,4 @@ sources: - daily_run_airbyte tables: - name: COUNTRY_POPULATIONS - description: 'Raw population information from Github Datasets repository' + description: 'country pupolations' From bcdbde60acb25c78056ffb1efd3d0ea0a979066b Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Thu, 14 Nov 2024 11:10:12 -0800 Subject: [PATCH 4/4] add missing desc --- transform/models/L1_staging/loans/_loans_noel.yml | 2 +- transform/models/L1_staging/loans/stg_noel_noel_source.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/models/L1_staging/loans/_loans_noel.yml b/transform/models/L1_staging/loans/_loans_noel.yml index a7e4726..2dcf586 100644 --- a/transform/models/L1_staging/loans/_loans_noel.yml +++ b/transform/models/L1_staging/loans/_loans_noel.yml @@ -5,4 +5,4 @@ sources: database: RAW tables: - name: NOEL_SOURCE - description: '' + description: 'country data' diff --git a/transform/models/L1_staging/loans/stg_noel_noel_source.yml b/transform/models/L1_staging/loans/stg_noel_noel_source.yml index b2a71df..2892c6a 100644 --- a/transform/models/L1_staging/loans/stg_noel_noel_source.yml +++ b/transform/models/L1_staging/loans/stg_noel_noel_source.yml @@ -2,7 +2,7 @@ version: 2 models: - name: stg_noel_noel_source - description: '' + description: 'source description' columns: - name: airbyte_raw_id - name: airbyte_extracted_at