qf-migration-tweaks: prioritize dbs, updated knm_maintenance name, co…

…nsolidate number migrations (2600hz#2951)
serg-i-o · Dec 9, 2016 · 6e85c96 · 6e85c96
1 parent 0991e87
commit 6e85c96
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 26 deletions.
diff --git a/applications/crossbar/src/modules_v2/cb_phone_numbers_v2.erl b/applications/crossbar/src/modules_v2/cb_phone_numbers_v2.erl
@@ -327,7 +327,7 @@ classified_number(Context, Number, Classifier) ->
 -spec post(cb_context:context(), path_token()) -> cb_context:context().
 post(Context, ?FIX) ->
     AccountDb = cb_context:account_db(Context),
-    'ok' = knm_maintenance:fix_account_numbers(AccountDb),
+    'ok' = kazoo_number_manager_maintenance:migrate(AccountDb),
     summary(Context);
 post(Context, ?CHECK) ->
     Numbers = cb_context:req_value(Context, ?COLLECTION_NUMBERS),

diff --git a/core/kazoo_apps/src/kapps_maintenance.erl b/core/kazoo_apps/src/kapps_maintenance.erl
@@ -22,7 +22,6 @@
 -export([find_invalid_acccount_dbs/0]).
 -export([refresh/0, refresh/1
         ,refresh_account_db/1
-        ,refresh_numbers_db/1
         ]).
 -export([blocking_refresh/0
         ,blocking_refresh/1
@@ -100,7 +99,7 @@ rebuild_token_auth(Pause) ->
 -spec migrate_to_4_0() -> no_return.
 migrate_to_4_0() ->
     %% Number migration
-    knm_maintenance:migrate(),
+    kazoo_number_manager_maintenance:migrate(),
     %% Voicemail migration
     kazoo_voicemail_maintenance:migrate(),
     no_return.
@@ -175,7 +174,7 @@ refresh(Databases, Pause) ->
 
 refresh([], _, _) -> 'no_return';
 refresh([Database|Databases], Pause, Total) ->
-    io:format("refreshing database (~p/~p) '~s'~n"
+    io:format("(~p/~p) refreshing database '~s'~n"
              ,[length(Databases) + 1, Total, Database]),
     _ = refresh(Database),
     _ = case Pause < 1 of
@@ -187,10 +186,11 @@ refresh([Database|Databases], Pause, Total) ->
 -spec get_databases() -> ne_binaries().
 get_databases() ->
     {'ok', Databases} = kz_datamgr:db_info(),
-    ?KZ_SYSTEM_DBS
-        ++ [Db || Db <- Databases,
-                  not lists:member(Db, ?KZ_SYSTEM_DBS)
-           ].
+    lists:sort(fun get_database_sort/2, Databases).
+
+-spec get_database_sort(ne_binary(), ne_binary()) -> boolean().
+get_database_sort(Db1, Db2) ->
+    kzs_util:db_priority(Db1) < kzs_util:db_priority(Db2).
 
 refresh(?KZ_CONFIG_DB) ->
     kz_datamgr:db_create(?KZ_CONFIG_DB),
@@ -285,7 +285,7 @@ refresh(Database) when is_binary(Database) ->
     case kz_datamgr:db_classification(Database) of
         'account' -> refresh_account_db(Database);
         'modb' -> kazoo_modb:refresh_views(Database);
-        'numbers' -> refresh_numbers_db(Database);
+        'numbers' -> kazoo_number_manager_maintenance:refresh_numbers_db(Database);
         'system' ->
             kz_datamgr:db_create(Database),
             'ok';
@@ -323,21 +323,6 @@ maybe_remove_invalid_notify_doc(<<"notification">>, _, JObj) ->
     'ok';
 maybe_remove_invalid_notify_doc(_Type, _Id, _Doc) -> 'ok'.
 
-%%--------------------------------------------------------------------
-%% @public
-%% @doc
-%%
-%% @end
-%%--------------------------------------------------------------------
--spec refresh_numbers_db(ne_binary()) -> 'ok'.
-refresh_numbers_db(<<?KNM_DB_PREFIX, Suffix/binary>>) ->
-    NumberDb = <<?KNM_DB_PREFIX_ENCODED, Suffix/binary>>,
-    {'ok',_} = kz_datamgr:revise_doc_from_file(NumberDb
-                                              ,'kazoo_number_manager'
-                                              ,<<"views/numbers.json">>
-                                              ),
-    'ok'.
-
 %%--------------------------------------------------------------------
 %% @private
 %% @doc

diff --git a/core/kazoo_data/src/kzs_util.erl b/core/kazoo_data/src/kzs_util.erl
@@ -8,6 +8,7 @@
 -module(kzs_util).
 
 -export([db_classification/1
+        ,db_priority/1
         ,map_keys_to_atoms/1
         ]).
 
@@ -71,6 +72,57 @@ db_classification(_Database) ->
     lager:debug("unknown database classification : ~p", [erlang:process_info(self(),current_stacktrace)]),
     'undefined'.
 
+%%------------------------------------------------------------------------------
+%% @public
+%% @doc
+%% @end
+%%------------------------------------------------------------------------------
+-spec db_priority(text()) -> non_neg_integer().
+db_priority(Db) when not is_binary(Db) ->
+    db_priority(kz_util:to_binary(Db));
+db_priority(?KZ_CONFIG_DB) -> 0;
+db_priority(?KZ_DATA_DB) -> 1;
+db_priority(?KZ_OFFNET_DB) -> 2;
+db_priority(?KZ_ACCOUNTS_DB) -> 3;
+db_priority(?KZ_SIP_DB) -> 4;
+db_priority(?KZ_AUTH_DB) -> 5;
+db_priority(?KZ_WEBHOOKS_DB) -> 6;
+db_priority(?KZ_RATES_DB) -> 7;
+db_priority(?KZ_ACDC_DB) -> 8;
+db_priority(?KZ_FAXES_DB) -> 9;
+db_priority(?KZ_SCHEMA_DB) -> 10;
+db_priority(?KZ_SERVICES_DB) -> 11;
+db_priority(?KZ_PORT_REQUESTS_DB) -> 12;
+db_priority(?KZ_TASKS_DB) -> 13;
+db_priority(?KZ_DEDICATED_IP_DB) -> 14;
+db_priority(?KZ_ALERTS_DB) -> 15;
+db_priority(?KZ_MEDIA_DB) -> 16;
+db_priority(?KZ_OAUTH_DB) -> 17;
+db_priority(?KZ_TOKEN_DB) -> 18;
+db_priority(<<?KNM_DB_PREFIX, _/binary>>) -> 19;
+db_priority(<<?KNM_DB_PREFIX_ENCODED, _/binary>>) -> 19;
+db_priority(<<?KNM_DB_PREFIX_encoded, _/binary>>) -> 19;
+db_priority(<<"numbers/", _/binary>>) -> 20;
+db_priority(<<"numbers%2F", _/binary>>) -> 20;
+db_priority(<<"numbers%2f", _/binary>>) -> 20;
+db_priority(?MATCH_ACCOUNT_UNENCODED(_AccountId)) -> 21;
+db_priority(?MATCH_ACCOUNT_encoded(_AccountId)) -> 21;
+db_priority(?MATCH_ACCOUNT_ENCODED(_AccountId)) -> 21;
+db_priority(?MATCH_MODB_SUFFIX_UNENCODED(_A,_B,_Rest,_Year,_Month)) -> 22;
+db_priority(?MATCH_MODB_SUFFIX_ENCODED(_A,_B,_Rest,_Year,_Month)) -> 22;
+db_priority(?MATCH_MODB_SUFFIX_encoded(_A,_B,_Rest,_Year,_Month)) -> 22;
+db_priority(?MATCH_MODB_SUFFIX_RAW(_Account,_Year,_Month)) -> 22;
+db_priority(?MATCH_RESOURCE_SELECTORS_UNENCODED(_AccountId)) -> 23;
+db_priority(?MATCH_RESOURCE_SELECTORS_encoded(_AccountId)) -> 23;
+db_priority(?MATCH_RESOURCE_SELECTORS_ENCODED(_AccountId)) -> 23;
+db_priority(?MATCH_RESOURCE_SELECTORS_RAW(_AccountId)) -> 23;
+db_priority(_Database) -> 24.
+
+%%------------------------------------------------------------------------------
+%% @public
+%% @doc
+%% @end
+%%------------------------------------------------------------------------------
 -spec map_keys_to_atoms(map()) -> map().
 map_keys_to_atoms(Map) ->
     maps:fold(fun map_keys_to_atoms_fold/3, #{}, Map).

diff --git a/...oo_number_manager/src/knm_maintenance.erl → .../src/kazoo_number_manager_maintenance.erl b/...oo_number_manager/src/knm_maintenance.erl → .../src/kazoo_number_manager_maintenance.erl
@@ -7,10 +7,13 @@
 %%% @contributors
 %%%   Pierre Fenoll
 %%%-------------------------------------------------------------------
--module(knm_maintenance).
+-module(kazoo_number_manager_maintenance).
 
 -include("knm.hrl").
 
+-export([refresh_numbers_dbs/0
+        ,refresh_numbers_db/1
+        ]).
 -export([fix_account_numbers/1
         ,fix_accounts_numbers/1
         ]).
@@ -36,7 +39,44 @@
         end
        ).
 
-%% API
+%%--------------------------------------------------------------------
+%% @public
+%% @doc
+%%
+%% @end
+%%--------------------------------------------------------------------
+-spec refresh_numbers_dbs() -> 'ok'.
+refresh_numbers_dbs() ->
+    {'ok', Databases} = kz_datamgr:db_info(),
+    NumberDbs = [Db
+                 || Db <- Databases,
+                    kzs_util:db_classification(Db) =:= 'numbers'
+                        orelse kzs_util:db_classification(Db) =:= 'system_numbers'
+               ],
+    refresh_numbers_dbs(NumberDbs, length(NumberDbs)).
+
+-spec refresh_numbers_dbs(ne_binaries(), non_neg_integer()) -> 'ok'.
+refresh_numbers_dbs([], _) -> 'ok';
+refresh_numbers_dbs([NumberDb|NumberDbs], Total) ->
+    ?LOG("(~p/~p) updating number db ~s", [length(NumberDbs) + 1, Total, NumberDb]),
+    _ = refresh_numbers_db(NumberDb),
+    refresh_numbers_dbs(NumberDbs, Total).
+
+-spec refresh_numbers_db(ne_binary()) -> 'ok'.
+refresh_numbers_db(<<?KNM_DB_PREFIX_ENCODED, _/binary>> = NumberDb) ->
+    {'ok',_} = kz_datamgr:revise_doc_from_file(NumberDb
+                                              ,'kazoo_number_manager'
+                                              ,<<"views/numbers.json">>
+                                              ),
+    'ok';
+refresh_numbers_db(<<?KNM_DB_PREFIX, Suffix/binary>>) ->
+    NumberDb = <<?KNM_DB_PREFIX_ENCODED, Suffix/binary>>,
+    refresh_numbers_db(NumberDb);
+refresh_numbers_db(<<"+", Suffix/binary>>) ->
+    refresh_numbers_db(Suffix);
+refresh_numbers_db(Suffix) ->
+    NumberDb = <<?KNM_DB_PREFIX_ENCODED, Suffix/binary>>,
+    refresh_numbers_db(NumberDb).
 
 %% @public
 -spec fix_accounts_numbers([ne_binary()]) -> 'ok'.
@@ -90,6 +130,7 @@ fix_account_numbers(Account = ?NE_BINARY) ->
 
 -spec migrate() -> 'ok'.
 migrate() ->
+    _ = refresh_numbers_dbs(),
     AccountDbs = kapps_util:get_all_accounts(),
     foreach_pause_in_between(?TIME_BETWEEN_ACCOUNTS_MS, fun migrate/1, AccountDbs),
     erase(callflow_DIDs),