From ac6c3f53171de1680febc42b56975bec43c0e77a Mon Sep 17 00:00:00 2001
From: Sanil Manandhar <sanilmanandhar@gmail.com>
Date: Tue, 29 Oct 2024 10:12:36 +0545
Subject: [PATCH 1/2] 1583-transactions-not-migrated-to-iati-publisher *[x]
Updated transaction migrate code *[x] Created command to check migrations of
transactions and migrate if needed
---
.../Commands/CheckMigratedTransactions.php | 328 ++++++++++++++++++
.../Commands/MigrateOrganizationCommand.php | 48 +--
app/Helpers/general.php | 2 +-
.../Forms/MultilevelSubElementForm.php | 2 +-
app/IATI/Traits/CommandInputTrait.php | 58 ++++
app/IATI/Traits/FillDefaultValuesTrait.php | 39 ++-
6 files changed, 420 insertions(+), 57 deletions(-)
create mode 100644 app/Console/Commands/CheckMigratedTransactions.php
create mode 100644 app/IATI/Traits/CommandInputTrait.php
diff --git a/app/Console/Commands/CheckMigratedTransactions.php b/app/Console/Commands/CheckMigratedTransactions.php
new file mode 100644
index 0000000000..e626b8d059
--- /dev/null
+++ b/app/Console/Commands/CheckMigratedTransactions.php
@@ -0,0 +1,328 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\IATI\Models\Activity\Activity;
+use App\IATI\Services\Activity\TransactionService;
+use App\IATI\Services\Organization\OrganizationService;
+use App\IATI\Traits\CommandInputTrait;
+use App\IATI\Traits\MigrateActivityTrait;
+use App\IATI\Traits\MigrateActivityTransactionTrait;
+use App\IATI\Traits\MigrateGeneralTrait;
+use App\IATI\Traits\MigrateOrganizationTrait;
+use App\IATI\Traits\MigrateSettingTrait;
+use Illuminate\Console\Command;
+use Illuminate\Database\DatabaseManager;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Facades\DB;
+
+/**
+ * Class CheckMigratedTransactions.
+ */
+class CheckMigratedTransactions extends Command
+{
+ use MigrateSettingTrait;
+ use MigrateGeneralTrait;
+ use MigrateOrganizationTrait;
+ use MigrateActivityTrait;
+ use MigrateActivityTransactionTrait;
+ use CommandInputTrait;
+
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'command:check-migrated-transactions';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Checks migrated transactions and updates if required.';
+
+ /**
+ * MigrateOrganizationCommand Constructor.
+ *
+ * @return void
+ */
+ public function __construct(
+ protected DB $db,
+ protected DatabaseManager $databaseManager,
+ protected OrganizationService $organizationService,
+ protected TransactionService $transactionService,
+ ) {
+ parent::__construct();
+ }
+
+ /**
+ * Execute the console command.
+ *
+ * @return void
+ */
+ public function handle(): void
+ {
+ try {
+// $aidstreamOrganizationIds = [2310,
+// 2427,
+// 1543,
+// 1070,
+// 1585,
+// 2509,
+// 576,
+// 2505,
+// 869,
+// 259,
+// 466,
+// 1686,
+// 1302,
+// 2202,
+// 2441,
+// 2254,
+// 1408,
+// 2562,
+// 1592,
+// 1117,
+// 2568,
+// 5,
+// 1009,
+// 2107,
+// 1386,
+// 2550,
+// 1891,
+// 2073,
+// 1979,
+// 1720,
+// 2241,
+// 1064,
+// 217,
+// 1426,
+// 2586
+// 1578];
+
+ $aidstreamOrganizationIdString = $this->askValid(
+ 'Please enter the organization ids which you want to migrate separated by comma (Compulsory)',
+ 'aidstreamOrganizationIdString',
+ ['required']
+ );
+
+ $aidstreamOrganizationIds = explode(',', $aidstreamOrganizationIdString);
+
+ foreach ($aidstreamOrganizationIds as $key => $aidstreamOrganizationId) {
+ $aidstreamOrganizationIds[$key] = (int) $aidstreamOrganizationId;
+ }
+
+ foreach ($aidstreamOrganizationIds as $aidstreamOrganizationId) {
+ $aidStreamOrganization = $this->db::connection('aidstream')->table('organizations')->where(
+ 'id',
+ $aidstreamOrganizationId
+ )->first();
+
+ if (!$aidStreamOrganization) {
+ logger()->channel('migration')->error(
+ "Organization with id {$aidstreamOrganizationId} does not exist in aidstream."
+ );
+
+ $this->error("Organization with id {$aidstreamOrganizationId} does not exist in aidstream.");
+ continue;
+ }
+
+ $aidStreamOrganizationSetting = $this->db::connection('aidstream')->table('settings')->where(
+ 'organization_id',
+ $aidstreamOrganizationId
+ )->first();
+
+ $aidstreamPublisherId = $aidStreamOrganization->user_identifier;
+
+ if ($aidStreamOrganizationSetting) {
+ $aidstreamPublisherId = $this->getSettingsPublisherId(
+ $aidStreamOrganizationSetting,
+ $aidStreamOrganization->user_identifier
+ );
+ }
+
+ $iatiOrganization = $this->organizationService->getOrganizationByPublisherId(
+ strtolower($aidstreamPublisherId)
+ );
+
+ if (!$iatiOrganization) {
+ logger()->channel('migration')->error(
+ "Organization with aidstream id {$aidstreamOrganizationId} publisher id {$aidstreamPublisherId} does not exist in IATI Publisher."
+ );
+
+ $this->error(
+ "Organization with aidstream id {$aidstreamOrganizationId} publisher id {$aidstreamPublisherId} does not exist in IATI Publisher."
+ );
+ }
+
+ $this->logInfo("Checking for organization with id: {$aidstreamOrganizationId}");
+ // Checking for activities
+ $aidstreamActivities = $this->db::connection('aidstream')->table('activity_data')->where(
+ 'organization_id',
+ $aidstreamOrganizationId
+ )->get();
+
+ if (count($aidstreamActivities)) {
+ foreach ($aidstreamActivities as $aidstreamActivity) {
+ $aidstreamActivityIdentifier = Arr::get(
+ json_decode(
+ $aidstreamActivity->identifier,
+ true,
+ 512,
+ JSON_THROW_ON_ERROR
+ ),
+ 'activity_identifier'
+ );
+
+ $this->info(
+ "Checking for activity with identifier {$aidstreamActivityIdentifier} for organization {$aidStreamOrganization->name} with id $aidStreamOrganization->id"
+ );
+// $this->logInfo(
+// "Checking for activity with identifier {$aidstreamActivityIdentifier} for organization {$aidStreamOrganization->name} with id $aidStreamOrganization->id"
+// );
+
+ $iatiActivity = Activity::whereRaw(
+ "(iati_identifier->>'activity_identifier')::text ilike ?",
+ $aidstreamActivityIdentifier
+ )->with('transactions')->first();
+
+ if (!$iatiActivity) {
+ logger()->channel('migration')->error(
+ "Activity with identifier {$aidstreamActivityIdentifier} does not exist in IATI Publisher."
+ );
+
+ $this->error(
+ "Activity with identifier {$aidstreamActivityIdentifier} does not exist in IATI Publisher."
+ );
+
+ continue;
+ }
+
+ $aidstreamActivityTransactions = $this->db::connection('aidstream')->table(
+ 'activity_transactions'
+ )->where(
+ 'activity_id',
+ $aidstreamActivity->id
+ )->get();
+
+ $iatiActivityTransactions = $iatiActivity->transactions->pluck('transaction');
+
+ $aidstreamActivityTransactionsCount = count($aidstreamActivityTransactions);
+ $iatiActivityTransactionsCount = count($iatiActivityTransactions);
+ $defaultValues = json_encode([$iatiActivity->default_field_values]);
+
+ if ($aidstreamActivityTransactionsCount !== $iatiActivityTransactionsCount) {
+ $this->logInfo(
+ "AidStream Organization Id: {$aidstreamOrganizationId} \n
+ IATI Organization Id: {$iatiOrganization->id} \n
+ Publisher Id: {$iatiOrganization->publisher_id} \n
+ AidStream Activity Id: {$aidstreamActivity->id} \n
+ IATI Activity Id: {$iatiActivity->id} \n
+ Activity Identifier: {$aidstreamActivityIdentifier} \n
+ AidStream Activity Transactions Count: {$aidstreamActivityTransactionsCount} \n
+ IATI Activity Transactions Count: {$iatiActivityTransactionsCount} \n"
+ );
+
+ $this->info('Updating transaction data.');
+ $existingIatiTransactions = $iatiActivityTransactions->toArray();
+
+ foreach ($aidstreamActivityTransactions as $aidstreamTransaction) {
+ $requiredTransactionData = [
+ 'activity_id' => $iatiActivity->id,
+ 'transaction' => $this->getRequiredTransactionData(
+ $aidstreamTransaction,
+ $defaultValues
+ ),
+ 'migrated_from_aidstream' => true,
+ 'created_at' => $aidstreamTransaction->created_at,
+ 'updated_at' => $aidstreamTransaction->updated_at,
+ ];
+
+ if (!$this->checkArrayPresent($existingIatiTransactions, $requiredTransactionData['transaction'])) {
+ $this->info(
+ "Creating transaction data for AidStream Transaction Id: $aidstreamTransaction->id"
+ );
+ $this->transactionService->create($requiredTransactionData);
+ $this->info(
+ "Created transaction data for AidStream Transaction Id: $aidstreamTransaction->id"
+ );
+ }
+ }
+ }
+
+ $this->info(
+ "Checking completed for activity with identifier {$aidstreamActivityIdentifier} for organization {$aidStreamOrganization->name} with id $aidStreamOrganization->id"
+ );
+ }
+ }
+
+ $this->logInfo("Checking completed for organization with id: {$aidstreamOrganizationId}");
+ }
+ } catch (\Exception $e) {
+ logger()->channel('migration')->error($e->getMessage());
+
+ $this->error($e->getMessage());
+ }
+ }
+
+ /**
+ * @param $aidstreamTransaction
+ * @param $defaultValues
+ *
+ * @return array
+ *
+ * @throws \JsonException
+ */
+ public function getRequiredTransactionData($aidstreamTransaction, $defaultValues): array
+ {
+ $newData = $this->getNewTransactionData($aidstreamTransaction->transaction);
+
+ return $this->populateDefaultFields(
+ $newData,
+ $defaultValues
+ );
+ }
+
+ /**
+ * @param $allArray
+ * @param $search
+ *
+ * @return bool
+ */
+ public function checkArrayPresent($allArray, $search): bool
+ {
+ $secondArray = Arr::dot($search);
+
+ foreach ($allArray as $array) {
+ $firstArray = Arr::dot($array);
+
+ if ($this->checkArrayIdentical($firstArray, $secondArray)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * @param $firstArray
+ * @param $secondArray
+ *
+ * @return bool
+ */
+ public function checkArrayIdentical($firstArray, $secondArray): bool
+ {
+ foreach ($firstArray as $key => $value) {
+ if (!array_key_exists($key, $secondArray)) {
+ return false;
+ }
+
+ if ($value !== $secondArray[$key]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
diff --git a/app/Console/Commands/MigrateOrganizationCommand.php b/app/Console/Commands/MigrateOrganizationCommand.php
index 0547c73706..363bb4ebe3 100644
--- a/app/Console/Commands/MigrateOrganizationCommand.php
+++ b/app/Console/Commands/MigrateOrganizationCommand.php
@@ -22,6 +22,7 @@
use App\IATI\Services\Publisher\PublisherService;
use App\IATI\Services\Setting\SettingService;
use App\IATI\Services\User\UserService;
+use App\IATI\Traits\CommandInputTrait;
use App\IATI\Traits\LogFunctionTrait;
use App\IATI\Traits\MigrateActivityPublishedTrait;
use App\IATI\Traits\MigrateActivityResultsTrait;
@@ -43,7 +44,6 @@
use Illuminate\Database\DatabaseManager;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
-use Illuminate\Support\Facades\Validator;
/**
* Class MigrateUserCommand.
@@ -65,6 +65,7 @@ class MigrateOrganizationCommand extends Command
use TrackMigrationErrorTrait;
use MigrateProUserTrait;
use LogFunctionTrait;
+ use CommandInputTrait;
/**
* The name and signature of the console command.
@@ -366,51 +367,6 @@ public function handle(): void
}
}
- /**
- * Ask input from user and return value.
- *
- * @param $question
- * @param $field
- * @param $rules
- *
- * @return string
- */
- protected function askValid($question, $field, $rules): string
- {
- $value = $this->ask($question);
- $message = $this->validateInput($rules, $field, $value);
-
- if ($message) {
- $this->error($message);
-
- return $this->askValid($question, $field, $rules);
- }
-
- return $value;
- }
-
- /**
- * Validates input given by user.
- *
- * @param $rules
- * @param $fieldName
- * @param $value
- *
- * @return string|null
- */
- protected function validateInput($rules, $fieldName, $value): ?string
- {
- $validator = Validator::make([
- $fieldName => $value,
- ], [
- $fieldName => $rules,
- ]);
-
- return $validator->fails()
- ? $validator->errors()->first($fieldName)
- : null;
- }
-
/**
* Publish files to registry.
*
diff --git a/app/Helpers/general.php b/app/Helpers/general.php
index 4dbe76f54a..31dea2f8c4 100644
--- a/app/Helpers/general.php
+++ b/app/Helpers/general.php
@@ -1270,7 +1270,7 @@ function refreshIndicatorDeprecationStatusMap(array $indicators): array
'description' => doesDescriptionHaveDeprecatedCode(Arr::get($indicators, 'description', [])),
'document_link' => doesDocumentLinkHaveDeprecatedCode(Arr::get($indicators, 'document_link', [])),
'reference' => doesReferenceHaveDeprecatedCode(Arr::get($indicators, 'reference', [])),
- 'baseline' => doesBaselineHaveDeprecatedCode(Arr::get($indicators, 'baseline', []))];
+ 'baseline' => doesBaselineHaveDeprecatedCode(Arr::get($indicators, 'baseline', [])), ];
}
}
diff --git a/app/IATI/Elements/Forms/MultilevelSubElementForm.php b/app/IATI/Elements/Forms/MultilevelSubElementForm.php
index 3af5010a9f..46c2202c9a 100644
--- a/app/IATI/Elements/Forms/MultilevelSubElementForm.php
+++ b/app/IATI/Elements/Forms/MultilevelSubElementForm.php
@@ -63,7 +63,7 @@ public function buildForm(): void
'is_collapsable' => Arr::get($sub_element, 'is_collapsable', ''),
'label_indicator' => Arr::get($sub_element, 'label_indicator', ),
'wrapper' => ['class' => $this->getBaseFormWrapperClasses()],
- 'dynamic_wrapper' => ['class' => $this->getBaseFormDynamicWrapperClasses($sub_element, $element)]],
+ 'dynamic_wrapper' => ['class' => $this->getBaseFormDynamicWrapperClasses($sub_element, $element)], ],
]
)->add('add_to_collection', 'button', [
'label' => generateAddAdditionalLabel($element['name'], $this->getData(sprintf('sub_elements.%s.name', $name))),
diff --git a/app/IATI/Traits/CommandInputTrait.php b/app/IATI/Traits/CommandInputTrait.php
new file mode 100644
index 0000000000..43a09be536
--- /dev/null
+++ b/app/IATI/Traits/CommandInputTrait.php
@@ -0,0 +1,58 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\IATI\Traits;
+
+use Illuminate\Support\Facades\Validator;
+
+/**
+ * Class MigrateGeneralTrait.
+ */
+trait CommandInputTrait
+{
+ /**
+ * Ask input from user and return value.
+ *
+ * @param $question
+ * @param $field
+ * @param $rules
+ *
+ * @return string
+ */
+ protected function askValid($question, $field, $rules): string
+ {
+ $value = $this->ask($question);
+ $message = $this->validateInput($rules, $field, $value);
+
+ if ($message) {
+ $this->error($message);
+
+ return $this->askValid($question, $field, $rules);
+ }
+
+ return $value;
+ }
+
+ /**
+ * Validates input given by user.
+ *
+ * @param $rules
+ * @param $fieldName
+ * @param $value
+ *
+ * @return string|null
+ */
+ protected function validateInput($rules, $fieldName, $value): ?string
+ {
+ $validator = Validator::make([
+ $fieldName => $value,
+ ], [
+ $fieldName => $rules,
+ ]);
+
+ return $validator->fails()
+ ? $validator->errors()->first($fieldName)
+ : null;
+ }
+}
diff --git a/app/IATI/Traits/FillDefaultValuesTrait.php b/app/IATI/Traits/FillDefaultValuesTrait.php
index 9c39c8313b..0fe42181f5 100644
--- a/app/IATI/Traits/FillDefaultValuesTrait.php
+++ b/app/IATI/Traits/FillDefaultValuesTrait.php
@@ -9,6 +9,7 @@
use App\IATI\Models\Activity\Period;
use App\IATI\Models\Activity\Result;
use App\IATI\Models\Activity\Transaction;
+use App\IATI\Models\Organization\Organization;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Arr;
@@ -146,16 +147,10 @@ public function store(array $data): Model
if (
Arr::get($data, 'migrated_from_aidstream', false) &&
- !$this->model->getModel() instanceof Activity
+ !$this->model->getModel() instanceof Activity &&
+ !$this->model->getModel() instanceof Organization
) {
- foreach ($data as $key => $datum) {
- $this->model->{$key} = $datum;
- }
-
- $this->model->setTouchedRelations([]);
- $this->model->save();
-
- return $this->model;
+ return $this->saveMigratedData($data, $this->model);
}
$data['default_field_values'] = $defaultFieldValues;
@@ -163,6 +158,32 @@ public function store(array $data): Model
return $this->model->create($data);
}
+ /**
+ * @param $data
+ * @param $model
+ *
+ * @return Model
+ */
+ public function saveMigratedData($data, $model): Model
+ {
+ $touchedRelations = $model->getTouchedRelations();
+ $parentModels = [];
+
+ foreach ($touchedRelations as $relation) {
+ // Ensure the relation is a valid method
+ if (method_exists($model, $relation)) {
+ $relatedModel = $model->$relation()->getRelated();
+ $parentModels[] = get_class($relatedModel);
+ }
+ }
+
+ $model->withoutTouchingOn($parentModels, function () use ($data, $model) {
+ return $model->create($data);
+ });
+
+ return $model;
+ }
+
/**
* Overriding base Repository class's update method.
* Modified to populate default field values on update.
From 18c101a0b6eba36ee004fbdc0b80084a4744e983 Mon Sep 17 00:00:00 2001
From: Sanil Manandhar <sanilmanandhar@gmail.com>
Date: Tue, 5 Nov 2024 10:55:19 +0545
Subject: [PATCH 2/2] Created command to check and migrate transactions
---
.../Commands/CheckMigratedTransactions.php | 79 ++++++++++---------
1 file changed, 40 insertions(+), 39 deletions(-)
diff --git a/app/Console/Commands/CheckMigratedTransactions.php b/app/Console/Commands/CheckMigratedTransactions.php
index e626b8d059..851215b73a 100644
--- a/app/Console/Commands/CheckMigratedTransactions.php
+++ b/app/Console/Commands/CheckMigratedTransactions.php
@@ -102,7 +102,7 @@ public function handle(): void
// 1578];
$aidstreamOrganizationIdString = $this->askValid(
- 'Please enter the organization ids which you want to migrate separated by comma (Compulsory)',
+ 'Please enter the organization ids for which you want to check and migrate transactions separated by comma (Compulsory)',
'aidstreamOrganizationIdString',
['required']
);
@@ -208,48 +208,49 @@ public function handle(): void
$iatiActivityTransactions = $iatiActivity->transactions->pluck('transaction');
- $aidstreamActivityTransactionsCount = count($aidstreamActivityTransactions);
- $iatiActivityTransactionsCount = count($iatiActivityTransactions);
+// $aidstreamActivityTransactionsCount = count($aidstreamActivityTransactions);
+// $iatiActivityTransactionsCount = count($iatiActivityTransactions);
$defaultValues = json_encode([$iatiActivity->default_field_values]);
- if ($aidstreamActivityTransactionsCount !== $iatiActivityTransactionsCount) {
- $this->logInfo(
- "AidStream Organization Id: {$aidstreamOrganizationId} \n
- IATI Organization Id: {$iatiOrganization->id} \n
- Publisher Id: {$iatiOrganization->publisher_id} \n
- AidStream Activity Id: {$aidstreamActivity->id} \n
- IATI Activity Id: {$iatiActivity->id} \n
- Activity Identifier: {$aidstreamActivityIdentifier} \n
- AidStream Activity Transactions Count: {$aidstreamActivityTransactionsCount} \n
- IATI Activity Transactions Count: {$iatiActivityTransactionsCount} \n"
- );
-
- $this->info('Updating transaction data.');
- $existingIatiTransactions = $iatiActivityTransactions->toArray();
-
- foreach ($aidstreamActivityTransactions as $aidstreamTransaction) {
- $requiredTransactionData = [
- 'activity_id' => $iatiActivity->id,
- 'transaction' => $this->getRequiredTransactionData(
- $aidstreamTransaction,
- $defaultValues
- ),
- 'migrated_from_aidstream' => true,
- 'created_at' => $aidstreamTransaction->created_at,
- 'updated_at' => $aidstreamTransaction->updated_at,
- ];
-
- if (!$this->checkArrayPresent($existingIatiTransactions, $requiredTransactionData['transaction'])) {
- $this->info(
- "Creating transaction data for AidStream Transaction Id: $aidstreamTransaction->id"
- );
- $this->transactionService->create($requiredTransactionData);
- $this->info(
- "Created transaction data for AidStream Transaction Id: $aidstreamTransaction->id"
- );
- }
+// if ($aidstreamActivityTransactionsCount !== $iatiActivityTransactionsCount) {
+// $this->logInfo(
+// "AidStream Organization Id: {$aidstreamOrganizationId} \n
+// IATI Organization Id: {$iatiOrganization->id} \n
+// Publisher Id: {$iatiOrganization->publisher_id} \n
+// AidStream Activity Id: {$aidstreamActivity->id} \n
+// IATI Activity Id: {$iatiActivity->id} \n
+// Activity Identifier: {$aidstreamActivityIdentifier} \n
+// AidStream Activity Transactions Count: {$aidstreamActivityTransactionsCount} \n
+// IATI Activity Transactions Count: {$iatiActivityTransactionsCount} \n"
+// );
+
+ $this->info("Checking and updating transaction data for activity with identifier {$aidstreamActivityIdentifier}.");
+ $existingIatiTransactions = $iatiActivityTransactions->toArray();
+
+ foreach ($aidstreamActivityTransactions as $aidstreamTransaction) {
+ $requiredTransactionData = [
+ 'activity_id' => $iatiActivity->id,
+ 'transaction' => $this->getRequiredTransactionData(
+ $aidstreamTransaction,
+ $defaultValues
+ ),
+ 'migrated_from_aidstream' => true,
+ 'created_at' => $aidstreamTransaction->created_at,
+ 'updated_at' => $aidstreamTransaction->updated_at,
+ ];
+
+ if (!$this->checkArrayPresent($existingIatiTransactions, $requiredTransactionData['transaction'])) {
+ $this->logInfo(
+ "Transaction with aidstream id {$aidstreamTransaction->id} does not exist for aidstream activity id {$aidstreamActivity->id}.
+ Creating transaction data for AidStream Transaction."
+ );
+ $this->transactionService->create($requiredTransactionData);
+ $this->logInfo(
+ "Created transaction data for AidStream Transaction Id: {$aidstreamTransaction->id}"
+ );
}
}
+// }
$this->info(
"Checking completed for activity with identifier {$aidstreamActivityIdentifier} for organization {$aidStreamOrganization->name} with id $aidStreamOrganization->id"