Skip to content

Commit

Permalink
Merge pull request #1786 from RotherOSS/issue-#1776-user_settings_again
Browse files Browse the repository at this point in the history 
Issue #1776 user settings again
  • Loading branch information
@bschmalhofer
bschmalhofer authored May 18, 2022
2 parents 101d094 + 6f511b1 commit b6632ae
Show file tree
Hide file tree
Showing 8 changed files with 271 additions and 197 deletions.
126 changes: 80 additions & 46 deletions Kernel/Config/Defaults.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2391,15 +2391,17 @@ sub SyncWithS3 {
CHECK_SYNC:
while (1) {

# run a blocking GET request to S3
my %Name2Properties = $StorageS3Object->ListObjects(
Prefix => "$FilesPrefix/",
# run a blocking GET request to S3, getting all keys below the prefix
# The keys are the pathes of files relative to Kernel/Config/Files
my %SubPath2Properties = $StorageS3Object->ListObjects(
Prefix => "$FilesPrefix/",
Delimiter => '',
);

# Package events are not handled here as the whole web server is restarted when
# a package has changed. See Plack::Handler::SyncWithS3 which is activated in entrypoint.sh.
my $EventFileName = 'event_package.json';
if ( exists $Name2Properties{$EventFileName} ) {
if ( exists $SubPath2Properties{$EventFileName} ) {

# gather info about the local event file
my $Stat = stat "$Self->{Home}/Kernel/Config/Files/$EventFileName";
Expand All @@ -2408,64 +2410,96 @@ sub SyncWithS3 {
last CHECK_SYNC unless $Stat;

# info about the event file in S3
my $Properties = $Name2Properties{$EventFileName};
my $Properties = $SubPath2Properties{$EventFileName};

# do not sync ZZZ*.pm files when the local event file differs from the version in S3
last CHECK_SYNC unless $Stat->size == $Properties->{Size};
last CHECK_SYNC unless int($Stat->mtime) == int($Properties->{Mtime});
}

# check the fixed list of ZZZ files, including the dynamic configs for unit testing
my @OutdatedZZZFilenames;
my %FileIsRelevant = map
{ $_ => 1 }
( qw(ZZZAAuto.pm ZZZACL.pm ZZZProcessManagement.pm), $Param{ExtraFileNames}->@* );
ZZZFILENAME:
for my $ZZZFileName ( keys %Name2Properties ) {

# skip the not relevant objects
if (
( ! $FileIsRelevant{$ZZZFileName} )
&&
$ZZZFileName !~ m/ZZZZUnitTest.*\.pm$/
)
{
next ZZZFILENAME;
}
my @OutdatedFiles;
{
# Files that need to be synced are recognised by patterns. Currently there are two cases.
my @SyncFilePatterns = (
qr'ZZZZUnitTest.*\.pm$', # files that are used in the unit tests
qr'User/\d+\.pm$', # user specific overrides of the SysConfig
);

# gather info about the local file
my $Stat = stat "$Self->{Home}/Kernel/Config/Files/$ZZZFileName";
# The hardcoded list of ZZZ files is treated in special way, as they are never deleted in the file system.
my %FileIsRelevant = map
{ $_ => 1 }
( qw(ZZZAAuto.pm ZZZACL.pm ZZZProcessManagement.pm), $Param{ExtraFileNames}->@* );

# fetch from S3 when container just started, and the files don't exist yet in the file system
if ( ! $Stat ) {
push @OutdatedZZZFilenames, $ZZZFileName;
# Essential gather/take like in Syntax::Keyword::Gather
my @CandidateOutdatedFiles;
SUB_PATH:
for my $SubPath ( sort keys %SubPath2Properties ) {

next ZZZFILENAME;
}
# skip the not relevant objects
if ( $FileIsRelevant{$SubPath} ) {
push @CandidateOutdatedFiles, $SubPath;

# either size of modified time must have changed
my $Properties = $Name2Properties{$ZZZFileName};
if ( $Stat->size != $Properties->{Size} ) {
push @OutdatedZZZFilenames, $ZZZFileName;
next SUB_PATH;
}

next ZZZFILENAME;
for my $Pattern ( @SyncFilePatterns ) {
if ( $SubPath =~ $Pattern ) {
push @CandidateOutdatedFiles, $SubPath;

next SUB_PATH;
}
}
}

# timestamp check does not consider differences within one second
if ( int($Stat->mtime) != int($Properties->{Mtime}) ) {
push @OutdatedZZZFilenames, $ZZZFileName;
SUB_PATH:
for my $SubPath ( @CandidateOutdatedFiles ) {

# gather info about the local file
my $Stat = stat "$Self->{Home}/Kernel/Config/Files/$SubPath";

# fetch from S3 when container just started, and the files don't exist yet in the file system
if ( ! $Stat ) {
push @OutdatedFiles, $SubPath;

next ZZZFILENAME;
next SUB_PATH;
}

# either size of modified time must have changed
my $Properties = $SubPath2Properties{$SubPath};
if ( $Stat->size != $Properties->{Size} ) {
push @OutdatedFiles, $SubPath;

next SUB_PATH;
}

# timestamp check does not consider differences within one second
if ( int($Stat->mtime) != int($Properties->{Mtime}) ) {
push @OutdatedFiles, $SubPath;

next SUB_PATH;
}
}
}

# find files in the file system that have been discarded in the S3 storage
my @ObsoleteZZZFilepathes = grep
{ !$Name2Properties{ basename($_) } }
glob "$Self->{Home}/Kernel/Config/Files/ZZZZUnitTest*.pm";
my @ObsoleteFiles;
{
# files that are used in the unit tests
push @ObsoleteFiles, grep
{ !$SubPath2Properties{ basename($_) } }
glob "$Self->{Home}/Kernel/Config/Files/ZZZZUnitTest*.pm";

# user specific overrides of the SysConfig
# note that POSIX does not allow to match multiple digits, so we need an extra filter
push @ObsoleteFiles, grep
{ !$SubPath2Properties{ basename($_) } }
grep
{ m!/\d+\.pm^! }
glob "$Self->{Home}/Kernel/Config/Files/User/[0-9]*.pm";
}

# go on reading the config files when all files are up to date
last CHECK_SYNC unless ( @OutdatedZZZFilenames || @ObsoleteZZZFilepathes );
last CHECK_SYNC unless ( @OutdatedFiles || @ObsoleteFiles );

# not sure whether flock is the best choice here, especially when running in Gazelle
# https://www.perl.com/article/2/2015/11/4/Run-only-one-instance-of-a-program-at-a-time/
Expand All @@ -2486,17 +2520,17 @@ sub SyncWithS3 {
}

# we got an exclusive lock, now do the work and update from S3
for my $ZZZFileName ( @OutdatedZZZFilenames ) {
my $FilePath = join '/', $FilesPrefix, $ZZZFileName;
my $Location = "$Self->{Home}/Kernel/Config/Files/$ZZZFileName";
for my $FileName ( @OutdatedFiles ) {
my $FilePath = join '/', $FilesPrefix, $FileName;
my $Location = "$Self->{Home}/Kernel/Config/Files/$FileName";
$StorageS3Object->SaveObjectToFile(
Key => $FilePath,
Location => $Location,
);
}

# we got an exclusive lock, now do the work and delete the obsolete files
for my $Location ( @ObsoleteZZZFilepathes ) {
for my $Location ( @ObsoleteFiles ) {

# ignore errors as we doublecheck anyways
unlink $Location;
Expand Down
1 change: 1 addition & 0 deletions Kernel/System/Console/Command/Maint/Config/Rebuild.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ sub Run {
);

$Self->PrintError("There was a problem writing XML to DB.");

return $Self->ExitCodeError();
}

Expand Down
1 change: 1 addition & 0 deletions Kernel/System/Storage/S3.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,7 @@ sub SaveObjectToFile {
$Self->{UserAgent}->start($Transaction);

# Do not use the Kernel::System::Main in Kernel/Config/Defaults
make_path( dirname( $Param{Location} ) );
$Transaction->result->save_to( $Param{Location} );

# Touch the downloaded file to the value of LastModified from S3, e.g. 'Sat, 23 Oct 2021 11:15:14 GMT'.
Expand Down
34 changes: 17 additions & 17 deletions Kernel/System/SysConfig.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@

package Kernel::System::SysConfig;

use v5.24;
use strict;
use warnings;
use v5.24;
use namespace::autoclean;
use utf8;

Expand Down Expand Up @@ -76,17 +76,17 @@ sub new {

$Self->{ConfigObject} = $Kernel::OM->Get('Kernel::Config');

# get home directory
$Self->{Home} = $Self->{ConfigObject}->Get('Home');
# get home directory and whether the S3 backend is active
$Self->{Home} = $Self->{ConfigObject}->Get('Home');
$Self->{UseS3Backend} = $Kernel::OM->Get('Kernel::Config')->Get('Storage::S3::Active') ? 1 : 0;

# Kernel::Config is loaded because it was loaded by $Kernel::OM above.
$Self->{ConfigDefaultObject} = Kernel::Config->new( Level => 'Default' );
$Self->{ConfigObject} = Kernel::Config->new( Level => 'First' );
$Self->{ConfigClearObject} = Kernel::Config->new( Level => 'Clear' );

# Load base files.
my $BaseDir = $Self->{Home} . '/Kernel/System/SysConfig/Base/';

my $BaseDir = $Self->{Home} . '/Kernel/System/SysConfig/Base/';
my $MainObject = $Kernel::OM->Get('Kernel::System::Main');

FILENAME:
Expand Down Expand Up @@ -3660,12 +3660,11 @@ sub ConfigurationDeploy {
$EffectiveValueStrg = $LastDeployment{EffectiveValueStrg};
}

if ( $ENV{OTOBO_SYNC_WITH_S3} ) {

my $StorageS3Object = Kernel::System::Storage::S3->new();
if ( $Self->{UseS3Backend} ) {

# only write to S3, no extra copy in the file system
my $S3Key = $StorageS3Object->StoreObject(
my $StorageS3Object = Kernel::System::Storage::S3->new();
my $S3Key = $StorageS3Object->StoreObject(
Key => $TargetPath,
Content => $EffectiveValueStrg,
);
Expand Down Expand Up @@ -3728,6 +3727,7 @@ sub ConfigurationDeployList {
=head2 ConfigurationDeploySync()
Updates C<ZZZAAuto.pm> to the latest deployment found in the database.
This method also updates the user configuration files.
my $Success = $SysConfigObject->ConfigurationDeploySync();
Expand All @@ -3753,8 +3753,10 @@ sub ConfigurationDeploySync {
TRY:
for my $Try ( 1 .. 40 ) {
$CleanupSuccess = $SysConfigDBObject->DeploymentListCleanup();

last TRY if !$CleanupSuccess;
last TRY if $CleanupSuccess == 1;

sleep .5;
}
if ( $CleanupSuccess != 1 ) {
Expand All @@ -3773,14 +3775,15 @@ sub ConfigurationDeploySync {
Priority => 'error',
Message => "No deployments found in Database!",
);

return;
}

if ( $CurrentDeploymentID ne $LastDeployment{DeploymentID} ) {

# Write latest deployment to ZZZAAuto.pm
my $PMFileContent = $LastDeployment{EffectiveValueStrg};
if ( $ENV{OTOBO_SYNC_WITH_S3} ) {
if ( $Self->{UseS3Backend} ) {

# only write to S3
my $StorageS3Object = Kernel::System::Storage::S3->new();
Expand All @@ -3791,9 +3794,6 @@ sub ConfigurationDeploySync {
);

return unless $Success;

# then update the file system from S3
$Kernel::OM->Get('Kernel::Config')->SyncWithS3();
}
else {
my $Success = $Self->_FileWriteAtomic(
Expand All @@ -3805,12 +3805,12 @@ sub ConfigurationDeploySync {
}
}

# Sync also user specific settings (if available).
return 1 unless $Self->can('UserConfigurationDeploySync'); # OTOBO Community Solution

# TODO: also sync the UserSettigs via S3
# Sync also user specific settings, always available in OTOBO
$Self->UserConfigurationDeploySync();

# then update the file system from S3,
$Kernel::OM->Get('Kernel::Config')->SyncWithS3();

return 1;
}

Expand Down
Loading

0 comments on commit b6632ae

Please sign in to comment.