From 342c6458f3ced88aa351e8807364f18b35c272ec Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Thu, 6 Mar 2025 19:04:17 +0530 Subject: [PATCH 01/10] Fix red index on close for remote enabled clusters (#17521) Signed-off-by: Ashish Singh --- .../org/opensearch/index/engine/ReadOnlyEngine.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java index 1852f2fa92b74..5d42a7b830de0 100644 --- a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java @@ -40,6 +40,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; @@ -166,7 +167,7 @@ public ReadOnlyEngine( } protected void ensureMaxSeqNoEqualsToGlobalCheckpoint(final SeqNoStats seqNoStats) { - if (requireCompleteHistory == false) { + if (requireCompleteHistory == false || isClosedRemoteIndex()) { return; } // Before 3.0 the global checkpoint is not known and up to date when the engine is created after @@ -187,6 +188,14 @@ protected void ensureMaxSeqNoEqualsToGlobalCheckpoint(final SeqNoStats seqNoStat } } + /** + * Returns true if this is a remote store index (included if migrating as well) which is closed. + */ + private boolean isClosedRemoteIndex() { + return this.engineConfig.getIndexSettings().isAssignedOnRemoteNode() + && this.engineConfig.getIndexSettings().getIndexMetadata().getState() == IndexMetadata.State.CLOSE; + } + protected boolean assertMaxSeqNoEqualsToGlobalCheckpoint(final long maxSeqNo, final long globalCheckpoint) { assert maxSeqNo == globalCheckpoint : "max seq. no. [" + maxSeqNo + "] does not match [" + globalCheckpoint + "]"; return true; From 73882054afcdb74244c07c5be1f54a629ffd0bc2 Mon Sep 17 00:00:00 2001 From: Divyansh Pandey <98746046+pandeydivyansh1803@users.noreply.github.com> Date: Thu, 6 Mar 2025 20:21:20 +0530 Subject: [PATCH 02/10] Update validator for index update request (#17529) Signed-off-by: Divyansh Pandey Co-authored-by: Divyansh Pandey --- ...AllocationDeciderRemoteStoreEnabledIT.java | 71 +++++++++++++++++++ .../metadata/MetadataCreateIndexService.java | 3 +- .../MetadataUpdateSettingsService.java | 32 ++++++++- 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/ShardsLimitAllocationDeciderRemoteStoreEnabledIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/ShardsLimitAllocationDeciderRemoteStoreEnabledIT.java index 401db7790de92..72c849cb395af 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/ShardsLimitAllocationDeciderRemoteStoreEnabledIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/ShardsLimitAllocationDeciderRemoteStoreEnabledIT.java @@ -8,6 +8,8 @@ package org.opensearch.cluster.routing.allocation.decider; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; @@ -99,6 +101,75 @@ public void testIndexPrimaryShardLimit() throws Exception { }); } + public void testUpdatingIndexPrimaryShardLimit() throws Exception { + // Create first index with primary shard limit + Settings firstIndexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 4)) // 4 shards, 0 replicas + .put(INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey(), 1) + .build(); + + // Create first index + createIndex("test1", firstIndexSettings); + + // Update the index settings to set INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest("test1"); + Settings updatedSettings = Settings.builder().put(INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey(), 1).build(); + updateSettingsRequest.settings(updatedSettings); + + AcknowledgedResponse response = client().admin().indices().updateSettings(updateSettingsRequest).actionGet(); + + assertTrue(response.isAcknowledged()); + + // Create second index + createIndex("test2", remoteStoreIndexSettings(0, 4)); + + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().get().getState(); + + // Check total number of shards (8 total: 4 from each index) + assertEquals("Total shards should be 8", 8, state.getRoutingTable().allShards().size()); + + // Count assigned and unassigned shards for test1 + int test1AssignedShards = 0; + int test1UnassignedShards = 0; + Map nodePrimaryCount = new HashMap<>(); + + // Check test1 shard distribution + for (IndexShardRoutingTable shardRouting : state.routingTable().index("test1")) { + for (ShardRouting shard : shardRouting) { + if (shard.assignedToNode()) { + test1AssignedShards++; + // Count primaries per node for test1 + String nodeId = shard.currentNodeId(); + nodePrimaryCount.merge(nodeId, 1, Integer::sum); + } else { + test1UnassignedShards++; + } + } + } + + // Check test2 shard assignment + int test2UnassignedShards = 0; + for (IndexShardRoutingTable shardRouting : state.routingTable().index("test2")) { + for (ShardRouting shard : shardRouting) { + if (!shard.assignedToNode()) { + test2UnassignedShards++; + } + } + } + + // Assertions + assertEquals("test1 should have 3 assigned shards", 3, test1AssignedShards); + assertEquals("test1 should have 1 unassigned shard", 1, test1UnassignedShards); + assertEquals("test2 should have no unassigned shards", 0, test2UnassignedShards); + + // Verify no node has more than one primary shard of test1 + for (Integer count : nodePrimaryCount.values()) { + assertTrue("No node should have more than 1 primary shard of test1", count <= 1); + } + }); + } + public void testClusterPrimaryShardLimitss() throws Exception { // Update cluster setting to limit primary shards per node updateClusterSetting(CLUSTER_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey(), 1); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index a81fe01f0e7f4..2bdd31b23aee3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1847,7 +1847,8 @@ public static void validateRefreshIntervalSettings(Settings requestSettings, Clu } /** - * Validates {@code index.routing.allocation.total_primary_shards_per_node} is only set for remote store enabled cluster + * Validates the {@code index.routing.allocation.total_primary_shards_per_node} setting during index creation. + * Ensures this setting is only specified for remote store enabled clusters. */ // TODO : Update this check for SegRep to DocRep migration on need basis public static void validateIndexTotalPrimaryShardsPerNodeSetting(Settings indexSettings) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java index eb10fd5d04288..469bec7220721 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -43,6 +43,7 @@ import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlock; import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; @@ -78,12 +79,12 @@ import static org.opensearch.action.support.ContextPreservingActionListener.wrapPreservingContext; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateIndexTotalPrimaryShardsPerNodeSetting; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateOverlap; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateRefreshIntervalSettings; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogDurabilitySettings; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex; import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findComponentTemplate; +import static org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING; import static org.opensearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX; import static org.opensearch.index.IndexSettings.same; @@ -140,7 +141,7 @@ public void updateSettings( validateRefreshIntervalSettings(normalizedSettings, clusterService.getClusterSettings()); validateTranslogDurabilitySettings(normalizedSettings, clusterService.getClusterSettings(), clusterService.getSettings()); - validateIndexTotalPrimaryShardsPerNodeSetting(normalizedSettings); + validateIndexTotalPrimaryShardsPerNodeSetting(normalizedSettings, clusterService); final int defaultReplicaCount = clusterService.getClusterSettings().get(Metadata.DEFAULT_REPLICA_COUNT_SETTING); Settings.Builder settingsForClosedIndices = Settings.builder(); @@ -549,4 +550,31 @@ private void validateSearchReplicaCountSettings(Settings requestSettings, Index[ } } } + + /** + * Validates the 'index.routing.allocation.total_primary_shards_per_node' setting during index settings update. + * Ensures this setting can only be modified for existing indices in remote store enabled clusters. + */ + public static void validateIndexTotalPrimaryShardsPerNodeSetting(Settings indexSettings, ClusterService clusterService) { + // Get the setting value + int indexPrimaryShardsPerNode = INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.get(indexSettings); + + // If default value (-1), no validation needed + if (indexPrimaryShardsPerNode == -1) { + return; + } + + // Check if remote store is enabled + boolean isRemoteStoreEnabled = clusterService.state() + .nodes() + .getNodes() + .values() + .stream() + .allMatch(DiscoveryNode::isRemoteStoreNode); + if (!isRemoteStoreEnabled) { + throw new IllegalArgumentException( + "Setting [" + INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey() + "] can only be used with remote store enabled clusters" + ); + } + } } From cb869c06861c63aa338120dcb33d1b3470c54155 Mon Sep 17 00:00:00 2001 From: Sandeep Kumawat <2025sandeepkumawat@gmail.com> Date: Fri, 7 Mar 2025 09:29:39 +0530 Subject: [PATCH 03/10] Writable warm replica replication/recovery (#17390) Signed-off-by: Sandeep Kumawat --- .../replication/SegmentReplicationBaseIT.java | 2 - .../WarmIndexSegmentReplicationIT.java | 1665 +++++++++++++++++ .../SegmentReplicationUsingRemoteStoreIT.java | 2 - .../allocation/IndexMetadataUpdater.java | 3 +- .../allocator/RemoteShardsBalancer.java | 5 +- .../index/engine/NRTReplicationEngine.java | 9 +- .../opensearch/index/shard/IndexShard.java | 6 +- .../index/store/CompositeDirectory.java | 55 +- .../store/remote/utils/FileTypeUtils.java | 4 +- .../store/remote/utils/cache/LRUCache.java | 13 +- .../remote/utils/cache/SegmentedCache.java | 7 +- .../replication/SegmentReplicationTarget.java | 7 + .../index/store/CompositeDirectoryTests.java | 12 +- 13 files changed, 1773 insertions(+), 17 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java index 8b4913fcb2704..ac2862806c858 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java @@ -49,8 +49,6 @@ import java.util.stream.Collectors; import static java.util.Arrays.asList; -import static org.opensearch.test.OpenSearchIntegTestCase.client; -import static org.opensearch.test.OpenSearchTestCase.assertBusy; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; public class SegmentReplicationBaseIT extends OpenSearchIntegTestCase { diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java new file mode 100644 index 0000000000000..d7f1c2209f798 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java @@ -0,0 +1,1665 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.replication; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; +import org.opensearch.action.admin.cluster.stats.ClusterStatsResponse; +import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.flush.FlushRequest; +import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.get.MultiGetResponse; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchType; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.termvectors.TermVectorsRequestBuilder; +import org.opensearch.action.termvectors.TermVectorsResponse; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.Preference; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.lease.Releasable; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.set.Sets; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexModule; +import org.opensearch.index.ReplicationStats; +import org.opensearch.index.SegmentReplicationPerGroupStats; +import org.opensearch.index.SegmentReplicationPressureService; +import org.opensearch.index.SegmentReplicationShardStats; +import org.opensearch.index.codec.CodecService; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.Node; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.junit.annotations.TestLogging; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.transport.TransportService; +import org.opensearch.transport.client.Requests; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.index.query.QueryBuilders.boolQuery; +import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.index.query.QueryBuilders.matchQuery; +import static org.opensearch.index.query.QueryBuilders.rangeQuery; +import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * This class runs Segment Replication Integ test suite with partial locality indices (warm indices). + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) +public class WarmIndexSegmentReplicationIT extends SegmentReplicationBaseIT { + + protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path absolutePath; + + @Before + private void setup() { + internalCluster().startClusterManagerOnlyNode(); + } + + private static String indexOrAlias() { + return randomBoolean() ? INDEX_NAME : "alias"; + } + + @Override + public Settings indexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name()) + .build(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + if (absolutePath == null) { + absolutePath = randomRepoPath().toAbsolutePath(); + } + ByteSizeValue cacheSize = new ByteSizeValue(16, ByteSizeUnit.GB); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) + .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize.toString()) + .build(); + } + + @Override + protected Settings featureFlagSettings() { + Settings.Builder featureSettings = Settings.builder(); + featureSettings.put(FeatureFlags.TIERED_REMOTE_INDEX, true); + return featureSettings.build(); + } + + @Override + protected boolean addMockIndexStorePlugin() { + return false; + } + + protected boolean warmIndexSegmentReplicationEnabled() { + return true; + } + + @After + public void teardown() throws Exception { + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); + for (String nodeName : internalCluster().getNodeNames()) { + FileCache fileCache = internalCluster().getInstance(Node.class, nodeName).fileCache(); + if (fileCache != null) { + fileCache.clear(); + } + } + clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get(); + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17526") + public void testRestartPrimary_NoReplicas() throws Exception { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureYellow(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), primary); + + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + if (randomBoolean()) { + flush(INDEX_NAME); + } else { + refresh(INDEX_NAME); + } + FileCache fileCache = internalCluster().getInstance(Node.class, primary).fileCache(); + internalCluster().restartNode(primary); + ensureYellow(INDEX_NAME); + assertDocCounts(1, primary); + fileCache.prune(); + } + + public void testPrimaryStopped_ReplicaPromoted() throws Exception { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForSearchableDocs(1, primary, replica); + + // index another doc but don't refresh, we will ensure this is searchable once replica is promoted. + client().prepareIndex(INDEX_NAME).setId("2").setSource("bar", "baz").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + FileCache fileCache1 = internalCluster().getInstance(Node.class, primary).fileCache(); + // stop the primary node - we only have one shard on here. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final ShardRouting replicaShardRouting = getShardRoutingForNodeName(replica); + assertNotNull(replicaShardRouting); + assertTrue(replicaShardRouting + " should be promoted as a primary", replicaShardRouting.primary()); + // new primary should have at least the doc count from the first set of segments. + assertBusy(() -> { + final SearchResponse response = client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(); + assertTrue(response.getHits().getTotalHits().value() >= 1); + }, 1, TimeUnit.MINUTES); + + // assert we can index into the new primary. + client().prepareIndex(INDEX_NAME).setId("3").setSource("bar", "baz").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); + + // start another node, index another doc and replicate. + String nodeC = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + client().prepareIndex(INDEX_NAME).setId("4").setSource("baz", "baz").get(); + refresh(INDEX_NAME); + waitForSearchableDocs(4, nodeC, replica); + verifyStoreContent(); + fileCache1.prune(); + } + + public void testRestartPrimary() throws Exception { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), primary); + + final int initialDocCount = 1; + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + FileCache fileCache = internalCluster().getInstance(Node.class, primary).fileCache(); + waitForSearchableDocs(initialDocCount, replica, primary); + internalCluster().restartNode(primary); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), replica); + + flushAndRefresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount, replica, primary); + verifyStoreContent(); + fileCache.prune(); + } + + public void testCancelPrimaryAllocation() throws Exception { + // this test cancels allocation on the primary - promoting the new replica and recreating the former primary as a replica. + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + final int initialDocCount = 1; + + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForSearchableDocs(initialDocCount, replica, primary); + + final IndexShard indexShard = getIndexShard(primary, INDEX_NAME); + client().admin() + .cluster() + .prepareReroute() + .add(new CancelAllocationCommand(INDEX_NAME, indexShard.shardId().id(), primary, true)) + .execute() + .actionGet(); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), replica); + + flushAndRefresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount, replica, primary); + verifyStoreContent(); + } + + public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { + final String nodeA = internalCluster().startDataAndSearchNodes(1).get(0); + final String nodeB = internalCluster().startDataAndSearchNodes(1).get(0); + final Settings settings = Settings.builder() + .put(indexSettings()) + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(new ArrayList<>(CODECS) { + { + add(CodecService.LUCENE_DEFAULT_CODEC); + } + })) + .build(); + createIndex(INDEX_NAME, settings); + ensureGreen(INDEX_NAME); + + final int initialDocCount = scaledRandomIntBetween(0, 10); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount, nodeA, nodeB); + + final int additionalDocCount = scaledRandomIntBetween(0, 10); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + + flushAndRefresh(INDEX_NAME); + waitForSearchableDocs(expectedHitCount, nodeA, nodeB); + + ensureGreen(INDEX_NAME); + verifyStoreContent(); + } + } + + public void testIndexReopenClose() throws Exception { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureGreen(INDEX_NAME); + + final int initialDocCount = scaledRandomIntBetween(1, 10); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + flush(INDEX_NAME); + waitForSearchableDocs(initialDocCount, primary, replica); + } + logger.info("--> Closing the index "); + client().admin().indices().prepareClose(INDEX_NAME).get(); + + logger.info("--> Opening the index"); + client().admin().indices().prepareOpen(INDEX_NAME).get(); + ensureGreen(INDEX_NAME); + waitForSearchableDocs(initialDocCount, primary, replica); + verifyStoreContent(); + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17526") + public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()); + ensureGreen(INDEX_NAME); + + // Index a doc to create the first set of segments. _s1.si + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").get(); + // Flush segments to disk and create a new commit point (Primary: segments_3, _s1.si) + flushAndRefresh(INDEX_NAME); + assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 1); + + // Index to create another segment + client().prepareIndex(INDEX_NAME).setId("2").setSource("foo", "bar").get(); + + // Force a merge here so that the in memory SegmentInfos does not reference old segments on disk. + client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).setFlush(false).get(); + refresh(INDEX_NAME); + + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + ); + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 2); + assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 2); + + client().prepareIndex(INDEX_NAME).setId("3").setSource("foo", "bar").get(); + refresh(INDEX_NAME); + waitForSearchableDocs(3, primaryNode, replicaNode); + assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); + assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); + if (!warmIndexSegmentReplicationEnabled()) { + verifyStoreContent(); + } + } + + /** + * This tests that the max seqNo we send to replicas is accurate and that after failover + * the new primary starts indexing from the correct maxSeqNo and replays the correct count of docs + * from xlog. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17527") + public void testReplicationPostDeleteAndForceMerge() throws Exception { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + final int initialDocCount = scaledRandomIntBetween(1, 10); + for (int i = 0; i < initialDocCount; i++) { + client().prepareIndex(INDEX_NAME).setId(String.valueOf(i)).setSource("foo", "bar").get(); + } + refresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount, primary, replica); + + final int deletedDocCount = randomIntBetween(1, initialDocCount); + for (int i = 0; i < deletedDocCount; i++) { + client(primary).prepareDelete(INDEX_NAME, String.valueOf(i)).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + } + client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).setFlush(false).get(); + + // randomly flush here after the force merge to wipe any old segments. + if (randomBoolean()) { + flush(INDEX_NAME); + } + + final IndexShard primaryShard = getIndexShard(primary, INDEX_NAME); + final IndexShard replicaShard = getIndexShard(replica, INDEX_NAME); + assertBusy( + () -> assertEquals( + primaryShard.getLatestReplicationCheckpoint().getSegmentInfosVersion(), + replicaShard.getLatestReplicationCheckpoint().getSegmentInfosVersion() + ) + ); + + // add some docs to the xlog and drop primary. + final int additionalDocs = randomIntBetween(1, 5); + for (int i = initialDocCount; i < initialDocCount + additionalDocs; i++) { + client().prepareIndex(INDEX_NAME).setId(String.valueOf(i)).setSource("foo", "bar").get(); + } + // Drop the primary and wait until replica is promoted. + FileCache fileCache1 = internalCluster().getInstance(Node.class, primary).fileCache(); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + + final ShardRouting replicaShardRouting = getShardRoutingForNodeName(replica); + assertNotNull(replicaShardRouting); + assertTrue(replicaShardRouting + " should be promoted as a primary", replicaShardRouting.primary()); + refresh(INDEX_NAME); + final long expectedHitCount = initialDocCount + additionalDocs - deletedDocCount; + // waitForSearchableDocs(initialDocCount, replica, primary); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + + int expectedMaxSeqNo = initialDocCount + deletedDocCount + additionalDocs - 1; + assertEquals(expectedMaxSeqNo, replicaShard.seqNoStats().getMaxSeqNo()); + + // index another doc. + client().prepareIndex(INDEX_NAME).setId(String.valueOf(expectedMaxSeqNo + 1)).setSource("another", "doc").get(); + refresh(INDEX_NAME); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount + 1); + fileCache1.clear(); + } + + public void testScrollWithConcurrentIndexAndSearch() throws Exception { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureGreen(INDEX_NAME); + final List> pendingIndexResponses = new ArrayList<>(); + final List> pendingSearchResponse = new ArrayList<>(); + final int searchCount = randomIntBetween(1, 2); + final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); + + for (int i = 0; i < searchCount; i++) { + pendingIndexResponses.add( + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(i)) + .setRefreshPolicy(refreshPolicy) + .setSource("field", "value" + i) + .execute() + ); + flush(INDEX_NAME); + forceMerge(); + } + + final SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setIndices(INDEX_NAME) + .setRequestCache(false) + .setScroll(TimeValue.timeValueDays(1)) + .setSize(10) + .get(); + + for (int i = searchCount; i < searchCount * 2; i++) { + pendingIndexResponses.add( + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(i)) + .setRefreshPolicy(refreshPolicy) + .setSource("field", "value" + i) + .execute() + ); + } + flush(INDEX_NAME); + forceMerge(); + client().prepareClearScroll().addScrollId(searchResponse.getScrollId()).get(); + + assertBusy(() -> { + client().admin().indices().prepareRefresh().execute().actionGet(); + assertTrue(pendingIndexResponses.stream().allMatch(ActionFuture::isDone)); + assertTrue(pendingSearchResponse.stream().allMatch(ActionFuture::isDone)); + }, 1, TimeUnit.MINUTES); + verifyStoreContent(); + waitForSearchableDocs(INDEX_NAME, 2 * searchCount, List.of(primary, replica)); + } + + public void testMultipleShards() throws Exception { + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + final String nodeA = internalCluster().startDataAndSearchNodes(1).get(0); + final String nodeB = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME, indexSettings); + ensureGreen(INDEX_NAME); + + final int initialDocCount = scaledRandomIntBetween(1, 10); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount, nodeA, nodeB); + + final int additionalDocCount = scaledRandomIntBetween(0, 10); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + + flushAndRefresh(INDEX_NAME); + waitForSearchableDocs(expectedHitCount, nodeA, nodeB); + + ensureGreen(INDEX_NAME); + if (!warmIndexSegmentReplicationEnabled()) { + verifyStoreContent(); + } + } + } + + public void testReplicationAfterForceMerge() throws Exception { + performReplicationAfterForceMerge(false, SHARD_COUNT * (1 + REPLICA_COUNT)); + } + + public void testReplicationAfterForceMergeOnPrimaryShardsOnly() throws Exception { + performReplicationAfterForceMerge(true, SHARD_COUNT); + } + + private void performReplicationAfterForceMerge(boolean primaryOnly, int expectedSuccessfulShards) throws Exception { + final String nodeA = internalCluster().startDataAndSearchNodes(1).get(0); + final String nodeB = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureGreen(INDEX_NAME); + + final int initialDocCount = scaledRandomIntBetween(0, 10); + final int additionalDocCount = scaledRandomIntBetween(0, 10); + final int expectedHitCount = initialDocCount + additionalDocCount; + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + + flush(INDEX_NAME); + waitForSearchableDocs(initialDocCount, nodeA, nodeB); + + // Index a second set of docs so we can merge into one segment. + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + waitForSearchableDocs(expectedHitCount, nodeA, nodeB); + + // Perform force merge only on the primary shards. + final ForceMergeResponse forceMergeResponse = client().admin() + .indices() + .prepareForceMerge(INDEX_NAME) + .setPrimaryOnly(primaryOnly) + .setMaxNumSegments(1) + .setFlush(false) + .get(); + assertThat(forceMergeResponse.getFailedShards(), is(0)); + assertThat(forceMergeResponse.getSuccessfulShards(), is(expectedSuccessfulShards)); + refresh(INDEX_NAME); + if (!warmIndexSegmentReplicationEnabled()) { + verifyStoreContent(); + } + } + } + + /** + * This test verifies that segment replication does not fail for closed indices + */ + public void testClosedIndices() { + List nodes = new ArrayList<>(); + // start 1st node so that it contains the primary + nodes.add(internalCluster().startDataAndSearchNodes(1).get(0)); + createIndex(INDEX_NAME, super.indexSettings()); + ensureYellowAndNoInitializingShards(INDEX_NAME); + // start 2nd node so that it contains the replica + nodes.add(internalCluster().startDataAndSearchNodes(1).get(0)); + ensureGreen(INDEX_NAME); + + logger.info("--> Close index"); + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + + logger.info("--> waiting for allocation to have shards assigned"); + waitForRelocation(ClusterHealthStatus.GREEN); + } + + /** + * This test validates the primary node drop does not result in shard failure on replica. + * @throws Exception when issue is encountered + */ + public void testNodeDropWithOngoingReplication() throws Exception { + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex( + INDEX_NAME, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.refresh_interval", -1) + .build() + ); + ensureYellow(INDEX_NAME); + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState(); + // Get replica allocation id + final String replicaAllocationId = state.routingTable() + .index(INDEX_NAME) + .shardsWithState(ShardRoutingState.STARTED) + .stream() + .filter(routing -> routing.primary() == false) + .findFirst() + .get() + .allocationId() + .getId(); + DiscoveryNode primaryDiscovery = state.nodes().resolveNode(primaryNode); + + CountDownLatch blockFileCopy = new CountDownLatch(1); + MockTransportService primaryTransportService = ((MockTransportService) internalCluster().getInstance( + TransportService.class, + primaryNode + )); + primaryTransportService.addSendBehavior( + internalCluster().getInstance(TransportService.class, replicaNode), + (connection, requestId, action, request, options) -> { + if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK)) { + fail("File copy should not happen for warm index replica shards"); + } + connection.sendRequest(requestId, action, request, options); + } + ); + final int docCount = scaledRandomIntBetween(1, 10); + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + // Refresh, this should trigger round of segment replication + refresh(INDEX_NAME); + blockFileCopy.countDown(); + FileCache fileCache = internalCluster().getInstance(Node.class, primaryNode).fileCache(); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); + ensureYellow(INDEX_NAME); + assertBusy(() -> { assertDocCounts(docCount, replicaNode); }); + state = client().admin().cluster().prepareState().execute().actionGet().getState(); + // replica now promoted as primary should have same allocation id + final String currentAllocationID = state.routingTable() + .index(INDEX_NAME) + .shardsWithState(ShardRoutingState.STARTED) + .stream() + .filter(routing -> routing.primary()) + .findFirst() + .get() + .allocationId() + .getId(); + assertEquals(currentAllocationID, replicaAllocationId); + fileCache.prune(); + } + + public void testCancellation() throws Exception { + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()); + ensureYellow(INDEX_NAME); + + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + + final SegmentReplicationSourceService segmentReplicationSourceService = internalCluster().getInstance( + SegmentReplicationSourceService.class, + primaryNode + ); + final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME); + + CountDownLatch latch = new CountDownLatch(1); + + MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance( + TransportService.class, + primaryNode + )); + mockTransportService.addSendBehavior( + internalCluster().getInstance(TransportService.class, replicaNode), + (connection, requestId, action, request, options) -> { + if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK)) { + fail("File copy should not happen for warm index replica shards"); + } + connection.sendRequest(requestId, action, request, options); + } + ); + + final int docCount = scaledRandomIntBetween(0, 10); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(docCount); + waitForDocs(docCount, indexer); + + flush(INDEX_NAME); + } + segmentReplicationSourceService.beforeIndexShardClosed(primaryShard.shardId(), primaryShard, indexSettings()); + latch.countDown(); + assertDocCounts(docCount, primaryNode); + } + + @TestLogging(reason = "Getting trace logs from replication package", value = "org.opensearch.indices.replication:TRACE") + public void testDeleteOperations() throws Exception { + final String nodeA = internalCluster().startDataAndSearchNodes(1).get(0); + final String nodeB = internalCluster().startDataAndSearchNodes(1).get(0); + + createIndex(INDEX_NAME); + ensureGreen(INDEX_NAME); + final int initialDocCount = scaledRandomIntBetween(1, 5); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount, nodeA, nodeB); + + final int additionalDocCount = scaledRandomIntBetween(0, 2); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + waitForSearchableDocs(expectedHitCount, nodeA, nodeB); + + ensureGreen(INDEX_NAME); + + Set ids = indexer.getIds(); + assertFalse(ids.isEmpty()); + String id = ids.toArray()[0].toString(); + client(nodeA).prepareDelete(INDEX_NAME, id).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + refresh(INDEX_NAME); + waitForSearchableDocs(expectedHitCount - 1, nodeA, nodeB); + verifyStoreContent(); + } + } + + public void testUpdateOperations() throws Exception { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureYellow(INDEX_NAME); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + final int initialDocCount = scaledRandomIntBetween(1, 5); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount, asList(primary, replica)); + + final int additionalDocCount = scaledRandomIntBetween(0, 5); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + waitForSearchableDocs(expectedHitCount, asList(primary, replica)); + + Set ids = indexer.getIds(); + String id = ids.toArray()[0].toString(); + UpdateResponse updateResponse = client(primary).prepareUpdate(INDEX_NAME, id) + .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz") + .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL) + .get(); + assertFalse("request shouldn't have forced a refresh", updateResponse.forcedRefresh()); + assertEquals(2, updateResponse.getVersion()); + + refresh(INDEX_NAME); + verifyStoreContent(); + assertSearchHits(client(primary).prepareSearch(INDEX_NAME).setQuery(matchQuery("foo", "baz")).get(), id); + assertSearchHits(client(replica).prepareSearch(INDEX_NAME).setQuery(matchQuery("foo", "baz")).get(), id); + } + } + + public void testDropPrimaryDuringReplication() throws Exception { + final int replica_count = 6; + final Settings settings = Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replica_count) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME, settings); + final List dataNodes = internalCluster().startDataAndSearchNodes(6); + ensureGreen(INDEX_NAME); + + int initialDocCount = scaledRandomIntBetween(5, 10); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + // don't wait for replication to complete, stop the primary immediately. + FileCache fileCache = internalCluster().getInstance(Node.class, primaryNode).fileCache(); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); + ensureYellow(INDEX_NAME); + + // start another replica. + dataNodes.add(internalCluster().startDataAndSearchNodes(1).get(0)); + ensureGreen(INDEX_NAME); + waitForSearchableDocs(initialDocCount, dataNodes); + + // index another doc and refresh - without this the new replica won't catch up. + String docId = String.valueOf(initialDocCount + 1); + client().prepareIndex(INDEX_NAME).setId(docId).setSource("foo", "bar").get(); + + flushAndRefresh(INDEX_NAME); + waitForSearchableDocs(initialDocCount + 1, dataNodes); + verifyStoreContent(); + fileCache.prune(); + } + } + + @TestLogging(reason = "Getting trace logs from replication package", value = "org.opensearch.indices.replication:TRACE") + public void testReplicaHasDiffFilesThanPrimary() throws Exception { + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()); + ensureYellow(INDEX_NAME); + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME); + IndexWriterConfig iwc = newIndexWriterConfig().setOpenMode(IndexWriterConfig.OpenMode.APPEND); + + // create a doc to index + int numDocs = 2 + random().nextInt(10); + + List docs = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO)); + doc.add( + new TextField( + "body", + TestUtil.randomRealisticUnicodeString(random()), + random().nextBoolean() ? Field.Store.YES : Field.Store.NO + ) + ); + doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random())))); + docs.add(doc); + } + // create some segments on the replica before copy. + try (IndexWriter writer = new IndexWriter(replicaShard.store().directory(), iwc)) { + for (Document d : docs) { + writer.addDocument(d); + } + writer.flush(); + writer.commit(); + } + + final SegmentInfos segmentInfos = SegmentInfos.readLatestCommit(replicaShard.store().directory()); + replicaShard.finalizeReplication(segmentInfos); + ensureYellow(INDEX_NAME); + + final int docCount = scaledRandomIntBetween(10, 20); + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + // Refresh, this should trigger round of segment replication + refresh(INDEX_NAME); + } + ensureGreen(INDEX_NAME); + waitForSearchableDocs(docCount, primaryNode, replicaNode); + // ToDo: verifyStoreContent() needs to be fixed for warm indices + if (!warmIndexSegmentReplicationEnabled()) { + verifyStoreContent(); + } + final IndexShard replicaAfterFailure = getIndexShard(replicaNode, INDEX_NAME); + assertNotEquals(replicaAfterFailure.routingEntry().allocationId().getId(), replicaShard.routingEntry().allocationId().getId()); + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/17527") + public void testPressureServiceStats() throws Exception { + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + createIndex(INDEX_NAME); + ensureYellow(INDEX_NAME); + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + int initialDocCount = scaledRandomIntBetween(10, 20); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + + // get shard references. + final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME); + final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME); + logger.info("Replica aid {}", replicaShard.routingEntry().allocationId()); + logger.info("former primary aid {}", primaryShard.routingEntry().allocationId()); + + // fetch pressure stats from the Primary's Node. + SegmentReplicationPressureService pressureService = internalCluster().getInstance( + SegmentReplicationPressureService.class, + primaryNode + ); + + // Fetch pressure stats from the Replica's Node we will assert replica node returns nothing until it is promoted. + SegmentReplicationPressureService replicaNode_service = internalCluster().getInstance( + SegmentReplicationPressureService.class, + replicaNode + ); + + final Map shardStats = pressureService.nodeStats().getShardStats(); + assertEquals("We should have stats returned for the replication group", 1, shardStats.size()); + + SegmentReplicationPerGroupStats groupStats = shardStats.get(primaryShard.shardId()); + Set replicaStats = groupStats.getReplicaStats(); + assertAllocationIdsInReplicaShardStats(Set.of(replicaShard.routingEntry().allocationId().getId()), replicaStats); + + assertTrue(replicaNode_service.nodeStats().getShardStats().isEmpty()); + + // drop the primary, this won't hand off pressure stats between old/new primary. + FileCache fileCache = internalCluster().getInstance(Node.class, primaryNode).fileCache(); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + + assertTrue("replica should be promoted as a primary", replicaShard.routingEntry().primary()); + assertEquals( + "We should have stats returned for the replication group", + 1, + replicaNode_service.nodeStats().getShardStats().size() + ); + // after the primary is dropped and replica is promoted we won't have a replica assigned yet, so stats per replica should return + // empty. + replicaStats = replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats(); + assertTrue(replicaStats.isEmpty()); + + // start another replica. + String replicaNode_2 = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + final IndexShard secondReplicaShard = getIndexShard(replicaNode_2, INDEX_NAME); + final String second_replica_aid = secondReplicaShard.routingEntry().allocationId().getId(); + waitForSearchableDocs(initialDocCount, replicaNode_2); + + assertEquals( + "We should have stats returned for the replication group", + 1, + replicaNode_service.nodeStats().getShardStats().size() + ); + replicaStats = replicaNode_service.nodeStats().getShardStats().get(replicaShard.shardId()).getReplicaStats(); + assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), replicaStats); + final SegmentReplicationShardStats replica_entry = replicaStats.stream().findFirst().get(); + assertEquals(replica_entry.getCheckpointsBehindCount(), 0); + + // test a checkpoint without any new segments + flush(INDEX_NAME); + assertBusy(() -> { + assertEquals(1, replicaNode_service.nodeStats().getShardStats().size()); + final Set shardStatsSet = replicaNode_service.nodeStats() + .getShardStats() + .get(replicaShard.shardId()) + .getReplicaStats(); + assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), shardStatsSet); + final SegmentReplicationShardStats stats = shardStatsSet.stream().findFirst().get(); + assertEquals(0, stats.getCheckpointsBehindCount()); + }); + fileCache.prune(); + } + } + + private void assertAllocationIdsInReplicaShardStats(Set expected, Set replicaStats) { + assertEquals(expected, replicaStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet())); + } + + /** + * Tests a scroll query on the replica + * @throws Exception when issue is encountered + */ + public void testScrollCreatedOnReplica() throws Exception { + // create the cluster with one primary node containing primary shard and replica node containing replica shard + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + prepareCreate( + INDEX_NAME, + Settings.builder() + .put(indexSettings()) + // we want to control refreshes + .put("index.refresh_interval", -1) + ).get(); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + client().prepareIndex(INDEX_NAME) + .setId(String.valueOf(0)) + .setSource(jsonBuilder().startObject().field("field", 0).endObject()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + refresh(INDEX_NAME); + + assertBusy( + () -> assertEquals( + getIndexShard(primary, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion(), + getIndexShard(replica, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion() + ) + ); + + // opens a scrolled query before a flush is called. + // this is for testing scroll segment consistency between refresh and flush + SearchResponse searchResponse = client(replica).prepareSearch() + .setQuery(matchAllQuery()) + .setIndices(INDEX_NAME) + .setRequestCache(false) + .setPreference("_only_local") + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) + .addSort("field", SortOrder.ASC) + .setSize(10) + .setScroll(TimeValue.timeValueDays(1)) + .get(); + + final IndexShard replicaShard = getIndexShard(replica, INDEX_NAME); + SegmentInfos latestSegmentInfos = getLatestSegmentInfos(replicaShard); + final Set snapshottedSegments = new HashSet<>(latestSegmentInfos.files(false)); + logger.info("Segments {}", snapshottedSegments); + + // index more docs and force merge down to 1 segment + for (int i = 1; i < 5; i++) { + client().prepareIndex(INDEX_NAME) + .setId(String.valueOf(i)) + .setSource(jsonBuilder().startObject().field("field", i).endObject()) + .get(); + refresh(INDEX_NAME); + } + // create new on-disk segments and copy them out. + assertBusy(() -> { + assertEquals( + getIndexShard(primary, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion(), + getIndexShard(replica, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion() + ); + }); + + // force merge and flush. + client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).setFlush(true).get(); + // wait for replication to complete + assertBusy(() -> { + assertEquals( + getIndexShard(primary, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion(), + getIndexShard(replica, INDEX_NAME).getLatestReplicationCheckpoint().getSegmentInfosVersion() + ); + }); + logger.info("Local segments after force merge and commit {}", getLatestSegmentInfos(replicaShard).files(false)); + + // Test stats + logger.info("--> Collect all scroll query hits"); + long scrollHits = 0; + do { + scrollHits += searchResponse.getHits().getHits().length; + searchResponse = client(replica).prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueDays(1)).get(); + assertAllSuccessful(searchResponse); + } while (searchResponse.getHits().getHits().length > 0); + assertEquals(1, scrollHits); + + client(replica).prepareClearScroll().addScrollId(searchResponse.getScrollId()).get(); + final Set filesAfterClearScroll = Arrays.stream(replicaShard.store().directory().listAll()).collect(Collectors.toSet()); + // there should be no active readers, snapshots, or on-disk commits containing the snapshotted files, check that they have been + // deleted. + Set latestCommitSegments = new HashSet<>(replicaShard.store().readLastCommittedSegmentsInfo().files(false)); + assertEquals( + "Snapshotted files are no longer part of the latest commit", + Collections.emptySet(), + Sets.intersection(latestCommitSegments, snapshottedSegments) + ); + assertEquals( + "All snapshotted files should be deleted", + Collections.emptySet(), + Sets.intersection(filesAfterClearScroll, snapshottedSegments) + ); + } + + /** + * This tests that if a primary receives docs while a replica is performing round of segrep during recovery + * the replica will catch up to latest checkpoint once recovery completes without requiring an additional primary refresh/flush. + */ + public void testPrimaryReceivesDocsDuringReplicaRecovery() throws Exception { + final List nodes = new ArrayList<>(); + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + nodes.add(primaryNode); + final Settings settings = Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); + createIndex(INDEX_NAME, settings); + ensureGreen(INDEX_NAME); + // start a replica node, initially will be empty with no shard assignment. + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + nodes.add(replicaNode); + + // index a doc. + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", randomInt()).get(); + refresh(INDEX_NAME); + + CountDownLatch latch = new CountDownLatch(1); + // block replication + try (final Releasable ignored = blockReplication(List.of(replicaNode), latch)) { + // update to add replica, initiating recovery, this will get stuck at last step + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + ); + ensureYellow(INDEX_NAME); + // index another doc while blocked, this would not get replicated to replica. + client().prepareIndex(INDEX_NAME).setId("2").setSource("foo2", randomInt()).get(); + refresh(INDEX_NAME); + } + ensureGreen(INDEX_NAME); + waitForSearchableDocs(2, nodes); + } + + public void testIndexWhileRecoveringReplica() throws Exception { + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + assertAcked( + prepareCreate(INDEX_NAME).setMapping( + jsonBuilder().startObject() + .startObject("_routing") + .field("required", true) + .endObject() + .startObject("properties") + .startObject("online") + .field("type", "boolean") + .endObject() + .startObject("ts") + .field("type", "date") + .field("ignore_malformed", false) + .field("format", "epoch_millis") + .endObject() + .startObject("bs") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + ) + ); + ensureYellow(INDEX_NAME); + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + + client().prepareIndex(INDEX_NAME) + .setId("1") + .setRouting("Y") + .setSource("online", false, "bs", "Y", "ts", System.currentTimeMillis() - 100, "type", "s") + .get(); + client().prepareIndex(INDEX_NAME) + .setId("2") + .setRouting("X") + .setSource("online", true, "bs", "X", "ts", System.currentTimeMillis() - 10000000, "type", "s") + .get(); + client().prepareIndex(INDEX_NAME) + .setId("3") + .setRouting(randomAlphaOfLength(2)) + .setSource("online", false, "ts", System.currentTimeMillis() - 100, "type", "bs") + .get(); + client().prepareIndex(INDEX_NAME) + .setId("4") + .setRouting(randomAlphaOfLength(2)) + .setSource("online", true, "ts", System.currentTimeMillis() - 123123, "type", "bs") + .get(); + refresh(); + ensureGreen(INDEX_NAME); + waitForSearchableDocs(4, primaryNode, replicaNode); + + SearchResponse response = client().prepareSearch(INDEX_NAME) + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) + .setQuery( + boolQuery().must(termQuery("online", true)) + .must( + boolQuery().should( + boolQuery().must(rangeQuery("ts").lt(System.currentTimeMillis() - (15 * 1000))).must(termQuery("type", "bs")) + ) + .should( + boolQuery().must(rangeQuery("ts").lt(System.currentTimeMillis() - (15 * 1000))).must(termQuery("type", "s")) + ) + ) + ) + .setVersion(true) + .setFrom(0) + .setSize(100) + .setExplain(true) + .get(); + assertNoFailures(response); + } + + /** + * Tests whether segment replication supports realtime get requests and reads and parses source from the translog to serve strong reads. + */ + public void testRealtimeGetRequestsSuccessful() { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + // refresh interval disabled to ensure refresh rate of index (when data is ready for search) doesn't affect realtime get + assertAcked( + prepareCreate(INDEX_NAME).setSettings(Settings.builder().put("index.refresh_interval", -1).put(indexSettings())) + .addAlias(new Alias("alias")) + ); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + final String id = routingKeyForShard(INDEX_NAME, 0); + + GetResponse response = client(replica).prepareGet(indexOrAlias(), "1").get(); + assertFalse(response.isExists()); + + // index doc 1 + client().prepareIndex(indexOrAlias()).setId("1").setSource("foo", "bar").get(); + + // non realtime get 1 + response = client().prepareGet(indexOrAlias(), "1").setRealtime(false).get(); + assertFalse(response.isExists()); + + // realtime get 1 + response = client(replica).prepareGet(indexOrAlias(), "1").get(); + assertTrue(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + assertThat(response.getSourceAsMap().get("foo").toString(), equalTo("bar")); + + // index doc 2 + client().prepareIndex(indexOrAlias()).setId("2").setSource("foo2", "bar2").setRouting(id).get(); + + // realtime get 2 (with routing) + response = client(replica).prepareGet(indexOrAlias(), "2").setRouting(id).get(); + assertTrue(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + assertThat(response.getSourceAsMap().get("foo2").toString(), equalTo("bar2")); + } + + public void testRealtimeGetRequestsUnsuccessful() { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + assertAcked( + prepareCreate(INDEX_NAME).setSettings( + Settings.builder().put("index.refresh_interval", -1).put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + ).addAlias(new Alias("alias")) + ); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + final String id = routingKeyForShard(INDEX_NAME, 0); + final String routingOtherShard = routingKeyForShard(INDEX_NAME, 1); + + // index doc 1 + client().prepareIndex(indexOrAlias()).setId("1").setSource("foo", "bar").setRouting(id).get(); + + // non realtime get 1 + GetResponse response = client().prepareGet(indexOrAlias(), "1").setRealtime(false).get(); + assertFalse(response.isExists()); + + // realtime get 1 (preference = _replica) + response = client(replica).prepareGet(indexOrAlias(), "1").setPreference(Preference.REPLICA.type()).get(); + assertFalse(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + + // realtime get 1 (with routing set) + response = client(replica).prepareGet(INDEX_NAME, "1").setRouting(routingOtherShard).get(); + assertFalse(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + } + + /** + * Tests whether segment replication supports realtime MultiGet requests and reads and parses source from the translog to serve strong reads. + */ + public void testRealtimeMultiGetRequestsSuccessful() { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + // refresh interval disabled to ensure refresh rate of index (when data is ready for search) doesn't affect realtime multi get + assertAcked( + prepareCreate(INDEX_NAME).setSettings( + Settings.builder().put("index.refresh_interval", -1).put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + ).addAlias(new Alias("alias")) + ); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + final String id = routingKeyForShard(INDEX_NAME, 0); + + // index doc 1 + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").get(); + + // index doc 2 + client().prepareIndex(INDEX_NAME).setId("2").setSource("foo2", "bar2").setRouting(id).get(); + + // multi get non realtime 1 + MultiGetResponse mgetResponse = client().prepareMultiGet() + .add(new MultiGetRequest.Item(INDEX_NAME, "1")) + .add(new MultiGetRequest.Item("nonExistingIndex", "1")) + .setRealtime(false) + .get(); + assertThat(mgetResponse.getResponses().length, is(2)); + + assertThat(mgetResponse.getResponses()[0].getIndex(), is(INDEX_NAME)); + assertFalse(mgetResponse.getResponses()[0].isFailed()); + assertFalse(mgetResponse.getResponses()[0].getResponse().isExists()); + + // multi get realtime 1 + mgetResponse = client(replica).prepareMultiGet() + .add(new MultiGetRequest.Item(INDEX_NAME, "1")) + .add(new MultiGetRequest.Item(INDEX_NAME, "2").routing(id)) + .add(new MultiGetRequest.Item("nonExistingIndex", "1")) + .get(); + + assertThat(mgetResponse.getResponses().length, is(3)); + assertThat(mgetResponse.getResponses()[0].getIndex(), is(INDEX_NAME)); + assertFalse(mgetResponse.getResponses()[0].isFailed()); + assertThat(mgetResponse.getResponses()[0].getResponse().getSourceAsMap().get("foo").toString(), equalTo("bar")); + + assertThat(mgetResponse.getResponses()[1].getIndex(), is(INDEX_NAME)); + assertFalse(mgetResponse.getResponses()[1].isFailed()); + assertThat(mgetResponse.getResponses()[1].getResponse().getSourceAsMap().get("foo2").toString(), equalTo("bar2")); + + assertThat(mgetResponse.getResponses()[2].getIndex(), is("nonExistingIndex")); + assertTrue(mgetResponse.getResponses()[2].isFailed()); + assertThat(mgetResponse.getResponses()[2].getFailure().getMessage(), is("no such index [nonExistingIndex]")); + } + + public void testRealtimeMultiGetRequestsUnsuccessful() { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + assertAcked( + prepareCreate(INDEX_NAME).setSettings( + Settings.builder().put("index.refresh_interval", -1).put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + ).addAlias(new Alias("alias")) + ); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + + final String id = routingKeyForShard(INDEX_NAME, 0); + final String routingOtherShard = routingKeyForShard(INDEX_NAME, 1); + + // index doc 1 + client().prepareIndex(indexOrAlias()).setId("1").setSource("foo", "bar").setRouting(id).get(); + + // realtime multi get 1 (preference = _replica) + MultiGetResponse mgetResponse = client(replica).prepareMultiGet() + .add(new MultiGetRequest.Item(INDEX_NAME, "1")) + .setPreference(Preference.REPLICA.type()) + .add(new MultiGetRequest.Item("nonExistingIndex", "1")) + .get(); + assertThat(mgetResponse.getResponses().length, is(2)); + assertThat(mgetResponse.getResponses()[0].getIndex(), is(INDEX_NAME)); + assertFalse(mgetResponse.getResponses()[0].getResponse().isExists()); + + assertThat(mgetResponse.getResponses()[1].getIndex(), is("nonExistingIndex")); + assertTrue(mgetResponse.getResponses()[1].isFailed()); + + // realtime multi get 1 (routing set) + mgetResponse = client(replica).prepareMultiGet() + .add(new MultiGetRequest.Item(INDEX_NAME, "1").routing(routingOtherShard)) + .add(new MultiGetRequest.Item("nonExistingIndex", "1")) + .get(); + assertThat(mgetResponse.getResponses().length, is(2)); + assertThat(mgetResponse.getResponses()[0].getIndex(), is(INDEX_NAME)); + // expecting failure since we explicitly route request to a shard on which it doesn't exist + assertFalse(mgetResponse.getResponses()[0].getResponse().isExists()); + assertThat(mgetResponse.getResponses()[1].getIndex(), is("nonExistingIndex")); + assertTrue(mgetResponse.getResponses()[1].isFailed()); + + } + + /** + * Tests whether segment replication supports realtime termvector requests and reads and parses source from the translog to serve strong reads. + */ + public void testRealtimeTermVectorRequestsSuccessful() throws IOException { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + XContentBuilder mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", "text") + .field("term_vector", "with_positions_offsets_payloads") + .field("analyzer", "tv_test") + .endObject() + .endObject() + .endObject(); + // refresh interval disabled to ensure refresh rate of index (when data is ready for search) doesn't affect realtime termvectors + assertAcked( + prepareCreate(INDEX_NAME).setMapping(mapping) + .addAlias(new Alias("alias")) + .setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.analysis.analyzer.tv_test.tokenizer", "standard") + .put("index.refresh_interval", -1) + .putList("index.analysis.analyzer.tv_test.filter", "lowercase") + ) + ); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + final String id = routingKeyForShard(INDEX_NAME, 0); + + TermVectorsResponse response = client(replica).prepareTermVectors(indexOrAlias(), "1").get(); + assertFalse(response.isExists()); + + // index doc 1 + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(1)) + .setSource(jsonBuilder().startObject().field("field", "the quick brown fox jumps over the lazy dog").endObject()) + .execute() + .actionGet(); + + // non realtime termvectors 1 + response = client().prepareTermVectors(indexOrAlias(), Integer.toString(1)).setRealtime(false).get(); + assertFalse(response.isExists()); + + // realtime termvectors 1 + TermVectorsRequestBuilder resp = client().prepareTermVectors(indexOrAlias(), Integer.toString(1)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setRealtime(true) + .setSelectedFields(); + response = resp.execute().actionGet(); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + assertThat("doc id: " + 1 + " doesn't exists but should", response.isExists(), equalTo(true)); + Fields fields = response.getFields(); + assertThat(fields.size(), equalTo(1)); + + // index doc 2 with routing + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(2)) + .setRouting(id) + .setSource(jsonBuilder().startObject().field("field", "the quick brown fox jumps over the lazy dog").endObject()) + .execute() + .actionGet(); + + // realtime termvectors 2 with routing + resp = client().prepareTermVectors(indexOrAlias(), Integer.toString(2)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setRouting(id) + .setSelectedFields(); + response = resp.execute().actionGet(); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + assertThat("doc id: " + 1 + " doesn't exists but should", response.isExists(), equalTo(true)); + fields = response.getFields(); + assertThat(fields.size(), equalTo(1)); + + } + + public void testRealtimeTermVectorRequestsUnSuccessful() throws IOException { + final String primary = internalCluster().startDataAndSearchNodes(1).get(0); + XContentBuilder mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", "text") + .field("term_vector", "with_positions_offsets_payloads") + .field("analyzer", "tv_test") + .endObject() + .endObject() + .endObject(); + // refresh interval disabled to ensure refresh rate of index (when data is ready for search) doesn't affect realtime termvectors + assertAcked( + prepareCreate(INDEX_NAME).setMapping(mapping) + .addAlias(new Alias("alias")) + .setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.analysis.analyzer.tv_test.tokenizer", "standard") + .put("index.refresh_interval", -1) + .putList("index.analysis.analyzer.tv_test.filter", "lowercase") + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + ) + ); + final String replica = internalCluster().startDataAndSearchNodes(1).get(0); + ensureGreen(INDEX_NAME); + final String id = routingKeyForShard(INDEX_NAME, 0); + final String routingOtherShard = routingKeyForShard(INDEX_NAME, 1); + + // index doc 1 + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(1)) + .setSource(jsonBuilder().startObject().field("field", "the quick brown fox jumps over the lazy dog").endObject()) + .setRouting(id) + .execute() + .actionGet(); + + // non realtime termvectors 1 + TermVectorsResponse response = client().prepareTermVectors(indexOrAlias(), Integer.toString(1)).setRealtime(false).get(); + assertFalse(response.isExists()); + + // realtime termvectors (preference = _replica) + TermVectorsRequestBuilder resp = client(replica).prepareTermVectors(indexOrAlias(), Integer.toString(1)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setPreference(Preference.REPLICA.type()) + .setRealtime(true) + .setSelectedFields(); + response = resp.execute().actionGet(); + + assertFalse(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + + // realtime termvectors (with routing set) + resp = client(replica).prepareTermVectors(indexOrAlias(), Integer.toString(1)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setRouting(routingOtherShard) + .setSelectedFields(); + response = resp.execute().actionGet(); + + assertFalse(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + + } + + public void testReplicaAlreadyAtCheckpoint() throws Exception { + final List nodes = new ArrayList<>(); + final String primaryNode = internalCluster().startDataAndSearchNodes(1).get(0); + nodes.add(primaryNode); + final Settings settings = Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); + createIndex(INDEX_NAME, settings); + ensureGreen(INDEX_NAME); + // start a replica node, initially will be empty with no shard assignment. + final String replicaNode = internalCluster().startDataAndSearchNodes(1).get(0); + nodes.add(replicaNode); + final String replicaNode2 = internalCluster().startDataAndSearchNodes(1).get(0); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)) + ); + ensureGreen(INDEX_NAME); + + // index a doc. + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", randomInt()).get(); + refresh(INDEX_NAME); + waitForSearchableDocs(1, primaryNode, replicaNode, replicaNode2); + + FileCache fileCache = internalCluster().getInstance(Node.class, primaryNode).fileCache(); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + IndexShard replica_1 = getIndexShard(replicaNode, INDEX_NAME); + IndexShard replica_2 = getIndexShard(replicaNode2, INDEX_NAME); + // wait until a replica is promoted & finishes engine flip, we don't care which one + AtomicReference primary = new AtomicReference<>(); + assertBusy(() -> { + assertTrue("replica should be promoted as a primary", replica_1.routingEntry().primary() || replica_2.routingEntry().primary()); + primary.set(replica_1.routingEntry().primary() ? replica_1 : replica_2); + }); + + FlushRequest request = new FlushRequest(INDEX_NAME); + request.force(true); + primary.get().flush(request); + + assertBusy(() -> { + assertEquals( + replica_1.getLatestReplicationCheckpoint().getSegmentInfosVersion(), + replica_2.getLatestReplicationCheckpoint().getSegmentInfosVersion() + ); + }); + + assertBusy(() -> { + ClusterStatsResponse clusterStatsResponse = client().admin().cluster().prepareClusterStats().get(); + ReplicationStats replicationStats = clusterStatsResponse.getIndicesStats().getSegments().getReplicationStats(); + assertEquals(0L, replicationStats.maxBytesBehind); + assertEquals(0L, replicationStats.maxReplicationLag); + assertEquals(0L, replicationStats.totalBytesBehind); + }); + fileCache.prune(); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index 23864c35ad154..8e8f008158b9c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -16,8 +16,6 @@ import java.nio.file.Path; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; - /** * This class runs Segment Replication Integ test suite with remote store enabled. */ diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java index e673c1409a869..6bcbf303adb94 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java @@ -242,7 +242,8 @@ private IndexMetadata.Builder updateInSyncAllocations( allocationId = RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID; } else { assert (recoverySource instanceof RecoverySource.SnapshotRecoverySource - || recoverySource instanceof RecoverySource.RemoteStoreRecoverySource) : recoverySource; + || recoverySource instanceof RecoverySource.RemoteStoreRecoverySource + || recoverySource instanceof RecoverySource.ExistingStoreRecoverySource) : recoverySource; allocationId = updates.initializedPrimary.allocationId().getId(); } // forcing a stale primary resets the in-sync allocations to the singleton set with the stale id diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java index 7999faece52ca..e0543b396728e 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/RemoteShardsBalancer.java @@ -33,6 +33,8 @@ import java.util.Queue; import java.util.Set; +import static org.opensearch.action.admin.indices.tiering.TieringUtils.isPartialIndex; + /** * A {@link RemoteShardsBalancer} used by the {@link BalancedShardsAllocator} to perform allocation operations * for remote shards within the cluster. @@ -345,7 +347,8 @@ private void unassignIgnoredRemoteShards(RoutingAllocation routingAllocation) { // Remote shards do not have an existing store to recover from and can be recovered from an empty source // to re-fetch any shard blocks from the repository. if (shard.primary()) { - if (RecoverySource.Type.SNAPSHOT.equals(shard.recoverySource().getType()) == false) { + if (RecoverySource.Type.SNAPSHOT.equals(shard.recoverySource().getType()) == false + && isPartialIndex(allocation.metadata().getIndexSafe(shard.index())) == false) { unassignedShard = shard.updateUnassigned(shard.unassignedInfo(), RecoverySource.EmptyStoreRecoverySource.INSTANCE); } } diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index d759423ce5a55..7f3010ff0937a 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -368,6 +368,11 @@ public boolean shouldPeriodicallyFlush() { @Override public void flush(boolean force, boolean waitIfOngoing) throws EngineException { ensureOpen(); + // Skip flushing for indices with partial locality (warm indices) + // For these indices, we don't need to commit as we will sync from the remote store on re-open + if (engineConfig.getIndexSettings().isStoreLocalityPartial()) { + return; + } // readLock is held here to wait/block any concurrent close that acquires the writeLock. try (final ReleasableLock lock = readLock.acquire()) { ensureOpen(); @@ -442,7 +447,9 @@ protected final void closeNoLock(String reason, CountDownLatch closedLatch) { latestSegmentInfos.changed(); } try { - commitSegmentInfos(latestSegmentInfos); + if (engineConfig.getIndexSettings().isStoreLocalityPartial() == false) { + commitSegmentInfos(latestSegmentInfos); + } } catch (IOException e) { // mark the store corrupted unless we are closing as result of engine failure. // in this case Engine#failShard will handle store corruption. diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index bd47a664b729d..173268997895e 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -5142,7 +5142,9 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runn } else { storeDirectory = store.directory(); } - copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal, onFileSync); + if (indexSettings.isStoreLocalityPartial() == false) { + copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal, onFileSync); + } if (remoteSegmentMetadata != null) { final SegmentInfos infosSnapshot = store.buildSegmentInfos( @@ -5158,7 +5160,7 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runn } } assert Arrays.stream(store.directory().listAll()).filter(f -> f.startsWith(IndexFileNames.SEGMENTS)).findAny().isEmpty() - : "There should not be any segments file in the dir"; + || indexSettings.isStoreLocalityPartial() : "There should not be any segments file in the dir"; store.commitSegmentInfos(infosSnapshot, processedLocalCheckpoint, processedLocalCheckpoint); } syncSegmentSuccess = true; diff --git a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java index 1c9a8b98fb86e..4733fbcdb0494 100644 --- a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java @@ -34,8 +34,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.lucene.index.IndexFileNames.SEGMENTS; /** * Composite Directory will contain both local and remote directory @@ -74,12 +78,37 @@ public CompositeDirectory(Directory localDirectory, Directory remoteDirectory, F ); } + /** + * Returns names of all files stored in local directory + * @throws IOException in case of I/O error + */ + private String[] listLocalFiles() throws IOException { + ensureOpen(); + logger.trace("Composite Directory[{}]: listLocalOnly() called", this::toString); + return localDirectory.listAll(); + } + + /** + * Returns a list of names of all block files stored in the local directory for a given file, + * including the original file itself if present. + * + * @param fileName The name of the file to search for, along with its associated block files. + * @return A list of file names, including the original file (if present) and all its block files. + * @throws IOException in case of I/O error while listing files. + */ + private List listBlockFiles(String fileName) throws IOException { + return Stream.of(listLocalFiles()) + .filter(file -> file.equals(fileName) || file.startsWith(fileName + FileTypeUtils.BLOCK_FILE_IDENTIFIER)) + .collect(Collectors.toList()); + } + /** * Returns names of all files stored in this directory in sorted order * Does not include locally stored block files (having _block_ in their names) and files pending deletion * * @throws IOException in case of I/O error */ + // TODO: https://github.com/opensearch-project/OpenSearch/issues/17527 @Override public String[] listAll() throws IOException { ensureOpen(); @@ -105,6 +134,7 @@ public String[] listAll() throws IOException { * Currently deleting only from local directory as files from remote should not be deleted as that is taken care by garbage collection logic of remote directory * @param name the name of an existing file. * @throws IOException in case of I/O error + * @throws NoSuchFileException when file does not exist in the directory */ @Override public void deleteFile(String name) throws IOException { @@ -115,7 +145,21 @@ public void deleteFile(String name) throws IOException { } else if (Arrays.asList(listAll()).contains(name) == false) { throw new NoSuchFileException("File " + name + " not found in directory"); } else { - fileCache.remove(getFilePath(name)); + List blockFiles = listBlockFiles(name); + if (blockFiles.isEmpty()) { + // Remove this condition when this issue is addressed. + // TODO: https://github.com/opensearch-project/OpenSearch/issues/17526 + logger.debug("The file [{}] or its block files do not exist in local directory", name); + } else { + for (String blockFile : blockFiles) { + if (fileCache.get(getFilePath(blockFile)) == null) { + logger.debug("The file [{}] exists in local but not part of FileCache, deleting it from local", blockFile); + localDirectory.deleteFile(blockFile); + } else { + fileCache.remove(getFilePath(blockFile)); + } + } + } } } @@ -254,6 +298,15 @@ public IndexInput openInput(String name, IOContext context) throws IOException { public void close() throws IOException { ensureOpen(); logger.trace("Composite Directory[{}]: close() called", this::toString); + String[] localFiles = listLocalFiles(); + for (String localFile : localFiles) { + // Delete segments_N file with ref count 1 created during index creation on replica shards + // TODO: https://github.com/opensearch-project/OpenSearch/issues/17534 + if (localFile.startsWith(SEGMENTS)) { + fileCache.remove(getFilePath(localFile)); + } + } + fileCache.prune(); localDirectory.close(); } diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java b/server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java index e78480bd500ee..ca0e6652f5ea4 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/FileTypeUtils.java @@ -18,12 +18,14 @@ @ExperimentalApi public class FileTypeUtils { + public static String BLOCK_FILE_IDENTIFIER = "_block_"; + public static boolean isTempFile(String name) { return name.endsWith(".tmp"); } public static boolean isBlockFile(String name) { - return name.contains("_block_"); + return name.contains(BLOCK_FILE_IDENTIFIER); } public static boolean isExtraFSFile(String name) { diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java index 7f7d42e8fbce8..6e30a39e27bb1 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/LRUCache.java @@ -310,13 +310,20 @@ public CacheStats stats() { public void logCurrentState() { lock.lock(); try { - String allFiles = "\n"; + final StringBuilder allFiles = new StringBuilder("\n"); for (Map.Entry> entry : data.entrySet()) { String path = entry.getKey().toString(); String file = path.substring(path.lastIndexOf('/')); - allFiles += file + " [RefCount: " + entry.getValue().refCount + " , Weight: " + entry.getValue().weight + " ]\n"; + allFiles.append(file) + .append(" [RefCount: ") + .append(entry.getValue().refCount) + .append(" , Weight: ") + .append(entry.getValue().weight) + .append(" ]\n"); + } + if (allFiles.length() > 1) { + logger.trace(() -> "Cache entries : " + allFiles); } - logger.trace("Cache entries : " + allFiles); } finally { lock.unlock(); } diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java index 9ff6ddb1fb667..ecf9bb2ead0d2 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/cache/SegmentedCache.java @@ -190,8 +190,11 @@ public CacheStats stats() { public void logCurrentState() { int i = 0; for (RefCountedCache cache : table) { - logger.trace("SegmentedCache " + i); - ((LRUCache) cache).logCurrentState(); + if (cache.size() > 0) { + final int segmentIndex = i; + logger.trace(() -> "SegmentedCache " + segmentIndex); + ((LRUCache) cache).logCurrentState(); + } i++; } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 64bd73ebb4611..6922ade22b714 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Set; @@ -202,6 +203,12 @@ public void startReplication(ActionListener listener, BiConsumer getFiles(CheckpointInfoResponse checkpointInfo) throws IOException { cancellableThreads.checkForCancel(); state.setStage(SegmentReplicationState.Stage.FILE_DIFF); + + // Return an empty list for warm indices, In this case, replica shards don't require downloading files from remote storage + // as replicas will sync all files from remote in case of failure. + if (indexShard.indexSettings().isStoreLocalityPartial()) { + return Collections.emptyList(); + } final Store.RecoveryDiff diff = Store.segmentReplicationDiff(checkpointInfo.getMetadataMap(), indexShard.getSegmentMetadataMap()); // local files final Set localFiles = Set.of(indexShard.store().directory().listAll()); diff --git a/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java index d5628cfab9ee7..43340be680a86 100644 --- a/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java @@ -39,8 +39,16 @@ public class CompositeDirectoryTests extends BaseRemoteSegmentStoreDirectoryTest private FSDirectory localDirectory; private CompositeDirectory compositeDirectory; - private final static String[] LOCAL_FILES = new String[] { "_1.cfe", "_2.cfe", "_0.cfe_block_7", "_0.cfs_block_7", "temp_file.tmp" }; + private final static String[] LOCAL_FILES = new String[] { + "_1.cfe", + "_1.cfe_block_0", + "_1.cfe_block_1", + "_2.cfe", + "_0.cfe_block_7", + "_0.cfs_block_7", + "temp_file.tmp" }; private final static String FILE_PRESENT_LOCALLY = "_1.cfe"; + private final static String BLOCK_FILE_PRESENT_LOCALLY = "_1.cfe_block_0"; private final static String FILE_PRESENT_IN_REMOTE_ONLY = "_0.si"; private final static String NON_EXISTENT_FILE = "non_existent_file"; private final static String NEW_FILE = "new_file"; @@ -67,9 +75,11 @@ public void testListAll() throws IOException { public void testDeleteFile() throws IOException { assertTrue(existsInCompositeDirectory(FILE_PRESENT_LOCALLY)); + assertTrue(existsInLocalDirectory(BLOCK_FILE_PRESENT_LOCALLY)); // Delete the file and assert that it no more is a part of the directory compositeDirectory.deleteFile(FILE_PRESENT_LOCALLY); assertFalse(existsInCompositeDirectory(FILE_PRESENT_LOCALLY)); + assertFalse(existsInCompositeDirectory(BLOCK_FILE_PRESENT_LOCALLY)); // Reading deleted file from directory should result in NoSuchFileException assertThrows(NoSuchFileException.class, () -> compositeDirectory.openInput(FILE_PRESENT_LOCALLY, IOContext.DEFAULT)); } From 588f46d731587bebd54511dc2df21a2f4ffb9f32 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 7 Mar 2025 11:28:47 +0530 Subject: [PATCH 04/10] Change IOContext from READONCE to DEFAULT to avoid WrongThreadException (#17502) --------- Signed-off-by: Sachin Kale --- .../index/shard/RemoteStoreRefreshListener.java | 2 +- .../opensearch/index/store/RemoteDirectory.java | 1 + .../index/store/RemoteDirectoryTests.java | 4 ++-- .../test/store/MockFSDirectoryFactory.java | 16 ++++++++++++++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 702928771f131..8ace4848806d7 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -459,7 +459,7 @@ private void uploadNewSegments( batchUploadListener.onFailure(ex); }); statsListener.beforeUpload(src); - remoteDirectory.copyFrom(storeDirectory, src, IOContext.READONCE, aggregatedListener, isLowPriorityUpload()); + remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener, isLowPriorityUpload()); } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index 949113cce87d6..38d1b4d1252cc 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -383,6 +383,7 @@ private void uploadBlob( ActionListener listener, boolean lowPriorityUpload ) throws Exception { + assert ioContext != IOContext.READONCE : "Remote upload will fail with IoContext.READONCE"; long expectedChecksum = calculateChecksumOfChecksum(from, src); long contentLength; try (IndexInput indexInput = from.openInput(src, ioContext)) { diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java index cc8c6085978a8..ea6e6e538caa5 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java @@ -92,7 +92,7 @@ public void testCopyFrom() throws IOException, InterruptedException { storeDirectory, filename, filename, - IOContext.READONCE, + IOContext.DEFAULT, () -> postUploadInvoked.set(true), new ActionListener<>() { @Override @@ -130,7 +130,7 @@ public void testCopyFromWithException() throws IOException, InterruptedException storeDirectory, filename, filename, - IOContext.READONCE, + IOContext.DEFAULT, () -> postUploadInvoked.set(true), new ActionListener<>() { @Override diff --git a/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java b/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java index 9c42ea2672601..d8279170ddd92 100644 --- a/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java +++ b/test/framework/src/main/java/org/opensearch/test/store/MockFSDirectoryFactory.java @@ -36,7 +36,10 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.CheckIndex; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.tests.store.BaseDirectoryWrapper; import org.apache.lucene.tests.store.MockDirectoryWrapper; @@ -203,6 +206,19 @@ public synchronized void crash() throws IOException { public Set getPendingDeletions() throws IOException { return in.getPendingDeletions(); } + + // In remote store feature, the upload flow is async and IndexInput can be opened and closed + // by different threads, so we always use IOContext.DEFAULT. + // But MockDirectoryWrapper throws an exception if segments_N fil is opened with any IOContext other than READONCE. + // Following change is temporary override to avoid the test failures. We should fix the multiple thread access + // in remote store upload flow. + @Override + public synchronized IndexInput openInput(String name, IOContext context) throws IOException { + if (name.startsWith(IndexFileNames.SEGMENTS)) { + context = IOContext.READONCE; + } + return super.openInput(name, context); + } } static final class CloseableDirectory implements Closeable { From c48efd0fd59e598c983b59d952f41f7b69341812 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Fri, 7 Mar 2025 12:50:03 +0530 Subject: [PATCH 05/10] Fix flaky tests in RemoteStoreStatsIT (#17535) Signed-off-by: Ashish Singh --- .../remotestore/RemoteStoreStatsIT.java | 241 +++++++++--------- .../remote/RemoteSegmentTransferTracker.java | 48 ++++ 2 files changed, 166 insertions(+), 123 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 86d586cd17146..b8986f5a3bf9a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -21,6 +21,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; +import org.opensearch.common.SetOnce; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.remote.RemoteSegmentTransferTracker; @@ -261,92 +262,59 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce // - Assert that download stats == upload stats // - Repeat this step for random times (between 5 and 10) - // Create index with 1 pri and 1 replica and refresh interval disabled - createIndex( - INDEX_NAME, - Settings.builder().put(remoteStoreIndexSettings(1, 1)).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1).build() - ); - ensureGreen(INDEX_NAME); - - // Manually invoke a refresh - refresh(INDEX_NAME); - - // Get zero state values - // Extract and assert zero state primary stats - RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - RemoteSegmentTransferTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) - .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) - .collect(Collectors.toList()) - .get(0) - .getSegmentStats(); - logger.info( - "Zero state primary stats: {}ms refresh time lag, {}b bytes lag, {}b upload bytes started, {}b upload bytes failed , {} uploads succeeded, {} upload byes succeeded.", - zeroStatePrimaryStats.refreshTimeLagMs, - zeroStatePrimaryStats.bytesLag, - zeroStatePrimaryStats.uploadBytesStarted, - zeroStatePrimaryStats.uploadBytesFailed, - zeroStatePrimaryStats.totalUploadsSucceeded, - zeroStatePrimaryStats.uploadBytesSucceeded - ); - assertTrue( - zeroStatePrimaryStats.totalUploadsStarted == zeroStatePrimaryStats.totalUploadsSucceeded - && zeroStatePrimaryStats.totalUploadsSucceeded == 1 - ); - assertTrue( - zeroStatePrimaryStats.uploadBytesStarted == zeroStatePrimaryStats.uploadBytesSucceeded - && zeroStatePrimaryStats.uploadBytesSucceeded > 0 - ); - assertTrue(zeroStatePrimaryStats.totalUploadsFailed == 0 && zeroStatePrimaryStats.uploadBytesFailed == 0); + // Prepare settings with single replica + Settings.Builder settings = Settings.builder() + .put(remoteStoreIndexSettings(1, 1)) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1); - // Extract and assert zero state replica stats - RemoteSegmentTransferTracker.Stats zeroStateReplicaStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) - .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) - .collect(Collectors.toList()) - .get(0) - .getSegmentStats(); - assertTrue( - zeroStateReplicaStats.directoryFileTransferTrackerStats.transferredBytesStarted == 0 - && zeroStateReplicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded == 0 - ); + // Retrieve zero state stats + SetOnce zeroStatePrimaryStats = prepareZeroStateStats(settings, false); - // Index documents + // Iteration logic for (int i = 1; i <= randomIntBetween(5, 10); i++) { indexSingleDoc(INDEX_NAME); - // Running Flush & Refresh manually flushAndRefresh(INDEX_NAME); ensureGreen(INDEX_NAME); + waitForReplication(); - // Poll for RemoteStore Stats assertBusy(() -> { RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - // Iterate through the response and extract the relevant segment upload and download stats + + // Existing validation logic List primaryStatsList = Arrays.stream(response.getRemoteStoreStats()) .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) - .collect(Collectors.toList()); + .toList(); assertEquals(1, primaryStatsList.size()); + List replicaStatsList = Arrays.stream(response.getRemoteStoreStats()) .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) - .collect(Collectors.toList()); + .toList(); assertEquals(1, replicaStatsList.size()); - RemoteSegmentTransferTracker.Stats primaryStats = primaryStatsList.get(0).getSegmentStats(); - RemoteSegmentTransferTracker.Stats replicaStats = replicaStatsList.get(0).getSegmentStats(); - // Assert Upload syncs - zero state uploads == download syncs + + RemoteSegmentTransferTracker.Stats primaryStats = primaryStatsList.getFirst().getSegmentStats(); + RemoteSegmentTransferTracker.Stats replicaStats = replicaStatsList.getFirst().getSegmentStats(); + + // Existing assertions assertTrue(primaryStats.totalUploadsStarted > 0); assertTrue(primaryStats.totalUploadsSucceeded > 0); + assertTrue(replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted > 0); + assertTrue( - replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted > 0 - && primaryStats.uploadBytesStarted - - zeroStatePrimaryStats.uploadBytesStarted >= replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted + primaryStats.uploadBytesStarted - zeroStatePrimaryStats + .get().uploadBytesStarted >= replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted ); + + assertTrue(replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded > 0); + assertTrue( - replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded > 0 - && primaryStats.uploadBytesSucceeded - - zeroStatePrimaryStats.uploadBytesSucceeded >= replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded + primaryStats.uploadBytesSucceeded - zeroStatePrimaryStats + .get().uploadBytesSucceeded >= replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded ); + // Assert zero failures assertEquals(0, primaryStats.uploadBytesFailed); assertEquals(0, replicaStats.directoryFileTransferTrackerStats.transferredBytesFailed); - }, 60, TimeUnit.SECONDS); + }); } } @@ -361,76 +329,42 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr // - Assert that download stats == upload stats // - Repeat this step for random times (between 5 and 10) - // Create index + // Get number of data nodes int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); - createIndex( - INDEX_NAME, - Settings.builder() - .put(remoteStoreIndexSettings(dataNodeCount - 1, 1)) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) - .build() - ); - ensureGreen(INDEX_NAME); - // Manually invoke a refresh - refresh(INDEX_NAME); - - // Get zero state values - // Extract and assert zero state primary stats - RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - RemoteSegmentTransferTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) - .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) - .collect(Collectors.toList()) - .get(0) - .getSegmentStats(); - logger.info( - "Zero state primary stats: {}ms refresh time lag, {}b bytes lag, {}b upload bytes started, {}b upload bytes failed , {} uploads succeeded, {} upload byes succeeded.", - zeroStatePrimaryStats.refreshTimeLagMs, - zeroStatePrimaryStats.bytesLag, - zeroStatePrimaryStats.uploadBytesStarted, - zeroStatePrimaryStats.uploadBytesFailed, - zeroStatePrimaryStats.totalUploadsSucceeded, - zeroStatePrimaryStats.uploadBytesSucceeded - ); - assertTrue( - zeroStatePrimaryStats.totalUploadsStarted == zeroStatePrimaryStats.totalUploadsSucceeded - && zeroStatePrimaryStats.totalUploadsSucceeded == 1 - ); - assertTrue( - zeroStatePrimaryStats.uploadBytesStarted == zeroStatePrimaryStats.uploadBytesSucceeded - && zeroStatePrimaryStats.uploadBytesSucceeded > 0 - ); - assertTrue(zeroStatePrimaryStats.totalUploadsFailed == 0 && zeroStatePrimaryStats.uploadBytesFailed == 0); + // Prepare settings with multiple replicas + Settings.Builder settings = Settings.builder() + .put(remoteStoreIndexSettings(dataNodeCount - 1, 1)) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1); - // Extract and assert zero state replica stats - List zeroStateReplicaStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) - .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) - .collect(Collectors.toList()); - zeroStateReplicaStats.forEach(stats -> { - assertTrue( - stats.getSegmentStats().directoryFileTransferTrackerStats.transferredBytesStarted == 0 - && stats.getSegmentStats().directoryFileTransferTrackerStats.transferredBytesSucceeded == 0 - ); - }); + // Retrieve zero state stats + SetOnce zeroStatePrimaryStats = prepareZeroStateStats(settings, true); + // Get current nodes in cluster int currentNodesInCluster = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); + + // Iteration logic for (int i = 0; i < randomIntBetween(5, 10); i++) { indexSingleDoc(INDEX_NAME); - // Running Flush & Refresh manually flushAndRefresh(INDEX_NAME); + ensureGreen(INDEX_NAME); + waitForReplication(); assertBusy(() -> { RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + + // Validate total and successful shards assertEquals(currentNodesInCluster, response.getSuccessfulShards()); - long uploadsStarted = 0, uploadsSucceeded = 0, uploadsFailed = 0; - long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; - List downloadBytesStarted = new ArrayList<>(), downloadBytesSucceeded = new ArrayList<>(), downloadBytesFailed = - new ArrayList<>(); - // Assert that stats for primary shard and replica shard set are equal - for (RemoteStoreStats eachStatsObject : response.getRemoteStoreStats()) { - RemoteSegmentTransferTracker.Stats stats = eachStatsObject.getSegmentStats(); - if (eachStatsObject.getShardRouting().primary()) { + long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; + List downloadBytesStarted = new ArrayList<>(); + List downloadBytesSucceeded = new ArrayList<>(); + List downloadBytesFailed = new ArrayList<>(); + + // Collect stats for primary and replica shards + for (RemoteStoreStats statsObject : response.getRemoteStoreStats()) { + RemoteSegmentTransferTracker.Stats stats = statsObject.getSegmentStats(); + if (statsObject.getShardRouting().primary()) { uploadBytesStarted = stats.uploadBytesStarted; uploadBytesSucceeded = stats.uploadBytesSucceeded; uploadBytesFailed = stats.uploadBytesFailed; @@ -441,17 +375,78 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr } } - assertEquals(0, uploadsFailed); + // Assertions assertEquals(0, uploadBytesFailed); for (int j = 0; j < response.getSuccessfulShards() - 1; j++) { - assertTrue(uploadBytesStarted - zeroStatePrimaryStats.uploadBytesStarted > downloadBytesStarted.get(j)); - assertTrue(uploadBytesSucceeded - zeroStatePrimaryStats.uploadBytesSucceeded > downloadBytesSucceeded.get(j)); + assertTrue(uploadBytesStarted - zeroStatePrimaryStats.get().uploadBytesStarted > downloadBytesStarted.get(j)); + assertTrue(uploadBytesSucceeded - zeroStatePrimaryStats.get().uploadBytesSucceeded > downloadBytesSucceeded.get(j)); assertEquals(0, (long) downloadBytesFailed.get(j)); } - }, 60, TimeUnit.SECONDS); + }); + } + } + + // New helper method to validate zero state primary stats + private void validateZeroStatePrimaryStats(RemoteSegmentTransferTracker.Stats primaryStats) { + logger.info("Zero state primary stats: {}", primaryStats); + assertEquals(primaryStats.totalUploadsStarted, primaryStats.totalUploadsSucceeded); + assertTrue(primaryStats.totalUploadsSucceeded >= 1); + assertEquals(primaryStats.uploadBytesStarted, primaryStats.uploadBytesSucceeded); + assertTrue(primaryStats.uploadBytesSucceeded > 0); + assertEquals(0, primaryStats.totalUploadsFailed); + assertEquals(0, primaryStats.uploadBytesFailed); + } + + // helper method to validate zero state replica stats + private void validateZeroStateReplicaStats(RemoteStoreStatsResponse zeroStateResponse, boolean multipleShardsExpected) { + List zeroStateReplicaStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) + .toList(); + + if (multipleShardsExpected) { + zeroStateReplicaStats.forEach(stats -> { + assertEquals(0, stats.getSegmentStats().directoryFileTransferTrackerStats.transferredBytesStarted); + assertEquals(0, stats.getSegmentStats().directoryFileTransferTrackerStats.transferredBytesSucceeded); + }); + } else { + RemoteSegmentTransferTracker.Stats replicaStats = zeroStateReplicaStats.getFirst().getSegmentStats(); + assertEquals(0, replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted); + assertEquals(0, replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded); } } + // New helper method for common test setup and zero state stats retrieval + private SetOnce prepareZeroStateStats( + Settings.Builder additionalSettings, + boolean multipleShardsExpected + ) throws Exception { + SetOnce zeroStatePrimaryStats = new SetOnce<>(); + + // Create index with specified settings + createIndex(INDEX_NAME, additionalSettings.build()); + ensureGreen(INDEX_NAME); + + // Manually invoke a refresh + refresh(INDEX_NAME); + + assertBusy(() -> { + RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + + RemoteSegmentTransferTracker.Stats primaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) + .toList() + .getFirst() + .getSegmentStats(); + + validateZeroStatePrimaryStats(primaryStats); + validateZeroStateReplicaStats(zeroStateResponse, multipleShardsExpected); + + zeroStatePrimaryStats.set(primaryStats); + }); + + return zeroStatePrimaryStats; + } + public void testStatsOnShardRelocation() { setup(); // Scenario: diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java index a29bd1d840b43..4114a14b455e7 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java @@ -580,5 +580,53 @@ public int hashCode() { directoryFileTransferTrackerStats ); } + + @Override + public String toString() { + return "Stats{" + + "shardId=" + + shardId + + ", localRefreshClockTimeMs=" + + localRefreshClockTimeMs + + ", remoteRefreshClockTimeMs=" + + remoteRefreshClockTimeMs + + ", refreshTimeLagMs=" + + refreshTimeLagMs + + ", localRefreshNumber=" + + localRefreshNumber + + ", remoteRefreshNumber=" + + remoteRefreshNumber + + ", uploadBytesStarted=" + + uploadBytesStarted + + ", uploadBytesFailed=" + + uploadBytesFailed + + ", uploadBytesSucceeded=" + + uploadBytesSucceeded + + ", totalUploadsStarted=" + + totalUploadsStarted + + ", totalUploadsFailed=" + + totalUploadsFailed + + ", totalUploadsSucceeded=" + + totalUploadsSucceeded + + ", rejectionCount=" + + rejectionCount + + ", consecutiveFailuresCount=" + + consecutiveFailuresCount + + ", lastSuccessfulRemoteRefreshBytes=" + + lastSuccessfulRemoteRefreshBytes + + ", uploadBytesMovingAverage=" + + uploadBytesMovingAverage + + ", uploadBytesPerSecMovingAverage=" + + uploadBytesPerSecMovingAverage + + ", totalUploadTimeInMs=" + + totalUploadTimeInMs + + ", uploadTimeMovingAverage=" + + uploadTimeMovingAverage + + ", bytesLag=" + + bytesLag + + ", directoryFileTransferTrackerStats=" + + directoryFileTransferTrackerStats + + '}'; + } } } From 6b4597275fee56b28891272c2c8a216768687ed0 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Fri, 7 Mar 2025 15:12:39 +0530 Subject: [PATCH 06/10] Fix compilation issue for PR #17535 during backport (#17546) Signed-off-by: Ashish Singh --- .../opensearch/remotestore/RemoteStoreStatsIT.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index b8986f5a3bf9a..4053ce5f6c678 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -283,16 +283,16 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce // Existing validation logic List primaryStatsList = Arrays.stream(response.getRemoteStoreStats()) .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) - .toList(); + .collect(Collectors.toList()); assertEquals(1, primaryStatsList.size()); List replicaStatsList = Arrays.stream(response.getRemoteStoreStats()) .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) - .toList(); + .collect(Collectors.toList()); assertEquals(1, replicaStatsList.size()); - RemoteSegmentTransferTracker.Stats primaryStats = primaryStatsList.getFirst().getSegmentStats(); - RemoteSegmentTransferTracker.Stats replicaStats = replicaStatsList.getFirst().getSegmentStats(); + RemoteSegmentTransferTracker.Stats primaryStats = primaryStatsList.get(0).getSegmentStats(); + RemoteSegmentTransferTracker.Stats replicaStats = replicaStatsList.get(0).getSegmentStats(); // Existing assertions assertTrue(primaryStats.totalUploadsStarted > 0); @@ -409,7 +409,7 @@ private void validateZeroStateReplicaStats(RemoteStoreStatsResponse zeroStateRes assertEquals(0, stats.getSegmentStats().directoryFileTransferTrackerStats.transferredBytesSucceeded); }); } else { - RemoteSegmentTransferTracker.Stats replicaStats = zeroStateReplicaStats.getFirst().getSegmentStats(); + RemoteSegmentTransferTracker.Stats replicaStats = zeroStateReplicaStats.get(0).getSegmentStats(); assertEquals(0, replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted); assertEquals(0, replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded); } @@ -434,8 +434,8 @@ private SetOnce prepareZeroStateStats( RemoteSegmentTransferTracker.Stats primaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) - .toList() - .getFirst() + .collect(Collectors.toList()) + .get(0) .getSegmentStats(); validateZeroStatePrimaryStats(primaryStats); From ffa46cadcc1f8bef7b247d731024de9a279701e0 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Fri, 7 Mar 2025 19:39:12 +0530 Subject: [PATCH 07/10] Add test for stale writer correctness with pinned timestamp snapshots (#17547) Signed-off-by: Ashish Singh --- .../index/remote/RemoteStoreUtilsTests.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java index d134a2bad7b74..1a9b9092ee12e 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -993,6 +993,37 @@ public void testGetPinnedTimestampLockedFilesWithPinnedTimestampsDifferentPrefix assertEquals(0, metadataFilePinnedTimestampCache.size()); } + /** + * This test checks the case when a stale writer is uploading metadata files with higher timestamp, but lower primary + * term. + */ + public void testGetPinnedTimestampLockedFilesForDivergentWrites() { + setupRemotePinnedTimestampFeature(true); + + Map metadataFilePinnedTimestampCache = new HashMap<>(); + + // Pinned timestamp 7000 + // Primary Term - Timestamp in md file + // 6 - 7002 + // 3 - 6999 + // 4 - 6998 + // 5 - 6995 + // 5 - 6990 + Tuple, Set> metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + Map.of(7002L, 6L, 6999L, 3L, 6998L, 4L, 6995L, 5L, 6990L, 5L), + Set.of(4000L, 5000L, 6000L, 7000L), + metadataFilePinnedTimestampCache + ); + Map metadataFiles = metadataAndLocks.v1(); + Set implicitLockedFiles = metadataAndLocks.v2(); + + assertEquals(1, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(6995L))); + // Now we cache all the matches except the last one. + assertEquals(1, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(6995L), metadataFilePinnedTimestampCache.get(7000L)); + } + public void testFilterOutMetadataFilesBasedOnAgeFeatureDisabled() { setupRemotePinnedTimestampFeature(false); List metadataFiles = new ArrayList<>(); From 105aeb5eaed26332cc11c4655c552a3ffe769e01 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 7 Mar 2025 15:59:38 -0800 Subject: [PATCH 08/10] [Tiered Caching] Remove PLUGGABLE_CACHE feature flag (#17344) * Remove PLUGGABLE_CACHE feature flag Signed-off-by: Peter Alfonsi * changelog Signed-off-by: Peter Alfonsi * move changelog entry Signed-off-by: Peter Alfonsi * rerun gradle Signed-off-by: Peter Alfonsi * rerun gradle Signed-off-by: Peter Alfonsi * fix IT init failure Signed-off-by: Peter Alfonsi * rerun gradle Signed-off-by: Peter Alfonsi * rerun gradle Signed-off-by: Peter Alfonsi * rerun gradle Signed-off-by: Peter Alfonsi * rerun gradle Signed-off-by: Peter Alfonsi * rerun gradle Signed-off-by: Peter Alfonsi --------- Signed-off-by: Peter Alfonsi Signed-off-by: Peter Alfonsi Co-authored-by: Peter Alfonsi --- CHANGELOG-3.0.md | 1 + distribution/src/config/opensearch.yml | 4 -- .../tier/TieredSpilloverCacheBaseIT.java | 2 - .../tier/TieredSpilloverCachePlugin.java | 5 +- .../tier/TieredSpilloverCachePluginTests.java | 7 +- .../tier/TieredSpilloverCacheTests.java | 13 ---- .../opensearch/cache/EhcacheDiskCacheIT.java | 6 -- .../store/disk/EhCacheDiskCacheTests.java | 2 - .../CacheStatsAPIIndicesRequestCacheIT.java | 17 +---- .../indices/IndicesRequestCacheIT.java | 5 +- .../common/cache/service/CacheService.java | 14 ++-- .../cache/store/OpenSearchOnHeapCache.java | 12 +--- .../common/settings/ClusterSettings.java | 21 +++--- .../common/settings/FeatureFlagSettings.java | 1 - .../opensearch/common/util/FeatureFlags.java | 9 --- .../indices/IndicesRequestCache.java | 6 +- .../cache/service/CacheServiceTests.java | 35 ++-------- .../store/OpenSearchOnHeapCacheTests.java | 54 +++++++-------- .../indices/IndicesRequestCacheTests.java | 67 ++----------------- 19 files changed, 61 insertions(+), 220 deletions(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 99b636822fb72..fabf6645fb6eb 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove package org.opensearch.action.support.master ([#4856](https://github.com/opensearch-project/OpenSearch/issues/4856)) - Remove transport-nio plugin ([#16887](https://github.com/opensearch-project/OpenSearch/issues/16887)) - Remove deprecated 'gateway' settings used to defer cluster recovery ([#3117](https://github.com/opensearch-project/OpenSearch/issues/3117)) +- Remove FeatureFlags.PLUGGABLE_CACHE as the feature is no longer experimental ([#17344](https://github.com/opensearch-project/OpenSearch/pull/17344)) ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 1ac7c5299964e..5b3babd3acaef 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -122,10 +122,6 @@ ${path.logs} # #opensearch.experimental.optimization.datetime_formatter_caching.enabled: false # -# Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting. -# -#opensearch.experimental.feature.pluggable.caching.enabled: false -# # Gates the functionality of star tree index, which improves the performance of search aggregations. # #opensearch.experimental.feature.composite_index.star_tree.enabled: true diff --git a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java index 01371ca8eeefb..75895b1fc4c11 100644 --- a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java +++ b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java @@ -12,14 +12,12 @@ import org.opensearch.common.cache.settings.CacheSettings; import org.opensearch.common.cache.store.OpenSearchOnHeapCache; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchIntegTestCase; public class TieredSpilloverCacheBaseIT extends OpenSearchIntegTestCase { public Settings defaultSettings(String onHeapCacheSizeInBytesOrPercentage, int numberOfSegments) { return Settings.builder() - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put( CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java index d1d033fae8cd2..3c4079fefd612 100644 --- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java +++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java @@ -12,7 +12,6 @@ import org.opensearch.common.cache.ICache; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.plugins.CachePlugin; import org.opensearch.plugins.Plugin; @@ -64,9 +63,7 @@ public List> getSettings() { ); settingList.add(TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(cacheType)); settingList.add(TOOK_TIME_DISK_TIER_POLICY_CONCRETE_SETTINGS_MAP.get(cacheType)); - if (FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings)) { - settingList.add(DISK_CACHE_ENABLED_SETTING_MAP.get(cacheType)); - } + settingList.add(DISK_CACHE_ENABLED_SETTING_MAP.get(cacheType)); settingList.add( TieredSpilloverCacheSettings.TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(cacheType.getSettingPrefix()) ); diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java index 4a96ffe2069ec..54aba3504f42f 100644 --- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java +++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java @@ -10,7 +10,6 @@ import org.opensearch.common.cache.ICache; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchTestCase; import java.util.Map; @@ -24,10 +23,8 @@ public void testGetCacheFactoryMap() { assertEquals(TieredSpilloverCachePlugin.TIERED_CACHE_SPILLOVER_PLUGIN_NAME, tieredSpilloverCachePlugin.getName()); } - public void testGetSettingsWithFeatureFlagOn() { - TieredSpilloverCachePlugin tieredSpilloverCachePlugin = new TieredSpilloverCachePlugin( - Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE_SETTING.getKey(), true).build() - ); + public void testGetSettings() { + TieredSpilloverCachePlugin tieredSpilloverCachePlugin = new TieredSpilloverCachePlugin(Settings.builder().build()); assertFalse(tieredSpilloverCachePlugin.getSettings().isEmpty()); } } diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java index c74eb371709f6..2dc115b73c378 100644 --- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java +++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java @@ -30,7 +30,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.env.NodeEnvironment; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -183,7 +182,6 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 1) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .build(); String storagePath = getStoragePath(settings); ICache tieredSpilloverICache = new TieredSpilloverCache.TieredSpilloverCacheFactory().create( @@ -283,7 +281,6 @@ public void testComputeIfAbsentWithSegmentedCache() throws Exception { CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .build(); String storagePath = getStoragePath(settings); ICache tieredSpilloverICache = new TieredSpilloverCache.TieredSpilloverCacheFactory().create( @@ -406,7 +403,6 @@ public void testWithFactoryCreationWithOnHeapCacheNotPresent() { CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .build(); IllegalArgumentException ex = assertThrows( @@ -491,7 +487,6 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put( TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() @@ -1276,7 +1271,6 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put( TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() @@ -2160,7 +2154,6 @@ public void testWithInvalidSegmentNumber() throws Exception { TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 1) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 3) .build(); String storagePath = getStoragePath(settings); @@ -2226,7 +2219,6 @@ public void testWithVeryLowDiskCacheSize() throws Exception { ).getKey(), 1L ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 2) .build(); String storagePath = getStoragePath(settings); @@ -2285,7 +2277,6 @@ public void testTieredCacheDefaultSegmentCount() { CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .build(); String storagePath = getStoragePath(settings); @@ -2419,7 +2410,6 @@ public void testSegmentSizesWhenUsingFactory() { ).getKey(), heapSizeFromImplSetting + "b" ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put( TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), numSegments @@ -2466,7 +2456,6 @@ public void testSegmentSizesWhenNotUsingFactory() { CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") // The size setting from the OpenSearchOnHeapCache implementation should not be honored .put( OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES.getConcreteSettingForNamespace( @@ -2697,7 +2686,6 @@ private TieredSpilloverCache initializeTieredSpilloverCache( CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put(settings) .build() ) @@ -2750,7 +2738,6 @@ private CacheConfig getCacheConfig( CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") .put(settings) .build() ) diff --git a/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java b/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java index d61cdaa89e093..1968c57ddb940 100644 --- a/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java +++ b/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java @@ -26,7 +26,6 @@ import org.opensearch.common.cache.settings.CacheSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.cache.request.RequestCacheStats; import org.opensearch.index.query.QueryBuilders; @@ -71,11 +70,6 @@ protected Collection> nodePlugins() { return Arrays.asList(EhcacheCachePlugin.class); } - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.PLUGGABLE_CACHE, "true").build(); - } - private Settings defaultSettings(long sizeInBytes, TimeValue expirationTime) { if (expirationTime == null) { expirationTime = TimeValue.MAX_VALUE; diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java index 4e879af052c15..2f58bb5df0ebe 100644 --- a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java +++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java @@ -26,7 +26,6 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; @@ -1221,7 +1220,6 @@ private EhcacheDiskCache setupMaxSizeTest(long maxSizeFromSettin MockRemovalListener listener = new MockRemovalListener<>(); try (NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) { Settings settings = Settings.builder() - .put(FeatureFlags.PLUGGABLE_CACHE, true) .put( CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), EhcacheDiskCache.EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java index c7f85c7233869..3f947b36e28de 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java @@ -8,8 +8,6 @@ package org.opensearch.indices; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; @@ -24,7 +22,6 @@ import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -34,13 +31,10 @@ import org.opensearch.index.cache.request.RequestCacheStats; import org.opensearch.index.query.QueryBuilders; import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; import org.opensearch.transport.client.Client; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,16 +44,7 @@ // Use a single data node to simplify logic about cache stats across different shards. @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) -public class CacheStatsAPIIndicesRequestCacheIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { - public CacheStatsAPIIndicesRequestCacheIT(Settings settings) { - super(settings); - } - - @ParametersFactory - public static Collection parameters() { - return Arrays.asList(new Object[] { Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE, "true").build() }); - } - +public class CacheStatsAPIIndicesRequestCacheIT extends OpenSearchIntegTestCase { /** * Test aggregating by indices, indices+shards, shards, or no levels, and check the resulting stats * are as we expect. diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index cbc453eb5c0b0..dc72291e95184 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -56,7 +56,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.NodeEnvironment; @@ -110,9 +109,7 @@ public IndicesRequestCacheIT(Settings settings) { public static Collection parameters() { return Arrays.asList( new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() }, - new Object[] { Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE, "true").build() }, - new Object[] { Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE, "false").build() } + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } ); } diff --git a/server/src/main/java/org/opensearch/common/cache/service/CacheService.java b/server/src/main/java/org/opensearch/common/cache/service/CacheService.java index da006264094d2..8fc0fc7d07cac 100644 --- a/server/src/main/java/org/opensearch/common/cache/service/CacheService.java +++ b/server/src/main/java/org/opensearch/common/cache/service/CacheService.java @@ -18,7 +18,6 @@ import org.opensearch.common.cache.store.config.CacheConfig; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import java.util.HashMap; import java.util.Map; @@ -47,10 +46,9 @@ public CacheService(Map cacheStoreTypeFactories, Setting public ICache createCache(CacheConfig config, CacheType cacheType) { String storeName = getStoreNameFromSetting(cacheType, settings); - if (!pluggableCachingEnabled(cacheType, settings)) { - // Condition 1: In case feature flag is off, we default to onHeap. - // Condition 2: In case storeName is not explicitly mentioned, we assume user is looking to use older - // settings, so we again fallback to onHeap to maintain backward compatibility. + if (!storeNamePresent(cacheType, settings)) { + // In case storeName is not explicitly mentioned, we assume user is looking to use older + // settings, so we fallback to onHeap to maintain backward compatibility. // It is guaranteed that we will have this store name registered, so // should be safe. storeName = OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME; @@ -73,11 +71,11 @@ public NodeCacheStats stats(CommonStatsFlags flags) { } /** - * Check if pluggable caching is on, and if a store type is present for this cache type. + * Check if a store type is present for this cache type. */ - public static boolean pluggableCachingEnabled(CacheType cacheType, Settings settings) { + public static boolean storeNamePresent(CacheType cacheType, Settings settings) { String storeName = getStoreNameFromSetting(cacheType, settings); - return FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings) && storeName != null && !storeName.isBlank(); + return storeName != null && !storeName.isBlank(); } private static String getStoreNameFromSetting(CacheType cacheType, Settings settings) { diff --git a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java index e1039c5d9ee55..f3a496f07b3e8 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java +++ b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java @@ -29,7 +29,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeValue; import java.util.List; @@ -182,7 +181,7 @@ public static class OpenSearchOnHeapCacheFactory implements Factory { public ICache create(CacheConfig config, CacheType cacheType, Map cacheFactories) { Map> settingList = OpenSearchOnHeapCacheSettings.getSettingListForCacheType(cacheType); Settings settings = config.getSettings(); - boolean statsTrackingEnabled = statsTrackingEnabled(config.getSettings(), config.getStatsTrackingEnabled()); + boolean statsTrackingEnabled = config.getStatsTrackingEnabled(); ICacheBuilder builder = new Builder().setDimensionNames(config.getDimensionNames()) .setStatsTrackingEnabled(statsTrackingEnabled) .setExpireAfterAccess(((TimeValue) settingList.get(EXPIRE_AFTER_ACCESS_KEY).get(settings))) @@ -197,7 +196,7 @@ public ICache create(CacheConfig config, CacheType cacheType, /* Use the cache config value if present. This can be passed down from the TieredSpilloverCache when creating individual segments, - but is not passed in from the IRC if pluggable caching is on. + but is not passed in from the IRC if a store name setting is present. */ builder.setMaximumWeightInBytes(config.getMaxSizeInBytes()); } else { @@ -209,7 +208,7 @@ public ICache create(CacheConfig config, CacheType cacheType, builder.setNumberOfSegments(-1); // By default it will use 256 segments. } - if (!CacheService.pluggableCachingEnabled(cacheType, settings)) { + if (!CacheService.storeNamePresent(cacheType, settings)) { // For backward compatibility as the user intent is to use older settings. builder.setMaximumWeightInBytes(config.getMaxSizeInBytes()); builder.setExpireAfterAccess(config.getExpireAfterAccess()); @@ -223,11 +222,6 @@ public ICache create(CacheConfig config, CacheType cacheType, public String getCacheName() { return NAME; } - - private boolean statsTrackingEnabled(Settings settings, boolean statsTrackingEnabledConfig) { - // Don't track stats when pluggable caching is off, or when explicitly set to false in the CacheConfig - return FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings) && statsTrackingEnabledConfig; - } } /** diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index b4b85e0a9d367..c1ce90aaa8efa 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -810,7 +810,16 @@ public void apply(Settings value, Settings current, Settings previous) { ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING, // Thread pool Settings - ThreadPool.CLUSTER_THREAD_POOL_SIZE_SETTING + ThreadPool.CLUSTER_THREAD_POOL_SIZE_SETTING, + + // Tiered caching settings + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE), + OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ), + OpenSearchOnHeapCacheSettings.EXPIRE_AFTER_ACCESS_SETTING.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ) ) ) ); @@ -831,16 +840,6 @@ public void apply(Settings value, Settings current, Settings previous) { TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING, TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING ), - List.of(FeatureFlags.PLUGGABLE_CACHE), - List.of( - CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE), - OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES.getConcreteSettingForNamespace( - CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() - ), - OpenSearchOnHeapCacheSettings.EXPIRE_AFTER_ACCESS_SETTING.getConcreteSettingForNamespace( - CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() - ) - ), List.of(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL), List.of(SearchReplicaAllocationDecider.SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 6753bb8eac083..b1d13b1ae8693 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -35,7 +35,6 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.PLUGGABLE_CACHE_SETTING, FeatureFlags.STAR_TREE_INDEX_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 4be45aed70023..f0b26d562c52b 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -56,12 +56,6 @@ public class FeatureFlags { */ public static final String TIERED_REMOTE_INDEX = "opensearch.experimental.feature.tiered_remote_index.enabled"; - /** - * Gates the functionality of pluggable cache. - * Enables OpenSearch to use pluggable caches with respective store names via setting. - */ - public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled"; - /** * Gates the functionality of background task execution. */ @@ -87,8 +81,6 @@ public class FeatureFlags { public static final Setting TIERED_REMOTE_INDEX_SETTING = Setting.boolSetting(TIERED_REMOTE_INDEX, false, Property.NodeScope); - public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); - public static final Setting READER_WRITER_SPLIT_EXPERIMENTAL_SETTING = Setting.boolSetting( READER_WRITER_SPLIT_EXPERIMENTAL, false, @@ -137,7 +129,6 @@ public class FeatureFlags { TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, - PLUGGABLE_CACHE_SETTING, STAR_TREE_INDEX_SETTING, APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java index 4f42cd8fe8672..8f1dcbc9b8ee8 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java @@ -233,9 +233,9 @@ CacheConfig getCacheConfig(Settings settings, NodeEnvironme .setClusterSettings(clusterService.getClusterSettings()) .setStoragePath(nodeEnvironment.nodePaths()[0].path.toString() + "/request_cache"); - if (!CacheService.pluggableCachingEnabled(CacheType.INDICES_REQUEST_CACHE, settings)) { - // If pluggable caching is not enabled, use the max size based on the IRC setting into the config. - // If pluggable caching is enabled, cache implementations instead determine their own sizes based on their own implementation + if (!CacheService.storeNamePresent(CacheType.INDICES_REQUEST_CACHE, settings)) { + // If a store name is absent, use the max size based on the IRC setting into the config. + // If a store name is present, cache implementations instead determine their own sizes based on their own implementation // size settings. configBuilder.setMaxSizeInBytes(sizeInBytes); } diff --git a/server/src/test/java/org/opensearch/common/cache/service/CacheServiceTests.java b/server/src/test/java/org/opensearch/common/cache/service/CacheServiceTests.java index b355161f6f310..6abc062d258ea 100644 --- a/server/src/test/java/org/opensearch/common/cache/service/CacheServiceTests.java +++ b/server/src/test/java/org/opensearch/common/cache/service/CacheServiceTests.java @@ -17,7 +17,6 @@ import org.opensearch.common.cache.store.config.CacheConfig; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.plugins.CachePlugin; import org.opensearch.test.OpenSearchTestCase; @@ -30,7 +29,6 @@ import static org.mockito.Mockito.when; public class CacheServiceTests extends OpenSearchTestCase { - public void testWithCreateCacheForIndicesRequestCacheType() { CachePlugin mockPlugin1 = mock(CachePlugin.class); ICache.Factory factory1 = mock(ICache.Factory.class); @@ -50,38 +48,15 @@ public void testWithCreateCacheForIndicesRequestCacheType() { ); CacheConfig config = mock(CacheConfig.class); ICache mockOnHeapCache = mock(OpenSearchOnHeapCache.class); - when(onHeapCacheFactory.create(eq(config), eq(CacheType.INDICES_REQUEST_CACHE), any(Map.class))).thenReturn(mockOnHeapCache); - - ICache ircCache = cacheService.createCache(config, CacheType.INDICES_REQUEST_CACHE); - assertEquals(mockOnHeapCache, ircCache); - } - - public void testWithCreateCacheForIndicesRequestCacheTypeWithFeatureFlagTrue() { - CachePlugin mockPlugin1 = mock(CachePlugin.class); - ICache.Factory factory1 = mock(ICache.Factory.class); - ICache.Factory onHeapCacheFactory = mock(OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.class); - Map factoryMap = Map.of( - "cache1", - factory1, - OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, - onHeapCacheFactory - ); - when(mockPlugin1.getCacheFactoryMap()).thenReturn(factoryMap); - - Setting indicesRequestCacheSetting = CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE); - CacheService cacheService = new CacheService( - factoryMap, - Settings.builder().put(indicesRequestCacheSetting.getKey(), "cache1").put(FeatureFlags.PLUGGABLE_CACHE, "true").build() - ); - CacheConfig config = mock(CacheConfig.class); - ICache mockOnHeapCache = mock(OpenSearchOnHeapCache.class); when(factory1.create(eq(config), eq(CacheType.INDICES_REQUEST_CACHE), any(Map.class))).thenReturn(mockOnHeapCache); + ICache otherMockOnHeapCache = mock(OpenSearchOnHeapCache.class); + when(onHeapCacheFactory.create(eq(config), eq(CacheType.INDICES_REQUEST_CACHE), any(Map.class))).thenReturn(otherMockOnHeapCache); ICache ircCache = cacheService.createCache(config, CacheType.INDICES_REQUEST_CACHE); assertEquals(mockOnHeapCache, ircCache); } - public void testWithCreateCacheForIndicesRequestCacheTypeWithFeatureFlagTrueAndStoreNameIsNull() { + public void testWithCreateCacheForIndicesRequestCacheTypeWithStoreNameNull() { CachePlugin mockPlugin1 = mock(CachePlugin.class); ICache.Factory factory1 = mock(ICache.Factory.class); ICache.Factory onHeapCacheFactory = mock(OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.class); @@ -93,7 +68,7 @@ public void testWithCreateCacheForIndicesRequestCacheTypeWithFeatureFlagTrueAndS ); when(mockPlugin1.getCacheFactoryMap()).thenReturn(factoryMap); - CacheService cacheService = new CacheService(factoryMap, Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE, "true").build()); + CacheService cacheService = new CacheService(factoryMap, Settings.builder().build()); CacheConfig config = mock(CacheConfig.class); ICache mockOnHeapCache = mock(OpenSearchOnHeapCache.class); when(onHeapCacheFactory.create(eq(config), eq(CacheType.INDICES_REQUEST_CACHE), any(Map.class))).thenReturn(mockOnHeapCache); @@ -149,6 +124,6 @@ public void testWithCreateCacheWithInvalidStoreNameAssociatedForCacheType() { IllegalArgumentException.class, () -> cacheService.createCache(config, CacheType.INDICES_REQUEST_CACHE) ); - assertEquals("No store name: [opensearch_onheap] is registered for cache type: INDICES_REQUEST_CACHE", ex.getMessage()); + assertEquals("No store name: [cache] is registered for cache type: INDICES_REQUEST_CACHE", ex.getMessage()); } } diff --git a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java index 5a989ad8ab777..e4f74d619a6a3 100644 --- a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java +++ b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java @@ -22,7 +22,6 @@ import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; @@ -40,7 +39,7 @@ public void testStats() throws Exception { MockRemovalListener listener = new MockRemovalListener<>(); int maxKeys = between(10, 50); int numEvicted = between(10, 20); - OpenSearchOnHeapCache cache = getCache(maxKeys, listener, true, true); + OpenSearchOnHeapCache cache = getCache(maxKeys, listener, true); // When the pluggable caches setting is on, we should get stats as expected from cache.stats(). @@ -82,49 +81,44 @@ public void testStats() throws Exception { } } - public void testStatsWithoutPluggableCaches() throws Exception { - // When the pluggable caches setting is off, or when we manually set statsTrackingEnabled = false in the config, + public void testWithoutStatsTracking() throws Exception { + // When we manually set statsTrackingEnabled = false in the config, // we should get all-zero stats from cache.stats(), but count() should still work. MockRemovalListener listener = new MockRemovalListener<>(); int maxKeys = between(10, 50); int numEvicted = between(10, 20); - OpenSearchOnHeapCache pluggableCachesOffCache = getCache(maxKeys, listener, false, true); - OpenSearchOnHeapCache manuallySetNoopStatsCache = getCache(maxKeys, listener, true, false); - List> caches = List.of(pluggableCachesOffCache, manuallySetNoopStatsCache); - - for (OpenSearchOnHeapCache cache : caches) { - int numAdded = maxKeys + numEvicted; - for (int i = 0; i < numAdded; i++) { - ICacheKey key = getICacheKey(UUID.randomUUID().toString()); - cache.computeIfAbsent(key, getLoadAwareCacheLoader()); + OpenSearchOnHeapCache manuallySetNoopStatsCache = getCache(maxKeys, listener, false); + int numAdded = maxKeys + numEvicted; + for (int i = 0; i < numAdded; i++) { + ICacheKey key = getICacheKey(UUID.randomUUID().toString()); + manuallySetNoopStatsCache.computeIfAbsent(key, getLoadAwareCacheLoader()); - assertEquals(Math.min(maxKeys, i + 1), cache.count()); - ImmutableCacheStatsHolder stats = cache.stats(); - assertZeroStats(cache.stats()); - } + assertEquals(Math.min(maxKeys, i + 1), manuallySetNoopStatsCache.count()); + ImmutableCacheStatsHolder stats = manuallySetNoopStatsCache.stats(); + assertZeroStats(manuallySetNoopStatsCache.stats()); } } - public void testWithCacheConfigSizeSettings_WhenPluggableCachingOff() { - // The "pluggable caching off" case can happen when the PLUGGABLE_CACHE setting is false, or if the store name is blank. - // The cache should get its size from the config, not the setting, in either case. - Settings.Builder settingsBuilder = Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE, false); + public void testWithCacheConfigSizeSettings_WhenStoreNameBlank() { + // If the store name is blank, the cache should get its size from the config, not the setting. long maxSizeFromSetting = between(1000, 2000); long maxSizeFromConfig = between(3000, 4000); - OpenSearchOnHeapCache onHeapCache = setupMaxSizeTest(settingsBuilder, maxSizeFromSetting, maxSizeFromConfig, true); - assertEquals(maxSizeFromConfig, onHeapCache.getMaximumWeight()); - Settings.Builder storeNameBlankSettingsBuilder = Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE, true); - onHeapCache = setupMaxSizeTest(storeNameBlankSettingsBuilder, maxSizeFromSetting, maxSizeFromConfig, true); + Settings.Builder storeNameBlankSettingsBuilder = Settings.builder(); + OpenSearchOnHeapCache onHeapCache = setupMaxSizeTest( + storeNameBlankSettingsBuilder, + maxSizeFromSetting, + maxSizeFromConfig, + true + ); assertEquals(maxSizeFromConfig, onHeapCache.getMaximumWeight()); } - public void testWithCacheConfigSettings_WhenPluggableCachingOn() { - // When pluggable caching is on, the cache should get its size from the config if present, and otherwise should get it from the + public void testWithCacheConfigSettings_WhenStoreNameNotBlank() { + // When the store name is not blank, the cache should get its size from the config if present, and otherwise should get it from the // setting. Settings.Builder settingsBuilder = Settings.builder() - .put(FeatureFlags.PLUGGABLE_CACHE, true) .put( CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME @@ -178,7 +172,6 @@ private void assertZeroStats(ImmutableCacheStatsHolder stats) { private OpenSearchOnHeapCache getCache( int maxSizeKeys, MockRemovalListener listener, - boolean pluggableCachesSetting, boolean statsTrackingEnabled ) { ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(); @@ -189,7 +182,6 @@ private OpenSearchOnHeapCache getCache( .getKey(), maxSizeKeys * keyValueSize + "b" ) - .put(FeatureFlags.PLUGGABLE_CACHE, pluggableCachesSetting) .build(); CacheConfig cacheConfig = new CacheConfig.Builder().setKeyType(String.class) @@ -207,7 +199,7 @@ private OpenSearchOnHeapCache getCache( public void testInvalidateWithDropDimensions() throws Exception { MockRemovalListener listener = new MockRemovalListener<>(); int maxKeys = 50; - OpenSearchOnHeapCache cache = getCache(maxKeys, listener, true, true); + OpenSearchOnHeapCache cache = getCache(maxKeys, listener, true); List> keysAdded = new ArrayList<>(); diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java index d60bbd62bdd13..a60e9b381a8d3 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java @@ -67,7 +67,6 @@ import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.bytes.AbstractBytesReference; import org.opensearch.core.common.bytes.BytesReference; @@ -199,58 +198,6 @@ public void testBasicOperationsCache() throws Exception { assertEquals(0, cache.numRegisteredCloseListeners()); } - public void testBasicOperationsCacheWithFeatureFlag() throws Exception { - threadPool = getThreadPool(); - Settings settings = Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.PLUGGABLE_CACHE, "true").build(); - cache = getIndicesRequestCache(settings); - writer.addDocument(newDoc(0, "foo")); - DirectoryReader reader = getReader(writer, indexShard.shardId()); - - // initial cache - IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard); - Loader loader = new Loader(reader, 0); - BytesReference value = cache.getOrCompute(entity, loader, reader, getTermBytes()); - assertEquals("foo", value.streamInput().readString()); - ShardRequestCache requestCacheStats = indexShard.requestCache(); - assertEquals(0, requestCacheStats.stats().getHitCount()); - assertEquals(1, requestCacheStats.stats().getMissCount()); - assertEquals(0, requestCacheStats.stats().getEvictions()); - assertFalse(loader.loadedFromCache); - assertEquals(1, cache.count()); - - // cache hit - entity = new IndicesService.IndexShardCacheEntity(indexShard); - loader = new Loader(reader, 0); - value = cache.getOrCompute(entity, loader, reader, getTermBytes()); - assertEquals("foo", value.streamInput().readString()); - requestCacheStats = indexShard.requestCache(); - assertEquals(1, requestCacheStats.stats().getHitCount()); - assertEquals(1, requestCacheStats.stats().getMissCount()); - assertEquals(0, requestCacheStats.stats().getEvictions()); - assertTrue(loader.loadedFromCache); - assertEquals(1, cache.count()); - assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length()); - assertEquals(1, cache.numRegisteredCloseListeners()); - - // Closing the cache doesn't modify an already returned CacheEntity - if (randomBoolean()) { - reader.close(); - } else { - indexShard.close("test", true, true); // closed shard but reader is still open - cache.clear(entity); - } - cache.cacheCleanupManager.cleanCache(); - assertEquals(1, requestCacheStats.stats().getHitCount()); - assertEquals(1, requestCacheStats.stats().getMissCount()); - assertEquals(0, requestCacheStats.stats().getEvictions()); - assertTrue(loader.loadedFromCache); - assertEquals(0, cache.count()); - assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt()); - - IOUtils.close(reader); - assertEquals(0, cache.numRegisteredCloseListeners()); - } - public void testCacheDifferentReaders() throws Exception { threadPool = getThreadPool(); cache = getIndicesRequestCache(Settings.EMPTY); @@ -856,8 +803,8 @@ public void testAddingToCleanupKeyToCountMapWorksAppropriatelyWithMultipleThread assertFalse(concurrentModificationExceptionDetected.get()); } - public void testCacheMaxSize_WhenPluggableCachingOff() throws Exception { - // If pluggable caching is off, the IRC should put a max size value into the cache config that it uses to create its cache. + public void testCacheMaxSize_WhenStoreNameAbsent() throws Exception { + // If a store name is absent, the IRC should put a max size value into the cache config that it uses to create its cache. threadPool = getThreadPool(); long cacheSize = 1000; Settings settings = Settings.builder().put(INDICES_CACHE_QUERY_SIZE.getKey(), cacheSize + "b").build(); @@ -871,12 +818,11 @@ public void testCacheMaxSize_WhenPluggableCachingOff() throws Exception { allowDeprecationWarning(); } - public void testCacheMaxSize_WhenPluggableCachingOn() throws Exception { - // If pluggable caching is on, and a store name is present, the IRC should NOT put a max size value into the cache config. + public void testCacheMaxSize_WhenStoreNamePresent() throws Exception { + // If and a store name is present, the IRC should NOT put a max size value into the cache config. threadPool = getThreadPool(); Settings settings = Settings.builder() .put(INDICES_CACHE_QUERY_SIZE.getKey(), 1000 + "b") - .put(FeatureFlags.PLUGGABLE_CACHE, true) .put( CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME @@ -953,10 +899,7 @@ public void testClosingIndexWipesStats() throws Exception { } threadPool = getThreadPool(); - Settings settings = Settings.builder() - .put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.001%") - .put(FeatureFlags.PLUGGABLE_CACHE, true) - .build(); + Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.001%").build(); try (NodeEnvironment env = newNodeEnvironment(settings)) { cache = new IndicesRequestCache(settings, (shardId -> { IndexService indexService = null; From 659e32d8331dc8300170e8b193ca92ca9706d376 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 7 Mar 2025 20:14:09 -0800 Subject: [PATCH 09/10] Add bwc version 2.19.2 (#17484) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Craig Perkins Signed-off-by: Andrew Ross Co-authored-by: opensearch-ci-bot <83309141+opensearch-ci-bot@users.noreply.github.com> --- .ci/bwcVersions | 1 + README.md | 2 +- libs/core/src/main/java/org/opensearch/Version.java | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 8ddc5e5811d7a..2167875966fb2 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -44,4 +44,5 @@ BWC_VERSION: - "2.18.1" - "2.19.0" - "2.19.1" + - "2.19.2" - "2.20.0" diff --git a/README.md b/README.md index bf9fef148df3e..77b9e09b9e1a7 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) -[![2.19.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.19.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.19.1") +[![2.19.2 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.19.2)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.19.2") [![2.18.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.1") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 5adde08a40d54..35bd1f2051b04 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -115,6 +115,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_18_1 = new Version(2180199, org.apache.lucene.util.Version.LUCENE_9_12_1); public static final Version V_2_19_0 = new Version(2190099, org.apache.lucene.util.Version.LUCENE_9_12_1); public static final Version V_2_19_1 = new Version(2190199, org.apache.lucene.util.Version.LUCENE_9_12_1); + public static final Version V_2_19_2 = new Version(2190299, org.apache.lucene.util.Version.LUCENE_9_12_1); public static final Version V_2_20_0 = new Version(2200099, org.apache.lucene.util.Version.LUCENE_9_12_1); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0); public static final Version CURRENT = V_3_0_0; From 9bef705d321d83b37f92292662c80a5e0cb4ba66 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Sat, 8 Mar 2025 19:41:25 -0500 Subject: [PATCH 10/10] Update Gradle to 8.13 (#17345) Signed-off-by: Andriy Redko --- build.gradle | 2 +- gradle/code-coverage.gradle | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 43583 -> 43705 bytes gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew | 5 ++--- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 679f7b9299248..187574da9e62a 100644 --- a/build.gradle +++ b/build.gradle @@ -721,7 +721,7 @@ subprojects { reporting { reports { testAggregateTestReport(AggregateTestReport) { - testType = TestSuiteType.UNIT_TEST + testSuiteName = "test" } } } diff --git a/gradle/code-coverage.gradle b/gradle/code-coverage.gradle index eb27dd1a76634..1e41f12e1cc48 100644 --- a/gradle/code-coverage.gradle +++ b/gradle/code-coverage.gradle @@ -38,7 +38,7 @@ if (System.getProperty("tests.coverage")) { reporting { reports { testCodeCoverageReport(JacocoCoverageReport) { - testType = TestSuiteType.UNIT_TEST + testSuiteName = "test" } } } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index a4b76b9530d66f5e68d973ea569d8e19de379189..9bbc975c742b298b441bfb90dbc124400a3751b9 100644 GIT binary patch delta 34744 zcmXuJV_+R@)3u$(Y~1X)v28cDZQE*`9qyPrXx!Mg8{4+s*nWFo&-eX5|IMs5>pW(< z=OJ4cAZzeZfy=9lI!r-0aXh8xKdlGq)X)o#ON+mC6t7t0WtgR!HN%?__cvdWdtQC< zrFQ;?l@%CxY55`8y(t7?1P_O7(6pv~(~l!kHB;z2evtUsGHzEDL+y4*no%g#AsI~i zJ%SFMv{j__Yaxnn2NtDK+!1XZX`CB}DGMIT{#8(iAk*`?VagyHx&|p8npkmz=-n!f z3D+^yIjP`D&Lfz500rpq#dJE`vM|-N7=`uN0z86BpiMcCOCS^;6CUG4o1I)W{q6Gv z1vZB6+|7An``GNoG7D!xJGJd_Qv(M-kdVdsIJ?CrXFEH^@Ts83}QX}1%P6KQFNz^-=) z<|qo#qmR!Nonr$p*Uu1Jo2c~KLTrvc*Yw%L+`IL}y|kd+t{NCrXaP=7C00CO?=pgp z!fyr#XFfFXO6z2TP5P1W{H_`$PKzUiGtJd!U52%yAJf}~tgXF`1#}@y`cZl9y{J-A zyUA&-X)+^N?W=2Fm_ce2w$C6>YWp7MgXa{7=kwwy9guBx26=MnPpuSt zB4}vo3{qxa+*{^oHxe7;JMNMp>F`iNv>0!MsFtnb+5eEZ$WI z0M9}rA&cgQ^Q8t_ojofiHaKuhvIB{B9I}3`Dsy3vW8ibigX}Kc912|UZ1uhH?RuHU=i&ePe2w%65)nBkHr7Bx5WwMZj%1B53sUEj0bxI( zEbS%WOUw)3-B0`-m0!{mk7Q%={B#7C^Si>C04@P|qm7$Oxn3ki)G_oNQBTh6CN6d_kt@UKx1Ezdo5)J0Gdf@TcW|{ zdz1V?a>zldA7_5*Pjn6kDj|sbUqt-7X z5+oajeC}*6oi~vxZ#Ac&85cYcC$5OKUnYPv$Y~>H@)mnTtALo*>>5&=0QMr5{5?S; zCDF=RI@94n(!~sa`4Y{JLxgcvRqMM&T!}rRd~Kl#_X4Z&85;})o4W*g>?TaAVXSWB zeY#!8qz^hmC6FERsjTnC)1Xu1UPd7_LfuNvuVqF8(}Jfar=T-K9iChEuZi-FH(P%u zzLrjpq|?}8?g1Vnw^&{eqw~QY0f*9c71&*<5#9f5JlhJmG~IuV*8~nEBLr`KrvOvs zkOLdlZ58K?u>1{vAU0CtT>Il<I{Q8#A!lO7#73V&iN13;oV?Hl?N5xDK63)Rp3%5reb&3n5OQ|9H zDpYEI%JQXcrs^o*SCFY~iYf-VM<`7Tl@+kQS3tfR-fyH_JDaz5SYEMU-bTCLQ=JVG ze?ZPcj95Tci|bVvSZk3^enqQ?pIcZn24V=YT{cf-L|P&{-%%^ql$)^Vu~)Ida=h$bZAMQEi$MM|&b zY8;D;aEba_`W^=VdKfttW)h_zjRA&0A^T*tF*%+}TZQCOvFqKUu=xf1Bx@T?&~S(J zopXniA?s%}Q4p9~F(Ty{8wt$l4oHeT(#U6sAu4>Q+~a;}I>0>??v*wfke}0TwPaeE zj3gWtfNlD{jRgy7;S9PS?su5pnobi%Zoe0LVpw%`<)V=yT~Ht_UUXIna4YUa;p=-T4df6^;bz%;@|$F zK;s9#K@9hqZCST!66N0uPB+FT*kq22%ovtJ%<9ArE%hcX^!(Lz;3?kCZ@Ak*MThjTOKU&t+uJdN*6t$;DDmh zFStdHO>r)8L@qO}K@H~7Z);#f6WU{@Icn7Tc^|IZ`;K^ek9eCWdync`kWCt2s%D-k zE$wyPCui$@gJJ9Q`CtixbMF(GiCCbm`ut(~ce-G|Ji|PZ3~DHlG`Asn;skVhnu0r_ zgGbdmfl|er`87x@uYmd8A+!-3V95GE4&_^9N@hp4SC4 zeFU+Z3Ou&G! zlvZy|iHIIX3X2-Yb7YJ#{SYE9lCoixO+}(|u+H@Z6Rz-l1eZ7{I;vk+Y7kP7ev>hG zv|(I<4?N{EXMSvRgUhbQhDoP1&A;SEUGGep8*!@4u)fNbl3%cts<&=m5<5pi7M-HQ zPS#svbXWu2n&m*K6jL#@xm3VSMJxnxve5J6w1qGv`2>5<6F!uzGVHP1A(_xI7CWlX zm6*wpT@dmQ&pAlm`r~T;)>m5HK^H^cM`pCSoh{;-CE43rMkg<;HnZaCHfMq1LoN0S z%%7|$y~&k6wpiY@rsdCY9ZDh%9W6Pf=2^p=;iv-Ah^ACxwK3VmI}SMNneTa9n%biL z#GoojRHxa}R2zOo!G@<8M-B6vNp?)@_>#mYku#pe{O~t?~}1 zE8`)=BstIRk5W*xZw@2=89@ds?eQ~mxzkrA`y<$oR8bmaUw=rE%lFmzHY&aY8?<-N zp1|bb$(XrOMmiYy{pH#)D1GOmv5aj_?waU~*h~s{VZ&H_PhoXYz`C8Pss{ymY_hPG zt{NY&nPMH#FRvwR+T0(Xo2#T6;=oFmRgA9b-HVY72d|~YF+6v$F%sY0 zS#^LF7sTj>Itvyi!~){Hit*~3imOG*Xh51qLz+!W~`vUBVeZZ5&k34SD%Ha%5#aclSzMfoGWjiq9#rl}j zOf*8NY>VN(`W!DxaBgjBzj3oUAVlLY{R}tiZZ0o>K$vwr?+eggZ!q74m2t?lkvm9z zAmL2=W$jQJL>SSrbIOibe734A(K^B8`M@uao!`E$p+9D!rBea8Oxb|p5r3o4##G8K zMr0I9y&`21{@m=Bi+4tTJ-xy(DB_mG$kYv+qw&VBM(A9^wP9;Yo*6{#5tMpfa;m2FC+%l@ zk_cKXg-d&YUIj3(x{)aNwYGYjSHiOQK2K#yWt$vQomhbnF;Qhkxl`+;i{&+t{PrY` zp5r28&|UvmUK|&Jlv>oX4>XE87Zns?fiE6c;VP7BixT*6n}Zsbv$wd{gXyrE&Sd zhRlv!-{%~xv6yNvx@3^@JEa$={&giRpqZG>`{93 zEjM}YI1i6JSx$DJa&NWcl0M;igxX;est*nz=W16zMfJ0#+s{>Eo>bxmCi)m*43hU1 z;FL43I}nWszjSS%*F1UYt^)4?D6&pDEt1(atK(DKY1pAkNMG`a>_ec;KiT z^xMBBZ9i=;!_hNGlYp^uR0FW^lcBrs_c3ZvhcctW4*T^-DD^OU{{hK8yHahyGyCK& zL0>f0XW|wvi4f`bNTfO+P*Ao^L@8~ezagtl%l z{(2uo71sT3rKTQ-L#Y5Rsy#x)Eo+HQranZmk;r_Hf7WWkRq&QmP{?}do0X=;3U_UYspffJl7v*Y&GnW;M7$C-5ZlL*MU|q*6`Lvx$g^ z6>MRgOZ>~=OyR3>WL0pgh2_ znG)RNd_;ufNwgQ9L6U@`!5=xjzpK_UfYftHOJ)|hrycrpgn-sCKdQ{BY&OEV3`roT|=4I#PT@q`6Lx=Lem2M&k4ghOSjXPH5<%cDd>`!rE} z5;hyRQ|6o>*}@SFEzb7b%5iY}9vOMRGpIQqt%%m)iSpQ@iSAU+A{CmB^&-04fQlV9 z14~oE=?j{b{xE*X^1H)eezKTE27;-=UfNvQZ0kZ+m76{6xqAyTrEB&Oe`Mx{4N;}5 zXp%ojp}JYx6PE}Z`IBO3qWsZEfVPa4EEz0vnsFNkQ!kG8tcec&)k$+s&XmPErROoNxeTh9fATBk)w1g|9*~&S!%r0u6+FTn}dK-qa7cfK~tkJlV zMi{BX!>lQsZhSQUWAf(M6+McPrv>)j<*T&hC!*?qq{@ABJWX z@!~2Y1rhy*Z|x`DZUBuyayz}Kv5Pzrh}1wiHT{9|fh`Wl%ao=lRSwEFl*wy6BZ%vo zrt9Ocbicd1q$a{F6`4#ZQ6vJa@`}IGz+xUr*=6TF^GR?`u{1to&gqJpwf$LN0?G&! zsLNiG+}M+c{*j-Q4I zO!=lj&~{29Os}hgEv`iJ1tU)dx}=ob>DHSHKX|FVu2Y#pO|SsigHRgg4?!FX2>b3W z`m}xI<#_02adGka0TuAIg89kS?>*lKyI)T)Pa)|12XfH;k9}#=dzH6TiciCNO->e9m>!W)l&4B zd74@>_LL9OuJ&v5e0)l7ME@xW)9K@*LUd1RY}Vs_${3YC%+LfSR^H+I=(7Szh2nKB z_8bMoty|M+k9A|hGURVePvMf0XY9NYOiC@h^MLs-X@(8PV4zI7A155!RnZrBE9R1> zuI4E`=JTxyJ#d`!(9_s?T2jxEM*E`){wGI`DBFIz%ouW`Y0cKDfXAGN{};aMpLRvZ zu`PZ-3(+Tsh?UKAr)TQQ;2Jz(kv8{R#!c9Tyeev55@5@Ng*c4-ZQ6vC?o#5>6{;?gVfAIr-+^g>3b$}13U^~?gce6s6k-4ulnzWlFpq}*)2 zd0!wP{2>3U+zYiPaNr+-6O`J;M2Cb`H5hjDXw(1oKK!?dN#Y~ygl{H2|9$( zVg7`gf9*O%Db^Bm6_d808Q!r%K;IUSa(r^hW`w)~)m<)kJ(>{IbCs-LkKJ5Qk~Ujv z|5`OBU>lb7(1IAMvx%~sj+&>%6+_-Pj&OOMzMrkXW}gMmCPOw5zddR}{r9blK&1(w z^6?`m=qMI=B*p~LklFLvlX{LflRXecS#lV$LVwi$+9F8zyE29LgL> zW6R-6z&3x-zL({$nMnbhu|plRO8S_EavN?EKrr+c&Tt;Mk)NC0e|cvyXk%VKb5VIc z;|DN^5)t^}tr&-2q)SbwrF>=k$moYK;yA{Q1!I940KmPvg_Ogb81w$_)i3FgFWG+MS?k=BpkVGk-bRhBF;xJ}wnGN{)?gbry^3=P1@$k^#z9*@tmmB+TZ|L@3#3Z+x z8hJE({GEeEWj#+MnUSN^~c!=G+yW^j=cfN_0!}%(J-f1`G}w^}xi!T8BJDOCri{mGBU? zsKXxeN*=L#<-p_aj6cHtYWMJ+;F`HLeW5cpmeVAhFfy+Y=0rIqqyJ-NRIu-aE*Mvr zVnC-RDR`d1nnQu|^S79I>%9=bPNx1JLOJnB**Y`2WCq zctq<)Cq2^Z%=$*&;QxX30;642;y+=mlMLec6{KA208FQ~_S&tiFQW zp2{C3nyrmgkh+HRmG+$_y19m~0z~b`Mo+m6)Qq82p5)Z6ePn&B=!*twk7Rz%zzm-R z>Qj!PE3XMBY)N-xO(=VpO6=Cky5kpl}fQztM7QzvG#a}5$>2$f5w|}b8=3E)cNQw<%e1xAEwaRHu zhHCGB4Uzs6x3A=7uUBC0({&iNH{!7JgQHVa+ zKfQItwD}sd;587x?M_hzpR|TKtTH^4{`G7*87o_wJrFlmrEjk=jvA z6xBPKYjFB9{0Sj0rBL-z9BuBY_3c||UjVgv2kqw2m<@4#>zfx&8Uhq8u+)q68y+P~ zLT;>P#tv|UD62Nvl`H+UVUXPoFG3>Wt-!sX*=4{XxV|GSC+alg10pP~VaA>^}sRr1I4~ zffa2?H+84k=_w8oc8CQ4Ak-bhjCJIsbX{NQ1Xsi*Ad{!x=^8D6kYup?i~Kr;o`d=$ z*xal=(NL$A?w8d;U8P=`Q;4mh?g@>aqpU}kg5rnx7TExzfX4E=ozb0kFcyc?>p6P# z5=t~3MDR*d{BLI~7ZZG&APgBa4B&r^(9lJO!tGxM7=ng?Py&aN;erj&h``@-V8OA> z=sQ4diM!6K=su^WMbU@R%Tj@%jT5prt8I39 zd3t`Tcw$2G!3;f!#<>>SQ<>g6}Q{xB|sx_%QKm2`NxN|Zl%?Ck6Lu_EMC?*eRxdgS!3zYU#OnO~0&UFei zmP3k9!70^O24j5;G-fH6%T}X{EdO(%*+7ThlNGAh;l?$&{eZ-l`j281o@47x+6Z*DC`R2CkPo{1Behvlt!4${0Q?fBx)iIw$Ky zI#xvxKs1U`uMgeZg5fD>s5AYH*n=+UaRzS?ogn6WwBPK3Gib5@Jj!sZN^tm>M&*r@ zjbBoF7uXJU2MW~JK3%Xa3R}3zsP7qHEqbnC%eKsJ51+% zVAT-eRHwD)0YlfK2&rN549*};CJ8I;dj8rD^PR(>#n?Jccsqx&wF#We;Auv9Vm%-} z3HjpBGp$t5^S$XhJmYAP0q_qM@^#D}NM1FmCCyo;F|wv3_ci@$MA<3An0Aa|>_M&S z%qGjO@w{NI$VKyDF@w5W*6XK~5S`S$@ABWh@uaFIBq~VqOl99dhS}?}3N#JizIfYYt`ZKK0i_e#E;P0)VXh-V!w+qX%^-I0^ok>HAm5)tbBZlYov@XkUL zU}l}NDq{%pc=rmBC>Xi>Y5j9N2WrO58FxmLTZ=$@Fn3>(8~6sbkJ;;Uw!F8zXNoF@ zpW;OS^aL|+aN@xwRNj^&9iX;XxRUuPo`ti>k3Hi3cugt`C(EwuQ&d2lyfO` ze!0fi{eHhU1yN+o%J22|{prPvPOs1S?1eUuGUkR zmzMlCXZtW)ABWasAn53}?BqtPMJ*g>L1i6{$HmoEb@h(kILnMp(2!H!rG?MNH`1V0 zotb`;u#Yz0BZrT1ffVTCV!?{L^z8q11_21ptR0ITbOcaZ!mlWhC_AZb>?2IDV|b_y z9lVt3)0d@W=lNp1ArE;h_;DDQX^_;WtsSIO<;Ly&(#O~Xw$R0~W|xdQk*Y(b2=vLV zt8HX8=;#;$=y}!;Qku2HJbGEzF`2_~&i$&ogHUe5vhx}FLR}K_Mp)J{n*Va2<|pk$ z4tI(7v3A%Z7Z0|ZWw#7%$U#*mv+`Ujlh^N(t63xFt_%*WoJ^oq!U0j+Bx`<>q!J&0sWy4&{@#*BOr-s ztZ68f;l0UT3wf@RRC}_ufMr6rQ69Woa@1sZ50Ww|{yfp8!7rMOh_POTE;|zamq+4OObJ-VeTK|D|h?mfR$^lA{E7pk8DRDz*j&r<&fR>GaG*d zYaJ*q5#n251XIpR6F1o-w>LZ)Cb6Ma^6tCfcOItn1o;$#H?^jqOd(PA)B3HaTlJK zw!~?nh-v-_WBi5*B=IuTZOX2sa{1I!#%VMd5eGe1VcL6 zQ!aDft}>TjlwzEJ9Kr6MWh1MoNNWr$5_?z9BJ=>^_M59+CGj=}Ln)NrZ;Fja%!0oU zAg07?Nw&^fIc9udtYSulVBb-USUpElN!VfpJc>kPV`>B3S$7`SO$B21eH8mymldT} zxRNhSd-uFb&1$^B)%$-O(C$#Ug&+KvM;E9xA=CE*?PIa5wDF_ibV2lMo(Zygl8QK5 zPgH1R(6)1XT9GZ6^ol$p>4UH@5-KV66NF$AH-qOb>-b~+*7)DYsUe&Is0yTx=pn8N zs&2Z4fZ1Wk=dz>AXIfd%>ad=rb-Womi{nVVTfd26+mCx`6ukuQ?gjAROtw&Tuo&w$|&=rEzNzwpuy0 zsqq)r5`=Mst4=HCtEV^^8%+Dv2x+_}4v7qEXSjKf%dOhGh~(FDkBW<~+z&*#4T>r@ z>i7T5TGc96MfD%hr~nK9!%r{Ns9=7fui)N%GN8MvuIrox)(0nNg2{McUIC6nq>dD+ zNvX69vvf=Pw1@x}^K{@%UCL734;&AVta#($&l2E|*VUaKW@h`X*L*;1Kl4tajl}GQ z$K>;*$3y1(<^32Cg8ugi^ZII=I&ina>q@GC&~gQ#Z88(nOj;*j z1{hyEq|R_0v7LZNKB|3jqZPqZOuUG(SuM^Z>0@mzsKqVbRrkTz#TRZ0sTQ|%XiYcE zEE5{9jEB+2Sdga|veYSFZEzOuepHGusAO#pg&R(%Ob@V0Lw;AfQJ{aLUJxnbe`q(m zadg^fXYiWr+mm2akb*J?y`w(!KAL8OfFD!mVWiWrgScgp9^yoh3lNNUxd?YyvgUL z>+!2VXP7Fzq zYQ?(9-r*?N*cJCK&)pbYzuv%R{b;TB_wC1V3nO#12V0ucgp);>!N=;G=l;({KZF>) zNAo=0m|3Zu*PNLa-2v=3r5>-hVI_xYdz0m*f-zUW_=eDqiM3j4MPnS~eIRNdw466? z)yxHI@6d7gL2Qj<_@72W{GDyINBy%X6X&_cF1(##v^}87YGZ87HgfH$&epf>Jlia4 zw53K1M6=Px@YCVTUk!%_MjyBeaWy7c40i47-3B{voi|&|7aXza!(OB~E)U;f>5Wd3&@#UP~gkM*qmK=aeZ zkP}gn%JmKK34}KdEu)4E2~qN)EnAhj>)4dbq&RbLu$BD&kJSoIvr$3A#S%P~l$l1A z!96hNdtFXsta!b+enJ@G;6rv-Rd=IQ_llL#tSGk-mpQi(mhop;lObiTQIARXw~&d> zVuCSG$T&zi?#&PT-fP)`*-d@gc;+tOPDaUA*6>RIrf67& zpZ<1ie#4rJ3HEu>v7sF={4;oXv?_MwEI-^o-Lr@rW%%cd0TR2q`p=rkMOKYzOs&^$ z=xW*e)6p-B(0Ek7w8+!@Cks9>$_#zi44MLyL9X?{sDlihX%V;$%a;wd&RL*XGcb$` zvU}#qxz8wAT)*NQ+lXO>AI`^r7B&IQ3J&{cVNn0aWa)(!fQtV+mm~`vsH24+xI|q{ z4ce$OB1hrqGLn;H#=~Rx%T#b|hN`d6SXt=;Jd=DNX3LO9R8xLX@6p3>SnZO7M+96a z1s=zJKd%qy0#GWLeFgc~?fsCw^$6lG;B*54&@n#>q$#nRSr?2GA4YaSSl5~B2k}R_ zfJE-$C~{O_6Rh6BJbWFuoaeXEI!Q-YSA9EvSG_sjB~-*hf_PM~mJ6BL+IcaF)8$+; z*4A4W&+_Mn6~tF|M8Sz57BxO=W9ZJrNPtdhME>$sS6)etinxj{YkK){@Q${`Vc~dX zLT4UYjwuC>dH8AAjQb{Ji>eMvJ5rH-4a(K{4EyLrCDtta)u#>`V_AvyS?Y(;FRT8L ze`JXZP4s~Quq$m=6NI@}`( z`>o3kbSApxcHP;1Mds3&41!_0r619~@AQr9TW*Swk`Q1JNmIk%nKm(ZbZMHEi z4n%vC0MuAKNz2njKLk~w|6u!|y7FN!SXk5=7>^^p-R4w7R;~G!v<{>H3%SC-?>8jAP&ka=owuQ$sKwU4e8EVyc6V2IpBR56HthbwJ*XdwnwrW4 zcR7oGg7kCmj(q{#ka1d85mRVIo0`1v3+B--4RXv$hGb545y#j7bmu0*>BLnTRZ+mp z29%AP8Id+57Q(6`ep^<tq}GO1dvJ*8~jxjiH0quR*Poy%N3@c8rhlO6YR@LBk%l zux{&bK~LvKYq%d;Tzl|VS=?rkBUD-j$YY-xX)z`zUfH^&($ZYco(Xc1tr|9rwx}=- zk`E2Wwkh*HIVsWej-nJ6HNH)7rWDlB0@`{QG*0)&P+~Ng{m^kG#J*^p`drM(`dnd& z9$U+FH=rXh2py-N$l_0)@|JY;X1hVL`@}qxNi@Zy5hI)@(af%=1cl~L3{fxZWys9G-hLv z*%jvhoba^ePB8YL)`%d%=t6Yh*c5p1S7`+BPjOD*#q4~gv#bn0wOaf_K0SiGC{jp8 zAc_Vk31hKTSUiEU7XNk7`D}S-RUrYb<7%)k+tV0zZ7(}vQN@0C5EI<=$$qW}m7f7I zk>dMLd+kSjN4{OaxBJ^_h?FayJ`Yr)3eC$jdk1@jEzVT=a?{BSjp?&?qPX=xO!ttw zN_s#<#Ve(0i_|cRa=MC2=8MonmoT5)UtF&Wr9-b2ng>>zv{8$*UcIBIXSZ3)x727q zy{r>bdOh?E;ZI(^io=P3`o*tLdsjkjM!rGae!v5QH<3-OBW(XcRhvM!(b)Yas?oK? z$5)Y*YS^_d9H-ZP^_iVooK6EE1(akYvmNkXQGH1`kXg()p94|_F8B@_ABt*7QTmYk z47RyNSjX8nMW&@VZIQ`1WB%-*W4oN#|M}EKDCC_@HQ9!BenOQ{0{i#>IaQkyU-HOT z#8ueeQdKezCP`+p0{|o?!axX6WB@{OJTR;qfs(;uKp@Kjq4Dr)^>R9T+^$ohEYKB= zQx_P+t?e3z}3#W ztf10?br2MbSVn%*3!j2QFu;=K)-ueTmgyYq;%9HjJL_W=dV$#21FIjyv}d3@oIy+c z?IcrTw17F6oYGMQA=66yCh`48DJb}^Q?8r3Lei%QJ!qpxnt5`aP%aJL9ltY7#;qzq)qdoGzpYx=gz7Lz$JJZ4?^Nr`!1MK@k z47M)#_%Bezu?xD<{tFcQ{{@OiDQRGst}MJJdOtp%(wvCymmU}NKvIK%z%RysueJ$h zMe(J;-iblcWW>90Ptma{$`%AUZi8_y>pQy*1GpoiiS>`GK9%)TGXC!$FDO5REO0l^ z&lv``tj^Y#F@DP6&qSkCYO-b8O*XVx^8O@0D}Wv-tbz7`pYOlCS4pVmi!~|4dv-5i^8laoUpk zxH@-rdRED~DyWrZO2290e;bISH8z$=kcmp_ct)+edl012<`vnqx}D^FD$twK8)RpVW@yMvk8CRc&d*ku^a#%~2|u>f%{up2Q6x9Mdt&e&@t?_bEXURy{+@>{ zJjDZB-f~7aGc%-QXc7g4fF1tUfP-hsa@qS*#N2_g3675xMqbzyQnC~pK_jH^3k}w%a6jCW!C?MU zo{9eUxt*=#6(neNmoNf#hiRNdGBu|Q(@9s7|H`J*IMWuCEyE4;3IJtKS-n7f+C1=O z89gY4%6N}DeX%EYz8B!^9f5Sf8V2S}yTJ>r+}=RsLXtADv|&$w!dxTz4oSIuz=8S> ze%G>2|5coCh@K)cA(h6O>kRSfAQt>H_fE#}H@p)v`Tw>aulOfNhyS)7=rI4b9Co$DH=Jd$I?iu%Tq!e%aPW7DXN#iTjDG0TqkpLrhBBzR8`k zD7XbvwV1f*5U7kBxrIxHO}NcgSmCK*P*zt<4FpS5V5@~j2g+wGN-WtIbV``U0-3X< z(0T||f@~2Ebo3UuxzrdG=FuH~6+|7!VsYU$0Z;OEL^Mr^S^zSSbYwE3A~U-vOJDyUDUStXfD%K9;#`BD_z>Zb zYj83mc+8KTgEK6`Y;^Q6ku|@W3|m*M55gt8^^WdrxGslExn_2O8$_a0M&&_Be0KPA zDd|?nYAOvUkTJUXZ7l2Ml&#rK04@AJabu&@g=pIr~b;eo^(8BT(?FunH$AF3j*ZiHB%C({8I)tTa3VRkn) z=9uW|9))}J#GUqRh<&w4yL15QpK%2bM)-YYq2tcqZmh#_)@tYAn7$!Z+6(FhAPs2p z^%a8A6xo5O-hgk)a=r7#iC9Sn=%vgrQsl}WCq)N+4q*=_VT+ac3I+*3lJQ&#epf@`!?G!7S(!aZGWqpGk8(*`ig}*V&iyhzH;xtxA$y_N z>)-lw)z%-mcQ3s#`hcb*fp;U`yikM&{Z0^!k1?*j(d(dK9Vw#6o;HRAhEj6!& zxJ$%z@#hubu+iCATwZBgyl$DO;-%^6*lhP|m`wV*S9e%1oP-d7}LFzNb-nbg&b zLeV~*+>vogxCnjjqMaj6y1jn;s7GQLf{ZSY20O#1YGg;yjg-{KM81iL;0{|;LN@@* z6ST#KrKAJTzEMTb{1d?&eNzE47+;ZFtJ8pB_U~EkOk=`-6MB) zTaU^zm3`7P2kZ;D_=u#Q2t;SHzo8P1xqM5!?7^WSE#u5XoolRV{Q}doTaC)1S08Zy7GJ?pd&8Jjw z`*_`ev(<+Ra2R&CQf7cb97~c^x3voFRhQSEV_1pF(I!QUWEkUh<2Uq?3Cz9FxIKeB|n?CuVkX7tAhr<4Ej#%Cq?uB5e^<(Tu{>54T z!(6b8DmhS=>>S)e9h|J%5}ljxfXIRDVa(%*0*xTQ{+ zUjroY*#_U^>b1Teuc$T-egClH97?IE<0#OhF0Y9ByTKPxej00P`|jMJVCqxQ>44F0 z6StS1JT#Ng(}>CWNb0uNM*qkV5JF(s$Hm`S`+O2LRS#bpUMgwU)x`e2u1#H8woa1YGZIsxydK5$JP$cfI67I1 zBE?jjeY6QO_arp9gg1v9k)(iTssRJl7=WdW!5$tkQ-3&w4c|W=|Bh|HOKy{C>%J3@ zZ|8r+H6nd{{iLE~*`b<}mmrmA{8WRDdlJ%rL%W#To}q01jQ%5ZNy@MC_fzCo_!q8x zb46H1v;|CrZ;mdn-6=g>sqK$5H<)H5rH0*n+c!YnE5YQcu{wHPyVztNP`)K`bv3XO ziFeTQst%KJAd9G3SLmUQ|V9fRRc;+ zPd%sGo1p@XsJh&z8?psQ1@NnY|!@p3%Mm9gi!S*yNThSTSi>xCoEGLx%T*dPC_ zK3J4iwp-OZ&1%b#}32cNRbgvhDTdd7->2vcnO3Mt%o zR22P|KlOg^Lw}@|mzlgUh+KF7hZA-R_k=AFARuTl!02E$Fun#45CtF|+z(y&M--)~ zkX(>sZe#6y_I>oP0}9KH=o`);bPVMO1Tg8k$trp`n2F7Ga^3Z^)#GsOamw&Zg{k!R z#))|f#dP=GU6 zM#KYRBI_eOICiiDR%oBa@n|ggpZJs>v7kQ|)(*x)4xxl6;d76Fl^)QGde*sDZnRit zpWm`UgACR9MH}@~KMp!Y^x#))Vw2>dEk%BKQY#ne{MWqyu__rdoOP0@hS7`G*TR#L zKP;$iLuM2_a){&S^B&D>F@2K;u0F-emkql27M7pe;`+bWflrlI6l9i)&m!9 zKWFwavy<&Bo0Kl4Wl3ARX|f3|khWV=npfMjo3u0yW&5B^b|=Zw-JP&I+cv0p1uCG| z3tkm1a=nURe4rq`*qB%GQMYwPaSWuNfK$rL>_?LeS`IYFZsza~WVW>x%gOxnvRx z*+DI|8n1eKAd%MfOd>si)x&xwi?gu4uHlk~b)mR^xaN%tF_YS3`PXTOwZ^2D9%$Urcby(HWpXn)Q`l!( z7~B_`-0v|36B}x;VwyL(+LqL^S(#KO-+*rJ%orw!fW>yhrco2DwP|GaST2(=ha0EE zZ19qo=BQLbbD5T&9aev)`AlY7yEtL0B7+0ZSiPda4nN~5m_3M9g@G++9U}U;kH`MO+ zQay!Ks-p(j%H||tGzyxHJ2i6Z)>qJ43K#WK*pcaSCRz9rhJS8)X|qkVTTAI)+G?-CUhe%3*J+vM3T=l2Gz?`71c#Z>vkG;A zuZ%vF)I?Bave3%9GUt}zq?{3V&`zQGE16cF8xc#K9>L^p+u?0-go3_WdI?oXJm@Ps6m_FK9%;;epp{iCXIh1z3D?~<4AhPkZ^c-4Z}mO zp@Sa4T#L5>h5BGOn|LS(TA@KB1^r67<@Qp!Vz2yF573JoDBug@iPQ=tr2+7*HcE3(5`Q%{A2 zp%psJG}nJ3lQR>^#z-QI>~|DG_2_261`HHDVmM&*2h2e|uG(OXl?228C|G32{9e%Onc=sVwIVZ=g2{K5s0>v2}V&CZi1_2LA=x)v|&YrWGaH zEe3L=lw}aSiEdWu&2-C5U0O~MpQ2Hj-U8)KQrLg0Wd|XyOt&Gc+g8oC4%@84Q6i;~ zUD^(7ILW`xAcSq1{tW_H3V};43Qpy=%}6HgWDX*C(mPbTgZ`b#A1n`J`|P_^ zx}DxFYEfhc*9DOGsB|m6m#OKsf?;{9-fv{=aPG1$)qI2n`vZ(R8tkySy+d9K1lag&7%F>R(e|_M^wtOmO}n{57Qw z_vv`gm^%s{UN#wnolnujDm_G>W|Bf7g-(AmgR@NtZ2eh!Qb2zWnb$~{NW1qO zOTcT2Y7?BIUmW`dIxST86w{i29$%&}BAXT16@Jl@frJ+a&w-axF1}39sPrZJ3aEbt zugKOG^x537N}*?=(nLD0AKlRpFN5+rz4Uc@PUz|z!k0T|Q|Gq?$bX?pHPS7GG|tpo z&U5}*Zofm%3vR!Q0%370n6-F)0oiLg>VhceaHsY}R>WW2OFytn+z*ke3mBmT0^!HS z{?Ov5rHI*)$%ugasY*W+rL!Vtq)mS`qS@{Gu$O)=8mc?!f0)jjE=p@Ik&KJ_`%4rb z1i-IUdQr3{Zqa|IQA0yz#h--?B>gS@PLTLt6F=3=v*e6s_6w`a%Y2=WmZ&nvqvZtioX0@ykkZ- zm~1cDi>knLm|k~oI5N*eLWoQ&$b|xXCok~ue6B1u&ZPh{SE*bray2(AeBLZMQN#*k zfT&{(5Tr1M2FFltdRtjY)3bk;{gPbHOBtiZ9gNYUs+?A3#)#p@AuY)y3dz(8Dk?cL zCoks}DlcP97juU)dKR8D(GN~9{-WS|ImophC>G;}QVazzTZ6^z91{5<+mRYFhrQeg z|Kn=LOySHXZqU8F1`dXWOJ?NViPE%&FB1@$8!ntuI?)geXh|#JJC1+G^n$h4F)g-P z4WJMPQn{p=fQtw0)}uk;u*&O2z+G5?iW_=1kTy(!AJzj}de{a9WHY+*SqJ7`={VTi)3NK|)*W3PUT#5a$D6oyqH%5zjdO$5 zICHx_V;1Z)4A(rT6aasvZ{{r`HnxK7^fMLS1{;H{o<8j5hz*F@WkKQmDI*Q%Kf$Mo!EpQ)=HV^lsj9KSz->ROVIrXAI0!Q?WUosf8t6CR*rl382^sU3q@($L~E zC(AoyIjS&2(el|I$ za*8oAtqGQs+O~huhBCOFw(^b&bol)FWsp15Sra3v%&#wXz*!kSi!sV>mhe(I=_Zxmz&E1>i6=yB*_X4M#ktdNg7_G}MVRGQ z7^zX=+mQ}1xtg7JN9E(QI&?4}=tP2#z2<7N%zf9rxzynL~!MgNpRvXaU69c*^X2(c?$=h&o~Fvv z06*{JdsM!gF$KALcW(}@Q&Alo`@3h!H3j^@5rFMp8l6-q!cb?1iS$oZfU+}A2< z)&2ZoL34kkSnbf=4>qd%guV7zM1p=amds@nhpkK7mRJlb?9zYI&?4ftd8+RvAYdk~CGE?#q!Bv= zbv1U(iVppMjz8~#Q+|Qzg4qLZ`D&RlZDh_GOr@SyE+h)n%I=lThPD;HsPfbNCEF{k zD;(61l99D=ufxyqS5%Vut1xOqGImJeufdwBLvf7pUVhHb`8`+K+G9 z>llAJ&Yz^XE0;ErC#SR#-@%O3X5^A_t2Kyaba-4~$hvC_#EaAd{YEAr)E*E92q=tk zV;;C}>B}0)oT=NEeZjg^LHx}p zic<&Fy$hApNZFROZbBJ@g_Jp>@Gn*Vg{XhVs!-LSmQL#^6Bh-iT+7Dn)vRT+0ti(1 zYyOQu{Vmgyvx3Tuxk5HG!x2a+(#>q7#Xji%f&ZxT@A*$m8~z`DDl?{&1=gKHThhqt zSBmSpx#kQc$Dh6W76k!dHlhS6V2(R4jj!#3(W?oQfEJB+-dxZOV?gj++sK_7-?qEM1^V z=Sxex)M5X+P{^{c^h3!k*jCU>7pYQ}gsEf>>V^n1+ji40tL#-AxLjHx42bchIx9Z< zz`>51CG4Iboc%m0DAfvd3@b}vv4%oRoYZpZ*dW?+yTcduQlxreAz&6V(Tac9Xw3_` zNotT9g&r{F_{!Xb%hDPJqn`CWqDwai4M@7F4CQ?@C{H~rqxXwD(MFpB4!uljQmH~( zTXJJj3MEVHkt7r8!^R;bp!H=&%-OG&ONKIOgLJtng(VD0u9%2LuXKe7h$?9lQ^#cL zOo}gOx^+ixt2Izmb6{J`u0VexU0j}8Is+?LWLGvQ66Pg0ax4n^G+xW-rwp&fIZ0}l zI?y~wn^6o3{jj*VSEQ}tBVn1#sVTQB(l&Gf(sriC0DKR8#{);Sgb5%k`%l#BfM#W| zfN5C8APnl5w%nrNi{BWrDgudYAZLGEQKTzz^rV(Bst!UI7|8?nB_w}@?_pYX_G?9i zgK?yo0}({MC^6DiO!bB88kijN>+BCQ8v!rg{Y zz$`Hf$tB*WdxSPHMMkJ{&p0(l zyXx|^X_VUQBdh9)?_2P1TViiYqy+91$zg%3%OjzWyY=X^f7I)2-34bDVCEhECAi z^YqS9x@(kD(Bto;VDKfgIo z-)s_q)d2mr4O;DTUTgjOe4f51kd6T9`xa6_AUP*N{jz%!Z0E!Dqq}JlfPZ2EyGN*E zoPHJ^rT;z^0vaI03Z(WcdHTh1suHxs?;>yWLj~GlkAQ#jSWq|nUE}m()bBZ1`Rh^o zO`d+Ar$33kry+En{&JjrML}&gUj3pUFE58(t|p~g@k3p&-uvoFzpGktUMnQ6RxDA& zibYl_A!{@9au^_fB@6;1XHLORS}C(Hi&J8=@>Kw66&QJD@w>_I1XJuBW3_vn?f~bb zTv3_J^W1+E?921QNo!MQiLHISD9?+dP0BsAK+yB?l009uXXMOteoGX;?5I|RG_v#B zf~l?TPy3zGkT`N>WlZRa=k7Vdbz-66IQ979fX!i7Wen@lu-oEcweu$76ZXrc&JWRf z!tLRg2JqNG{;`-H@L` zKHfgY-Lve@vsPT7B0@716|Z$Z-Z{!WV;qGHV!`h!S>b)rZpc`9J))^79ey;7@-=zZ zjys+j=U6maKhDddqZ}XQffIbFYn)R657nRGEG#j`M-Gni4deWVXcr=HoNok4SKTPT zIW&LDw*WrceS&Wj^l1|q_VHWu{Pt**e2;MKxqf%Gt#e^JAKy{jQz4T)LUa6XN40EO zCKLskF@9&B?+PnEe(xB+KN|M<@$&ZP{jM;DemSl!tAG2{Iisge|}6`>*BENm!G2E!s_XsaUit2`a&pfn!ggt)wG<~No zFFD~p(1PRvhIRZaPhi})MXmEm6+(X?Aw+GxB}7gAxHKo)H7d=m&r6ljuG2KX{&D9A zNUe9Q=^7yych#S!-Q!YKbbka8)p==Am-8`N5_Qz~j7dxLQeaeCHYTma$)Fy}ORKS4 z5sf%}(j`4U=~Aq(!-|ZRRXvQijeGJ^%cq3itmW;FI)JsU8k4pNmCazDyH9@=bqwS9 zq)y8?KhH}MpVTd^>?u+Cs!&l|6KH<*pikOqr$wK%YZ7(>z%vWLb^+m&cCQ+h_MDo+ zaXmPW7CD|K$-d&cg$&GVPEi#)hPjGYx|SBxatca)&Ig?*6~uiQKE)tF7l+ci4JvbZ>vQo}1mB?m;{w?j6>1xBD9F+2p#Y zP3U>vfnMicQVHdhK1yDCfacJHG?$*GdGs93XO$LkB~?nFAfNOoRY`xRs9JiG7CM&D zd5!=ra;zY~qn6HhG|^&58(rYoNlP4qwA7KN3mvymz;PR0%5d!IoDF1vxVxNS5wG&fEt`JYIGi>i=Fq;YUc>8aXv_wIKNAm zI$xs8oUc$5M((w)<+NMQ6{7X7iz)2tqz$eebh#@<&91|=(KSq0xZX>fTn|!v{~LlTjaOXR{3kxDZfD5rHpl>gbmAU z@|wOa$t%grx`7}nA|ePPsN0Y)k&2=Mc4?uE@gW0-f>S_2bO;VnKt&W3k$KKdvZh@& z*WWKa@7#~`b#Kuyw9kqd zj%CMuQ9ESPc-)MbM#7}YUL)ZP_L{+siDWcU?e8%n3A4VsFYJpNeLjn2bT>CI3NCJ< zwecm{{XNM@ga#75hHnwEW-M&QOfzo9!Zfi7EH$DX3S}9p>0NY#8jZt#!W_KUc?R>k@Ky-w6=+Da+_s0GJldl zF|P?(31@{B7bweeajQGYky;y%9NZK$oyN7RTWNn&2`?k9Jytjwmk||M(3Z!M&NOYw zT}t~sPOp`iw~(CAw<+U2uUl%xEN7WOyk@N3`M9ikM-q9|HZC|6CJ8jAUA zst!H<<<&6(6Zvbpj!BrzUo!>VHN3A3vo$EF5-6b1Q~ajXENB~lhUA@|>x6=N0u#cf zv&w(qgG`^+5=HoNur`2lvR~b&P zjumO|P8X;=d`c+z1YJlY7&H@Dz-Rts$X0IYE9kSIlqGZ7utSx^+ z2hOEC-eXviWZXQ9;$Va+WlHlU%y|f~w(|)o@(5J0o|3MQ2O@+B<@r*H4*65)(r^JT zq+<*b06XMGclsEElst5dEfFJ;AQfYhRt}O0CVKdGh4Tk3-(^-{kukZb*3oM$ZffpG zMs;jtk2ZjAsn%mND4R~OS73JDbj^Q440{oS&4<@VUYMInc0xxy?FE@$J_^n)b|gY+ zOj;8Pk^)6$w9nbnMms3RSr6q(9wP_)v01|=P}UbkXoS_1#FCl?>&9cjCHOS!yEJqiGd`83Nj00{X6dHFN84%)I^*MZ=*Ihw5FxD0YSJHV{j!9v(DT#k7##q~$ z87Dig!k3EiMO;k|9XhYz8cGVPukGe$N5@yNtQgngIs(U-9QZ2c^1uxg$A}#co1|!Z zzB|+=CrR6lxT%N&|8??u1*Z?CRaGbp6;&#}$uQEzu(M6Tdss;dZl=hPN*%ZG@^9f* zig-F9Wi2cjmjWEC+i?dU`nP`xymRwO$9K3IY`|SvRL^9Jg6|TlJNEL9me$rRD1MJ| z>27?VB1%1i)w5-V-5-nCMyMszfCx0@xjILKpFhA4*}fl9HYZ~jTYYU@{12DS2OXo0 z_u+ot_~UfZNaN>@w4Es$Ye>i&qhgqtxJf9xi6El-@UNPeQ>aXcYVxOUA--x3v1 z3e=7+%#m@}QuMTjN3n--=-{@rNtyYdYS@LJ(G?*np*HILbUeo)+l8N#+F-;^(8w>i z8Q6til8Y^NG7_qa*-n2|4}(k<-HF~R0v*cP7bxlTWNJ1s6#Rz!N zCYesAbm(}4qp%-;B%AF-LyS5Q6@Q|V&Y2ar$uWn(?UstqXy;5$ZOCC_?L$F z@o#dk--?Co{)CGEP^73Kb_^>`G8sAN)M@iNKQLBj>QAcHjIw0!1 zl6{UYd;|bA+CcC#3IGYysWLa4!KA}CsEV#c)JpJcF~NX9mrX2WwItXv+s%I2>x#v) zy%5xDSB`&bU!9COR@6LwbI|OQ&5mf&L^GGZnOXEOLshxOs;Y;ikp^M(l-^>J(o0NIdbt5`(fTq>p%?cG z;%aHXhv=-@!20#xf*q)++kt8IJ5cG{ff?Sy9hfzQIroA8N>Git>3xOUNhe8nUspSV z`GL0DK}<_w!3gRCwOvD~m+Zn6jxTMde<_?egr$S1OySh6XsS!0Wh)wJPX+xd11YQ= zMq7X2tU;U;Xx|ObfO}%y{pchi>ryaM2zAy50_$ltt(ew6h#CF@+U74D#H@hdQ=dX_ z=OChf#oerWnu~l=x>~Mog;wwL7Nl^Iw=e}~8;XZ%co+bp)3O z{Mryc`*3ryyIC*S%Zu;8Y_D3bFAn%8NTYv?y_%Q4zR-DvE(Q*~>ec+JSA76q7D#_w zFR&HI@z>V`9-)xr*ME%7~<$Ykd?U8uZ~EqUe&AlGDqP{uUvna zvy#q%0y2VKf%UxO(ZC2ECkuzLyY#6cJTru6Q`qZQQ+VF1`jr8+bHIwcJg}=iko8FE zDt(bW8pbOr>?{5KLASE=YFFv&(&IM|P6@wK(5#jhxh@Pe7u_QKd{x@L_-HM=1`rX8`BDds3pf+|$)DBqpXrDP>JcOxubC$Dy60;8(mfG^6yXE(+N*UWMW? zA~?H-#B7S@URtmlHC|7dnB!Lqc0vjGi`-tNgQ8uO67%USUuhq}WcpRIpksgNqrx{V z>QkbTfi6_2l0TUk5SXdbPt}D^kwXm^fm04 z^i66Xn0`pLmnhX(P0|TezLiFcQ{E0~v*cmmAR2|PETl7Ls>OakCexUmie^yDw3ccuqd5(wV_6?YM+ zegsV{M=^n{F2a}~qL}DfhDok9nC!X$C9WV!U15~DF2xl0YLvS#K!rPqsqS7(b8m## zZA(3F3H0v&0Z>Z^2u=i$A;aa9-FaPq+e!m55QhI)wY9F+db;s$6+CraswhRp8$lEl zK|$~`-A=dB?15xkFT_5GZ{dXqUibh$lsH=z5gEwL{Q2fjNZvnQ-vDf4Uf{9czi8aM zO&Q!$+;Vr_pzYS&Ac<0?Wu}tYi;@J__n)1+zBq-Wa3ZrY|-n%;+_{BHn|APLH8qfZ}ZXXee!oA>_rzc+m4JD1L)i(VEV-##+;VR(`_BX|7?J@w}DMF>dQQU2}9yj%!XlJ+7xu zIfcB_n#gK7M~}5mjK%ZXMBLy#M!UMUrMK^dti7wUK3mA;FyM@9@onhp=9ppXx^0+a z7(K1q4$i{(u8tiYyW$!Bbn6oV5`vTwt6-<~`;D9~Xq{z`b&lCuCZ~6vv9*bR3El1- zFdbLR<^1FowCbdGTI=6 z$L96-7^dOw5%h5Q7W&>&!&;Mn2Q_!R$8q%hXb#KUj|lRF+m8fk1+7xZPmO|he;<1L zsac`b)EJ~7EpH$ntqD?q8u;tBAStwrzt+K>nq0Mc>(;G;#%f-$?9kmw=}g1wDm#OQM0@K7K=BR+dhUV`*uus`*ND&2x<wG1HL5>74*j@^8Jn_YA_uTKbCF<(bN-6P0vID7dbLE1xY%jjOZPtc z2-(JHfiJCYX>+!y8B2Fm({k0cWxASSs+u_ov64=P?sTYo&rYDDXH?fxvxb>b^|M;q z%}uJ?X5}V30@O1vluQ2hQy*NBwd}kGo8BE>42WYjZn#(~NPFpjeuet!0YO{7M+Et4 zK+vY}8zNGM)1X58C@IM67?0@^Gy_2zq62KcgNW)S%~!UX1LIg~{{L&cVH^pxv&RS8 z7h5Dqhv+b?!UT{rMg#O##tHOouVIW{%W|QnHnAUyjkuZ(R@l7FPsbEG&X{YTZxd6? zGc~wOFg0-e2%mI+LeRc9Mi3vb*?iSmEU7hC;l7%nHAo*ucCtc$edXLFXlD(Sys;Aj z`;iBG;@fw21qcpYFGU6D0@j_)KD&L`tcGuKP_k_u+uZ@Sh<3$bA}GmGrYql z`YBOYe}rLeq-7bVTG?6wpk_57A#-P&*=D9tDbG+8N86Ovlm%$~Fhhg1!#<%uJPW4P+L>rOa{&N2gbFd3Fh-nnA8 zlL@IrHd6K33HFYag|7^pP;EZ&_CU5|tx*P)T5w<3xsYB7C+*ZJvZ7o_)pdFg0Mq37s%lo=)Pp+u-bBo85|bFx@z znXN$P1N#N~1jF)^LHc?61qH?2r$7+}^DzU=b4Sh0ILA`+DkZGwe8`w6RaaLOy2{+; z*G-qRoS@LWVrj2g$m_QBE_9ft8J2%>-hNdge!7N;!t-RmW$Sx$dLFwX06)v6%V+3+ zI_SpK&${J_g&{nfAAf~@mBoJzd1aB-d!go}pMC=xBXEb1?t=6Z2khtQWf04f1vH2D zAzR~Tj#erum;iqZ)uy9mW#IE(g6{gBs0m8`Hho^9SLk>6WYl=|`BSI?aM#~0G0T@g zhZQIE7P486_X7pDDlh!Lpxdh5G=KJg4;1hc2-bl zI9c0tmCMY}Qn=5b(4Vqv{|sKKb)cXA9B?~>}U6*`p`RQ9+ELmfJLHahw z(?8R{AQudS8<=zg^lz2qD}8im+_uhWqYUr=fMT#sIo${8zZfe2N&j7)tPfNL^8Z2} z6)v8;x|<$fDzHr5?L0g@AOmYTwm%3~HQmw+c~!W5LEVM>2|z;BF)jd7U&jQ>xPb5h zeEn5a91wogI=6UL`b7g^&v-q5Y#V}Z4=>PWem5wViJ&4Bv3xeU=0-BSSJgLq4+X0GzB+;^$X5GmqzaR*xhkIN?DGhN6_q3Am7=yuN- zb_|MEpaRpI;Cvp9%i(}%s}RtlP5ojEwsLfL7&QhevV-Nsj0eq<1@D5yAlgMl5n&O9 zX|Vqp%RY4oNyRFF7sWu6%!Dt0yWz|+d4`L7CrbsM*o^`YllRPf2_m#~2I3w7AEh+I zzBIIu%uA#2wR>--P{=o&yasGhV$95c?|JRlO>qdUDA33j5IN=@U7M#9+aa>fFb^X45 z?2QBBpdyCETfk(qrO_G9QH{AF(1{Qg6c9(jWVU>`9kPNV#kqZxKsnG@ z%?+|N3y9-DUAf>)sBX#CYB(Ss;o`eS>0TYtk8(ugt>(!)?E#S%6uC82XIZqAYlIHH zMHZAe8xkWHvSk$;54;FuF~4*RSLzf()!C1J`J>iHkKBN2e70b?Xqa3NOvAB(w2*)%usxAitdXR zXsosCjl0P-*iH$V%MrP>2!E3ZHl@yU_+CN1fffNwny;LnWvPf(q;(3vd z)}hwfgz-(OR5H?(nx==K>;(!(<@t9;uhDT<@L}{HO(kEVmC@_oXQ(0S**-;H@pAPM zql=DME;|u{PV`eSkr1cw8-cy+VdH~Tho_^5PQzI5hn0Vy#^@BR|0?|QZJ6^W2bop9*@$1i0N4&+iqmgc&o1yom5?K6W zxbL!%ch!H^B7N{Ew#U$ikDm9zAzzB|J{M9$Mf%ALP$`-!(j_?i*`%M1k~*I7dLkp< z=!h>iQXd~_`k9coWTEF$u+PukkXqb;1zKnw?ZnMCAU$*2j^CZL_F4f6AMEu3*y|O1 zH*on~MrSW(JZQTj(qC~jzsPRd?74SC6t~&Ho{fJ*H*AMvXXx@p@_Al3UkBY^gXE8Bdj+ z^csKuPu+aSU<4<E+ z*bM#6<ud+wQMn*g0ivOoLF2sMG zMX|YA+;yTTVpqi0qIi@1?JkN$!q*sv^Y<6UyZ3E5ufmiwQi z%d*cc_c?mG&n@>~qR-1dx7`0aeM9!S<^Jm^0J+aC`obd`xi4Gp$3(a6bIbj-cuMM7 zii;+o|1H4kBUC4nix*$<2{av@xW8pXsPUVs;6 zJVT3+(1xAt?9Q3@Iqyu)%%8u%egjy8DR6vr^rrerZ%S*Q{Fc6`FJH6}@8{p6nQo%F$e3uUKnOSQ}Q)_}#>H zIS{p_QQ;x^w&N3pj&F1Hkiv+)I9^?SyjnF{bf|wGg%C(Lf+V!)h2xUId=T2E9mcN1L$QF^ z5g2*u_)h#xV5qoL+7?I^OWPS_a6JtT*$mPcAHy(mJmUtoz)Z1zp0^RJebf|pVGWIs zQB0nO8D@fneP+6d6PT}AA2UVLt7UKlb7PprygKtn-5>!^V1XRwIrG!}4+mn=`W zBk<_rS~lAZls_hOj;GnnAs;L$9u zaRbuj_dhXN_<^afP)`ndO!qW}o+exVj;Uj$zv1Tc32vVWmrHP`CoJ`Zxvp@$E4=rv z{Dp%8tK5(97c5fP{T{ZAA#Omvi%lqOVetgT%V6phEDiQ6oM7cL#+QIm<(v8kP)i30 z>q=X}6rk(Ww~ zN);x^iv)>V)F>R%WhPu8Gn7lW${nB1g?2dLWg6t73{<@%o=iq^d`ejx{msu;S`%=Y z2!BRo(WJ^CT4hqAYqXBuA|4G-hEb5yvQw2Bx7zVRpD;RR2ccOu@PhR3faoc zzJIZ5StRhvJT*c`VV6u>2x;0SlCBHsQ7n>YhA$6iQU$Rd`#A*0pf5UAX^2~Qi`Ky%f6RGsoueIc_WKEcM!=sZzkijF|}LFs~GM=v-1aFc3dl?tifz zSiqvXmL+l|5-?ahOL%3?PG<>&D{-(~{sG3$mZG!I^`lqCHWOSn}?5JWosiW?}R7Hz45Z6M; z|I3ZkC#9f+gJwObwvJ7+lKPKs9)HS$N-3eNAWZc~d`TP=sY$X_md=Li)LwW?#|kR6 zy$#RzQ>|l?27Kf`O2bZM(f5 zT<@B@DC9-<3~{+a6@$%* zbtze+^?#(ya}=}LbSblhT0Q6Rm4>3=gi)o*G!B_6$tq*ItV%e0&U6FU!uj0%!h9}S zX6NEZ9}oimg4WPW?76Hk0#QwuQj$)~3QJw+v|eX=>YZgbHMJs34ZXEzFL($9Pw6>L zDO8nGd&N^$GQH4GKq$+GsmsL%*AWQpwp1!JQ-AyUofV|o;~RKj0^!|%nF=P~ai{JL zHLCol`|FQ7a$D7+PR6Mx&`hnhg>;JWrBjTd0T_>aUBJK||PoA}xw zjpy>>3&$74TY?_p_n~D4+YZ_`VA~C};yEAv@pMP)u1z-biGn_klvcL6s zU`UFOa5WKV3&fLwP#~_QGqNI?vZjX9e_Ddmyv`La8Jre}B_kXk=J63Dn>GS%Nl7ty zD3D2o(^4iZ3mZc%E$ibOHj%F0n#U)zib4~{uoPZTL$0P|m2+KIQ#3oub%T7-d~5T@ z=GJh6j|NV-!5BPIEvv`*E?MCW0ZmUuQo58-cw|hMG8wK%_B(RtIFDydO?RP^e__!P zX;g|RlA4P24jtif(}ij>mC-fQG-YluEa|d!vZky=`ljZ$Ff1r&IZhWinz9xVW74RO zYid$XF*J6~9#4m@lhthw1!$|R%I2dC^$n%=%E!^TkD;QWai13pu*d@!Y6y9c-dw2l zpbj-&crkx2s<6ZhH|C13WnOqNe@}d^VDJ{l;le5kl8?)VY1pm@y|@qed$1aQ;y}@) zL?Jvc0$AuFD-SZv*SVC~K`>q0t1Aq34UJs|`lF_(@D?xDV66bu6ClOSK1t`Q>F~QK z56Cm(MI(a3aT7ypQO-6;vTAZ&m6Uwuwr6=LD-tLFL&h0P zIO1GPDmNp0`#UM72-bPfjP(o)4PIiAp{Ai!ThwhM9u`&DL*e7r45@}qS>??T@1^nnVwqpqQ|k{%dq*L zC>flElRbiyesX2Z>T19VbuXQiV{#@+&4oMF+fTiOA{>-6PSIjcOoKFS6iq+l;13qz z9r6xO;T=vS2R}50ccv2#o=Q|h+CAJH)AW%6InA}KX&=!}FH#s5e>yTlWkaW!*oqO6 z8SU{JVB)Hl0v zvZTX1MRnmt>R(Ase@{zh`Mq(VYx=EF{=B@5S3GzLuQCMxe}@eW>)Mz!MD4@r)31AQ z0&md9FQ^oyd75EqanI>gGg*_2aw+Y?TZJByZ%K~Lw>>z6cc`nDyCqzBkH{8`(LOG~ zi!9q#KEQ__ypNCak(H{r@CidzT+zgq{Y+dopW-YvxkPDIf8F?;VQslqQT}{=AzZ6F zxnZyS=YB7*X}^!B6yLBv)PF1Vi?pQN^vOp4KT@~m?Cor>*}GrNCrA8Eop<;|;99Y} zKl%=)R=@D=O1lzz203Idf@c;Io*aod|N(Ldvd&;<#t}{mYn$t?;DCw($YAa`5v;U*>3p2K6PL7 zys(f}dR3lZQ!YEl$O}x4oh@DO@qatRvqM}Vm)_j>J-94ELt=Krd$CtZ8|QKA>}ys5b|I0wKk~(gw@WTg-gz-E z-n{phQ@gf~i|(7xw!Vj%cOG@#m!2tdzIT#XUxY_=#kr=;#50FJdPiKX;<6g%q5bcD(S^wB;}3Jp@7< zZ8SLqRYg^%-#s)lqC8l`qOsgr%x+u3JE@b!)d9qQ{Pr~%n=KFw@&Ec@m*Rq_0JbiJ-FiiY_(H~OychZCO!23^?kxr zsb6t9-n)(!fBU=h#GNC%a*MbEeJ^QR$1+>KO}iv^@kf((?fv)jjy!#k$T;iB`fx9s zvzxcKJl2e6tM1)!{qv34mp6vCtlhS;y6DDUlXXfveK%ZiQ8{u;>;0mt%BNQ^#D=u4 zTW8me!45Xh8a%S}8iHk*; zc34jqTp|rTRNYt_aaJ*KIuAv!@??P}v9jPJZ-M46271&EMPA8~VY0rX2RK?0r?4_G z=%c8Lbe^oZLUeMavnp62{G3T(ETUTH>k3u~IlNU5tQh%hJ`)sE-+Mq6Yk?H9f)CP} zY_Lp}$-xIK5$7WgHUV@9%T1u`HvwI*i(Pa>H^(8RR7~s8;^31S^uMk^xyMjTmQSU{F9Y?c8LA z6*jEkA*0EOD@2*(y1`E9U7;!i9~1$43N=S==mjf!yh29?-XUURV9-M`*{~m^2y+-k vO&Z*)1cp)oP!FoJdnQj@>B$Ny9`3IcWx78NY!UY=EiM6G;6aIVL4^VU&1=uc delta 34727 zcmXV%Ra6`cvxO5Z$lx}3aCi6M?oM!bCpZ&qa2?#;f(LgPoZ#+m!6j&boByo)(og-+ zYgN^*s&7}fEx`25!_*O>gBqKvn~dOCN!``g&ecy%t0`n>G*p;ir0B{<{sUU9M>#WqH4lTN!~PgB@D;`rIdQ#hRw z?T|`wO^O=zovKDMVjuZHAeratT0Q-HK<95;BTTtc%A5Bo>Z{jfiz& z$W5u4#(O_eLYQDY_i&xqzVd#y&cR>MOQU@-w1GN((w{b+PM;=Y3ndBGVv|>|_=ZIC zB^E2+XVovHYl%!I#}4)Pma4)hM2Ly6E;&R5LmOnMf-Qz43>#K*j*LSWoYxxIR5Csm zuHXA8{`YgmqApC|BgY0wGwj-im6rmS^jrAbN8^PEIHj1WH#AVVuUA2HXj&Vm*QD^# zWX8+sR14XM!@6HrfzFpcC$ZXlhjA{{oq5cs&VRBUX2VwX$fdjO~`3n~1})#Bxr5Vh%KwFov=k zW;Jy5qsvC$lw>?*BsoPIo}YgJN>u)C^4Abbjx$NW@n5S8aN_T0BeAXWjz#dQ=3v*# zRQrjH1%R&krxBrfITop};aQdE=ZRgLN%n%+^y5BOs|pO6lg|I3prX{gSgQuRK%177 zlE#t+nHbT~VSO995imTaX&SCB&pgp`Izkg}-NV zI%~Z42T+^_9-gw;yOI&!oZf=H(Cot~)w4^gX&q(zg`7ekm4un&?FuaJQKIrLF$<_% zR;ok9K%L!NlTYgW8?uhX&TS?ojtu~oLm(`7iY<5Ci@V)7+gRHbb!o0OipVh)`vKW) zp9OVLDkaP@Sn!ZRa zpfwY36ct~JlEsS7_Dr%e0UL8^zRSsSv3K)+n$b@Xq9*^-p|AFj(*#}L-%5Z}D@Zl%y2gokn7l;Zr z3CK}pP8BDR1$L~R{R^BwKH~@v9m;O_$00a5MMXTe!u0FG^=2=_f-XZR!DQeQ`5S_$ zO>mOUF8Y-Wfl3P|Mk-VDsBp`X&=kMQl<>nt9$C)^A<4v@xtW>qn@`Z)`|gCedb?$A z^S(N0{?3!oy|^tx0p&<-D62OWo$gVhEodpMi;O#DM7P>i6bnTf$_=~8)PdQ+^h30pu>DfM=LQT20!&5)= zGdR6}f=YHb45NFG9?dd44$Dm~B6k3w1%E%atidmZ`Kaw4q&8yb+5=wqe`pXWH0J%);cCo710p3&(EMuAI{aKjT^Z!u)Eq~b?HpnrSE9ftF4Ibs#HFpuPR zyT$g5JIX12nSw?q!}IY^iHMikUh8V)gjx{JN@8Am6<$2Mz^mHY*_n$LNj)%w6Vs2|Kwpq;J=(VFf`y)>|;A@J@8mL zpw=k%oRd`%OdUL*1^Bd27^<|sYM9NqMxOfyc56FSDcG3u;oJKCAOsBvw)JlyBt5jT zQZ;fkKI1}9MJMtnCEG?ZUph^R-lV{%Av1S91fH#pacM-EI@93$Z)d@UUxu6ruJMHVl=>YjT8reRi0SjW8t!4qJkSw2EWvi_K%!>35@JDfw9#W$~G@9?4ubk&}M9<~>f3`r6~|Hun&D&#w^ zZ2xrK!I3O(3uNXz*JhWWdgESs3jPCOS_W_J;0ggAduavgNUuLi`PfS*0$=1$q$C-# z>ca0l=Pm+p9&+rJQNFKvb%8vn0!qW9SGnIO&tjv!kv980`FquGKanhc(YAwQTGx)(9c1fRnojjxST~<*=y|?=9V1w`t~7Ag$5h)P#FwB7FM=E`e^youj?Nh^d}|GOC7mPW z_H&16WtD5M9H)i@@=Vzo^f`%yIQZ-qGuCko?CP8h^B$X|UkaKazJe>9C00F82u$Iz zFOjPU5)>;*KBg9UezT$OL$aW(Ogut^COwjSO2!@-ZbW#lHVfb_k?7DlEGcbl^tn{p z#+go${sx^TPB3R5272wadT(x2lACj6Y4~LktAm z<+#pEqlksdo%9?Q29%rP9C+LM*WZM-N-e*wX85OOu}J7Zrt%9iGjxN358Fy5GGaNA zlr-b*b{4zqiK)A~_jjEnJhRaVOdID52{6I%oS^X6)EYS(>ZE6NKd-S?F}lIJNYkBz zX=;apb)xyAi#nMFCj#Ex($CGiR?oF|gei))16?8E-mB*}o2=$UtMDZxq+&Q?liP(n z&Ni8pBpgnCai7%!7$wG2n4{^JeW)f-h&_$4648~!d7<~p8apf5f~7e0n$lV_qbrLM zH6T|df(D0@=>WA5f5yN)2BIZFqObOK5I*vhD*2~PZSt*83>fM))aLjXIEokDF;KGw zZ_75?2$lhYW)I_!@r8QpYKr4p27lOeG~ESg#8)LE@pH;oozO*hv19;A7iT#2eow_h z8?gZtDstc~s|f{hFXH|~d~zQ~z_94FB&hp$n~Uv_DB!2y<6&VqZs>-fmUU^yuJGdJ zNCHP?2Q+FZr?J{^_M3`92rOWnrL2vymWZ&0dYxz>Kv&GXWgwxTKz)<+J43r&!q}II z1DmfLl8nu-xGa?TgsrX45d}j{QAC!m8iO1JU=|Pb8D@9FE-V0hJEA?F)srec5$GqD z8(`^KQozt$N;6ts8^+R_uiy|d8MO=#Jvd3z_#2aHXjF94XkEdq3myI_UvT|r>1&LP zU*Mm7Fk}T$qbutLyH`@m{L57Mlkq!hAMe>2-o(8*axogLh^b!!{|amH_{Hrdu!4kWol?jSB%l2>w;Jry$!mf_nbz9_B1#8bWJwL@w!No42F zZ!YAr(^WO;wuxHb`%ZD(qKIOW&)L%j)eAUf-WERo1D?D~FV`np( z5x$@RPj8}2Rbm<>mRjfuPFJ`nN>>ltyp;oE9#K9IU>+pE$;Cq!IYr!NXvc_-MDFXBXW=Z9LZM(k9}OKqEKn5 zMk4%l_POO{UM$2M+YvQV#N~$?Ycqe>LbTz9ur0(-Wp!^8a^GDh7h{U~8h980RG|9E z6RPnEU0ccY1fEIdJfnZ?3Nl4X0Ag>*m6>|oajhbexf9~a8(K`2Ys~o)z{jnuOj93V zg4L4K@x2Dewt5Bok=03M@JIhBSWy2hwxcxRv7ukj`8uYPGrMdH0q!`qHJ^xDQ_bLG ze*?ZCvMv^t`JI7rlqLPEo^WJ0b^>d@C~mI!Zv)-ljBg#u;uvw%ZXMqZsz8Mxdtvbh zbK^eGn90ynsgjzKUOl)O`l3#-uY%L?tj;+Edgz+awV132>9Z-?mj*}u ziM4~P{Pc$s;}v&zYF)Te5J7W2!$o`EH|~F3NfA2NjF&~?@K5S*f_mv2@wT};{Sj`b z%#^~iJN17>qQ6aej~{ubsrhkBAD`C(j7{y)+hU@!^SU03F0Vu6vU3+>!lN@MLR}42 zLOtGS+@f@~=id z8&aK=-2+Pz*y)te)kF3xgyS?qgp@L;G(tM1&#!4p&Z$yX2<+lj>VWT1tiO4`_h^}* zQ@WGd`H9t~sH>+NT2d{O5(~BeYjG#5=s&k0J)iACkpC8u;rFz@_E-w@s0bAs_;b>+ zeR6?5n@}4wjy}GSL@%#%!-~chg|$Q=CE38#Hj0u5P4^Y-V?j(=38#%L#%l4={T(Rq z=x*H|^!EG)+e-leqrbec5?(g)@Op(cHsVg4*>F$Xb=BheCE*5LdSmdwZ-MSJs@@i{5t){y; zxAVyon;`>Rns;YH^`c&M3QdxzNaJl(Byct8a9v38fkXaJ_<=8oe=(6%mZ}CJAQ}2r z#oHZ)q;H0pGydy~@02e)oeVW*rQaD_OLr+)29*|p(gAHd<9*JxBnu0W61lNr+cO_= zX$B`VmPwyz9?FV9j3-@v0D7Z1Z}O;#KZ!@Gm7ZeKORcLQsPN8= zAZRd8VWqow?b1Kp8!AiYk8acC$>6xHuUZWkNk~?EqKsUr2$iixV=zYwM9laPwn)(W z7b-$PlwKh6n5^&Rs$#s&98P1ch#7FGNN6yU!Nwzcesp2Ylw~C1F@G^YA!PF|a$MJ+ z{!r?468ju$sWQLL=o~SYP|CBJ7(3`;c^t;TL4ScL$Pvv>N+5iugRLdmL zaD(CzY&3J+N)7MS)Jw`U8u*IevtEAUKN4~AiL82B$4Bl5oK#No3jGEW-o4`>c%G#8 z!h<$iX*efTk1lnM-d*7Db6h_94Y@IcQg@UJ1-g76_d9@vHWB%F55WG&!4DAy{K)Xv zz~7iiiq(J#G*Jdb2F>RKFnc3y>bIwlQ_Jhzoc4h(EOVm|0C}@X1v`lf-*wuaH5_H)kg%$_&tAkc`-Mk_04t+f0A_7=y20O8`7#X)4WDMOUpG*Z~n ziH5Zevf@*c28LS>z60h(QH92FxJHOKTj&>ep>z##ag+Tm*{QU<#Sk`f3)1y<#hgNV zkGRx3`qggo)?FK!Vd`6U+lA@MVk3QlsjDj#M*^!8JsEqK;p+%l%NyiKg#EX^3GBuk zlh2;u`5~mtZgY!005*{*dmF!OsrxVg*Rpvf{ieqF1ZPV6Mm4vb&^x06M8jn4XO#a* zXJhi$qNRT@M;;!sLq`lbqmcnAsSvSakQ{XcfmP-CU5_ini_P>t3m1P+(5I3tq028F zE8xAnu-M!FQ{&(q8oC{RXMCqw5&ri5tvt$=P|_J!+#m6Iz;U2BaX7}7%E%i{`jgjM^OfP1@K6wN+iSJ-2z7%MfLBS2$+zC|(5j4tu zq@N1d5n}UyXF>Bz{_%qT2O=&{@hkb|g++>5oZPMe%j~Ee^;OCr)Y7u{V4m&Qf@%WD zEUKEu%teX>pmF5DMIP1!>pm1D);32{D-N5>U4W*9kTO|z(Tb#n-@+j!vWj-S8aRy<(xvQm zwZ-#hyB%RQf|G(r&oI7iZhf^pG13lCEWA>mk}rI8IFlm%*!~#7;2xQps>NS2$f@g2 z1EoM!1ML(HjM)=bp>Z>u=jEM5{Ir>yFJ{m8hLv-$1jxB4a{4HNUhk+Rj5-H8}G za~r&Uoh}bQzyC)f6#o3mEkwFNhaD8_~{CW03Dv2Tbl4{ zAFamTS$i&ZYWmae1aCxVNIKrj+u4g3%D96}iqw8~HBu+gFA&*oRP5Z`MikjjDgYjq zkf0&#_Xj->@bJ>!}JGl=t1|~ zGIx9!u63fRtm^?=^0z=^H2SZA43p1deVixbphteFyrqycaRq6DLy2$x4nxgB;-Dug zzoN<>vK7~UxLPDR{wE0ps6mN9MKC>dWM{~@#F)ne0*ExL**#VrA^|@km1xCtF`2N( ze{G#meS3J5(rIs2)mwi>518)j5=wQ+Q`|O{br)MyktYd}-u+5QYQmrBU2ckYE7#Z$ z>MgHjknqi-2`)(Z+pJ?ah4UMg*D%PFgHFMnKg?{GSZZ*f3V+g@129FH@79v%&$&v32_So*G$-3SIp6 zYTlLgF2}s>)U;QtdWf5P&xikI0p1eg2{G!w0+xXNuYf%n#X#fou8}EYvAw$zmrjK&OZkS!$REMr$*aG zyPPjsYd_SXp#Vt9NGI*R;-*4~Gz)&7!zq>hh7)i?8PzCAAv(pNcUGlPNf^OXS$=bx(V#ji2eMF6q{U@ z9?ldp%YEsl;)d%}_Qs81OX>!2>kyChh!-n0Xd@2C1cI2qkRk&b4)(?@KY|?%qMoYb zEi7l}n$O`v+T31;YZF(;FEwj`I8Dz*9fbKrE)8#&?joolVY~3YbZuJwfRt4-kCOM; zcm34HXKH>;a?joGLqjIBG|B??@rS`LSU(l!vxSyfKmGa^x5&S$gvrsrlVT0@Yw#bP z-3#zdbm1;n!DpT@>AnxkZ4llVa;h^fj?R3uN5?-F)SLb}a%TBE=HM5_U*{K=ddu;L7kJ## zqyyGh;WY5rpvMm)$*xZHv!CUlc{zU8huQp`KmQT*yq*ugOu_#Kt-kRa+ODx`Va(;{ zLMO*lsSV`U%+u>-R9GmwqgWulP#>jO9|V60TBE z5ONjntHY2V_MmDJHr3CyuL5X%IlQKbDRch~>EBrwAM? zvOJj&z#NzlWa*K*VEZgjP#cAQ-HRG&mC)aqyjY19GP$U zSKm`d_gXzrLE_^a!9R<~vT9n;>{y3F`!rB%M5psN(yv*%*}F{akxIj9`XBf6jg8a| z^a*Bnpt%;w7P)rXQ8ZkhEt)_RlV=QxL5Ub(IPe9H%T>phrx_UNUT(Tx_Ku09G2}!K($6 zk&bmp@^oUdf8qZpAqrEe`R@M|WEk$lzm$X=&;cRF7^D#Nd;~}a8z$(h7q%A88yb=# zVd1n3r|vPZuhe!9QR*ZtnjELX5i*NoXH%d1E1O1wmebT~HX0F~DbFxk=J^<v|BCiebRdAHYXxOo$YS#BHYecz?S6CX@AcF_k;#_IF+JIV*5|%lV=Y;Ql?=b^ zt}1qN)~qaKnz~KZRf9Aa7U5S&Opz~;SF2ojOSD3HP8WYTbvlEyYK~);#wr+UO8_Sl z$-Yx3B~JYU!uChjzf0v1TKYAtsRkH`QZeF8Q$_`7iPJ79{8V(jbX4T=-LF59vw>au zY6LS|t!~Zz>*ops1&9o5w z3lQx+lhgdg^4d0r-%q!s(A$J%XYhUx~)v|ptx_cU#?44pnz*s$G%3=wh_01 z5l7f$uM;P6oqhM8F|$4h0me5--syUE%vI)HuhLv@kL`s1eP@buw&}80Umf5QOXBlP zAY(8r9}paD1p*&Bir^3<@3Cc4Mr>EpoDHghr{U$hcD8$^OZ6bZS{UYhl_*Otp}Be} z-P^9U7tc!@aodKCp{~TV6o}?M9xG$hN$Kr>|7e~E4mJK>_yjrqF@Kk1;fHw1PP`UI z1Aoa$7yGRMrUVO0M9$rM;=Glzi>SO8!lqon9E_1^0b)CsR0%Nv-$st+be?a*qJkqI zUNaqi*6Y^E>qlHH+*M=aj?)y2r>RGkG?X;Rv!7JG6Uz=^g7B`jEKEvgUq)s3Fw|zFMdak((XwlUaSRN4hGMrH zn2xFaLH!t8txnTiQW;qUWd^m#<3zgCp(=5~i~xw9lU{R~o1qSo#Sh1_4W5(^hL%O9 zOauMH!uGL}u?hV!4V~#?F-<;)X<)4B$u1F4 zf=%}>{b#f`$Ixo^Du_42V6Wir?Muh`(!izQSV9Y3d-MCQT|9bs zIlCtJP7*;A%^1-=u(Laj97hG}uP6Hq0+DzAjB^|$CG(?e_adMTiO&^_9WwrW4H!ju zWEYrjLw<{fSyh-yiPOP{O;c|453fxkp`E;k&)d^wYK=ipbD_kG$u*Ro!kQJOppV5* zP4o#ab%r@RITbag_zHMKF5$z8fJd1L+D8G@m^`*H->XyF$E{x;d;A+T`A zR!1#O!ed)ai|TF054f1+K6 zTDH=fps}vL7=Yl3_R)o948I{CP*`f1v{E~-xX#PaLvb?#qQRElOF-pVuL>d8_�{ zSCu|?z-R)71@L#eM!y^Z6p;ZjzlW@gZzHJC3~O?Pk5QEa0q(aFy!-~pFZ%vBM{a0B zOfAZFmYc{!vg!PSF@l2U zJK`=N@CTmAO4Wuqv6k{SNl?~rs-CcW0VFIdAj^B2Wacs>M@3N&63=c06V6Rf2sR|QLucLaU zKEq5=F9zA=+3ZT|OlY$lIrFmvTV4H!iv+MxhtKJ%j}wlD3qAoT@g^}Cw`#0dsQnXX zETbS9p{IGl{fkz7ld(7^$~HEkkh7pv3NYi8<1qwOw!a|xaQ$TntGU7;01Z4?b9D8N zBh&aOYgatY!f;X<$(oO>v=8iOcEG%aUvS8Uu1du6!YK*G&VLOXlHRCKu=FF(IkNo_ z!128k!z=B?9(@872S5v{*=6WjNH3gAJAUYkC%^7Y;H4r>$kZZC%?&3E-qa#4n-YG$ z{5tlV`bCK=X~Idzr7&v8p)y!whKx;pP;V!X^4&igR1g*2j}8HyVC+>KqbPFthf}+i z5*V2^NBvmwfWIU)3;IBGEwFtYFWVWUoB2RyvL7S*E#d%FT_ytxM895Q4V_PCQh+>< zlu~L{SuQcQ?il+AeFdE87H!P8>HgIJjkGW8@`{o5wNd6uVn=dNX5$aDi14$pTSR=` z!YTmifM=Cy`Z=%xX-u&9>1bJBw3nKr0@mO&YfAp~^V^fzVJyvwMY(hM5 z=T^FaQL~&c{7fIT@FE@vI;GbS=Go0=v=3x<1AaB@b>U z;-hwvu#U||CUj!>9G3YgO6yQX+H)L6*ozXXaV=U_b`_DQWq#`f$?cZ;??y9(AcTLq zHrc9U_$w&NRKgWZ>e};_T#tf-g1TX#Ttj{JjKjCJqlf63U8$=~02ty9Nn3p2WX;CqqYS% zz5QZEArIj!d6Y0VI^JFWKudu=NFUPF=6TxRR|reQB5_2vIn)qBV}S3;MX1}04E3Mt z#5d$zK8z>OW^i7tXPB6e%UCqcK(le)>M}pUp6H17YHZ$`4urRAwERt6^`Bj>zwymc z6H+f|4zhQjlg1Gy%93Sw`uMScxrA;vQE~ta!zM?jz@&c;IxYkrPHXB+h4)S0@SIgF zdm{UTZqxJaxzBR!!`71;K*uco18U~X>AK&Pu-C&`R?B-Aj0=_$cxPzn{MlJK>ywJq zsw-Yj{^>7%vDCYw^iw(od$~o-Pz6ks8aQ}A1JFWnE@Ez_SYh@cOMFVY`?D$Y&Z~a1 zd>zg|c6+o8_xSfEUIvTsdiN&WOe=n|xS;8X;CYLvf)|=u($YtOu_6J z0tW_ukuKXj2f=f}eva;=T4k7`&zTqf{?>lGm&{Fe_;9R2b^^i}Krru0>ta|4^_A$H z7DO?PFho!p4A2C|$W~JYbWN&eW(4R;;Tmhz zkr;EbZ4D?Birca@{afZpp_|p2YAInGJ`1Fkz7A$droV0#{h=lZdX+xO4B%I?B_3ac z=7FCkf`P*_R`SaCnBPG1Jd|Abx!brVL zIt?Rv1@qnIGKpG7W-M54@Oi;BujL}Xdacfmc_9q?u&4#P2hPg`({??ZOOjRFnps_D z-f(IqU)UUW`f&U}`A@568jBEz<~CX~Yv+1et@-+dsV3RVrNTx?H9ht?VAAS0D1{G? zJbr4_B_Tqy_Ag;Xppzr)KXQ9QX}21eoMW|m_{|BBHJ*=OjhvNq(4HgLp`u-X3tw>X z9A?^?H5zIU4r9K*QM+{?cdUL9B5b=rk!&F@Nffz-w_pG9&x+7;!Am0;Llsa02xfYC z*PtggCwO@a;vLXCgarLHOaCqh;)QBGzd)|oeVtn=&wvyz)rOR3B)bLn=ZqpwZHq0G z#6YvZtco3reVEzgsfMR6A16B&XJA|n?MuIu8bp_){SA_{zu;H?8${rR&r^T3v9C(nb5F3yeC zBCfU1>1a`bLUbS{A0x;?CCtvBD58$7u3>y2A_P9vigNVLI2|Lin+b~C-EytjMOHW0NTui}pkxXdFdIJ$-J+Bm$%CN%mac~u zc65u)RMsVt!-|8Ysv6BvqDBlFKElp~B6L!lpd@XpeV9f#ZPtB*A?b!2cQ>(0KpkD3 zcX2g{WebJL!6EmdE>s!+V>?WUff2Qb1G0)SgHlNwmhKjxqoM~UZ>S=G#3}dZqbOgm zLQr$%IH~rG-VibZjQxA+wx_MOF@JC7m(z5WFp@?e-&dnA^W!f5(1q_mx7SHG&7Mjz zJ*FkzBLiO~YXM}_WN$-^LB=)#9j0}Ig(60{oTJ7L{`hY&|LX}pO&lXsa+ZJY)@FOggOhohsSKci~64T#~a*U>?#ib&8;moQD4mX2U+S(Fg|)$9R86W zITbI3PGBmng{xAMx7@wkfPyHgTBnY--U-MN(8g4;hg*?%-H-2y9+fMsROmUruu~DJ zD`y+zHt;&kEmb0pX<5f>5axt7b!mHhGZrk)cPJl8fFV}4Hof{DHc?nmlNe4OZlh%Hw~gDORC9fFH@ z(dp|iOIbEM2+*ogN5G5IIj5N6dcX2{rbl=|y=_lReUu(wdD=vfPY1!pN@X;H)!7M& zsVSTH?G;8EjqWqJgt8F#raa9{%Ig46>|d7k@)*edY9u$q-2MD_g(YtesUb(fF@ zeIca^`q$v%I*l@1*pSA^WwV15>IOc#+Fmv`%pKtg3<1=cn#Ja|#i_eqW9ZRn2w?3Zu_&o>0hrKEWdq=wCF&fL1pI33H z5NrC$5!#iQpC~h3&=-FwKV0nX1y6cWqW7`fBi39 zRr%M}*B_mXH{5;YJwIOwK9T9bU^f*OUt#~R;VnR}qpl2)y`p76Dk90bpUnmP%jt$sr^*lRURZhg{Jc|t% zzJ@`+8sVJPXQ1iJ<*|KHnVaNh6Bw9w7(H5d@A2z)pFDaQHfA+~;ft*Wl5TXgXt$X+ zw>HuHuNiPuH}l);i?tm23b}z`d*)Fc#9aSTR0**x64KPFxH=waD^aF`<3*U+;u(Jl z%Vml|ibUgNPW@Mu(3F&xqqX`Ywa;f)vz@_@ai=KchFb+T#v=)>bVeCp(|;s8%R{-yG(vI#MB|PpTf%;Q_dytxihYgUEEp*4UnBD2i zFzwhlAsbs^rvyOn1@$Y4a#xL*#mfe*-%9pKM;rMxBrQ{x6g=Z)-ac6r2QHFaIB3Cb z)MlIq>|a&HnWt;JF7aNioc_56#kOM7`*3HQOh2zj587o#jVvMmd0^Lq^}+G*kE4L@ zyr1bonUrLt{25*}164@vq#vyAHWXa=#coq+BP`G?NvJ{D6iI(?WK_#=?Sghj z1PAobWSn&T1JN2+aDKWLzLa-vkU}op+rSMu-^54o|YB$BNlXsc4)Pk+N;1Zjv_2G@*gdMul2v zus9!wq9-nM_j*C2j*4}T#EOpQH+mG;>6M45k1Bv!l)vdjfmgsSe9%ze*37SC0>9_L zi$J!Ziite+mT#sPW;8{9EdmpRcM_V2yctTOVr}V45Ya@X%iVpnLr%`<6JxcpQZJW7 z8cdPFktXB1WhRl~Hl4PUPw4E0+n*{!yDCO9mjal(#n-SeE6ATb`3BWpmcOoQtW0YC&i_4DFt9eMt#<$YtDl1dXA!$_EIQN?X#w1#3P}!YVg2_+D)GMjl zY@_EZ_ZKP?D)_w?>J6RZnB*Q7Ruv~$QHEOp7abg-XyAe)|FAORoics58~_N@dE!`8kvn*VMyv=fg8F zE;Y1gK-hU9#R`_&5n`$v&+@j=#2b-LIZsY&v=}NAOjfOB3*&2UItP}{OqgRpGh>_f zh%mJf#U&@U;;T#cyP}$M2?X^}$+%Xb$hdUMG3A`>ty6>%4yuP<(Yi8VcxH+@{t9(T zEf55zdju@GID-2&%(4Va<|Ra3khy_F5iqDnK(rPsYx`73WPueFWRJV)QFt_0MR4ew z^AAwRM+u8@ln#u7JFYkT)O+ zi#|KR&In+^((C^Qz6W~{byGrm-eEQBwWk;Gru$Vq&12PTBnehngdy#zSGdTlw| zntnZVw0Zw8@x6+gX%7C`9GLL`vpHbla6TX+B7XSrfgEy0hYHbGenBTju?E1^# zcPx@a{i?zW3ISa;V@%Kjgr2)Vx3UHv;v0j#v5i!do{bld!wDqWoiXLi;bP20NC_Q1 zWmLa5QI~_)A`d}#*aQ+SfANbQB7Qd!Ncl(>6 zheiX141UI3v(dtiSKg*zR;+|a*Uv_OU@_I@u$Sw%+tp%rqDxg~Va^*|OD%zXAYe6! z!Osuw69pNHQ-?@qEDa7bt^Ga?Xa(5g6(KJGSSDy#r$D2V;~$a?q6O+}b4^#6wsf5E zX_GK0Km%Z@vtZr~zNs08B zzlMH4(M*)#G5 zynvFiw~srA#@cLNhHk`!r@!W}8-+5UBM7C2P^oZ%kc0uzbTp>FHRO=xYa=v)0aQul z9UgNxrY#bF^%AFxsI;{sv#0ekRc8}5bc+e-tghcK-OU0FGl`O!q9lk-bQK3kz*s7? zV*U~Q9=~-fem_OJizGL{$4*=a7|@ZKwLY%#p@2?FP3Q>15nTl#b(ZW{k6q`Nx zOMonpItf;aZ4(|66znCH7E27N)R9I&GsIJ z*ClS8kTkcOvZ{S>Fv|`^GkxEX=rkW1(MQX6IyC;Za75_)p3!=|BF|6pLRsYUq@}YIj4k#cwM<(2dKCeZZpd6cJ$fz6 zXU8ca+ou~;k@S379zHDD8S5)O*BT7~{)Dj3LCoshK9dt=*UEKo$P_!yxozT=ZtBkj zev^`G~ zc4AoF3d|9i#^@>JywzuSvW7krJ{v(4IX&@ZU5})Jy)F_p647?_s=B2@mHHAWI5l=- znNFit0x5-AIV}8zv2z;Y-K9McGGqK{hU0@PjRaEJG*_X4Jo*Ua=DamQ8b7f09*Mazbhhn6LBj%&=C`Zw8uz@XoMbA z%j)N=G34Q-&zQal!IQE=*PWyC%Nzbkc?SQz^J9l> z3}_mkctbvtd6Vvr=Tx5dQ|k=lg-=zHk76OjP=g9IPH_%tWed^LXiY9Cazf??c$snr zz!4}Hl4G4@_xpkYJf2FXoKOO9-6J)oiWYVXuSJAY&Q`aFnV)5L@nU~x9O9VuEbZmm zRJHYpRyw?}bQVa47oYcRa)$0@{Whq+Eszd#|A;H146&zmxR5#?^3=Qdiij=KX-Bvd zk&plq0|^#&B~AjImXrDvvJ40$v(^a!JSp>w3$@6tFc)7&spiek=YVmKkS2(%uo;S; zqBCrWkh+zGsP=MQ_NEL>&43-zSnE7k>kbEB)jJWqRV5}k>J?*Rcn)jx=c`6*MZ~|i z%~^le&(UQK^+n_>?xxUQts<>aPR-TgOJSE6Uvk5ZUkP+>VveCD#mghIG(nOynL#Rs z2$vVgxk2{9-OsO=D`|Z%@x3w)&CjCgeKN0P_V|BE-c%IL`c-nXVk9#S-YNj3*P!-C z^7XvFA|Fc zQxCIu-q?|)UMe%sa3wKx=4brU5@->gWRLT4CltHUIy;}a|KrUJ{a?72odi_$Jtv~g zkQWC&u|Ui#HMR{#IS~nXxMkhhGSf zY@Od4)>#^qTHlZOA6ih(()g<+OnN3wb6{Q^(N3|JFQ>wk@M>uhX) zr)h?8eW=WL#|vUm?PV9~lwWnXh-FzzJ%!x>#?s)dgZwur=+ie)NL%H#f~c%;e2_O? ztRDfj%ldcOwjk(ny5_GYpz}QMZ&YY${hM|O2AyZWre5QzFI62O!>~tkqcDdtBY{-$ zuP(XeSh@3Xk*0o^Wa)qAsTKNxZe}ik_%)PtKt<$f>wWvxMo*99^R)3&;*5cJd|r=q^}Qw~=ZGkr7Dg^@4b4T-b$ zv#R2Xe!$2km%(4C))AfZ26hixuAF}-+f zZwfDSoMo+1_8Bu$7xPtlaoSMSxTLFO1~#1+>uc(Djj`l$TpKz(SF{%R8g%NC7!}{IaPsNc}&S&M`WZu4&tu*tTukwv8*!#C9^# z72CG$WMbR4ZQGgo=6>GqNB3UctM{K?)xCF}Rdo~rsc4{MqGT*X7Wi1f9D7k%cwP1a?U&RIrc`PKXV&fRKgI#_d$X(&SXS1O&!lRovJGQJQVg60S*AF9wDZ zh9=X$yV0h)E%*z&CuydVyRSQ+JH9@TQ=dpevf`7)2Bn*IUCx&ilfbHu<}m{SoElh7 z39m})DpJWpAR!Qp@x3%)%4JbzWB4LPxVLQRSboj0EXO)iCbQ->>+)1T{T~oy%}-k zZPiD;=v1*g?z+0TArLF-QXVcw-NDyEHfrSgjtgkt>ep=3P%Q6WnvrJt z+4RwtdR4Q#RUS7xS~!Qbs=E;lje z53Oy>LXWHQ$2v+95NE2^FeUsgp1y4FyvUw1VadDrg*G_B4otGbMYIlWq>so@%yJ!C zV+>DAk}AXSYO|>TXO$oecP3UZixgcI-#ccF znJq7up8Zjx1AN0)D-mL!udb@{XsbvCrCnAgur+f+WxIfw{$K!o4 zfn|*egR+@Cqfbd)SeHLedNl(erm}_}Clq=82-p7cA`8%vq@&iJlk<}*b;&T@mm@wX z}1cA((mK@yos zPW0ZW@JX#qtMNijTe@pH1gG4`^<{AR@h;s(T} z&3#(~u$Qi#%j!zW{ss#Xsm|DQOrmKNB0cK9N~^$rZJLyDEKoClR=V$R;aujtgT#1b zA`U4#ht`VKoHWuito?@~br1x@B1L^j>cuo=exM!L_g$Gz0SpZ^`C+o-yaA}LPlf0= z^n~1R7J(vVSULvS{$R8709Q#R@ZbWBjZyY(AbHaC(7|(oHtzZ@NbtoHn;_g=+H3fa zy!pe)r}Lf|tftQ|FMWp`rny9HZ;N&8jH3-LHf6@ zM&!|x^O%ZcPJiq#EK4mpID>Rd469b;u>zA+kvrUva9OQIDXPl_*T6IGn29GAYKQ0n zASA;!l#^KpqRw`sb%#}-2}Ud`ZK&<)htt;RIog2CA2(DI+sP*f^;yl%Jzz6%{0}^a#h=NyKLgPR? z+h)#g+PQn_^B*+snviZU(joHWllOKpV9D$p5IwQbsoi6pC_`)m%$bm~s>3~@oHT|MFt~;^&e$k z`!AZ@c$^%MzW3|Jt;kr?yNKC`4g;qphv-mowYqO~qxIDHG&T*1Il;sp@iK|H~; zRY8%8d5`6`s8oac%2s^AFKN^&{3cN##QttYZ`4w%O1kG)vS3r_nko@(3WSWY^hy%k zD_xZkb0hmkTBJdfu$mY-P*DN?TlRxM-eP1OB3FiJK5ogaE%S@t)Zzn*d&`8NQU6AL zC9qU0aDA(=vpOu~8PPvMOGiOGcbw0;i&OIZa_^2(khD z;&117LsI_yz=<&pOSpyG0=nv1z6nB$uqp6DxHM4~*{6ytIT39}>Z<;BowyqFU@THt z9tvb``MojCN=M7LPJs?9k>}02!$N}>-Hdf5sj+7zPsGcEpJ72v5=@DHxVbShM znTCaXY66l$r(TQRo{5JpXcn1GZ4$yFyu=I%t%@xcR3pUKP%~9_4y2j%Q(-)PkDfn} z9I;eUk*#9=IplZ{KjMiWV(J5dk%FI*g!Mq0g2h}Kb^c8wfG~@54Ml|sRB_zCI<@{6 z^>GrT2@cGf?mzHC4F8I^S9r33+|on(dnh|1Z>%)RxVYT~j~E*AoAP*jexWIP76myS zPmxHAcOLo4+KFvX7leBb75ClA;yi&nJL{!SU3@ zWMvA{qx5Pu{sRs@9^q`F3_ray9*Q&n76E5u$F_G0Tl}P{sn+HS)^78+pUqFXayKO{ zi^~-OJkHkEj&_t9g1Y0<`H^--_8B+x!zqT9=#17`5WUA@RUk-mPwZ;c+8RhB+N`=K znJs*ymvdg07$&iKn$G*Mk6>^D1*zhr9ipPUJ%R8Yk{s78rc=2jq zx?!bk{FtF%6OeF@OlMxwiOa{3JZqSunUzIK$Krxk3j28$=JhtBUVAPyC$e(tOs@2&>aIiai+vP@s~9CD!K+B*cxuJH5{ZoroEdkOb07;B!(&?FM&tYiDzMEi^#Kvu)$>mUMf_&sIXt9V z1`|{6PuR}`LE+?M@z!%&B1y|M_RaF73@U??hm`07>sJ^Y!2lLnd(8Vpp>y1ny1lr3 zl!y`Wp!J+)z{ok;P0$-LP(J+_fL&p*f0=;J+-ts3-7_(rS04#pN+)SQz)n%tOxR6_ z@iS9s7}z{TeV+AZUSI^TvB)a<)51kpw?}19ciIMhgxJi+fk$dzsUIxLVQ}Nw6>zz% zYtr38Z538+YKBWeW51rNm{Tpg2qKiX&!^s#!ve?C(NY6ft*#v{M7+r!kFvwni9Vg9 zVE>1ImnPXi@nY&lD&bwEzxTI{dNtF18pL$JC~#UVZdYp;{nAd(+?7ql2-I0p0a3h^ zdE7VU7KJ)trJ-z)KsCRt^QH%e#W!F~rPh@w4+*$@ zK4)>+_gDsG){RQP2XFWefCz@LxK4qr#%x=WmPy&Qi9cIKa_7gh__E4y=^U1@#vNfA=^ut28X2_ieyr<^WqKZ6Z-Or8MH|Ad<`?oNVuOc^D;a300H_ zM@89Pv5h{>T$*iPbD?^mIOFe&5u_Bf2CQ{5|AFdS+Fwi*XSv_QuaOXm*g$E@V6`8E zQRKWE^)Z_$Y0gO|a~q&cE+vcV=jv9uS%8|>#SnVFD4{g@06WNT*HBsw>2!tC0{d{{ z-?m)$6BB^p0Jsu~0e@^&+QoxKB>XGk((rAyZ?!zC_Y&)X*aR~{dd)P4=tBS}&bgS2 z{qy^PL8LkzJ@}LlCE)1?0?Rcsi(8&_kltfWR6M$DM zB@k7TLP~t7P?uK;Ts)*HwZe_wZDjbBZM%!6b?Jhxe7&{7sfsC;9!MX@l+!aDwGefQ z4x^TY#)Apr3tC6_!dw?x(%AL$?5VUr|4VvE0UoX+_onVuhyG zjno6xQ`GYfpa&yn`;1$$&NDY>HXLD&54al2@3A?CO|q4u_Avv9^NpXV^|y@IoDy42y31Z)~eiGpE6 zjFQWawJp?DvP0va!#N^er>_g=QN4?!$QgS^+?fbZUO$e-pB_^&i#<6xi*}@zikhr) zQ3p!O-n4OUat{Ysi^*BT_O2f8jyx#;l8S9XRMCoMZ2A)_ zX({EoS{qBU0kjhm%{)Y@gbA}dPEho2-^nP_{xyxl3R{(C!oi@~ily18z0RaLa0~`Q z-}?ov&mj*bb++L+Cn&la1{QW6ioeY&-ik0^fbt>FeFp7$E%vk?b`~WsQnvbzyglt2 z9`}pj;QLZOF2GfJW`1Ani=s|17tLg$8U+`!R+s>XANYrUg=l>KXV@4VJI=(f0lM4q zc{QF7gEfqt;%le{C3*5Z;l{WC zFSAqZwN$9H)7C|NkiQGy?ue@E(A}7Xg?|NcL2!wKV2fX9dAtshHJ||p-F=%=!ny8q z6#06TOF*fvSQIa|E4OQ!zt_m$j8YEAXLb#*=)p7dhKLDe#O1>ypGw~Mhuiss4SE&o zUCOJU9zDRJ%X0NAEI1iD47H_vlSGZkF~C$89(cGGOkm&MeNlaq=G0Z^LGoC#&+(5; zaLHJmE~eLwe)P>Soonm@y#9COv=j>${%>Y)XCS}#)W(vgsSVQX`2E(M^D$y3#n~@U zgV@DGaFc@HzP4;aOZH2b_Z$V?;5?hCMg* zn!6cCC{y}g^m+AoL?$;eAC=f(GWM_EJYNcPYf@{mDE%^ugN=T0ugCc2Ib$OHbSS~)R(7Omi zjZ9k3U(d1-{M$k<#<4`~+j1kbgN}?&yxq;C&cE~NugdUGNRR`qr}^`}2t-ziw}9Yu zND&z4NgN_teN~?NfvUpDyi>c_B^0D$$U%w_9IM8HxQLYy){J#zv$J|XC2k3T=4g!TR3r2+)_P(#EJsgpZU#ejJ820y9k*w+P@sqnB zl9o~obFSN-5jU6z9D=9cynbWie^HJCnF-Ek_hYH71W5_lcLsNLo|gKJBcNoqk5c#` ze{rg+LtS})^(X{gJxq+Am1Jg{hJ6adCBk8!+}{d>I_;u1kC3In1Oy{5Hv>zNHJZs5 znjAml*}FNZQo=Ul=BGBKuJg#6S6ZrlZyojk7hV6B@O&_H#+`Ni^H}s&=v1+EevijAm=O*FaVtKKpajjc} ztaO=b1DMn~BYxd*1Ljzw4}l3A@`qiyNuq=mV%qB(#Sat#fi05rT^EFLO~bNLgjSc> zSJeJCu>K0517vo(tmJk=ys?J>M|?&{ev!nS5H~cObS#1rSXcN(j8<2c>5`D6w2tf7 zjkvK{8I{la@AP+{l|PZ5ymZ+vIZ)x*a@lgzr?3`tKDAD@YKBNf+PeRun(}CTCE(QK$%Jyv^`vksei?l5pL8gQ{6s0E?fw#I?&W!G9 z+C)pZbxWvq8L3$`GAe}p$97nO+37R48}bxo#dEr&Qg2J#ZMnsBo=g#@IeASh%rv$3 zCyobcB()INWZIHZD`1NqVUEe;JpLx>!$#$~`lfTHjZNvIt*&KmP29<5qHD)>(a~>x zDT_5fVT~3K%Ybc3xNBC1#@T$N^+~ISZ6!Z%293?xQi>N0^`8#KfX@*0`rA@o@8FAT zsB`&GEUOCN_|)~=lHXT#bL%f2XZWAqP55N5u%n`YbLctRQH>0A*QR;vQFGqagnY+W1#k`J)!VJdJRaXokyH%~~(F{OUSN8mX&?MrQyK$stRrJN_8j?Wp zkvR4O{4Z^Vqxx%u2m=IUj^=*~`lcNV5Y9)}4C60QCd=D9OJJjRd!f6-KB(4iLqL0d z06RKXrX;z+KDpkwUBP~_lcJsC)qGnR83P3c9A(LFOs=@F++QC+{gdCcPuUTcIvlZ| z1hzapkd$@yJ+ayMyfQFU1*rdhojeGzLl{LMmVJLfqNj@w~3XBub!DJCFknUoW~z8qjLV2$^@+>HX1 zzkSZ4A3OtiiMH9G)F{x8-`pxn7O@+>p8bL7A}3@y3{7A@M8Vy*CAVFWIF!T1DH%dJu5FlvnwyLF0#cSdT1$M6# zZ18qzTQfAt9;sl^A2aK%_~@pCg>_Qp()DFxmpa6s=1SZ4*=uzdMYCjqo;X(5oMhv{ z(dB(zEBvvp#a1pisvEaXUh>{EKF)%>rO~fl_8B-_Ime(8ne*WlnsG* z=ur;WDhz}R_=p6&Me__0Dnqa)Vm(Gjshb;d)FwR&H(;EMbdzAFeKFCT-Ig4E$-4aK zGi-#-;?EInxP?iXbRq=$>IBkhmhdo$FOD!Kejf)(j0kQ2kZL;=o?Rn5)dp>0x9TTa zCPh;SH*Hd8zFU~s1yV6Aqabc3g)G)YP&0~_iN4(1;c@Mm-(~T@_R?w9F6{(DUIimi zp3cI_mO`0P?HWD-gKBwij}GDE1U1oqsx#4xf_P&!$(ge3=p}rPpg(z7QtSLwVp%wr z)b0###i4ADrG59KZ8H5jrgmQYIGWL*j+|7cc$#s65id0@KZnq(3&wC@I#!RvrVJD` zc}=SdM#lo1wY7qQ?%8r4UAkOF5s^!cBg2nM=0e+U=;dHNa8Rk z6OSdR1P^6%75kui(xcdvAns#PwNEUe)W6QKvx++Gk|I@P=%B{I!M1%mN#BD~Z&~S> z$J6!HZEokW811c=}jB3iJ%ga)vN0pvV7DdI!MQ|gk(^k^%8^T$}3nBR>8|jLy4Kc zE=NuJDc;yGJK4Q)RVO0FMbi#2d?W{tqrvP2@CjY;agYympLu+8SM^1Bm^UyXv=)A) z$BGy?QAf}MC3Q9vaj5ue2ht+%CG->!2?Xo*aAjdD>+D7_N2BVDezDXJyMf0#@!V-l zodn=f$EwhwvPjP_`FNCTC?>YxIjNyQ{JA`OmQ^H@t*Ugyq^(rOx@Jb)%18SEeuX)K#ChVAWHY=G3=!Nw39B8L}Up9V)+ma4^A&pH?m z!ZxP?A|Ow92k*S%zgJf&B;)6NY_3^}60 zB^*Tq4Y^#YePB|#FBZNY8^FhrqL)yz@kIB=2}87#%Sz7pTM@ebhNF*?h-zOlGaGfv zZQ6P7qKX#@;EeeS%nI0kqiA2Vr6}63Y&%v5y0ML^&*z*~kj@ok`vxQmDwUd}iS^e} z-?Z%5Rm&l#PM70=N&Wo!2i0KZ&gRQpo@dtJqbT)p_hI@y$KO)UOh{V+3hcj2VhIFR)|`=Pg4tx(@};;bTtOsuNyB$QXe9pmHv*L z1ben*Fi>HnWoMC*FSQmeJ=SCE7~L=5TdT2brdx>Lpwa+1d|$6We068K6Wxxe&F!baQ|&s7pR zl$NXuC6`oi3J}9TYEA17G5kP5aP5fSaDISnI#xzANK&8QAygL9p|IKcF>Js?yRHxU zXvzf=6iuHcb=PWBZ^DVxxF3fDUpU6wevU*hwgyKVtY3u>XIdUCa0x^aO19CqYHPS9 zu`dYUXsTy$uB%DR^04ViJd4h7l#|9UlYmL0#XJR0%{SPhqaVrB&z{5U&dg+Rrx@9o zO385wN^)BuxZOicKQ)$`=k7N#;9Rnz+VF@5%Y`gGshFy8Hw5qg1W|DShA!yJt9nJq z$TD$(FaiuiWu6WUWb_!WUy*ZE@V4svwd&C@-1t~Z{HSQZ`B<(gJ*A@AOX3QZPVwMQNTn>MiKs)cfbC0;XP9g$wQ(ssw*!|cIBS)~BQVg{XNM;6Q z;Z4vGuyho7&kMD)b8KPy{I)E0CA9=YS*^)sySa<+o{t^_`#Wr&9lM#6YQ7DV>6?p(hnyN`!Gj7pUlUK!ybM`VhCQNEdRJw0Ukd^J@oN^+6;{FFz;7a!3hiE!Py)C;^8Cbt>|>vA@hw*yV9$+*+F}_|C^C{ z^$4FY6yp6QXa@b-Xbg5FDP(X<&GfJpd+IZhw5H3X1pyX`UgqephJAD<7@yKcmyak{ zBe-1l&h}3?t;+`H{Z5<-0A-Ed?nmf4oZn+6q=JKLD0`|9;b#lCP+P-NR`c8`gG}~o za_Wop;jix$On;U>r}s_Z#~q-fxnlbMCTVSaw6-|ETsY)HQi$+ZohweoYG;J!#MmYU zJ-&E}<7=c5?zK`~6X1y;X3s^0gnjdu`^z8PyA=m4zB2}%OVJ>2-(KV1!c_UG5tvz;-b<-P>67PMe-{!%S$+ge-~q#h{~r!iBIm0yR$+-JIM$&8J3`IN$zZby7XCwIYN&KX**xR?3#I`P@$25sP73{J~Fr{&VSx zWjo4(!WZY0!WRLG+&5_hs+36ennIRCGszV{g{c&nVv<_CY*JB76~&P_B3|dIkxj~o zswLyq+@`s3IgBXdfGL(JNd6+zp~TOG2=b5kop^*4-kRP~>$H7FNTn$aAkWn2(`%K@ zrFm>^ze(m-JNeWHOSG8y%D)sDXEXClyF~dn{9#!|`|qY&trq!g^80r!*MCE+{w?so ziMQ>7@&6_Yxnljhy1zm7fOt$qRr3GE8*nPAj(P{1Ed#RkgKMS8Kldx-Y36B97IYsk z|9}y6IW9i}gPJn_ITCs#0(+!0^=F_B17!!Ja0Fejsus9etsKjEH{|gRobo=RabqWx z+E&({i>_*%E@=1X|NH^2N9Z7gBRCL{zZm~NrH23ixJRLXwVMH>*4=hnF@c(Vhz6L? zfp{Y5=prJH88g|6MHz78O^o71L#>V^fpA29VW_j}65@zQ*^j4uK+%Uk_aBf(U@o9> zNJyvCe618gc(S4%qX--Jg9r=UYJd}3g)VM{2sg3JVv3zB=}QO#SbJNpmK#M~YdHii zU{sg3c`hw~d2=^L3ugw$bl$tWmJOz@l-DIhqBt!HD{X}KbwYy==H+zrbaN?|>TEYr z0CKrru|C>d!2)@Ga^_fEG(5+9tE4#&&R_0^_9d@-J|c81x}VBM4}h2AIy2OFiy9l) z2iDN_TbnQHnDsiZ1q<~HtUsOfO(hHZK(R8@n&|X&-gme5v8YW}j;=D)lv_A@`oA1+ zNUKZ`vXjqpP>7Wn$t?Ru;6+8)qSGP}KP5OAm_7UIg5B&VzSzLZ|8a+!1NZ5<@uMGk zC%5@!@%x4*mY3luwenb&Jx8X{=A`6&qZX+C^T;Z}lVq*`rMsN|JN}nXopeTxk#y!Q z1;nHgX~8#Wp%Il5CkUX>H2{TkrZ7rd*OxBTr?aAamEB~ISQMB2*=}#sQIjND1HPa_ z`VzU_VYSd?wZLZglgn%4^}vuEa|9P^noEhB(MO`zY_m{qND#(h`HJd6D$kG_kme5{oszd&i( zEO$uPV&<4Nk5pW9Y~0A>hUeCvz*EBZtGT4R@XC&cP9DRNGq&SM(;Fuyixh&|s@)*| z@R`oGyCdd^huhWJ8piCIg>D{fJaRF-E(BkVkmZr9$R)jZlgrWyD^K@hc1=v&CD8pe z|GW*rcuG~5uTj?g8(^WxCdG#oo4vAFn|A@Rd|ExPvW?j!sPofTRq+M|eN6jwD!arC z+^(8p%`i9gjQ87zSIaT_w`yIkE5IZBJF{Y3?WWGaHoew93sB1j*FTe;A{Yecfk@wu zpS8McksjKqHCMF1dFHK)V52~|0NiRI9G!n8tyZOz2fMkVdBpl=JIpar9_Zchau!WviRC`DxWD%D3h_317BbUl44j1a4&^ zGs$RKV+L}b>ga6jc(uQI1uWd|5+t!4_96Io%_HvJhrg2uY)acmo&SFF&mSd9q|{jTx^fJvbGU$-P~^aGpDRPn#1$1;sIRL24$V+`egtex zE0k}VA5-#zF0nBs%l&y#BhpJ~zUqR^xco=d$&7V*PH zZ=(514Nu-@FP;;Wg?->1LF)jYHi}1_6XDz?5r0lRq0^lXaH8k<3vAvt#)oP8Jqopn zrAsa?bw*t^03OdK3HpRM0`p{7XB=%X>0D6C*+UeG(3y##xz;tUM1{^fo^F%pfTlLd z#?dCv%;ETjo#!e$C)Lv`iA+?t?z5~zU%{cd-;DX>v_MGiYDW9< zxgX|zu<79r0gb4~B!MrWUytBX=pu9m7rpvVIlw0`O1cN41Fb?v&Z6_1mp2eH4{GvQB3CrHZWyrJ;VnXLHO@%E zN}Lo;kSiq2fzh`?=X#gM-#%8;q(d{1S4eY6v`^npV%ZZaTx~x^K8$(CSiZ=xP0G{T zc0(O^50=d&>c_p$N43*lVIrBX3n(=G{Ivvw*be|0`dVQ&l^=&sB&pxb7BL=}$~X|` ztZcSIzQG9LxDz1?LIBcJ3y2zUcP~kNIxR=HnK=Z z$Wk>Vx#^8P+vXHHZAm8UFFR3!#hHtX@Y<}(s$-Omy#$v~zLk0N7ajAJ`o~JX()PFc zWrpRbuu*pK0Y{Qv34&GzdRHoS@k8)D4bmvj40_&)M`F5^D#&F=t-fRWF}}{L+uiU-6_d--48;;BRMD~TQn3cBij`+7B^`ye zsH$AndXoEoe5G+SztfZ>ycU7WwiDI7j(Hy<<)HI8pVpN-D@n?jWThZq|4u{WT}l92 zgM;60dekYz?-Rl2H}NbCJEz1jbe>FP6mCEO|JH z3_(<5pMGGP-K>)xQsP2Z@yxwywe=+~J8hr?y<61l@QJh!w3q+x(#_Sz9{Bx!pLVXL z{iT(lg=r-K!a?=*bUB9|;0w>|#mOz~OgdS&|qCbH}A(#|zMe z6uhN4%e@WH%s+CNx4`g<@yk+@jM2&i3I*YUczoxe{`UFds_i7|K$3OrDWvUK^)PS? z(^0gc@Mr-vEMRId6m`k1!K4hmkN3)Qk5^@QXnC&?+bWtOgAP#?ryk z-yqkXeE_ZvHcB`Ny#azmP1R>8^$}PRZmr+)@s90MQEgqYX4H|wG8~Ib$fDbyeKRg zCr8v{0HDv)uS^-HK1K0?s1#GqxSF3QK#JA|7|!-3K+AsTY$58G27<7Yzi!9C&IH3NshKKtMbEHyh%yHtJl3+Aey;Lh59(yqb??B4IeD zm9F)fMrB^tbIcgRMuM#3d^gvtS4S7aPR#7$h;)>PH|;*1>MMn6A&JiwkKa5Ur9(F% zL1dS_1Db1u`Yo_*JP-F_C^XB9Z1L%C4q+orHgXL8I1Qzx`W4jrt?5EU|8G;!NSzWeNG&Hjli{v-u-D zK|+c?Ehk)<>H{WSI-Kn-rf=uD{+^_AaB*JD!npc%U;;R6;)=QgB=CEuocaaljF4O^ zzh3^FZZYf2_(J=uj?=7+#$yjMqav7#SK`)IPa+SN+=qlo_e!s_>W_|fWSCEG>IbO+ z4~)$s6yV~rwtl@A73o)$Yk~A`&@)zpUu5o!>pQ^bK5JG@s%yBlD8XJoz4WyhRr{-` z?Y1%AV;Q(Y+WnWiWpoZI&hV+9#4!9`FijOI@(C?1UzJ^>n9lL#QAP-l!i{zRSv<6R z-q_H#O;B*_X_3TXT$HKUC@(K30Wj4E%Fq<+eqfFlpWALXdOM@zUE?2&^x{Qy^^Dtt z*Y?F&^c#zfut^`~ypB85(1^?KWviDYa?{pmRuWi<*D~0!==#k1&d;P@9dzR${4gPB zwpXZ4yV+KSPcXZie_65QSFS_9K!xMM7Tp>3_QvsJ%!ks=-y`(=P~s!T>LVL`=9Fn( zwrA;<@ShpH%kZK^?dCHz9;K;XWzc*$k8w!=)r;%MyJB`A{(L~!RKHz5kLw!7l}#vm zfdT(gIdpqd2PW;L{|mA*)jiC@ld6k!y~x7Vq+SD5%{FE28WGgeY&{kY))D6f*D25Q zZIKpb)^m&1>KPLxb=G4OC^kX6rCPowoo~yKCR>iMApU@GvgktHya9$ou^;6|xY1)2 z77Yy*2*QhNRl*Z61(u(lX+Cs`!LhAByn$as6T5%IiG(Yp|Eglf-rG+vBMiH zNSRL~4z>Ds_`*DKHWA$IFyjUaiNWXB=oRPVpNREz~ zJdb0>;6p5v6{Ap$$6i?8IF(M#@^o+V%BY6TpW3(m|8$-~te>WSGA)dn=IQI+0JCc+ z1Y5UG&yN3{fgyr)pIgpUQ2yMG@mf>~r-@em=hB4Fs zPb*keoJx*#qEzubR$|G;*rVNlJ}u6i+w3bM2#6>C|3n4uC`O>oe;pP>cTvtnX++y$ zFws|ab+tA7kWz5b7Keh1RemB!_9(Q5T@M&c7%-2FA?<6G&u6~%6Ya&Z<`zguZ-j1N zUEO57^4w-*X9xj--;nh%YI{#dM+)aj25BoK?+CuStuN0U+pt}!hZAcsK7(+$L-+A| zi75A`YLcPLxgP>|q589cvPj-(Q-~QFwVzNdrq#xNZy(E{6RzPeFY#v$sNQj|a;fsnxzI(QS z{VxM!EhB2fwQ1s@ODoItDdL!WmT2NhHhUwuspBfFUp5T@DIKRY>vG>{lLz)G7BuoJ zwpEerKA-82becp1o*+DJ>_L7^2=fnU_9O77RM<8@$jNktpD?X$roUS71EkVyD%j1m zi;9B(0p=z`tb2#kAf~F~b4j)G>2^Cov%uDKasoo}w8VVriKr*Tw%&Zqj7~!Sy7;1^ zYXoZCSciBN^qHn`ZBGtWsl93LukGbpBV!*@Rb@_{ngsW#*s99n=UBvfoEUa;`FK47AVK3Z(Kk(`VMK%yB0isQfAzy_3+`v+SvC`vx<*mRenZ{rYe)+FRhOGb8<>o1JfoC4lLp|Q8h!ZVWpYp z07yBY#DyLjqm#Ft%nC9?=7gD;Q5ew0z{kR7g;rohjNHvfHj3lzM9_A+B0g#t*@*@9 z{}HX0C=Zbt-1H1+v=)mJxzxka&}Zhp+WrDpM_JLG{nPm;I$-s3wqsAM49srLc&@FG zsSi5S^wPxDXRWkHj_AgJiOi0$SLF4XOF4+)uII;p@9csmNs#=Xu4Mh=zwZ!?83ZP2 zzXTmw?U#$InVqt;gQJO)TX9nQFNFeHunGU#0U(YKcfCc z84#4Am^@i|WI`3q8)xJJ+WL)Ocu)OW2EQ`trvMLoSx7zacwbm6zN#CgSZU@pQ&aCR zzPAo}yMO;2Yk{QA8Ljy|n6|eiR65#dv@I{WPE?jW&`jF2*oHy1oZ>3f(Lw{$22i%J z$ZZ{W>v0DF&zlND9Quc`Ob->B+m;Wh#&kr5&d1KptP&lKZ9ffd_z-{i1>s?(MC!Kc zlN4XC!04kblxYWJQI%0fNorJ=_(cb@oSD@zFgPu`gNv;sJ&Wo;RFc77Cbj}ZF(=}_ zh1nhC;t&HEzIbjDwXMUM;e~)lHeGv;tp?ha{OFqb#^J_IjDbO#@TZH90(P5p*I5hvP54 zxh0t^54jbYv)5d@)6zndct=vo?){V~T9*+g0?@lE_Ss9^nBNUh9nOK$dv>AWhxfFD z6#^xKpSd@D+*JeQIFJmZj}rJa8ls@5H2WI&ZSG5fxHg^_xoapOW%| zOow14uOw#3p6V1%SNXsjPT39#z4-#;Op=pZXA{=Qs?W9GHMIeh)t^7o0(woLngo8H z4+<`;3k_TF3ii8&u70}@15*aHJ6uf>^L}bt?G_vGHDOJ#Bov{K;>*h3QRG}&gQA@e z9uuwy{Gu;!pid-0$Sm*--v8_BhG$5_$izneQaowLRi9<@l0X3jTqMppT7(t&mgqZd zDr(dm2mtDIXaq9!9H6->&ZG}aZPHH0aT{I$=!SpgV87(Dkm)+bc$OZ3T-qn z!OMiD!w1mEJvir zW2aB4yS38ZKex_!?|*;5l|zc^%zwxkMacgz)ng?gr$HrASK=q_C1C*z{EtQAsZzj) zn*sykJ8fjxA4I<3d*+5lhOqoVgp!?FJjzN0Y?J=AZu#rr?qUAAdP^kq z!-%j2#;2oW!dx)?7og3^T15{9j>1Wj-ZG`KT3Kyn$y9=lHG4H9e)>KgFRGv=@ zc=wADdn#VCmndt<5**Fy^goF*{V1TuD`h;j(UT&s-&L=ek|zL~ziK8}$2jZC2=^h57nb&+Xj0;6SK0M{Not zdZz(j4-L_ilW$;OzN@|ih7mQU2i-~jJ|$tSoAseoPDM>*%W1v2)MgWKlT^6ZZHGNF z8c*EwJ6_0X#_|qDK*Y&GQL+Wb5n00*6lHD1u^afa915W- zT?Loj+aB5k@$jc%8FKd!@1QnC~E88_D_bL04aMukP?cxyVom601|3fVoQoI-RZwN7@6Q2ln#~spKR=Ry(6IxzC zF#%G+G2D|id5_3Z6hUrCG9IDR-DvGwThMI#;US{nZ6p)-TOnW1-kx0TTX2w&(1xm(aP0F71hR_K*TMY<5a+Phx^w{W=@t17gH^mSK(im&ZG=( zHY+&j8`#KC*)CXO1mRNQ2prSNvye;Fm5%5KQCx; z+dA2~9tVLR*2#}wl3kX<%G~y*mW&hYC(@b49;C3o^Z~v_7$_x*N|I|v`&i45IX|B1=4vaVd3PpNY;;~A ztC*Q@XS!v7{8;phXUsnbA-TMXmOWsCxte$qib6tBnljH_wrg(qy)J~r(YKJKiI^@L z32i1FU~UBL+>rPfVS4sWYUk4F-yrQH&d^$snQ+bh=Grrl*yp_Y6P_G42ksY7{XDy!@BpD zR7o?eFWUQz?llUyQc1AcFyYNn=wV8H2Y518w=C)>qG}Dt!QVs|`{G*hTt>yKL6|Aws-73L-7Tq6n*O^57tyDvcRy5%UYtiLUv~R9V`;&h>u37{T3v< zEBXKCudNlzz882L^h?Hd@5OHmzJA%W>qTRDqg3I?%i+B{zU6xQGfmPHm>A*ke=Wu%L&yh?jK4PyH&G0^GizJmh0C&7taf*Z*5)C+PrUhW`)J}iYwoBdLQi! zymZKrJCpl-q=9Zvghi#~YAfIYXmtHkldpVts$g2*daUr-xl%9PhOn4}vooBx z>sA*WndWYo;?1g_Qz?|5Q#tKlD@&m0iOKa%0)at}MK@K>9kr5nK3KR%deeuEts7sf z9Dg_AUd*L9mK#SdF{`(~aW#FXyi>J;`E;$gPED!!y#?=?Rxim}-+3Z4@##G+!MZhz z50xuMN%s8Om$^jdSm8%LMah3l>iHvAE_{D<+mdXX^!xL>&-kvnt+rg?s><9=mrW;J z&Qr=2>`l|(aq0Wtdz>+x-?%TZ)a{LWl(}xNs*L|lqZ_YV_D(#0Z&u%0rJSw3cc&kg zTTm!^QnsnpO-XUv+E03`riaII-*pXraqE>~$i|mBB|)aSMoyPc3anhatYF66U$rZK z@Pj%~f{}?Yf+zRPUCBB*p(;Xgvemp~mc!G9W=>u>PmIY$U~=F*naQ;RqLUx26kvti zt^R+WC=uynoD+HdCGWoQ!JlHzW4QPvi zy~J8z4dn~9WW=t+?#W_cFh)`QKm$p!HY@l>rpW?}M47_1;Syepv}BO) z$+1T4#Ch@z3~DGQ#h6Y$uviIrMFm75 z_%L*!57z*(4vNChmOzE>vXH}}85rgOPp3!q)hcU-$qx2Xliyn_gY1-rpH~bFEJqZh zgzZ5py}_#B$KL`~*`cTsa%7ln@8|(`KjI`-1_pf;RUXchA1oD}+`rUR8gbAhx`j5A z?=OvI1)s+^*>RaD(_NscOXVhOdMbiVM;w*|Je&{3bX^~yLfOd=mdVS&4_g5`R2N0j zt5C2L43-axH1|&#=Wr3=B#r3YSm5zuZm+d94eoZBHsE zKUgk1*`f-PT@V9^3=9e=25qVaDwLVLbA`MNVnm36K^{dBLpRu2{@vi5DT5dWK~EIW&pHfkaU4roNf6g>=uCr>T__Rcg`=}3c15@4P_ a%EQ2*fnt2> /dev/null && printf '%s -' "$PWD" ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -206,7 +205,7 @@ fi DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line.