From d0c0001f63b49ec70f5fc08b78478e97760afa01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20R?= Date: Wed, 8 Nov 2017 15:30:15 +0100 Subject: [PATCH] EZP-28179: [5.4] Support for selective indexing since date & no-purge (#114) * EZP-28179: [5.4] Improve index command with support for selective indexing since date Backporting some of the features added to re-index command in EZP-28176. First and formost no-purge and since options to be able to re index only part of the index * CS * Fix issues with usage of undefined logger service on symfony 2 --- bundle/Command/SolrCreateIndexCommand.php | 36 ++++++++++++++----- lib/Gateway/HttpClient/Stream.php | 15 ++++---- .../config/container/solr/services.yml | 2 +- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/bundle/Command/SolrCreateIndexCommand.php b/bundle/Command/SolrCreateIndexCommand.php index 3ac9aacf3..eac51b18c 100644 --- a/bundle/Command/SolrCreateIndexCommand.php +++ b/bundle/Command/SolrCreateIndexCommand.php @@ -15,9 +15,11 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputOption; use eZ\Publish\SPI\Persistence\Content\ContentInfo; use eZ\Publish\Core\Base\Exceptions\NotFoundException; use EzSystems\EzPlatformSolrSearchEngine\Handler as SolrSearchEngineHandler; +use DateTime; use RuntimeException; use PDO; @@ -29,6 +31,9 @@ protected function configure() ->setName('ezplatform:solr_create_index') ->setDescription('Indexes the configured database in configured Solr index') ->addArgument('bulk_count', InputArgument::OPTIONAL, 'Number of Content objects indexed at once', 5) + ->addOption('no-commit', null, InputOption::VALUE_NONE, 'Do not commit after each bulk iteration') + ->addOption('no-purge', null, InputOption::VALUE_NONE, 'Do not purge before indexing, hence rather refresh index') + ->addOption('since', null, InputOption::VALUE_OPTIONAL, 'Index changes since a given time, any format understood by DateTime. Implies "no-purge".') ->setHelp( <<%command.name% indexes current configured database in configured Solr storage. @@ -40,6 +45,8 @@ protected function execute(InputInterface $input, OutputInterface $output) { $this->logger = $this->getContainer()->get('logger'); + $commit = !$input->getOption('no-commit'); + $purge = !$input->getOption('no-purge'); $bulkCount = $input->getArgument('bulk_count'); if (!is_numeric($bulkCount) || (int)$bulkCount < 1) { throw new RuntimeException("'bulk_count' argument should be > 0, got '{$bulkCount}'"); @@ -61,9 +68,19 @@ protected function execute(InputInterface $input, OutputInterface $output) // Indexing Content $query = $databaseHandler->createSelectQuery(); + $where = $query->expr->eq('status', ContentInfo::STATUS_PUBLISHED); + if ($since = $input->getOption('since')) { + $date = new DateTime($since); + $where = [ + $where, + $query->expr->gte('modified', $date->getTimestamp()), + ]; + $purge = false; + } + $query->select('count(id)') ->from('ezcontentobject') - ->where($query->expr->eq('status', ContentInfo::STATUS_PUBLISHED)); + ->where($where); $stmt = $query->prepare(); $stmt->execute(); $totalCount = $stmt->fetchColumn(); @@ -71,13 +88,14 @@ protected function execute(InputInterface $input, OutputInterface $output) $query = $databaseHandler->createSelectQuery(); $query->select('id', 'current_version') ->from('ezcontentobject') - ->where($query->expr->eq('status', ContentInfo::STATUS_PUBLISHED)); - + ->where($where); $stmt = $query->prepare(); $stmt->execute(); - /** @var \EzSystems\EzPlatformSolrSearchEngine\Handler $searchHandler */ - $searchHandler->purgeIndex(); + if ($purge) { + $output->writeln('Purging index before starting re-indexing (use no-purge to skip this)..'); + $searchHandler->purgeIndex(); + } $output->writeln('Indexing Content...'); @@ -115,14 +133,16 @@ protected function execute(InputInterface $input, OutputInterface $output) if (!empty($documents)) { $searchHandler->bulkIndexDocuments($documents); + + if ($commit) { + // Make the bulk changes available for search + $searchHandler->commit(); + } } $progress->advance($k); } while (($i += $bulkCount) < $totalCount); - // Make changes available for search - $searchHandler->commit(); - $progress->finish(); $output->writeln(''); } diff --git a/lib/Gateway/HttpClient/Stream.php b/lib/Gateway/HttpClient/Stream.php index b38d2d07d..9e6c48c7b 100644 --- a/lib/Gateway/HttpClient/Stream.php +++ b/lib/Gateway/HttpClient/Stream.php @@ -21,7 +21,7 @@ class Stream implements HttpClient { /** - * @var \Psr\Log\LoggerInterface + * @var \Psr\Log\LoggerInterface|null */ private $logger; @@ -43,12 +43,12 @@ class Stream implements HttpClient /** * Stream constructor. * - * @param \Psr\Log\LoggerInterface $logger + * @param \Psr\Log\LoggerInterface|null $logger * @param int $timeout Timeout for connection in seconds. * @param int $retry Number of times to re-try connection. * @param int $retryWaitMs Time in milli seconds. */ - public function __construct(LoggerInterface $logger, $timeout = 10, $retry = 5, $retryWaitMs = 100) + public function __construct(LoggerInterface $logger = null, $timeout = 10, $retry = 5, $retryWaitMs = 100) { $this->logger = $logger; $this->connectionTimeout = $timeout; @@ -83,9 +83,12 @@ public function request($method, Endpoint $endpoint, $path, Message $message = n usleep($this->retryWaitMs * 1000); } while ($i < $this->connectionRetry); - $this->logger->error( - sprintf('Connection to %s failed, attempted %d times', $endpoint->getURL(), $this->connectionRetry) - ); + if ($this->logger instanceof LoggerInterface) { + $this->logger->error( + sprintf('Connection to %s failed, attempted %d times', $endpoint->getURL(), $this->connectionRetry) + ); + } + throw new ConnectionException($endpoint->getURL(), $path, $method); } diff --git a/lib/Resources/config/container/solr/services.yml b/lib/Resources/config/container/solr/services.yml index f9e97a1f7..72df50993 100644 --- a/lib/Resources/config/container/solr/services.yml +++ b/lib/Resources/config/container/solr/services.yml @@ -8,7 +8,7 @@ parameters: services: ezpublish.search.solr.gateway.client.http.stream: class: "%ezpublish.search.solr.gateway.client.http.stream.class%" - arguments: ["@logger"] + arguments: ["@?logger"] # Note: services tagged with 'ezpublish.search.solr.query.content.criterion_visitor' # are registered to this one using compilation pass