Refactored reindex command (#261)

* refactored reindex command

* added documentation for reindex-command

* added upgrade for index-rebuild

* fixed return of drop-index

* fixed description of 'clear' option

* fixed question output

Command/ReindexCommand.php

@@ -11,13 +11,16 @@
 
 namespace Sulu\Bundle\ArticleBundle\Command;
 
-use Sulu\Component\Content\Document\WorkflowStage;
+use PHPCR\Query\QueryResultInterface;
+use Sulu\Bundle\ArticleBundle\Document\Index\IndexerInterface;
+use Sulu\Component\HttpKernel\SuluKernel;
 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
 use Symfony\Component\Console\Helper\ProgressBar;
-use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Helper\QuestionHelper;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
 
 /**
  * Reindixes articles.
@@ -29,50 +32,185 @@ class ReindexCommand extends ContainerAwareCommand
      */
     public function configure()
     {
-        $this->setName('sulu:article:index-rebuild')
-            ->addArgument('locale', InputArgument::REQUIRED)
-            ->addOption('live', 'l', InputOption::VALUE_NONE)
-            ->addOption('clear', null, InputOption::VALUE_NONE);
+        $this->setName('sulu:article:reindex');
+        $this->setDescription('Rebuild elastic-search index for articles');
+        $this->setHelp('This command will load all articles and index them to elastic-search indexes.');
+        $this->addOption('drop', null, InputOption::VALUE_NONE, 'Drop and recreate index before reindex');
+        $this->addOption('clear', null, InputOption::VALUE_NONE, 'Clear all articles of index before reindex');
     }
 
     /**
      * {@inheritdoc}
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $sql2 = 'SELECT * FROM [nt:unstructured] AS a WHERE [jcr:mixinTypes] = "sulu:article"';
+        $context = $this->getContainer()->getParameter('sulu.context');
+        $startTime = microtime(true);
 
         $id = 'sulu_article.elastic_search.article_indexer';
-        if ($input->getOption('live')) {
+        if ($context === SuluKernel::CONTEXT_WEBSITE) {
             $id = 'sulu_article.elastic_search.article_live_indexer';
-            $sql2 = sprintf(
-                '%s AND [i18n:%s-state] = %s',
-                $sql2,
-                $input->getArgument('locale'),
-                WorkflowStage::PUBLISHED
-            );
         }
 
+        /** @var IndexerInterface $indexer */
         $indexer = $this->getContainer()->get($id);
-        $documentManager = $this->getContainer()->get('sulu_document_manager.document_manager');
-        $query = $documentManager->createQuery($sql2, $input->getArgument('locale'));
 
-        if ($input->getOption('clear')) {
-            $indexer->clear();
+        $output->writeln(sprintf('Reindex articles for the <comment>`%s`</comment> context' . PHP_EOL, $context));
+
+        if (!$this->dropIndex($indexer, $input, $output)) {
+            // Drop was canceled by user.
+
+            return;
+        }
+
+        $indexer->createIndex();
+        $this->clearIndex($indexer, $input, $output);
+
+        $webspaceManager = $this->getContainer()->get('sulu_core.webspace.webspace_manager');
+        $locales = $webspaceManager->getAllLocalizations();
+
+        foreach ($locales as $locale) {
+            $output->writeln(sprintf('<info>Locale "</info>%s<info>"</info>' . PHP_EOL, $locale->getLocale()));
+
+            $this->indexDocuments($locale->getLocale(), $indexer, $output);
+
+            $output->writeln(PHP_EOL);
         }
 
-        $result = $query->execute();
+        $output->writeln(
+            sprintf(
+                '<info>Index rebuild completed (</info>%ss %s</info><info>)</info>',
+                number_format(microtime(true) - $startTime, 2),
+                $this->humanBytes(memory_get_peak_usage())
+            )
+        );
+    }
 
-        $progessBar = new ProgressBar($output, count($result));
-        $progessBar->setFormat('debug');
+    /**
+     * Drop index if requested.
+     *
+     * @param IndexerInterface $indexer
+     * @param InputInterface $input
+     * @param OutputInterface $output
+     *
+     * @return bool
+     */
+    protected function dropIndex(IndexerInterface $indexer, InputInterface $input, OutputInterface $output)
+    {
+        if (!$input->getOption('drop')) {
+            return true;
+        }
+
+        if (!$input->getOption('no-interaction')) {
+            $output->writeln(
+                '<comment>ATTENTION</comment>: This operation drops and recreates the whole index and deletes the complete data.'
+            );
+            $output->writeln('');
+
+            $question = new ConfirmationQuestion('Are you sure you want to drop the index? [Y/n] ');
+
+            /** @var QuestionHelper $questionHelper */
+            $questionHelper = $this->getHelper('question');
+            if (!$questionHelper->ask($input, $output, $question)) {
+                return false;
+            }
+
+            $output->writeln('');
+        }
+
+        $indexer->dropIndex();
+
+        $context = $this->getContainer()->getParameter('sulu.context');
+        $output->writeln(
+            sprintf('Dropped and recreated index for the <comment>`%s`</comment> context' . PHP_EOL, $context)
+        );
+
+        return true;
+    }
+
+    /**
+     * Clear article-content of index.
+     *
+     * @param IndexerInterface $indexer
+     * @param InputInterface $input
+     * @param OutputInterface $output
+     */
+    protected function clearIndex(IndexerInterface $indexer, InputInterface $input, OutputInterface $output)
+    {
+        if (!$input->getOption('clear')) {
+            return;
+        }
+
+        $context = $this->getContainer()->getParameter('sulu.context');
+        $output->writeln(sprintf('Cleared index for the <comment>`%s`</comment> context', $context));
+        $indexer->clear();
+    }
+
+    /**
+     * Index documents for given locale.
+     *
+     * @param string $locale
+     * @param IndexerInterface $indexer
+     * @param OutputInterface $output
+     */
+    protected function indexDocuments($locale, IndexerInterface $indexer, OutputInterface $output)
+    {
+        $documents = $this->getDocuments($locale);
+        $count = count($documents);
+        if (0 === $count) {
+            $output->writeln('  No documents found');
+
+            return;
+        }
+
+        $progessBar = new ProgressBar($output, $count);
+        $progessBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%');
         $progessBar->start();
 
-        foreach ($result as $document) {
+        foreach ($documents as $document) {
             $indexer->index($document);
             $progessBar->advance();
         }
 
         $indexer->flush();
         $progessBar->finish();
     }
+
+    /**
+     * Query for documents with given locale.
+     *
+     * @param string $locale
+     *
+     * @return QueryResultInterface
+     */
+    protected function getDocuments($locale)
+    {
+        $propertyEncoder = $this->getContainer()->get('sulu_document_manager.property_encoder');
+        $documentManager = $this->getContainer()->get('sulu_document_manager.document_manager');
+
+        $sql2 = sprintf(
+            'SELECT * FROM [nt:unstructured] AS a WHERE [jcr:mixinTypes] = "sulu:article" AND [%s] IS NOT NULL',
+            $propertyEncoder->localizedSystemName('template', $locale)
+        );
+
+        return $documentManager->createQuery($sql2, $locale, ['load_ghost_content' => false])->execute();
+    }
+
+    /**
+     * Converts bytes into human readable.
+     *
+     * Inspired by http://jeffreysambells.com/2012/10/25/human-readable-filesize-php
+     *
+     * @param int $bytes
+     * @param int $dec
+     *
+     * @return string
+     */
+    protected function humanBytes($bytes, $dec = 2)
+    {
+        $size = ['b', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+        $factor = (int) floor((strlen($bytes) - 1) / 3);
+
+        return sprintf("%.{$dec}f", $bytes / pow(1024, $factor)) . $size[$factor];
+    }
 }

Document/Index/ArticleIndexer.php

@@ -436,4 +436,24 @@ public function index(ArticleDocument $document)
         $this->dispatchIndexEvent($document, $article);
         $this->manager->persist($article);
     }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function dropIndex()
+    {
+        $this->manager->dropIndex();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function createIndex()
+    {
+        if ($this->manager->indexExists()) {
+            return;
+        }
+
+        $this->manager->createIndex();
+    }
 }

Document/Index/IndexerInterface.php

@@ -52,4 +52,14 @@ public function remove($document);
      * Flushes index.
      */
     public function flush();
+
+    /**
+     * Drop and recreate elastic-search index.
+     */
+    public function dropIndex();
+
+    /**
+     * Drop and create elastic-search index.
+     */
+    public function createIndex();
 }

Resources/doc/README.md

@@ -9,5 +9,6 @@ This documentation covers basic-topics to install and use this bundle.
 * [Content-Types](content-types.md)
 * [ArticleViewDocument](article-view-document.md)
 * [Twig-Extensions](twig-extensions.md)
+* [Commands](commands.md)
 * [Author](author.md)
 * [Multi-Page](multi-page.md)

Resources/doc/commands.md

@@ -0,0 +1,37 @@
+# Commands
+
+## Reindex command
+
+The reindex command can be used to rebuild the elastic-search index of the articles.
+
+```bash
+bin/adminconsole sulu:article:reindex
+bin/websiteconsole sulu:article:reindex
+```
+
+The default behaviour of the command is to load all the articles and update the content of them. This includes that
+deleted articles (which was not removed correctly from index) will stay inside the index.
+
+To avoid this behaviour you can use the `--clear` option to remove all the articles from the index before reindex the
+existing ones.
+
+```bash
+bin/adminconsole sulu:article:reindex --clear
+bin/websiteconsole sulu:article:reindex --clear
+```
+
+Sometimes you want to update also the mapping of the index. To achieve this you can use the `--drop` option. This will
+drop and recreate the index with the current mapping.
+
+```bash
+bin/adminconsole sulu:article:reindex --drop
+bin/websiteconsole sulu:article:reindex --drop
+```
+
+This options will answer you the confirm this operation. To avoid this interaction you can use the `--no-interaction` 
+option.
+
+```bash
+bin/adminconsole sulu:article:reindex --drop --no-interaction
+bin/websiteconsole sulu:article:reindex --drop --no-interaction
+```

UPGRADE.md

@@ -2,19 +2,18 @@
 
 ## 0.8.0 (unreleased)
 
+### Reindex command
+
+The `sulu:article:index-rebuild` command was refactored and renamed to `sulu:article:reindex`. 
+See [Commands in documentation](Resources/doc/commands.md).
+
 ### NewIndex mapping has changed
 
 Recreate the index to update mapping (new `content_data` field) and reindex your articles:
 
 ```bash
-bin/adminconsole ongr:es:index:drop -m default --force
-bin/websiteconsole ongr:es:index:drop -m live --force
-
-bin/adminconsole ongr:es:index:create -m default
-bin/websiteconsole ongr:es:index:create -m live
-
-bin/adminconsole sulu:article:index-rebuild ###LOCALE###
-bin/websiteconsole sulu:article:index-rebuild ###LOCALE### --live
+bin/adminconsole sulu:article:reindex --drop --no-interactive
+bin/websiteconsole sulu:article:reindex --drop --no-interactive
 ```
 
 ## 0.7.0