Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: ezsystems/ezplatform-kernel
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v1.3.37
Choose a base ref
...
head repository: ezsystems/ezplatform-kernel
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 1.3
Choose a head ref
  • 2 commits
  • 4 files changed
  • 2 contributors

Commits on Oct 22, 2024

  1. IBX-8562: Command to remove duplicated entries after faulty IBX-5388 fix

    Nattfarinn authored Oct 22, 2024

    Verified

    This commit was created on GitHub.com and signed with GitHub’s verified signature.
    Copy the full SHA
    b19be8d View commit details

Commits on Feb 11, 2025

  1. IBX-9455: Upgraded Twig to ^3.19.0 (#411)

    Allow versions below 3.16 due to PHP 7 support requirement
    glye authored Feb 11, 2025
    Copy the full SHA
    553e0dd View commit details
2 changes: 1 addition & 1 deletion composer.json
Original file line number Diff line number Diff line change
@@ -50,7 +50,7 @@
"friendsofsymfony/http-cache-bundle": "^2.8",
"sensio/framework-extra-bundle": "^6.1",
"jms/translation-bundle": "^1.5",
"twig/twig": "^3.0",
"twig/twig": ">=3.0 <3.16 || ^3.19.0",
"twig/extra-bundle": "^3.0",
"friendsofsymfony/jsrouting-bundle": "^2.5",
"friendsofphp/proxy-manager-lts": "^1.0",
8 changes: 8 additions & 0 deletions eZ/Bundle/EzPublishCoreBundle/Resources/config/commands.yml
Original file line number Diff line number Diff line change
@@ -62,3 +62,11 @@ services:
$userHandler: '@ezpublish.spi.persistence.user_handler'
tags:
- { name: console.command }

Ibexa\Bundle\Core\Command\VirtualFieldDuplicateFixCommand:
autowire: true
autoconfigure: true
arguments:
$connection: '@ezpublish.persistence.connection'
tags:
- { name: console.command }
4 changes: 2 additions & 2 deletions eZ/Bundle/EzPublishDebugBundle/Twig/DebugTemplate.php
Original file line number Diff line number Diff line change
@@ -76,9 +76,9 @@ public function getSourceContext(): Source
return new Source('', '');
}

protected function doDisplay(array $context, array $blocks = []): string
protected function doDisplay(array $context, array $blocks = []): iterable
{
return '';
return [];
}

/**
258 changes: 258 additions & 0 deletions src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,258 @@
<?php

/**
* @copyright Copyright (C) Ibexa AS. All rights reserved.
* @license For full copyright and license information view LICENSE file distributed with this source code.
*/
declare(strict_types=1);

namespace Ibexa\Bundle\Core\Command;

use Doctrine\DBAL\Connection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
use Symfony\Component\Console\Style\SymfonyStyle;
use Symfony\Component\Stopwatch\Stopwatch;

final class VirtualFieldDuplicateFixCommand extends Command
{
private const DEFAULT_BATCH_SIZE = 10000;

private const MAX_ITERATIONS_UNLIMITED = -1;

private const DEFAULT_SLEEP = 0;

protected static $defaultName = 'ibexa:content:remove-duplicate-fields';

protected static $defaultDescription = 'Removes duplicate fields created as a result of faulty IBX-5388 performance fix.';

/** @var \Doctrine\DBAL\Connection */
private $connection;

public function __construct(
Connection $connection
) {
parent::__construct();

$this->connection = $connection;
}

public function configure(): void
{
$this->addOption(
'batch-size',
'b',
InputOption::VALUE_REQUIRED,
'Number of attributes affected per iteration',
self::DEFAULT_BATCH_SIZE
);

$this->addOption(
'max-iterations',
'i',
InputOption::VALUE_REQUIRED,
'Max iterations count (default or -1: unlimited)',
self::MAX_ITERATIONS_UNLIMITED
);

$this->addOption(
'sleep',
's',
InputOption::VALUE_REQUIRED,
'Wait between iterations, in milliseconds',
self::DEFAULT_SLEEP
);
}

protected function execute(InputInterface $input, OutputInterface $output): int
{
$style = new SymfonyStyle($input, $output);
$stopwatch = new Stopwatch(true);
$stopwatch->start('total', 'command');

$batchSize = (int)$input->getOption('batch-size');
if ($batchSize === 0) {
$style->warning('Batch size is set to 0. Nothing to do.');

return Command::INVALID;
}

$maxIterations = (int)$input->getOption('max-iterations');
if ($maxIterations === 0) {
$style->warning('Max iterations is set to 0. Nothing to do.');

return Command::INVALID;
}

$sleep = (int)$input->getOption('sleep');

$totalCount = $this->getDuplicatedAttributeTotalCount($style, $stopwatch);

if ($totalCount === 0) {
$style->success('Database is clean of attribute duplicates. Nothing to do.');

return Command::SUCCESS;
}

if ($input->isInteractive()) {
$confirmation = $this->askForConfirmation($style);
if (!$confirmation) {
$style->info('Confirmation rejected. Terminating.');

return Command::FAILURE;
}
}

$iteration = 1;
$totalDeleted = 0;
do {
$deleted = 0;
$stopwatch->start('iteration', 'sql');

$attributes = $this->getDuplicatedAttributesBatch($batchSize);
foreach ($attributes as $attribute) {
$attributeIds = $this->getDuplicatedAttributeIds($attribute);

if (!empty($attributeIds)) {
$iterationDeleted = $this->deleteAttributes($attributeIds);

$deleted += $iterationDeleted;
$totalDeleted += $iterationDeleted;
}
}

$style->info(
sprintf(
'Iteration %d: Removed %d duplicate database rows (total removed this execution: %d). [Debug %s]',
$iteration,
$deleted,
$totalDeleted,
$stopwatch->stop('iteration')
)
);

if ($maxIterations !== self::MAX_ITERATIONS_UNLIMITED && ++$iteration > $maxIterations) {
$style->warning('Max iterations count reached. Terminating.');

return self::SUCCESS;
}

// Wait, if needed, before moving to next iteration
usleep($sleep * 1000);
} while ($batchSize === count($attributes));

$style->success(sprintf(
'Operation successful. Removed total of %d duplicate database rows. [Debug %s]',
$totalDeleted,
$stopwatch->stop('total')
));

return Command::SUCCESS;
}

private function getDuplicatedAttributeTotalCount(
SymfonyStyle $style,
Stopwatch $stopwatch
): int {
$stopwatch->start('total_count', 'sql');
$query = $this->connection->createQueryBuilder()
->select('COUNT(a.id) as instances')
->groupBy('version', 'contentclassattribute_id', 'contentobject_id', 'language_id')
->from('ezcontentobject_attribute', 'a')
->having('instances > 1');

$count = $query->execute()->rowCount();

if ($count > 0) {
$style->warning(
sprintf(
'Found %d of affected attributes. [Debug: %s]',
$count,
$stopwatch->stop('total_count')
)
);
}

return $count;
}

/**
* @phpstan-return array<array{
* version: int,
* contentclassattribute_id: int,
* contentobject_id: int,
* language_id: int,
* }>
*/
private function getDuplicatedAttributesBatch(int $batchSize): array
{
$query = $this->connection->createQueryBuilder();

$query
->select('version', 'contentclassattribute_id', 'contentobject_id', 'language_id')
->groupBy('version', 'contentclassattribute_id', 'contentobject_id', 'language_id')
->from('ezcontentobject_attribute')
->having('COUNT(id) > 1')
->setFirstResult(0)
->setMaxResults($batchSize);

return $query->execute()->fetchAllAssociative();
}

/**
* @phpstan-param array{
* version: int,
* contentclassattribute_id: int,
* contentobject_id: int,
* language_id: int
* } $attribute
*
* @return int[]
*/
private function getDuplicatedAttributeIds(array $attribute): array
{
$query = $this->connection->createQueryBuilder();

$query
->select('id')
->from('ezcontentobject_attribute')
->andWhere('version = :version')
->andWhere('contentclassattribute_id = :contentclassattribute_id')
->andWhere('contentobject_id = :contentobject_id')
->andWhere('language_id = :language_id')
->orderBy('id', 'ASC')
// Keep the original attribute row, the very first one
->setFirstResult(1);

$query->setParameters($attribute);
$result = $query->execute()->fetchFirstColumn();

return array_map('intval', $result);
}

private function askForConfirmation(SymfonyStyle $style): bool
{
$style->warning('Operation is irreversible.');

return $style->askQuestion(
new ConfirmationQuestion(
'Proceed with deletion?',
false
)
);
}

private function deleteAttributes($ids): int
{
$query = $this->connection->createQueryBuilder();

$query
->delete('ezcontentobject_attribute')
->andWhere($query->expr()->in('id', $ids));

return (int)$query->execute();
}
}