Skip to content

Commit

Permalink
Fix archiving for range periods (matomo-org#22577)
Browse files Browse the repository at this point in the history 
* Fix archiving for range periods

* adds another test
  • Loading branch information
@sgiehl @jorgeuos
sgiehl authored and jorgeuos committed Oct 1, 2024
1 parent c5906fd commit 21859ce
Show file tree
Hide file tree
Showing 2 changed files with 157 additions and 59 deletions.
7 changes: 6 additions & 1 deletion core/CronArchive/ArchiveFilter.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
use Piwik\Date;
use Piwik\Period\Factory;
use Piwik\Period\Factory as PeriodFactory;
use Piwik\Period\Range;
use Piwik\Piwik;
use Piwik\Plugins\SegmentEditor\Model as SegmentEditorModel;
use Piwik\Segment;
Expand Down Expand Up @@ -89,7 +90,11 @@ public function filterArchive($archive)

if (!empty($this->skipSegmentsForToday)) {
$site = new Site($archive['idsite']);
$period = Factory::build($this->periodIdsToLabels[$archive['period']], $archive['date1']);
if ((int) $archive['period'] === Range::PERIOD_ID) {
$period = Factory::build($this->periodIdsToLabels[$archive['period']], "{$archive['date1']},{$archive['date2']}");
} else {
$period = Factory::build($this->periodIdsToLabels[$archive['period']], $archive['date1']);
}
$segment = new Segment($segment, [$archive['idsite']]);
if (Archive::shouldSkipArchiveIfSkippingSegmentArchiveForToday($site, $period, $segment)) {
return "skipping segment archives for today";
Expand Down
209 changes: 151 additions & 58 deletions tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@
use Piwik\CliMulti\RequestParser;
use Piwik\Common;
use Piwik\Config;
use Piwik\Period\Day;
use Piwik\Period\Factory;
use Piwik\Period\Month;
use Piwik\Period\Range;
use Piwik\Plugins\CustomDimensions;
use Piwik\Container\StaticContainer;
use Piwik\CronArchive;
Expand Down Expand Up @@ -616,12 +619,15 @@ public function testSkipSegmentsToday()
$segmentHash2 = (new Segment('browserCode==FF', [1]))->getHash();

$invalidations = [
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash2 . '.ExamplePlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null],
['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null],
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 2, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 3, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 4, 'name' => 'done' . $segmentHash2 . '.ExamplePlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 5, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => Month::PERIOD_ID, 'report' => null],
['idarchive' => 6, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-31', 'period' => Range::PERIOD_ID, 'report' => null],
['idarchive' => 7, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-02-02', 'date2' => '2018-03-04', 'period' => Range::PERIOD_ID, 'report' => null],
['idarchive' => 8, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => Day::PERIOD_ID, 'report' => null],
['idarchive' => 9, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-03', 'period' => Range::PERIOD_ID, 'report' => null],
];
shuffle($invalidations);

Expand All @@ -646,60 +652,84 @@ public function testSkipSegmentsToday()
}

$expectedInvalidationsFound = [
array (
array (
'idarchive' => '1',
'idsite' => '1',
'date1' => '2018-03-04',
'date2' => '2018-03-04',
'period' => '1',
'name' => 'done',
'report' => null,
'plugin' => null,
'segment' => '',
'ts_started' => null,
'status' => '0',
),
array (
'idarchive' => '1',
'idsite' => '1',
'date1' => '2018-03-03',
'date2' => '2018-03-03',
'period' => '1',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
),
),
array (
0 =>
array (
'idarchive' => '1',
'idsite' => '1',
'date1' => '2018-03-01',
'date2' => '2018-03-31',
'period' => '3',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
),
),
array (// end of idsite=1
),
[
[
'idarchive' => '8',
'idsite' => '1',
'date1' => '2018-03-04',
'date2' => '2018-03-04',
'period' => '1',
'name' => 'done',
'report' => null,
'plugin' => null,
'segment' => '',
'ts_started' => null,
'status' => '0',
],
[
'idarchive' => '3',
'idsite' => '1',
'date1' => '2018-03-03',
'date2' => '2018-03-03',
'period' => '1',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
],
],
[
[
'idarchive' => '5',
'idsite' => '1',
'date1' => '2018-03-01',
'date2' => '2018-03-31',
'period' => '3',
'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==IE',
'ts_started' => null,
'status' => '0',
],
],
[
[
'idarchive' => '9',
'name' => 'done',
'idsite' => '1',
'date1' => '2018-03-01',
'date2' => '2018-03-03',
'period' => '5',
'ts_started' => null,
'status' => '0',
'report' => null,
'plugin' => null,
'segment' => '',
],
],
[
[
'idarchive' => '7',
'name' => 'done3736b708e4d20cfc10610e816a1b2341',
'idsite' => '1',
'date1' => '2018-02-02',
'date2' => '2018-03-04',
'period' => '5',
'ts_started' => null,
'status' => '0',
'report' => null,
'plugin' => null,
'segment' => 'browserCode==FF',
],
],
[// end of idsite=1
],
];

try {
$this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations);
} catch (\Exception $ex) {
print "\nInvalidations inserted:\n" . var_export($invalidations, true) . "\n";
throw $ex;
}
$this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations);

// automated check for no duplicates
$invalidationDescs = [];
Expand All @@ -714,6 +744,69 @@ public function testSkipSegmentsToday()
$this->assertEquals($uniqueInvalidationDescs, $invalidationDescs, "Found duplicate archives being processed.");
}

/**
* @dataProvider getAllPeriods
*/
public function testSkipSegmentsTodayShouldSkipAllSegmentInvalidationsIfPeriodBeginsToday(string $periodType)
{
if ($periodType === 'range') {
$period = Factory::build($periodType, '2018-03-04,2018-04-03');
} else {
$period = Factory::build($periodType, '2018-03-04');
}

Date::$now = strtotime($period->getDateStart()->toString() . ' 01:00:00');

Fixture::createWebsite('2015-02-03');

Rules::setBrowserTriggerArchiving(false);
API::getInstance()->add('testegment', 'browserCode==IE', false, true);
Rules::setBrowserTriggerArchiving(true);

// force archiving so we don't skip those without visits
Piwik::addAction('Archiving.getIdSitesToArchiveWhenNoVisits', function (&$idSites) {
$idSites[] = 1;
});

$cronArchive = new MockCronArchive();
$cronArchive->init();

$archiveFilter = $this->makeTestArchiveFilter(null, null, null, false, true);

$queueConsumer = new QueueConsumer(
StaticContainer::get(LoggerInterface::class),
new FixedSiteIds([1]),
3,
24,
new Model(),
new SegmentArchiving(),
$cronArchive,
new RequestParser(true),
$archiveFilter
);

$segmentHash1 = (new Segment('browserCode==IE', [1]))->getHash();

$invalidations = [
['idarchive' => 1, 'name' => 'done' . $segmentHash1, 'idsite' => 1, 'date1' => $period->getDateStart()->toString(), 'date2' => $period->getDateEnd()->toString(), 'period' => $period::PERIOD_ID, 'report' => null],
];

$this->insertInvalidations($invalidations);

self::assertEmpty($queueConsumer->getNextArchivesToProcess());
}

public function getAllPeriods()
{
return [
['day'],
['week'],
['month'],
['year'],
['range'],
];
}

public function testMaxWebsitesToProcess()
{
Fixture::createWebsite('2021-11-16');
Expand Down

0 comments on commit 21859ce

Please sign in to comment.