Skip to content

Commit

Permalink
Allow date ranges to be passed to core:invalidate-report-data (matomo…
Browse files Browse the repository at this point in the history 
…-org#14450)
  • Loading branch information
@tsteur
Kate Butler authored and tsteur committed Oct 16, 2019
1 parent d262022 commit 8b2964a
Show file tree
Hide file tree
Showing 5 changed files with 375 additions and 28 deletions.
64 changes: 56 additions & 8 deletions core/Archive/ArchiveInvalidator.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,50 @@ public function markArchivesAsInvalidated(array $idSites, array $dates, $period,
return $invalidationInfo;
}

/**
* @param $idSites int[]
* @param $dates Date[]
* @param $period string
* @param $segment Segment
* @param bool $cascadeDown
* @return InvalidationResult
* @throws \Exception
*/
public function markArchivesOverlappingRangeAsInvalidated(array $idSites, array $dates, Segment $segment = null)
{
$invalidationInfo = new InvalidationResult();

$ranges = array();
foreach ($dates as $dateRange) {
$ranges[] = $dateRange[0] . ',' . $dateRange[1];
}
$periodsByType = array(Period\Range::PERIOD_ID => $ranges);

$invalidatedMonths = array();
$archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
foreach ($archiveNumericTables as $table) {
$tableDate = ArchiveTableCreator::getDateFromTableName($table);

$result = $this->model->updateArchiveAsInvalidated($table, $idSites, $periodsByType, $segment);
$rowsAffected = $result->rowCount();
if ($rowsAffected > 0) {
$invalidatedMonths[] = $tableDate;
}
}

foreach ($idSites as $idSite) {
foreach ($dates as $dateRange) {
$this->forgetRememberedArchivedReportsToInvalidate($idSite, $dateRange[0]);
$invalidationInfo->processedDates[] = $dateRange[0];
}
}

$archivesToPurge = new ArchivesToPurgeDistributedList();
$archivesToPurge->add($invalidatedMonths);

return $invalidationInfo;
}

/**
* @param string[][][] $periodDates
* @return string[][][]
Expand All @@ -230,21 +274,21 @@ private function getPeriodsToInvalidate($dates, $periodType, $cascadeDown)
{
$periodsToInvalidate = array();

foreach ($dates as $date) {
if ($periodType == 'range') {
$date = $date . ',' . $date;
}
if ($periodType == 'range') {
$rangeString = $dates[0] . ',' . $dates[1];
$periodsToInvalidate[] = Period\Factory::build('range', $rangeString);
return $periodsToInvalidate;
}

foreach ($dates as $date) {
$period = Period\Factory::build($periodType, $date);
$periodsToInvalidate[] = $period;

if ($cascadeDown) {
$periodsToInvalidate = array_merge($periodsToInvalidate, $period->getAllOverlappingChildPeriods());
}

if ($periodType != 'year'
&& $periodType != 'range'
) {
if ($periodType != 'year') {
$periodsToInvalidate[] = Period\Factory::build('year', $date);
}
}
Expand All @@ -264,7 +308,11 @@ private function getPeriodDatesByYearMonthAndPeriodType($periods)
$periodType = $period->getId();

$yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
$result[$yearMonth][$periodType][] = $date->toString();
$dateString = $date->toString();
if ($periodType == Period\Range::PERIOD_ID) {
$dateString = $period->getRangeString();
}
$result[$yearMonth][$periodType][] = $dateString;
}
return $result;
}
Expand Down
20 changes: 15 additions & 5 deletions core/DataAccess/Model.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,21 @@ public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeri
foreach ($datesByPeriodType as $periodType => $dates) {
$dateConditions = array();

foreach ($dates as $date) {
$dateConditions[] = "(date1 <= ? AND ? <= date2)";
$bind[] = $date;
$bind[] = $date;
if ($periodType == Period\Range::PERIOD_ID) {
foreach ($dates as $date) {
// Ranges in the DB match if their date2 is after the start of the search range and date1 is before the end
// e.g. search range is 2019-01-01 to 2019-01-31
// date2 >= startdate -> Ranges with date2 < 2019-01-01 (ended before 1 January) and are excluded
// date1 <= endate -> Ranges with date1 > 2019-01-31 (started after 31 January) and are excluded
$dateConditions[] = "(date2 >= ? AND date1 <= ?)";
$bind = array_merge($bind, explode(',', $date));
}
} else {
foreach ($dates as $date) {
$dateConditions[] = "(date1 <= ? AND ? <= date2)";
$bind[] = $date;
$bind[] = $date;
}
}

$dateConditionsSql = implode(" OR ", $dateConditions);
Expand Down Expand Up @@ -149,7 +160,6 @@ public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeri
return Db::query($sql, $bind);
}


public function getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan)
{
$query = "SELECT idarchive FROM " . $archiveTable . "
Expand Down
40 changes: 33 additions & 7 deletions plugins/CoreAdminHome/Commands/InvalidateReportData.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ protected function execute(InputInterface $input, OutputInterface $output)
$segments = $this->getSegmentsToInvalidateFor($input, $sites);

foreach ($periodTypes as $periodType) {
if ($periodType === 'range') {
continue; // special handling for range below
}
foreach ($dateRanges as $dateRange) {
foreach ($segments as $segment) {
$segmentStr = $segment ? $segment->getString() : '';
Expand All @@ -93,6 +96,31 @@ protected function execute(InputInterface $input, OutputInterface $output)
}
}
}

$periods = trim($input->getOption('periods'));
$isUsingAllOption = $periods === self::ALL_OPTION_VALUE;
if ($isUsingAllOption || in_array('range', $periodTypes)) {
$rangeDates = array();
foreach ($dateRanges as $dateRange) {
if ($isUsingAllOption
&& !Period::isMultiplePeriod($dateRange, 'day')) {
continue; // not a range, nothing to do... only when "all" option is used
}

$rangeDates[] = $this->getPeriodDates('range', $dateRange);
}
if (!empty($rangeDates)) {
foreach ($segments as $segment) {
$segmentStr = $segment ? $segment->getString() : '';
if ($dryRun) {
$dateRangeStr = implode($dateRanges, ';');
$output->writeln("Invalidating range periods overlapping $dateRangeStr [segment = $segmentStr]...");
} else {
$invalidator->markArchivesOverlappingRangeAsInvalidated($sites, $rangeDates, $segment);
}
}
}
}
}

private function getSitesToInvalidateFor(InputInterface $input)
Expand Down Expand Up @@ -123,19 +151,13 @@ private function getPeriodTypesToInvalidateFor(InputInterface $input)

if ($periods == self::ALL_OPTION_VALUE) {
$result = array_keys(Piwik::$idPeriods);
unset($result[4]); // remove 'range' period
return $result;
}

$periods = explode(',', $periods);
$periods = array_map('trim', $periods);

foreach ($periods as $periodIdentifier) {
if ($periodIdentifier == 'range') {
throw new \InvalidArgumentException(
"Invalid period type: invalidating range periods is not currently supported.");
}

if (!isset(Piwik::$idPeriods[$periodIdentifier])) {
throw new \InvalidArgumentException("Invalid period type '$periodIdentifier' supplied in --periods.");
}
Expand Down Expand Up @@ -165,13 +187,17 @@ private function getPeriodDates($periodType, $dateRange)
}

try {

$period = PeriodFactory::build($periodType, $dateRange);
} catch (\Exception $ex) {
throw new \InvalidArgumentException("Invalid date or date range specifier '$dateRange'", $code = 0, $ex);
}

$result = array();
if ($period instanceof Range) {
if ($periodType === 'range') {
$result[] = $period->getDateStart();
$result[] = $period->getDateEnd();
} elseif ($period instanceof Range) {
foreach ($period->getSubperiods() as $subperiod) {
$result[] = $subperiod->getDateStart();
}
Expand Down
126 changes: 125 additions & 1 deletion plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ public function getInvalidPeriodTypes()
{
return array(
array('cranberries'),
array('range'),
);
}

Expand Down Expand Up @@ -144,6 +143,131 @@ public function test_Command_InvalidatesCorrectSitesAndDates($dates, $periods, $
}
}

public function test_Command_InvalidateDateRange()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01,2019-01-09'),
'--periods' => 'range',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay());
}

public function test_Command_InvalidateDateRange_invalidDate()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01,2019-01--09'),
'--periods' => 'range',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertNotEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertContains("The date '2019-01-01,2019-01--09' is not a correct date range", $this->applicationTester->getDisplay());
}

public function test_Command_InvalidateDateRange_onlyOneDate()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01'),
'--periods' => 'range',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertNotEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertContains("The date '2019-01-01' is not a correct date range", $this->applicationTester->getDisplay());
}

public function test_Command_InvalidateDateRange_tooManyDatesInRange()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01,2019-01-09,2019-01-12,2019-01-15'),
'--periods' => 'range',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertNotEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertContains("The date '2019-01-01,2019-01-09,2019-01-12,2019-01-15' is not a correct date range", $this->applicationTester->getDisplay());
}

public function test_Command_InvalidateDateRange_multipleDateRanges()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01,2019-01-09', '2019-01-12,2019-01-15'),
'--periods' => 'range',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09;2019-01-12,2019-01-15", $this->applicationTester->getDisplay());
}

public function test_Command_InvalidateDateRange_invalidateAllPeriodTypesSkipsRangeWhenNotRangeDAte()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01'),
'--periods' => 'all',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertNotContains("range", $this->applicationTester->getDisplay());
$this->assertNotContains("Range", $this->applicationTester->getDisplay());
}

public function test_Command_InvalidateDateRange_invalidateAllPeriodTypes()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01,2019-01-09'),
'--periods' => 'all',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertContains("Invalidating day periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay());
$this->assertContains("Invalidating week periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay());
$this->assertContains("Invalidating month periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay());
$this->assertContains("Invalidating year periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay());
$this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay());
}

public function test_Command_InvalidateAll_multipleDateRanges()
{
$code = $this->applicationTester->run(array(
'command' => 'core:invalidate-report-data',
'--dates' => array('2019-01-01,2019-01-09', '2019-01-12,2019-01-13'),
'--periods' => 'all',
'--sites' => '1',
'--dry-run' => true,
'-vvv' => true,
));

$this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
$this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09;2019-01-12,2019-01-13 [segment = ]", $this->applicationTester->getDisplay());
}

public function getTestDataForSuccessTests()
{
return array(
Expand Down
Loading

0 comments on commit 8b2964a

Please sign in to comment.