diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php index 0dab3baa215..4f8bf5e976a 100644 --- a/core/Archive/ArchiveInvalidator.php +++ b/core/Archive/ArchiveInvalidator.php @@ -205,6 +205,50 @@ public function markArchivesAsInvalidated(array $idSites, array $dates, $period, return $invalidationInfo; } + /** + * @param $idSites int[] + * @param $dates Date[] + * @param $period string + * @param $segment Segment + * @param bool $cascadeDown + * @return InvalidationResult + * @throws \Exception + */ + public function markArchivesOverlappingRangeAsInvalidated(array $idSites, array $dates, Segment $segment = null) + { + $invalidationInfo = new InvalidationResult(); + + $ranges = array(); + foreach ($dates as $dateRange) { + $ranges[] = $dateRange[0] . ',' . $dateRange[1]; + } + $periodsByType = array(Period\Range::PERIOD_ID => $ranges); + + $invalidatedMonths = array(); + $archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE); + foreach ($archiveNumericTables as $table) { + $tableDate = ArchiveTableCreator::getDateFromTableName($table); + + $result = $this->model->updateArchiveAsInvalidated($table, $idSites, $periodsByType, $segment); + $rowsAffected = $result->rowCount(); + if ($rowsAffected > 0) { + $invalidatedMonths[] = $tableDate; + } + } + + foreach ($idSites as $idSite) { + foreach ($dates as $dateRange) { + $this->forgetRememberedArchivedReportsToInvalidate($idSite, $dateRange[0]); + $invalidationInfo->processedDates[] = $dateRange[0]; + } + } + + $archivesToPurge = new ArchivesToPurgeDistributedList(); + $archivesToPurge->add($invalidatedMonths); + + return $invalidationInfo; + } + /** * @param string[][][] $periodDates * @return string[][][] @@ -230,11 +274,13 @@ private function getPeriodsToInvalidate($dates, $periodType, $cascadeDown) { $periodsToInvalidate = array(); - foreach ($dates as $date) { - if ($periodType == 'range') { - $date = $date . ',' . $date; - } + if ($periodType == 'range') { + $rangeString = $dates[0] . ',' . $dates[1]; + $periodsToInvalidate[] = Period\Factory::build('range', $rangeString); + return $periodsToInvalidate; + } + foreach ($dates as $date) { $period = Period\Factory::build($periodType, $date); $periodsToInvalidate[] = $period; @@ -242,9 +288,7 @@ private function getPeriodsToInvalidate($dates, $periodType, $cascadeDown) $periodsToInvalidate = array_merge($periodsToInvalidate, $period->getAllOverlappingChildPeriods()); } - if ($periodType != 'year' - && $periodType != 'range' - ) { + if ($periodType != 'year') { $periodsToInvalidate[] = Period\Factory::build('year', $date); } } @@ -264,7 +308,11 @@ private function getPeriodDatesByYearMonthAndPeriodType($periods) $periodType = $period->getId(); $yearMonth = ArchiveTableCreator::getTableMonthFromDate($date); - $result[$yearMonth][$periodType][] = $date->toString(); + $dateString = $date->toString(); + if ($periodType == Period\Range::PERIOD_ID) { + $dateString = $period->getRangeString(); + } + $result[$yearMonth][$periodType][] = $dateString; } return $result; } diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 3f807538356..6b0d5e264e6 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -115,10 +115,21 @@ public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeri foreach ($datesByPeriodType as $periodType => $dates) { $dateConditions = array(); - foreach ($dates as $date) { - $dateConditions[] = "(date1 <= ? AND ? <= date2)"; - $bind[] = $date; - $bind[] = $date; + if ($periodType == Period\Range::PERIOD_ID) { + foreach ($dates as $date) { + // Ranges in the DB match if their date2 is after the start of the search range and date1 is before the end + // e.g. search range is 2019-01-01 to 2019-01-31 + // date2 >= startdate -> Ranges with date2 < 2019-01-01 (ended before 1 January) and are excluded + // date1 <= endate -> Ranges with date1 > 2019-01-31 (started after 31 January) and are excluded + $dateConditions[] = "(date2 >= ? AND date1 <= ?)"; + $bind = array_merge($bind, explode(',', $date)); + } + } else { + foreach ($dates as $date) { + $dateConditions[] = "(date1 <= ? AND ? <= date2)"; + $bind[] = $date; + $bind[] = $date; + } } $dateConditionsSql = implode(" OR ", $dateConditions); @@ -149,7 +160,6 @@ public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeri return Db::query($sql, $bind); } - public function getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan) { $query = "SELECT idarchive FROM " . $archiveTable . " diff --git a/plugins/CoreAdminHome/Commands/InvalidateReportData.php b/plugins/CoreAdminHome/Commands/InvalidateReportData.php index b8abd737f6d..75878a48f15 100644 --- a/plugins/CoreAdminHome/Commands/InvalidateReportData.php +++ b/plugins/CoreAdminHome/Commands/InvalidateReportData.php @@ -71,6 +71,9 @@ protected function execute(InputInterface $input, OutputInterface $output) $segments = $this->getSegmentsToInvalidateFor($input, $sites); foreach ($periodTypes as $periodType) { + if ($periodType === 'range') { + continue; // special handling for range below + } foreach ($dateRanges as $dateRange) { foreach ($segments as $segment) { $segmentStr = $segment ? $segment->getString() : ''; @@ -93,6 +96,31 @@ protected function execute(InputInterface $input, OutputInterface $output) } } } + + $periods = trim($input->getOption('periods')); + $isUsingAllOption = $periods === self::ALL_OPTION_VALUE; + if ($isUsingAllOption || in_array('range', $periodTypes)) { + $rangeDates = array(); + foreach ($dateRanges as $dateRange) { + if ($isUsingAllOption + && !Period::isMultiplePeriod($dateRange, 'day')) { + continue; // not a range, nothing to do... only when "all" option is used + } + + $rangeDates[] = $this->getPeriodDates('range', $dateRange); + } + if (!empty($rangeDates)) { + foreach ($segments as $segment) { + $segmentStr = $segment ? $segment->getString() : ''; + if ($dryRun) { + $dateRangeStr = implode($dateRanges, ';'); + $output->writeln("Invalidating range periods overlapping $dateRangeStr [segment = $segmentStr]..."); + } else { + $invalidator->markArchivesOverlappingRangeAsInvalidated($sites, $rangeDates, $segment); + } + } + } + } } private function getSitesToInvalidateFor(InputInterface $input) @@ -123,7 +151,6 @@ private function getPeriodTypesToInvalidateFor(InputInterface $input) if ($periods == self::ALL_OPTION_VALUE) { $result = array_keys(Piwik::$idPeriods); - unset($result[4]); // remove 'range' period return $result; } @@ -131,11 +158,6 @@ private function getPeriodTypesToInvalidateFor(InputInterface $input) $periods = array_map('trim', $periods); foreach ($periods as $periodIdentifier) { - if ($periodIdentifier == 'range') { - throw new \InvalidArgumentException( - "Invalid period type: invalidating range periods is not currently supported."); - } - if (!isset(Piwik::$idPeriods[$periodIdentifier])) { throw new \InvalidArgumentException("Invalid period type '$periodIdentifier' supplied in --periods."); } @@ -165,13 +187,17 @@ private function getPeriodDates($periodType, $dateRange) } try { + $period = PeriodFactory::build($periodType, $dateRange); } catch (\Exception $ex) { throw new \InvalidArgumentException("Invalid date or date range specifier '$dateRange'", $code = 0, $ex); } $result = array(); - if ($period instanceof Range) { + if ($periodType === 'range') { + $result[] = $period->getDateStart(); + $result[] = $period->getDateEnd(); + } elseif ($period instanceof Range) { foreach ($period->getSubperiods() as $subperiod) { $result[] = $subperiod->getDateStart(); } diff --git a/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php b/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php index 542b3117e56..ccbbba8d4d8 100644 --- a/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php +++ b/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php @@ -74,7 +74,6 @@ public function getInvalidPeriodTypes() { return array( array('cranberries'), - array('range'), ); } @@ -144,6 +143,131 @@ public function test_Command_InvalidatesCorrectSitesAndDates($dates, $periods, $ } } + public function test_Command_InvalidateDateRange() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01,2019-01-09'), + '--periods' => 'range', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay()); + } + + public function test_Command_InvalidateDateRange_invalidDate() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01,2019-01--09'), + '--periods' => 'range', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertNotEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertContains("The date '2019-01-01,2019-01--09' is not a correct date range", $this->applicationTester->getDisplay()); + } + + public function test_Command_InvalidateDateRange_onlyOneDate() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01'), + '--periods' => 'range', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertNotEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertContains("The date '2019-01-01' is not a correct date range", $this->applicationTester->getDisplay()); + } + + public function test_Command_InvalidateDateRange_tooManyDatesInRange() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01,2019-01-09,2019-01-12,2019-01-15'), + '--periods' => 'range', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertNotEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertContains("The date '2019-01-01,2019-01-09,2019-01-12,2019-01-15' is not a correct date range", $this->applicationTester->getDisplay()); + } + + public function test_Command_InvalidateDateRange_multipleDateRanges() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01,2019-01-09', '2019-01-12,2019-01-15'), + '--periods' => 'range', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09;2019-01-12,2019-01-15", $this->applicationTester->getDisplay()); + } + + public function test_Command_InvalidateDateRange_invalidateAllPeriodTypesSkipsRangeWhenNotRangeDAte() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01'), + '--periods' => 'all', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertNotContains("range", $this->applicationTester->getDisplay()); + $this->assertNotContains("Range", $this->applicationTester->getDisplay()); + } + + public function test_Command_InvalidateDateRange_invalidateAllPeriodTypes() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01,2019-01-09'), + '--periods' => 'all', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertContains("Invalidating day periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay()); + $this->assertContains("Invalidating week periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay()); + $this->assertContains("Invalidating month periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay()); + $this->assertContains("Invalidating year periods in 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay()); + $this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09 [segment = ]", $this->applicationTester->getDisplay()); + } + + public function test_Command_InvalidateAll_multipleDateRanges() + { + $code = $this->applicationTester->run(array( + 'command' => 'core:invalidate-report-data', + '--dates' => array('2019-01-01,2019-01-09', '2019-01-12,2019-01-13'), + '--periods' => 'all', + '--sites' => '1', + '--dry-run' => true, + '-vvv' => true, + )); + + $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); + $this->assertContains("Invalidating range periods overlapping 2019-01-01,2019-01-09;2019-01-12,2019-01-13 [segment = ]", $this->applicationTester->getDisplay()); + } + public function getTestDataForSuccessTests() { return array( diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index 6ae44105051..7bc718b373c 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -310,6 +310,34 @@ public function test_markArchivesAsInvalidated_MarksCorrectArchivesAsInvalidated $this->assertEquals($expectedIdArchives, $idArchives); } + /** + * @dataProvider getTestDataForMarkArchiveRangesAsInvalidated + */ + public function test_markArchivesAsInvalidated_MarksAllSubrangesOfRange($idSites, $dates, $segment, $expectedIdArchives) + { + $dates = array_map(array('Piwik\Date', 'factory'), $dates); + + $this->insertArchiveRowsForTest(); + + if (!empty($segment)) { + $segment = new Segment($segment, $idSites); + } + + /** @var ArchiveInvalidator $archiveInvalidator */ + $archiveInvalidator = self::$fixture->piwikEnvironment->getContainer()->get('Piwik\Archive\ArchiveInvalidator'); + $result = $archiveInvalidator->markArchivesOverlappingRangeAsInvalidated($idSites, array($dates), $segment); + + $this->assertEquals(array($dates[0]), $result->processedDates); + + $idArchives = $this->getInvalidatedArchives(); + + // Remove empty values (some new empty entries may be added each month) + $idArchives = array_filter($idArchives); + $expectedIdArchives = array_filter($expectedIdArchives); + + $this->assertEquals($expectedIdArchives, $idArchives); + } + public function getTestDataForMarkArchivesAsInvalidated() { // $idSites, $dates, $period, $segment, $cascadeDown, $expectedIdArchives @@ -462,10 +490,10 @@ public function getTestDataForMarkArchivesAsInvalidated() ), ), - // range period, one site, cascade = true + // range period, exact match array( array(1), - array('2015-01-02', '2015-03-05'), + array('2015-01-01', '2015-01-10'), 'range', null, true, @@ -473,10 +501,20 @@ public function getTestDataForMarkArchivesAsInvalidated() '2015_01' => array( '1.2015-01-01.2015-01-10.5.done.VisitsSummary', ), - '2015_03' => array( - '1.2015-03-04.2015-03-05.5.done.VisitsSummary', - '1.2015-03-05.2015-03-10.5.done3736b708e4d20cfc10610e816a1b2341.UserCountry', - ), + ), + ), + + // range period, overlapping a range in the DB + array( + array(1), + array('2015-01-02', '2015-03-05'), + 'range', + null, + true, + array( + '2015_01' => array( + '1.2015-01-01.2015-01-10.5.done.VisitsSummary', + ) ), ), @@ -532,6 +570,101 @@ public function getTestDataForMarkArchivesAsInvalidated() ); } + public function getTestDataForMarkArchiveRangesAsInvalidated() + { + // $idSites, $dates, $segment, $expectedIdArchives + return array( + // range period, has an exact match, also a match where DB end date = reference start date + array( + array(1), + array('2015-01-01', '2015-01-10'), + null, + array( + '2014_12' => array( + '1.2014-12-05.2015-01-01.5.done.VisitsSummary', + ), + '2015_01' => array( + '1.2015-01-01.2015-01-10.5.done.VisitsSummary', + ), + ), + ), + + // range period, overlapping range = a match + array( + array(1), + array('2015-01-02', '2015-03-05'), + null, + array( + '2015_01' => array( + '1.2015-01-01.2015-01-10.5.done.VisitsSummary', + ), + '2015_03' => array( + '1.2015-03-04.2015-03-05.5.done.VisitsSummary', + '1.2015-03-05.2015-03-10.5.done3736b708e4d20cfc10610e816a1b2341.UserCountry', + ), + ), + ), + + // range period, small range within the 2014-12-05 to 2015-01-01 range should cause it to be invalidated + array( + array(1), + array('2014-12-18', '2014-12-20'), + null, + array( + '2014_12' => array( + '1.2014-12-05.2015-01-01.5.done.VisitsSummary', + ), + ), + ), + + // range period, range that overlaps start of archived range + array( + array(1), + array('2014-12-01', '2014-12-05'), + null, + array( + '2014_12' => array( + '1.2014-12-05.2015-01-01.5.done.VisitsSummary', + ), + ), + ), + + // range period, large range that includes the smallest archived range (3 to 4 March) + array( + array(1), + array('2015-01-11', '2015-03-30'), + null, + array( + '2015_03' => array( + '1.2015-03-04.2015-03-05.5.done.VisitsSummary', + '1.2015-03-05.2015-03-10.5.done3736b708e4d20cfc10610e816a1b2341.UserCountry', + ), + ), + ), + + // range period, doesn't match any archived ranges + array( + array(1), + array('2014-12-01', '2014-12-04'), + null, + array( + ), + ), + + // three-month range period, there's a range archive for the middle month + array( + array(1), + array('2014-09-01', '2014-11-08'), + null, + array( + '2014_10' => array( + '1.2014-10-15.2014-10-20.5.done3736b708e4d20cfc10610e816a1b2341', + ), + ), + ), + ); + } + private function getInvalidatedIdArchives() { $result = array(); @@ -579,7 +712,13 @@ private function insertArchiveRowsForTest() } } - $rangePeriods = array('2015-03-04,2015-03-05', '2014-12-05,2015-01-01', '2015-03-05,2015-03-10', '2015-01-01,2015-01-10'); + $rangePeriods = array( + '2015-03-04,2015-03-05', + '2014-12-05,2015-01-01', + '2015-03-05,2015-03-10', + '2015-01-01,2015-01-10', + '2014-10-15,2014-10-20' + ); foreach ($rangePeriods as $dateRange) { $this->insertArchiveRow($idSite = 1, $dateRange, 'range'); }