Skip to content

Commit

Permalink
Merge branch '3.x-dev' into 4.x-dev
Browse files Browse the repository at this point in the history
  • Loading branch information
@sgiehl
sgiehl committed Mar 16, 2020
2 parents f8c78e5 + 7c1d705 commit e493fee
Show file tree
Hide file tree
Showing 82 changed files with 1,485 additions and 941 deletions.
10 changes: 10 additions & 0 deletions core/Access.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,16 @@ private function throwNoAccessException($message)

throw new NoAccessException($message);
}

/**
* Returns true if the current user is logged in or not.
*
* @return bool
*/
public function isUserLoggedIn()
{
return !empty($this->login);
}
}

/**
Expand Down
76 changes: 4 additions & 72 deletions core/Archive.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
use Piwik\Archive\ArchiveQueryFactory;
use Piwik\Archive\Parameters;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Archive\ArchiveInvalidator;
use Piwik\Container\StaticContainer;
use Piwik\DataAccess\ArchiveSelector;

Expand Down Expand Up @@ -167,11 +166,6 @@ class Archive implements ArchiveQuery
*/
private static $cache;

/**
* @var ArchiveInvalidator
*/
private $invalidator;

/**
* @param Parameters $params
* @param bool $forceIndexedBySite Whether to force index the result of a query by site ID.
Expand All @@ -183,8 +177,6 @@ public function __construct(Parameters $params, $forceIndexedBySite = false,
$this->params = $params;
$this->forceIndexedBySite = $forceIndexedBySite;
$this->forceIndexedByDate = $forceIndexedByDate;

$this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
}

/**
Expand Down Expand Up @@ -453,67 +445,6 @@ public static function createDataTableFromArchive($recordName, $idSite, $period,
return $dataTable;
}

private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet()
{
if (is_null(self::$cache)) {
self::$cache = Cache::getTransientCache();
}

$id = 'Archive.SiteIdsOfRememberedReportsInvalidated';

if (!self::$cache->contains($id)) {
self::$cache->save($id, array());
}

$siteIdsAlreadyHandled = self::$cache->fetch($id);
$siteIdsRequested = $this->params->getIdSites();

foreach ($siteIdsRequested as $index => $siteIdRequested) {
$siteIdRequested = (int) $siteIdRequested;

if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) {
unset($siteIdsRequested[$index]); // was already handled previously, do not do it again
} else {
$siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time
}
}

self::$cache->save($id, $siteIdsAlreadyHandled);

return $siteIdsRequested;
}

private function invalidatedReportsIfNeeded()
{
$siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet();

if (empty($siteIdsRequested)) {
return; // all requested site ids were already handled
}

$sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();

foreach ($sitesPerDays as $date => $siteIds) {
if (empty($siteIds)) {
continue;
}

$siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested);

if (empty($siteIdsToActuallyInvalidate)) {
continue; // all site ids that should be handled are already handled
}

try {
$this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false);
} catch (\Exception $e) {
Site::clearCache();
throw $e;
}
}

Site::clearCache();
}

/**
* Queries archive tables for data and returns the result.
Expand Down Expand Up @@ -638,8 +569,6 @@ private function getArchiveIds($archiveNames)
*/
private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
{
$this->invalidatedReportsIfNeeded();

$today = Date::today();

foreach ($this->params->getPeriods() as $period) {
Expand Down Expand Up @@ -856,8 +785,11 @@ public static function getPluginForReport($report)
*/
private function prepareArchive(array $archiveGroups, Site $site, Period $period)
{
// if cron archiving is running, we will invalidate in CronArchive, not here
$invalidateBeforeArchiving = !SettingsServer::isArchivePhpTriggered();

$parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment());
$archiveLoader = new ArchiveProcessor\Loader($parameters);
$archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving);

$periodString = $period->getRangeString();

Expand Down
23 changes: 16 additions & 7 deletions core/Archive/ArchiveInvalidator.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,12 @@ public function getAllRememberToInvalidateArchivedReportsLater()
// we do not really have to get the value first. we could simply always try to call set() and it would update or
// insert the record if needed but we do not want to lock the table (especially since there are still some
// MyISAM installations)
$values = Option::getLike($this->rememberArchivedReportIdStart . '%');
$values = Option::getLike('%' . $this->rememberArchivedReportIdStart . '%');

$all = [];
foreach ($values as $name => $value) {
$suffix = substr($name, strlen($this->rememberArchivedReportIdStart));
$suffix = substr($name, strpos($name, $this->rememberArchivedReportIdStart));
$suffix = str_replace($this->rememberArchivedReportIdStart, '', $suffix);
list($idSite, $dateStr) = explode('_', $suffix);

$all[$idSite][$dateStr] = $value;
Expand All @@ -101,7 +102,7 @@ public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
// we do not really have to get the value first. we could simply always try to call set() and it would update or
// insert the record if needed but we do not want to lock the table (especially since there are still some
// MyISAM installations)
$value = Option::getLike($key . '%');
$value = Option::getLike('%' . $key . '%');
}

// getLike() returns an empty array rather than 'false'
Expand All @@ -116,6 +117,7 @@ public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
$mykey = $this->buildRememberArchivedReportIdProcessSafe($idSite, $date->toString());
Option::set($mykey, '1');
Cache::clearCacheGeneral();
return $mykey;
}
}

Expand All @@ -133,16 +135,21 @@ private function getRememberedArchivedReportsOptionFromTracker($idSite, $dateStr

public function getRememberedArchivedReportsThatShouldBeInvalidated()
{
$reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%');
$reports = Option::getLike('%' . $this->rememberArchivedReportIdStart . '%_%');

$sitesPerDay = array();

foreach ($reports as $report => $value) {
$report = substr($report, strpos($report, $this->rememberArchivedReportIdStart));
$report = str_replace($this->rememberArchivedReportIdStart, '', $report);
$report = explode('_', $report);
$siteId = (int) $report[0];
$date = $report[1];

if (empty($siteId)) {
continue;
}

if (empty($sitesPerDay[$date])) {
$sitesPerDay[$date] = array();
}
Expand All @@ -169,14 +176,16 @@ private function buildRememberArchivedReportIdForSiteAndDate($idSite, $date)
// This version is multi process safe on the insert of a new date to invalidate.
private function buildRememberArchivedReportIdProcessSafe($idSite, $date)
{
$id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date);
$id = Common::getRandomString(4, 'abcdefghijklmnoprstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ') . '_';
$id .= $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date);
$id .= '_' . Common::getProcessId();

return $id;
}

public function forgetRememberedArchivedReportsToInvalidateForSite($idSite)
{
$id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%';
$id = $this->buildRememberArchivedReportIdForSite($idSite);
$this->deleteOptionLike($id);
Cache::clearCacheGeneral();
}
Expand All @@ -200,7 +209,7 @@ private function deleteOptionLike($id)
{
// we're not using deleteLike since it maybe could cause deadlocks see https://github.com/matomo-org/matomo/issues/15545
// we want to reduce number of rows scanned and only delete specific primary key
$keys = Option::getLike($id . '%');
$keys = Option::getLike('%' . $id . '%');

if (empty($keys)) {
return;
Expand Down
97 changes: 84 additions & 13 deletions core/ArchiveProcessor/Loader.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,18 @@
*/
namespace Piwik\ArchiveProcessor;

use Piwik\Archive\ArchiveInvalidator;
use Piwik\Cache;
use Piwik\Config;
use Piwik\Container\StaticContainer;
use Piwik\Context;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\Date;
use Piwik\Db;
use Piwik\Piwik;
use Piwik\Site;
use Psr\Log\LoggerInterface;

/**
* This class uses PluginsArchiver class to trigger data aggregation and create archives.
Expand All @@ -33,9 +38,28 @@ class Loader
*/
protected $params;

public function __construct(Parameters $params)
/**
* @var ArchiveInvalidator
*/
private $invalidator;

/**
* @var \Piwik\Cache\Cache
*/
private $cache;

/**
* @var LoggerInterface
*/
private $logger;

public function __construct(Parameters $params, $invalidateBeforeArchiving = false)
{
$this->params = $params;
$this->invalidateBeforeArchiving = $invalidateBeforeArchiving;
$this->invalidator = StaticContainer::get(ArchiveInvalidator::class);
$this->cache = Cache::getTransientCache();
$this->logger = StaticContainer::get(LoggerInterface::class);
}

/**
Expand Down Expand Up @@ -65,11 +89,19 @@ private function prepareArchiveImpl($pluginName)
{
$this->params->setRequestedPlugin($pluginName);

list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb();
if (!empty($idArchive)) {
list($idArchive, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb();
if (!empty($idArchive)) { // we have a usable idarchive (it's not invalidated and it's new enough)
return $idArchive;
}

// if there is an archive, but we can't use it for some reason, invalidate existing archives before
// we start archiving. if the archive is made invalid, we will correctly re-archive below.
if ($this->invalidateBeforeArchiving
&& $isAnyArchiveExists
) {
$this->invalidatedReportsIfNeeded();
}

/** @var ArchivingStatus $archivingStatus */
$archivingStatus = StaticContainer::get(ArchivingStatus::class);
$archivingStatus->archiveStarted($this->params);
Expand Down Expand Up @@ -170,20 +202,17 @@ protected function isArchivingForcedToTrigger()
*/
public function loadExistingArchiveIdFromDb()
{
$noArchiveFound = array(false, false, false);

if ($this->isArchivingForcedToTrigger()) {
return $noArchiveFound;
}
$this->logger->debug("Archiving forced to trigger for {$this->params}.");

$minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
$idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);

if (!$idAndVisits) {
return $noArchiveFound;
// return no usable archive found, and no existing archive. this will skip invalidation, which should
// be fine since we just force archiving.
return [false, false, false, false];
}

return $idAndVisits;
$minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
$result = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
return $result;
}

/**
Expand Down Expand Up @@ -242,4 +271,46 @@ private function getIdSitesToArchiveWhenNoVisits()

return $cache->fetch($cacheKey);
}

// public for tests
public function getReportsToInvalidate()
{
$sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();

foreach ($sitesPerDays as $dateStr => $siteIds) {
if (empty($siteIds)
|| !in_array($this->params->getSite()->getId(), $siteIds)
) {
unset($sitesPerDays[$dateStr]);
}

$date = Date::factory($dateStr);
if ($date->isEarlier($this->params->getPeriod()->getDateStart())
|| $date->isLater($this->params->getPeriod()->getDateEnd())
) { // date in list is not the current date, so ignore it
unset($sitesPerDays[$dateStr]);
}
}

return $sitesPerDays;
}

private function invalidatedReportsIfNeeded()
{
$sitesPerDays = $this->getReportsToInvalidate();
if (empty($sitesPerDays)) {
return;
}

foreach ($sitesPerDays as $date => $siteIds) {
try {
$this->invalidator->markArchivesAsInvalidated([$this->params->getSite()->getId()], array(Date::factory($date)), false, $this->params->getSegment());
} catch (\Exception $e) {
Site::clearCache();
throw $e;
}
}

Site::clearCache();
}
}
5 changes: 5 additions & 0 deletions core/ArchiveProcessor/Parameters.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,4 +258,9 @@ public function setIsRootArchiveRequest($isRootArchiveRequest)
{
$this->isRootArchiveRequest = $isRootArchiveRequest;
}

public function __toString()
{
return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}]";
}
}
Loading

0 comments on commit e493fee

Please sign in to comment.