Merge branch 3.x-dev into 4.x-dev (matomo-org#15543)

* Updates search engine and social definitions (matomo-org#15384) * updates device detector to latest release (matomo-org#15388) * updates device detector to latest release * updates tests * translation update (matomo-org#15389) * Fix Could not get the lock for ID, when creating a site (matomo-org#15401) * Lock key start * do not empty key lock Co-authored-by: Thomas Steur <[email protected]> * 3.13.1 * submodule updates * Use correct name in update available message (matomo-org#15423) * Fix removing user capabilities (matomo-org#15422) * Order of implode() args, avoid E_NOTICE in PHP7.4 (matomo-org#15428) * Fixes possible php warning in visitor log (matomo-org#15442) * silence is_executable call (matomo-org#15446) * Make sure geolocation admin experience is consistent if user is not using GeoIp2 plugin. (matomo-org#15447) * Fix referrers test. (matomo-org#15448) * Ensure to close visitor popover correctly (matomo-org#15443) * Fixes possible warning (matomo-org#15453) * Forward instance_id from local config when reseting config during tests. (matomo-org#15445) * Add event that allows plugins to disable archiving for certain periods/sites if they want. (matomo-org#15457) * Add event that allows plugins to disable archiving for certain periods/sites if they want. * apply review feedback * Fix possible warning for columns without index (matomo-org#15467) * Day range archiving issue (matomo-org#15462) * Improve lock ID check for max length (matomo-org#15407) Better patch for matomo-org#15401 which was merged last minute... This way it always works even when someone calls `acquireLock` directly instead of `execute` Pushing this for now into 3.x-dev but can also put it into 4.x-dev directly but then there might be merge conflicts when merging 3.x-dev into 4.x-dev * Use SameSite none for session token when embedded into iframe (matomo-org#15439) * Make sure tracking works in IE9 and lower (matomo-org#15480) * Mention Joomla install FAQ (matomo-org#15481) * Make sparklines work when mbstring extension is not installed (matomo-org#15489) 1) Too few arguments to function mb_strtolower(), 1 passed in matomo/vendor/davaxi/sparkline/src/Sparkline/StyleTrait.php on line 129 and exactly 2 expected 2) mb_strlen is not defined * update screenshots (matomo-org#15488) * 3.13.2-rc1 * Use safemode when running CLI commands (matomo-org#15472) * update icons submodule (matomo-org#15490) * update icons submodule * update UI tests * Fix possible undefined index notice (matomo-org#15502) * Use latest davaxi/sparkline release (matomo-org#15464) * translation update * submodule updates * Fix deprecation notice (matomo-org#15530) see matomo-org#15467 (comment) * 3.13.2-rc2 * update cache component (matomo-org#15536) * fixes copy dashboard to user for more than 100 users (matomo-org#15538) cherry picking matomo-org#15424 to fix matomo-org#15420 in 3.x-dev * Add missing return statement. (matomo-org#15539) * 3.13.2 * update tests * update tests Co-authored-by: Matthieu Aubry <[email protected]> Co-authored-by: Thomas Steur <[email protected]> Co-authored-by: Peter Upfold <[email protected]> Co-authored-by: diosmosis <[email protected]> Co-authored-by: Lukas Winkler <[email protected]>
alemonteiro · Feb 11, 2020 · 5d2dab7 · 5d2dab7
1 parent ab1e700
commit 5d2dab7
Show file tree

Hide file tree

Showing 68 changed files with 480 additions and 133 deletions.
diff --git a/composer.json b/composer.json
@@ -37,8 +37,8 @@
         "matomo/ini": "~2.0",
         "matomo/matomo-php-tracker": "~2.0",
         "matomo/network": "~2.0",
-        "matomo/referrer-spam-blacklist": "~1.0",
-        "matomo/searchengine-and-social-list": "~1.0",
+        "matomo/referrer-spam-blacklist": "~3.0",
+        "matomo/searchengine-and-social-list": "~3.0",
         "monolog/monolog": "~1.11",
         "mustangostang/spyc": "~0.6.0",
         "pear/pear_exception": "~1.0.0",

diff --git a/composer.lock b/composer.lock
diff --git a/console b/console
@@ -1,5 +1,8 @@
 #!/usr/bin/env php
 <?php
+
+use Piwik\FrontController;
+
 if (!defined('PIWIK_DOCUMENT_ROOT')) {
     define('PIWIK_DOCUMENT_ROOT', dirname(__FILE__) == '/' ? '' : dirname(__FILE__));
 }
@@ -23,5 +26,7 @@ if (!defined('PIWIK_ENABLE_ERROR_HANDLER') || PIWIK_ENABLE_ERROR_HANDLER) {
     Piwik\ExceptionHandler::setUp();
 }
 
+FrontController::setUpSafeMode();
+
 $console = new Piwik\Console();
 $console->run();
diff --git a/core/Archive.php b/core/Archive.php
@@ -649,10 +649,11 @@ private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
             foreach ($this->params->getIdSites() as $idSite) {
                 $site = new Site($idSite);
 
-                 if ($period->getLabel() === 'day'
+                if ($period->getLabel() === 'day'
                     && !$this->params->getSegment()->isEmpty()
                     && Common::getRequestVar('skipArchiveSegmentToday', 0, 'int')
-                    && $period->getDateStart()->toString() == Date::factory('now', $site->getTimezone())->toString()) {
+                    && $period->getDateStart()->toString() == Date::factory('now', $site->getTimezone())->toString()
+                ) {
 
                     Log::debug("Skipping archive %s for %s as segment today is disabled", $period->getLabel(), $period->getPrettyString());
                     continue;

diff --git a/core/ArchiveProcessor/PluginsArchiver.php b/core/ArchiveProcessor/PluginsArchiver.php
@@ -160,13 +160,15 @@ public function callAggregateAllPlugins($visits, $visitsConverted, $forceArchivi
                 try {
                     self::$currentPluginBeingArchived = $pluginName;
 
+                    $period = $this->params->getPeriod()->getLabel();
+
                     $timer = new Timer();
                     if ($this->shouldAggregateFromRawData) {
-                        Log::debug("PluginsArchiver::%s: Archiving day reports for plugin '%s'.", __FUNCTION__, $pluginName);
+                        Log::debug("PluginsArchiver::%s: Archiving $period reports for plugin '%s' from raw data.", __FUNCTION__, $pluginName);
 
                         $archiver->callAggregateDayReport();
                     } else {
-                        Log::debug("PluginsArchiver::%s: Archiving period reports for plugin '%s'.", __FUNCTION__, $pluginName);
+                        Log::debug("PluginsArchiver::%s: Archiving $period reports for plugin '%s' using reports for smaller periods.", __FUNCTION__, $pluginName);
 
                         $archiver->callAggregateMultipleReports();
                     }

diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php
@@ -226,9 +226,21 @@ public static function isArchivingDisabledFor(array $idSites, Segment $segment,
         return !$isArchivingEnabled;
     }
 
-    public static function isRequestAuthorizedToArchive()
+    public static function isRequestAuthorizedToArchive(Parameters $params = null)
     {
-        return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
+        $isRequestAuthorizedToArchive = Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
+
+        if (!empty($params)) {
+            /**
+             * @ignore
+             *
+             * @params bool &$isRequestAuthorizedToArchive
+             * @params Parameters $params
+             */
+            Piwik::postEvent('Archiving.isRequestAuthorizedToArchive', [&$isRequestAuthorizedToArchive, $params]);
+        }
+
+        return $isRequestAuthorizedToArchive;
     }
 
     public static function isBrowserTriggerEnabled()
@@ -293,11 +305,11 @@ public static function isSegmentPreProcessed(array $idSites, Segment $segment)
      *
      * @return string[]
      */
-    public static function getSelectableDoneFlagValues($includeInvalidated = true)
+    public static function getSelectableDoneFlagValues($includeInvalidated = true, Parameters $params = null)
     {
         $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
 
-        if (!Rules::isRequestAuthorizedToArchive()
+        if (!Rules::isRequestAuthorizedToArchive($params)
             && $includeInvalidated
         ) {
             //If request is not authorized to archive then fetch also invalidated archives

diff --git a/core/Archiver/Request.php b/core/Archiver/Request.php
@@ -31,7 +31,7 @@ class Request
      */
     public function __construct($url)
     {
-        $this->url = $url;
+        $this->setUrl($url);
     }
 
     public function before($callable)
@@ -68,9 +68,34 @@ public function setUrl($url)
     }
 
     public function changeDate($newDate)
+    {
+        $this->changeParam('date', $newDate);
+    }
+
+    public function makeSureDateIsNotSingleDayRange()
+    {
+        // TODO: revisit in matomo 4
+        // period=range&date=last1/period=range&date=previous1 can cause problems during archiving due to Parameters::isDayArchive()
+        if (preg_match('/[&?]period=range/', $this->url)) {
+            if (preg_match('/[&?]date=last1/', $this->url)) {
+                $this->changeParam('period', 'day');
+                $this->changeParam('date', 'today');
+            } else if (preg_match('/[&?]date=previous1/', $this->url)) {
+                $this->changeParam('period', 'day');
+                $this->changeParam('date', 'yesterday');
+            } else if (preg_match('/[&?]date=([^,]+),([^,&]+)/', $this->url, $matches)
+                && $matches[1] == $matches[2]
+            ) {
+                $this->changeParam('period', 'day');
+                $this->changeParam('date', $matches[1]);
+            }
+        }
+    }
+
+    public function changeParam($name, $newValue)
     {
         $url = $this->getUrl();
-        $url = preg_replace('/([&?])date=[^&]*/', '$1date=' . $newDate, $url);
+        $url = preg_replace('/([&?])' . preg_quote($name) . '=[^&]*/', '$1' . $name . '=' . $newValue, $url);
         $this->setUrl($url);
     }
 }
diff --git a/core/CliMulti/CliPhp.php b/core/CliMulti/CliPhp.php
@@ -86,7 +86,7 @@ private function isValidPhpType($path)
 
     private function getPhpCommandIfValid($path)
     {
-        if (!empty($path) && is_executable($path)) {
+        if (!empty($path) && @is_executable($path)) {
             if (0 === strpos($path, PHP_BINDIR) && $this->isValidPhpType($path)) {
                 return $path;
             }

diff --git a/core/Concurrency/Lock.php b/core/Concurrency/Lock.php
@@ -50,13 +50,6 @@ public function getAllAcquiredLockKeys()
 
     public function execute($id, $callback)
     {
-        if (Common::mb_strlen($id) > self::MAX_KEY_LEN) {
-            // Lock key might be too long for DB column, so we hash it but leave the start of the original as well
-            // to make it more readable
-            $md5Len = 32;
-            $id = Common::mb_substr($id, 0, self::MAX_KEY_LEN - $md5Len - 1) . md5($id);
-        }
-
         $i = 0;
         while (!$this->acquireLock($id)) {
             $i++;
@@ -76,6 +69,13 @@ public function acquireLock($id, $ttlInSeconds = 60)
     {
         $this->lockKey = $this->lockKeyStart . $id;
 
+        if (Common::mb_strlen($this->lockKey) > self::MAX_KEY_LEN) {
+            // Lock key might be too long for DB column, so we hash it but leave the start of the original as well
+            // to make it more readable
+            $md5Len = 32;
+            $this->lockKey = Common::mb_substr($id, 0, self::MAX_KEY_LEN - $md5Len - 1) . md5($id);
+        }
+
         $lockValue = substr(Common::generateUniqId(), 0, 12);
         $locked    = $this->backend->setIfNotExists($this->lockKey, $lockValue, $ttlInSeconds);
 

diff --git a/core/Console.php b/core/Console.php
@@ -77,6 +77,21 @@ public function renderException($e, $output)
     }
 
     public function doRun(InputInterface $input, OutputInterface $output)
+    {
+        try {
+            return $this->doRunImpl($input, $output);
+        } catch (\Exception $ex) {
+            try {
+                FrontController::generateSafeModeOutputFromException($ex);
+            } catch (\Exception $ex) {
+                // ignore, we re-throw the original exception, not a wrapped one
+            }
+
+            throw $ex;
+        }
+    }
+
+    private function doRunImpl(InputInterface $input, OutputInterface $output)
     {
         if ($input->hasParameterOption('--xhprof')) {
             Profiler::setupProfilerXHProf(true, true);

diff --git a/core/CronArchive.php b/core/CronArchive.php
@@ -1104,7 +1104,16 @@ private function archiveReportsFor($idSite, $period, $date, $archiveSegments, Ti
                     return Request::ABORT;
                 }
 
+                $urlBefore = $request->getUrl();
                 $request->changeDate($newDate);
+                $request->makeSureDateIsNotSingleDayRange();
+
+                // check again if we are already archiving the URL since we just changed it
+                if ($request->getUrl() !== $urlBefore
+                    && $self->isAlreadyArchivingSegment($request->getUrl(), $idSite, $period, $segment)
+                ) {
+                    return Request::ABORT;
+                }
 
                 $this->logArchiveWebsite($idSite, $period, $newDate);
             });
@@ -1972,9 +1981,17 @@ private function getUrlsWithSegment($idSite, $period, $date)
                     return Request::ABORT;
                 }
 
-                $url = $request->getUrl();
-                $url = preg_replace('/([&?])date=[^&]*/', '$1date=' . $newDate, $url);
+                $urlBefore = $request->getUrl();
+                $url = preg_replace('/([&?])date=[^&]*/', '$1date=' . $newDate, $urlBefore);
                 $request->setUrl($url);
+                $request->makeSureDateIsNotSingleDayRange();
+
+                // check again if we are already archiving the URL since we just changed it
+                if ($request->getUrl() !== $urlBefore
+                    && $self->isAlreadyArchivingSegment($request->getUrl(), $idSite, $period, $segment)
+                ) {
+                    return Request::ABORT;
+                }
 
                 $processedSegmentCount++;
                 $logger->info(sprintf(

diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php
@@ -71,7 +71,7 @@ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params
         $plugins = array("VisitsSummary", $requestedPlugin);
 
         $doneFlags      = Rules::getDoneFlags($plugins, $segment);
-        $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated);
+        $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated, $params);
 
         $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues);
 

diff --git a/core/FrontController.php b/core/FrontController.php
@@ -106,7 +106,7 @@ private static function generateSafeModeOutputFromError($lastError)
      * @param Exception $e
      * @return string
      */
-    private static function generateSafeModeOutputFromException($e)
+    public static function generateSafeModeOutputFromException($e)
     {
         StaticContainer::get(LoggerInterface::class)->error('Uncaught exception: {exception}', [
             'exception' => $e,

diff --git a/core/Plugin/Report.php b/core/Plugin/Report.php
@@ -496,8 +496,8 @@ protected function getMetricsDocumentation()
             } elseif ($metric instanceof Metric) {
                 $name = $metric->getName();
                 $metricDocs = $metric->getDocumentation();
-                if (empty($metricDocs)) {
-                    $metricDocs = @$translations[$name];
+                if (empty($metricDocs) && !empty($translations[$name])) {
+                    $metricDocs = $translations[$name];
                 }
 
                 if (!empty($metricDocs)) {
@@ -513,8 +513,8 @@ protected function getMetricsDocumentation()
             } elseif ($processedMetric instanceof Metric) {
                 $name = $processedMetric->getName();
                 $metricDocs = $processedMetric->getDocumentation();
-                if (empty($metricDocs)) {
-                    $metricDocs = @$translations[$name];
+                if (empty($metricDocs) && !empty($translations[$name])) {
+                    $metricDocs = $translations[$name];
                 }
 
                 if (!empty($metricDocs)) {

diff --git a/core/Session.php b/core/Session.php
@@ -171,6 +171,17 @@ public static function isSessionStarted()
         return self::$sessionStarted;
     }
 
+    public static function getSameSiteCookieValue()
+    {
+        $config = Config::getInstance();
+        $general = $config->General;
+        if (!empty($general['enable_framed_pages']) && ProxyHttp::isHttps()) {
+            return 'None';
+        }
+
+        return 'Lax';
+    }
+
     /**
      * Write cookie header.  Similar to the native setcookie() function but also supports
      * the SameSite cookie property.

diff --git a/core/Session/SessionAuth.php b/core/Session/SessionAuth.php
@@ -195,7 +195,7 @@ private function updateSessionExpireTime(SessionFingerprint $sessionFingerprint)
             $sessionParams['domain'],
             $sessionParams['secure'],
             $sessionParams['httponly'],
-            'lax'
+            Session::getSameSiteCookieValue()
         );
 
         // ...and we also update the expiration time stored server side so we can prevent expired sessions from being reused

diff --git a/core/Tracker/Model.php b/core/Tracker/Model.php
@@ -346,7 +346,7 @@ public function updateAction($idLinkVa, $valuesToUpdate)
 
         list($updateParts, $sqlBind) = $this->fieldsToQuery($valuesToUpdate);
 
-        $parts = implode($updateParts, ', ');
+        $parts = implode(', ', $updateParts);
         $table = Common::prefixTable('log_link_visit_action');
 
         $sqlQuery = "UPDATE $table SET $parts WHERE idlink_va = ?";

diff --git a/core/Tracker/Settings.php b/core/Tracker/Settings.php
@@ -40,7 +40,7 @@ public function getConfigId(Request $request, $ipAddress)
         $deviceDetector = StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($userAgent);
         $aBrowserInfo   = $deviceDetector->getClient();
 
-        if ($aBrowserInfo['type'] != 'browser') {
+        if (empty($aBrowserInfo['type']) || 'browser' !== $aBrowserInfo['type']) {
             // for now only track browsers
             unset($aBrowserInfo);
         }