Make matomo.js and matomo.php the default Tracking API endpoints used (…

…matomo-org#13596)

* Make matomo.js and matomo.php the default Tracking API endpoints used

* few fixes

* fix few tests

* update matomo php tracker in composer

* make sure to record install version

* do not overwrite ever install version

* trying to fix test

* more tweaks and fix tests

* prefer matomo endpoint in the ui, fix some tests

* file was still needed

* apply review feedback

* fix ui tests

.gitignore

@@ -61,9 +61,12 @@ php_errors.log
 /tests/resources/overlay-test-site-real/
 /tests/PHPUnit/proxy/libs
 /tests/PHPUnit/proxy/piwik.js
+/tests/PHPUnit/proxy/matomo.js
 /tests/PHPUnit/proxy/plugins
 /tests/PHPUnit/proxy/tests
+/tests/PHPUnit/proxy/misc
 /tests/resources/piwik.test.js
+/tests/resources/matomo.test.js
 /config/*.config.ini.php
 /Vagrantfile
 /misc/vagrant

CHANGELOG.md

@@ -11,6 +11,12 @@ The Product Changelog at **[matomo.org/changelog](https://matomo.org/changelog)*
 * It is now possible to queue a request on the JavaScript tracker using the method `queueRequest(requestUrl)`. This can be useful to group multiple tracking requests into one bulk request to reduce the number of tracking requests that are sent to your server making the tracking more efficient.
 * The output type "save on disk" in the API method `ScheduledReport.generateReport` has been replaced by the download output type.
 
+### Internal change
+* New Matomo installation will now use by default "matomo.js" and "matomo.php" as tracking endpoints. From Matomo 4.0 all installations will use "matomo.js" and "matomo.php" by default. We recommend you ensure those files can be accessed through the web and are not blocked.
+
+### Deprecations
+* The method `Piwik\SettingsPiwik::isPiwikInstalled()` has been deprecated and renamed to `isMatomoInstalled()`. It is still supported to use the method, but the method will be removed in Piwik 4.0.0
+
 ## Matomo 3.6.1
 
 ### New APIs

LEGALNOTICE

@@ -57,8 +57,9 @@ SEPARATELY LICENSED COMPONENTS AND LIBRARIES
 	Link:  https://github.com/matomo-org/piwik/blob/master/libs/jquery/truncate/
 	License:  New BSD
 
-	Name:  piwik.js - JavaScript tracker
-	Link:  https://github.com/matomo-org/piwik/blob/master/js/piwik.js
+	Name:  matomo.js & piwik.js - JavaScript tracker
+	Link:  https://github.com/matomo-org/matomo/blob/master/js/piwik.js
+	Link:  https://github.com/matomo-org/matomo/blob/master/js/matomo.js
 	License:  New BSD
 
 	Name:  PiwikTracker - server-side tracker (PHP)

composer.json

@@ -44,7 +44,7 @@
         "matomo/referrer-spam-blacklist": "~1.0",
         "matomo/searchengine-and-social-list": "~1.0",
         "tecnickcom/tcpdf": "~6.0",
-        "piwik/piwik-php-tracker": "^1.0",
+        "piwik/piwik-php-tracker": "^1.0.0",
         "composer/semver": "~1.3.0",
         "szymach/c-pchart": "^2.0",
         "geoip2/geoip2": "^2.8",

config/global.ini.php

@@ -138,7 +138,7 @@
 ; NOTE: you must also set [log] log_writers[] = "screen" to enable the profiler to print on screen
 enable_sql_profiler = 0
 
-; If set to 1, all requests to piwik.php will be forced to be 'new visitors'
+; If set to 1, all requests to matomo.php will be forced to be 'new visitors'
 tracker_always_new_visitor = 0
 
 ; if set to 1, all SQL queries will be logged using the DEBUG log level

core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php

@@ -74,6 +74,7 @@ protected function getPriorityOrder()
             'libs/',
             'js/',
             'piwik.js',
+            'matomo.js',
             'plugins/CoreHome/javascripts/require.js',
             'plugins/Morpheus/javascripts/piwikHelper.js',
             'plugins/Morpheus/javascripts/',

core/Db/Schema.php

@@ -141,6 +141,22 @@ public function createAnonymousUser()
         $this->getSchema()->createAnonymousUser();
     }
 
+    /**
+     * Records the Matomo version a user used when installing this Matomo for the first time
+     */
+    public function recordInstallVersion()
+    {
+        $this->getSchema()->recordInstallVersion();
+    }
+
+    /**
+     * Returns which Matomo version was used to install this Matomo for the first time.
+     */
+    public function getInstallVersion()
+    {
+        return $this->getSchema()->getInstallVersion();
+    }
+
     /**
      * Truncate all tables
      */

core/Db/Schema/Mysql.php

@@ -14,12 +14,17 @@
 use Piwik\Db\SchemaInterface;
 use Piwik\Db;
 use Piwik\DbHelper;
+use Piwik\Option;
+use Piwik\Plugins\Installation\Installation;
+use Piwik\Version;
 
 /**
  * MySQL schema
  */
 class Mysql implements SchemaInterface
 {
+    const OPTION_NAME_MATOMO_INSTALL_VERSION = 'install_version';
+
     private $tablesInstalled = null;
 
     /**
@@ -472,6 +477,28 @@ public function createAnonymousUser()
                     VALUES ( 'anonymous', '', 'anonymous', '[email protected]', 'anonymous', 0, '$now', '$now' );");
     }
 
+    /**
+     * Records the Matomo version a user used when installing this Matomo for the first time
+     */
+    public function recordInstallVersion()
+    {
+        if (!self::getInstallVersion()) {
+            Option::set(self::OPTION_NAME_MATOMO_INSTALL_VERSION, Version::VERSION);
+        }
+    }
+
+    /**
+     * Returns which Matomo version was used to install this Matomo for the first time.
+     */
+    public function getInstallVersion()
+    {
+        Option::clearCachedOption(self::OPTION_NAME_MATOMO_INSTALL_VERSION);
+        $version = Option::get(self::OPTION_NAME_MATOMO_INSTALL_VERSION);
+        if (!empty($version)) {
+            return $version;
+        }
+    }
+
     /**
      * Truncate all tables
      */

core/Db/SchemaInterface.php

@@ -59,6 +59,16 @@ public function createTables();
      */
     public function createAnonymousUser();
 
+    /**
+     * Records the Matomo version a user used when installing this Matomo for the first time
+     */
+    public function recordInstallVersion();
+
+    /**
+     * Returns which Matomo version was used to install this Matomo for the first time.
+     */
+    public function getInstallVersion();
+
     /**
      * Truncate all tables
      */

core/DbHelper.php

@@ -93,6 +93,31 @@ public static function createAnonymousUser()
         Schema::getInstance()->createAnonymousUser();
     }
 
+    /**
+     * Records the Matomo version a user used when installing this Matomo for the first time
+     */
+    public static function recordInstallVersion()
+    {
+        Schema::getInstance()->recordInstallVersion();
+    }
+
+    /**
+     * Returns which Matomo version was used to install this Matomo for the first time.
+     */
+    public static function getInstallVersion()
+    {
+        return Schema::getInstance()->getInstallVersion();
+    }
+
+    public static function wasMatomoInstalledBeforeVersion($version)
+    {
+        $installVersion = self::getInstallVersion();
+        if (empty($installVersion)) {
+            return true; // we assume yes it was installed
+        }
+        return true === version_compare($version, $installVersion, '>');
+    }
+
     /**
      * Create all tables
      */

core/FileIntegrity.php

@@ -366,7 +366,7 @@ protected static function getMessagesFilesMismatch($messages)
 
     protected static function isModifiedPathValid($path)
     {
-        if ($path === 'piwik.js') {
+        if ($path === 'piwik.js' || $path === 'matomo.js') {
             // we could have used a postEvent hook to enrich "\Piwik\Manifest::$files;" which would also benefit plugins
             // that want to check for file integrity but we do not want to risk to break anything right now. It is not
             // as trivial because piwik.js might be already updated, or updated on the next request. We cannot define

core/SettingsPiwik.php

@@ -213,6 +213,15 @@ public static function getPiwikUrl()
         return $url;
     }
 
+    /**
+     * @see SettingsPiwik::isPiwikInstalled
+     * @return bool
+     */
+    public static function isMatomoInstalled()
+    {
+        return self::isPiwikInstalled();
+    }
+
     /**
      * Return true if Piwik is installed (installation is done).
      * @return bool

core/Tracker/Db/Pdo/Mysql.php

@@ -112,7 +112,7 @@ public function connect()
         $this->connection = @new PDO($this->dsn, $this->username, $this->password, $this->mysqlOptions);
 
         // we may want to setAttribute(PDO::ATTR_TIMEOUT ) to a few seconds (default is 60) in case the DB is locked
-        // the piwik.php would stay waiting for the database... bad!
+        // the matomo.php would stay waiting for the database... bad!
         // we delete the password from this object "just in case" it could be printed
         $this->password = '';
 

core/Tracker/Db/Pdo/Pgsql.php

@@ -43,7 +43,7 @@ public function connect()
         $this->connection = new PDO($this->dsn, $this->username, $this->password, $config = array());
         $this->connection->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
         // we may want to setAttribute(PDO::ATTR_TIMEOUT ) to a few seconds (default is 60) in case the DB is locked
-        // the piwik.php would stay waiting for the database... bad!
+        // the matomo.php would stay waiting for the database... bad!
         // we delete the password from this object "just in case" it could be printed
         $this->password = '';
 

core/Tracker/TrackerCodeGenerator.php

@@ -9,6 +9,8 @@
 namespace Piwik\Tracker;
 
 use Piwik\Common;
+use Piwik\DbHelper;
+use Piwik\Option;
 use Piwik\Piwik;
 use Piwik\Plugins\CustomVariables\CustomVariables;
 use Piwik\Plugins\SitesManager\API as APISitesManager;
@@ -20,6 +22,17 @@
  */
 class TrackerCodeGenerator
 {
+    /**
+     * whether matomo.js|php should be forced over piwik.js|php
+     * @var bool
+     */
+    private $shouldForceMatomoEndpoint = false;
+
+    public function forceMatomoEndpoint()
+    {
+        $this->shouldForceMatomoEndpoint = true;
+    }
+
     /**
      * @param int $idSite
      * @param string $piwikUrl http://path/to/piwik/site/
@@ -134,7 +147,9 @@ public function generate(
             'optionsBeforeTrackerUrl' => $optionsBeforeTrackerUrl,
             'protocol'                => '//',
             'loadAsync'               => true,
-            'trackNoScript'           => $trackNoScript
+            'trackNoScript'           => $trackNoScript,
+            'matomoJsFilename'        => $this->getJsTrackerEndpoint(),
+            'matomoPhpFilename'       => $this->getPhpTrackerEndpoint(),
         );
 
         if (SettingsPiwik::isHttpsForced()) {
@@ -191,6 +206,34 @@ public function generate(
         return $jsCode;
     }
 
+    public function getJsTrackerEndpoint()
+    {
+        $name = 'matomo.js';
+        if ($this->shouldPreferPiwikEndpoint()) {
+            $name = 'piwik.js';
+        }
+        return $name;
+    }
+
+    public function getPhpTrackerEndpoint()
+    {
+        $name = 'matomo.php';
+        if ($this->shouldPreferPiwikEndpoint()) {
+            $name = 'piwik.php';
+        }
+        return $name;
+    }
+
+    public function shouldPreferPiwikEndpoint()
+    {
+        if ($this->shouldForceMatomoEndpoint) {
+            return false;
+        }
+
+        // only since 3.7.0 we use the default matomo.js|php... for all other installs we need to keep BC
+        return DbHelper::wasMatomoInstalledBeforeVersion('3.7.0-b1');
+    }
+
     private function getJavascriptTagOptions($idSite, $mergeSubdomains, $mergeAliasUrls)
     {
         try {

core/Tracker/Visit.php

@@ -31,7 +31,7 @@
  * views, time spent, etc.
  *
  * Whether a visit is NEW or KNOWN we also save the action in the DB.
- * One request to the piwik.php script is associated to one action.
+ * One request to the matomo.php script is associated to one action.
  *
  */
 class Visit implements VisitInterface

core/Updater.php

@@ -10,12 +10,12 @@
 
 use Piwik\Columns\Updater as ColumnUpdater;
 use Piwik\Container\StaticContainer;
+use Piwik\Plugins\Installation\ServerFilesGenerator;
 use Piwik\Updater\Migration;
 use Piwik\Updater\Migration\Db\Sql;
 use Piwik\Exception\MissingFilePermissionException;
 use Piwik\Updater\UpdateObserver;
 use Zend_Db_Exception;
-use Piwik\Plugins\Installation\ServerFilesGenerator;
 
 /**
  * Load and execute all relevant, incremental update scripts for Piwik core and plugins, and bump the component version numbers for completed updates.
@@ -325,7 +325,7 @@ public function update($componentName)
         $this->markComponentSuccessfullyUpdated($componentName, $updatedVersion);
 
         $this->executeListenerHook('onComponentUpdateFinished', array($componentName, $updatedVersion, $warningMessages));
-
+        ServerFilesGenerator::createHtAccessFiles();
         return $warningMessages;
     }
 

js/README.md

@@ -16,11 +16,11 @@ The js/ folder contains:
     * it supports deflate/gzip compression if your web server (e.g., Apache
       without mod_deflate or mod_gzip), shrinking the data transfer to 8K
 
-* js/index.php (or implicitly as "js/") can also act as a proxy to piwik.php
+* js/index.php (or implicitly as "js/") can also act as a proxy to matomo.php
 
 * If you are concerned about the impact of browser-based privacy filters which
   attempt to block tracking, you can change your tracking code to use "js/"
-  instead of "piwik.js" and "piwik.php", respectively.
+  instead of "piwik.js" and "matomo.php", respectively.
 
   Note that in order for [Page Overlay](https://matomo.org/docs/page-overlay/) to work, the Piwik tracker method `setAPIUrl()` needs to be called with its parameter pointing to the root directory of Piwik. E.g.:
 
@@ -44,7 +44,7 @@ The js/ folder contains:
 
   ```bash
   $ cd /path/to/piwik/js/
-  $ sed '/<DEBUG>/,/<\/DEBUG>/d' < piwik.js | sed 's/eval/replacedEvilString/' | java -jar yuicompressor-2.4.8.jar --type js --line-break 1000 | sed 's/replacedEvilString/eval/' | sed 's/^[/][*]/\/*!/' > piwik.min.js && cp piwik.min.js ../piwik.js
+  $ sed '/<DEBUG>/,/<\/DEBUG>/d' < piwik.js | sed 's/eval/replacedEvilString/' | java -jar yuicompressor-2.4.8.jar --type js --line-break 1000 | sed 's/replacedEvilString/eval/' | sed 's/^[/][*]/\/*!/' > piwik.min.js && cp piwik.min.js ../piwik.js && cp piwik.min.js ../matomo.js
   ```
 
   This will generate the minify /path/to/piwik/js/piwik.min.js and copy it to