Merge pull request FreshRSS#1921 from FreshRSS/dev

FreshRSS 1.11.1
Leepic · Jun 16, 2018 · 3306a16 · 3306a16
2 parents c012200 + 8fcacc8
commit 3306a16
Show file tree

Hide file tree

Showing 24 changed files with 386 additions and 329 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,20 @@
 # FreshRSS changelog
 
+## 2018-06-16 FreshRSS 1.11.1
+
+* Features
+	* Better support of `media:` tags such as thumbnails and descriptions (e.g. for YouTube) [#944](https://github.com/FreshRSS/FreshRSS/issues/944)
+* Extensions
+	* New extension mechanism allowing changing HTTP headers and other SimplePie parameters [#1924](https://github.com/FreshRSS/FreshRSS/pull/1924)
+	* Built-in extension to fix Tumblr feeds from European Union due to GDPR [#1894](https://github.com/FreshRSS/FreshRSS/issues/1894)
+* Bug fixing
+	* Fix bug in case of bad i18n in extensions [#1797](https://github.com/FreshRSS/FreshRSS/issues/1797)
+	* Fix extension callback for updated articles and PubSubHubbub [#1926](https://github.com/FreshRSS/FreshRSS/issues/1926)
+	* Fix regression in fetching full articles content [#1917](https://github.com/FreshRSS/FreshRSS/issues/1917)
+	* Fix several bugs in the new Fever API [#1930](https://github.com/FreshRSS/FreshRSS/issues/1930)
+	* Updated sharing to Mastodon [#1904](https://github.com/FreshRSS/FreshRSS/issues/1904)
+
+
 ## 2018-06-03 FreshRSS 1.11.0
 
 * API

diff --git a/app/Controllers/extensionController.php b/app/Controllers/extensionController.php
@@ -140,7 +140,7 @@ public function enableAction() {
 
 			if ($res === true) {
 				$ext_list = $conf->extensions_enabled;
-				array_push_unique($ext_list, $ext_name);
+				$ext_list[$ext_name] = true;
 				$conf->extensions_enabled = $ext_list;
 				$conf->save();
 
@@ -196,7 +196,11 @@ public function disableAction() {
 
 			if ($res === true) {
 				$ext_list = $conf->extensions_enabled;
-				array_remove($ext_list, $ext_name);
+				$legacyKey = array_search($ext_name, $ext_list, true);
+				if ($legacyKey !== false) {	//Legacy format FreshRSS < 1.11.1
+					unset($ext_list[$legacyKey]);
+				}
+				$ext_list[$ext_name] = false;
 				$conf->extensions_enabled = $ext_list;
 				$conf->save();
 

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
@@ -351,13 +351,20 @@ public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush
 							//This entry already exists and is unchanged. TODO: Remove the test with the zero'ed hash in FreshRSS v1.3
 							$oldGuids[] = $entry->guid();
 						} else {	//This entry already exists but has been updated
-							//Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() .
+							//Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->url() .
 								//', old hash ' . $existingHash . ', new hash ' . $entry->hash());
 							$mark_updated_article_unread = $feed->attributes('mark_updated_article_unread') !== null ? (
 									$feed->attributes('mark_updated_article_unread')
 								) : FreshRSS_Context::$user_conf->mark_updated_article_unread;
 							$needFeedCacheRefresh = $mark_updated_article_unread;
 							$entry->_isRead(FreshRSS_Context::$user_conf->mark_updated_article_unread ? false : null);	//Change is_read according to policy.
+
+							$entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry);
+							if ($entry === null) {
+								// An extension has returned a null value, there is nothing to insert.
+								continue;
+							}
+
 							if (!$entryDAO->inTransaction()) {
 								$entryDAO->beginTransaction();
 							}

diff --git a/app/Models/Entry.php b/app/Models/Entry.php
@@ -17,10 +17,11 @@ class FreshRSS_Entry extends Minz_Model {
 	private $hash = null;
 	private $is_read;	//Nullable boolean
 	private $is_favorite;
+	private $feedId;
 	private $feed;
 	private $tags;
 
-	public function __construct($feed = '', $guid = '', $title = '', $author = '', $content = '',
+	public function __construct($feedId = '', $guid = '', $title = '', $author = '', $content = '',
 	                            $link = '', $pubdate = 0, $is_read = false, $is_favorite = false, $tags = '') {
 		$this->_title($title);
 		$this->_author($author);
@@ -29,7 +30,7 @@ public function __construct($feed = '', $guid = '', $title = '', $author = '', $
 		$this->_date($pubdate);
 		$this->_isRead($is_read);
 		$this->_isFavorite($is_favorite);
-		$this->_feed($feed);
+		$this->_feedId($feedId);
 		$this->_tags(preg_split('/[\s#]/', $tags));
 		$this->_guid($guid);
 	}
@@ -75,10 +76,13 @@ public function isFavorite() {
 	}
 	public function feed($object = false) {
 		if ($object) {
-			$feedDAO = FreshRSS_Factory::createFeedDao();
-			return $feedDAO->searchById($this->feed);
-		} else {
+			if ($this->feed == null) {
+				$feedDAO = FreshRSS_Factory::createFeedDao();
+				$this->feed = $feedDAO->searchById($this->feedId);
+			}
 			return $this->feed;
+		} else {
+			return $this->feedId;
 		}
 	}
 	public function tags($inString = false) {
@@ -145,7 +149,14 @@ public function _isFavorite($value) {
 		$this->is_favorite = $value;
 	}
 	public function _feed($value) {
-		$this->feed = $value;
+		if ($value != null) {
+			$this->feed = $value;
+			$this->feedId = $this->feed->id();
+		}
+	}
+	private function _feedId($value) {
+		$this->feed = null;
+		$this->feedId = $value;
 	}
 	public function _tags($value) {
 		$this->hash = null;
@@ -179,23 +190,89 @@ public function isDay($day, $today) {
 		}
 	}
 
-	public function loadCompleteContent($pathEntries) {
+	private static function get_content_by_parsing($url, $path, $attributes = array()) {
+		require_once(LIB_PATH . '/lib_phpQuery.php');
+		$system_conf = Minz_Configuration::get('system');
+		$limits = $system_conf->limits;
+		$feed_timeout = empty($attributes['timeout']) ? 0 : intval($attributes['timeout']);
+
+		if ($system_conf->simplepie_syslog_enabled) {
+			syslog(LOG_INFO, 'FreshRSS GET ' . SimplePie_Misc::url_remove_credentials($url));
+		}
+
+		$ch = curl_init();
+		curl_setopt_array($ch, array(
+			CURLOPT_URL => $url,
+			CURLOPT_REFERER => SimplePie_Misc::url_remove_credentials($url),
+			CURLOPT_HTTPHEADER => array('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
+			CURLOPT_USERAGENT => FRESHRSS_USERAGENT,
+			CURLOPT_CONNECTTIMEOUT => $feed_timeout > 0 ? $feed_timeout : $limits['timeout'],
+			CURLOPT_TIMEOUT => $feed_timeout > 0 ? $feed_timeout : $limits['timeout'],
+			//CURLOPT_FAILONERROR => true;
+			CURLOPT_MAXREDIRS => 4,
+			CURLOPT_RETURNTRANSFER => true,
+		));
+		if (version_compare(PHP_VERSION, '5.6.0') >= 0 || ini_get('open_basedir') == '') {
+			curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);	//Keep option separated for open_basedir PHP bug 65646
+		}
+		if (defined('CURLOPT_ENCODING')) {
+			curl_setopt($ch, CURLOPT_ENCODING, '');	//Enable all encodings
+		}
+		curl_setopt_array($ch, $system_conf->curl_options);
+		if (isset($attributes['ssl_verify'])) {
+			curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, $attributes['ssl_verify'] ? 2 : 0);
+			curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, $attributes['ssl_verify'] ? true : false);
+		}
+		$html = curl_exec($ch);
+		$c_status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+		$c_error = curl_error($ch);
+		curl_close($ch);
+
+		if ($c_status != 200 || $c_error != '') {
+			Minz_Log::warning('Error fetching content: HTTP code ' . $c_status . ': ' . $c_error . ' ' . $url);
+		}
+
+		if ($html) {
+			$doc = phpQuery::newDocument($html);
+			$content = $doc->find($path);
+
+			foreach (pq('img[data-src]') as $img) {
+				$imgP = pq($img);
+				$dataSrc = $imgP->attr('data-src');
+				if (strlen($dataSrc) > 4) {
+					$imgP->attr('src', $dataSrc);
+					$imgP->removeAttr('data-src');
+				}
+			}
+
+			return trim(sanitizeHTML($content->__toString(), $url));
+		} else {
+			throw new Exception();
+		}
+	}
+
+	public function loadCompleteContent() {
 		// Gestion du contenu
 		// On cherche à récupérer les articles en entier... même si le flux ne le propose pas
-		if ($pathEntries) {
+		$feed = $this->feed(true);
+		if ($feed != null && trim($feed->pathEntries()) != '') {
 			$entryDAO = FreshRSS_Factory::createEntryDao();
-			$entry = $entryDAO->searchByGuid($this->feed, $this->guid);
+			$entry = $entryDAO->searchByGuid($this->feedId, $this->guid);
 
 			if ($entry) {
 				// l'article existe déjà en BDD, en se contente de recharger ce contenu
 				$this->content = $entry->content();
 			} else {
 				try {
 					// l'article n'est pas en BDD, on va le chercher sur le site
-					$this->content = get_content_by_parsing(
-						htmlspecialchars_decode($this->link(), ENT_QUOTES), $pathEntries,
-						$this->feed->attributes()
+					$fullContent = self::get_content_by_parsing(
+						htmlspecialchars_decode($this->link(), ENT_QUOTES),
+						$feed->pathEntries(),
+						$feed->attributes()
 					);
+					if ($fullContent != '') {
+						$this->content = $fullContent;
+					}
 				} catch (Exception $e) {
 					// rien à faire, on garde l'ancien contenu(requête a échoué)
 					Minz_Log::warning($e->getMessage());

diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
@@ -904,8 +904,8 @@ public function updateLastSeen($id_feed, $guids, $mtime = 0) {
 	}
 
 	public function countUnreadRead() {
-		$sql = 'SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id WHERE priority > 0'
-			. ' UNION SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id WHERE priority > 0 AND is_read=0';
+		$sql = 'SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id WHERE f.priority > 0'
+			. ' UNION SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id WHERE f.priority > 0 AND e.is_read=0';
 		$stm = $this->bd->prepare($sql);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_COLUMN, 0);
@@ -914,19 +914,24 @@ public function countUnreadRead() {
 		return array('all' => $all, 'unread' => $unread, 'read' => $all - $unread);
 	}
 	public function count($minPriority = null) {
-		$sql = 'SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id';
+		$sql = 'SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e';
 		if ($minPriority !== null) {
-			$sql = ' WHERE priority > ' . intval($minPriority);
+			$sql .= ' INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id';
+			$sql .= ' WHERE f.priority > ' . intval($minPriority);
 		}
 		$stm = $this->bd->prepare($sql);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_COLUMN, 0);
 		return $res[0];
 	}
 	public function countNotRead($minPriority = null) {
-		$sql = 'SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id WHERE is_read=0';
+		$sql = 'SELECT COUNT(e.id) AS count FROM `' . $this->prefix . 'entry` e';
 		if ($minPriority !== null) {
-			$sql = ' AND priority > ' . intval($minPriority);
+			$sql .= ' INNER JOIN `' . $this->prefix . 'feed` f ON e.id_feed=f.id';
+		}
+		$sql .= ' WHERE e.is_read=0';
+		if ($minPriority !== null) {
+			$sql .= ' AND f.priority > ' . intval($minPriority);
 		}
 		$stm = $this->bd->prepare($sql);
 		$stm->execute();

diff --git a/app/Models/Feed.php b/app/Models/Feed.php
@@ -286,6 +286,7 @@ public function load($loadDetails = false, $noCache = false) {
 				if (!$loadDetails) {	//Only activates auto-discovery when adding a new feed
 					$feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE);
 				}
+				Minz_ExtensionManager::callHook('simplepie_before_init', $feed, $this);
 				$mtime = $feed->init();
 
 				if ((!$mtime) || $feed->error()) {
@@ -355,24 +356,52 @@ public function loadEntries($feed) {
 
 			$content = html_only_entity_decode($item->get_content());
 
-			$elinks = array();
-			foreach ($item->get_enclosures() as $enclosure) {
-				$elink = $enclosure->get_link();
-				if ($elink != '' && empty($elinks[$elink])) {
-					$elinks[$elink] = '1';
-					$mime = strtolower($enclosure->get_type());
-					if (strpos($mime, 'image/') === 0) {
-						$content .= '<p class="enclosure"><img src="' . $elink . '" alt="" /></p>';
-					} elseif (strpos($mime, 'audio/') === 0) {
-						$content .= '<p class="enclosure"><audio preload="none" src="' . $elink
-							. '" controls="controls"></audio> <a download="" href="' . $elink . '">💾</a></p>';
-					} elseif (strpos($mime, 'video/') === 0) {
-						$content .= '<p class="enclosure"><video preload="none" src="' . $elink
-							. '" controls="controls"></video> <a download="" href="' . $elink . '">💾</a></p>';
-					} elseif (strpos($mime, 'application/') === 0 || strpos($mime, 'text/') === 0) {
-						$content .= '<p class="enclosure"><a download="" href="' . $elink . '">💾</a></p>';
-					} else {
-						unset($elinks[$elink]);
+			if ($item->get_enclosures() != null) {
+				$elinks = array();
+				foreach ($item->get_enclosures() as $enclosure) {
+					$elink = $enclosure->get_link();
+					if ($elink != '' && empty($elinks[$elink])) {
+						$content .= '<div class="enclosure">';
+
+						if ($enclosure->get_title() != '') {
+							$content .= '<p class="enclosure-title">' . $enclosure->get_title() . '</p>';
+						}
+
+						$enclosureContent = '';
+						$elinks[$elink] = true;
+						$mime = strtolower($enclosure->get_type());
+						$medium = strtolower($enclosure->get_medium());
+						if ($medium === 'image' || strpos($mime, 'image/') === 0) {
+							$enclosureContent .= '<p class="enclosure-content"><img src="' . $elink . '" alt="" /></p>';
+						} elseif ($medium === 'audio' || strpos($mime, 'audio/') === 0) {
+							$enclosureContent .= '<p class="enclosure-content"><audio preload="none" src="' . $elink
+								. '" controls="controls"></audio> <a download="" href="' . $elink . '">💾</a></p>';
+						} elseif ($medium === 'video' || strpos($mime, 'video/') === 0) {
+							$enclosureContent .= '<p class="enclosure-content"><video preload="none" src="' . $elink
+								. '" controls="controls"></video> <a download="" href="' . $elink . '">💾</a></p>';
+						} elseif ($medium != '' || strpos($mime, 'application/') === 0 || strpos($mime, 'text/') === 0) {
+							$enclosureContent .= '<p class="enclosure-content"><a download="" href="' . $elink . '">💾</a></p>';
+						} else {
+							unset($elinks[$elink]);
+						}
+
+						$thumbnailContent = '';
+						if ($enclosure->get_thumbnails() != null) {
+							foreach ($enclosure->get_thumbnails() as $thumbnail) {
+								if (empty($elinks[$thumbnail])) {
+									$elinks[$thumbnail] = true;
+									$thumbnailContent .= '<p><img class="enclosure-thumbnail" src="' . $thumbnail . '" alt="" /></p>';
+								}
+							}
+						}
+
+						$content .= $thumbnailContent;
+						$content .= $enclosureContent;
+
+						if ($enclosure->get_description() != '') {
+							$content .= '<pre class="enclosure-description">' . $enclosure->get_description() . '</pre>';
+						}
+						$content .= "</div>\n";
 					}
 				}
 			}
@@ -391,8 +420,11 @@ public function loadEntries($feed) {
 				$date ? $date : time()
 			);
 			$entry->_tags($tags);
-			// permet de récupérer le contenu des flux tronqués
-			$entry->loadCompleteContent($this->pathEntries());
+			$entry->_feed($this);
+			if ($this->pathEntries != '') {
+				// Optionally load full content for truncated feeds
+				$entry->loadCompleteContent();
+			}
 
 			$entries[] = $entry;
 			unset($item);

diff --git a/app/shares.php b/app/shares.php
@@ -120,11 +120,10 @@
 		'method' => 'GET',
 	),
 	'mastodon' => array(
-		'url' => '~URL~/api/v1/statuses',
-		'transform' => array(),
+		'url' => '~URL~/share?title=~TITLE~&url=~LINK~',
+		'transform' => array('rawurlencode'),
 		'form' => 'advanced',
-		'method' => 'POST',
-		'field' => 'status',
+		'method' => 'GET',
 	),
 	'pocket' => array(
 		'url' => 'https://getpocket.com/save?url=~LINK~&amp;title=~TITLE~',

diff --git a/config.default.php b/config.default.php
@@ -145,7 +145,9 @@
 	),
 
 	# List of enabled FreshRSS extensions.
-	'extensions_enabled' => array(),
+	'extensions_enabled' => array(
+		'Tumblr-GDPR' => true,
+	),
 
 	# Disable self-update,
 	'disable_update' => false,

diff --git a/constants.php b/constants.php
@@ -2,7 +2,7 @@
 //NB: Do not edit; use ./constants.local.php instead.
 
 //<Not customisable>
-define('FRESHRSS_VERSION', '1.11.0');
+define('FRESHRSS_VERSION', '1.11.1');
 define('FRESHRSS_WEBSITE', 'https://freshrss.org');
 define('FRESHRSS_WIKI', 'https://freshrss.github.io/FreshRSS/');
 

diff --git a/docs/en/developers/03_Backend/05_Extensions.md b/docs/en/developers/03_Backend/05_Extensions.md
@@ -366,6 +366,7 @@ The following events are available:
 - `entry_before_insert` (`function($entry) -> Entry | null`) : will be executed when a feed is refreshed and new entries will be imported into the database. The new entry (instance of FreshRSS_Entry) will be passed as parameter. 
 - `feed_before_insert` (`function($feed) -> Feed | null`) : will be executed when a new feed is imported into the database. The new feed (instance of FreshRSS_Feed) will be passed as parameter. 
 - `post_update` (`function(none) -> none`) : **TODO** add documentation
+- `simplepie_before_init` (`function($simplePie, $feed) -> none`) : **TODO** add documentation
 
 ### Writing your own configure.phtml