Save readme data that npm provides on publish

fix #100, fix #128

lib/index-web.js

@@ -2,6 +2,7 @@ var fs = require('fs')
 var marked = require('marked')
 var search = require('./search')
 var Handlebars = require('handlebars')
+var Error = require('http-errors')
 
 module.exports = function(app, config, storage) {
 	search.configureStorage(storage)
@@ -71,9 +72,10 @@ module.exports = function(app, config, storage) {
 		}
 	})
 
-	app.get('/-/readme/:package/:version', function(req, res, next) {
-		storage.get_readme(req.params.package, req.params.version, function(readme) {
-			res.send(marked(readme))
+	app.get('/-/readme/:package/:version?', function(req, res, next) {
+		storage.get_package(req.params.package, {req: req}, function(err, info) {
+			if (err) return next(err)
+			res.send(marked(info.readme || 'ERROR: No README data found!'))
 		})
 	})
 }

lib/index.js

@@ -289,6 +289,7 @@ module.exports = function(config_hash) {
 				if (err) return next(err)
 
 				var t2 = Object.keys(metadata.versions)[0]
+				metadata.versions[t2].readme = metadata.readme != null ? String(metadata.readme) : ''
 				create_version(t2, metadata.versions[t2], function(err) {
 					if (err) return next(err)
 

lib/local-storage.js

@@ -8,7 +8,6 @@ var fs = require('fs')
   , mystreams = require('./streams')
   , Logger = require('./logger')
   , info_file = 'package.json'
-  , targz = require('tar.gz')
   , search = require('./search');
 
 //
@@ -141,7 +140,8 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
 			if (data.versions[ver] == null) {
 				var verdata = newdata.versions[ver]
 
-				// why does anyone need to keep that in database?
+				// we don't keep readmes for package versions,
+				// only one readme per package
 				delete verdata.readme
 
 				change = true
@@ -189,6 +189,10 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
 				data._uplinks[up] = newdata._uplinks[up]
 			}
 		}
+		if (newdata.readme !== data.readme) {
+			data.readme = newdata.readme
+			change = true
+		}
 
 		if (change) {
 			self.logger.debug('updating package info')
@@ -204,7 +208,8 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
 Storage.prototype.add_version = function(name, version, metadata, tag, callback) {
 	var self = this
 	self.update_package(name, function updater(data, cb) {
-		// why does anyone need to keep that in database?
+		// keep only one readme per package
+		data.readme = metadata.readme
 		delete metadata.readme
 
 		if (data.versions[version] != null) {
@@ -386,37 +391,6 @@ Storage.prototype.add_tarball = function(name, filename) {
 	return stream
 }
 
-Storage.prototype.unpack_tarball = function(file, callback) {
-	new targz().extract(file + '.tgz', file, callback);
-};
-
-Storage.prototype.get_readme = function(name, version, callback) {
-	var self = this
-	  , storage = this.storage(name)
-	if (!storage) return callback('')
-
-	var fileName = storage.path + '/' + name + '-' + version
-
-	fs.exists(fileName, function(exists) {
-		if (exists) {
-			returnReadme();
-		} else {
-			self.unpack_tarball(fileName, function(err) {
-				returnReadme();
-			});
-		}
-	});
-
-	function returnReadme() {
-		var readmeFileName = fileName + '/package/README.md';
-
-		fs.readFile(readmeFileName, {encoding: 'utf8'}, function(err, data) {
-			if (err) return callback(err)
-			callback(data)
-		})
-	}
-}
-
 Storage.prototype.get_tarball = function(name, filename, callback) {
 	assert(utils.validate_name(filename))
 	var self = this

lib/storage.js

@@ -164,10 +164,6 @@ Storage.prototype.add_tarball = function(name, filename) {
 	return this.local.add_tarball(name, filename)
 }
 
-Storage.prototype.get_readme = function(name, version, callback) {
-	return this.local.get_readme(name, version, callback);
-};
-
 //
 // Get a tarball from a storage for {name} package
 //
@@ -297,7 +293,7 @@ Storage.prototype.get_package = function(name, options, callback) {
 
 		self._sync_package_with_uplinks(name, data, options, function(err, result, uplink_errors) {
 			if (err) return callback(err)
-			var whitelist = ['_rev', 'name', 'versions', 'dist-tags']
+			var whitelist = ['_rev', 'name', 'versions', 'dist-tags', 'readme']
 			for (var i in result) {
 				if (whitelist.indexOf(i) === -1) delete result[i]
 			}
@@ -543,7 +539,11 @@ Storage._merge_versions = function(local, up, config) {
 
 	// refresh dist-tags
 	for (var i in up['dist-tags']) {
-		utils.tag_version(local, up['dist-tags'][i], i, config || {})
+		var added = utils.tag_version(local, up['dist-tags'][i], i, config || {})
+		if (i === 'latest' && added) {
+			// if remote has more fresh package, we should borrow its readme
+			local.readme = up.readme
+		}
 	}
 }
 

package.yaml

@@ -46,9 +46,6 @@ dependencies:
   sinopia-htpasswd: '>= 0.4.3'
   http-errors: '~1.2.0' # ferver
 
-  # TODO: not really needed
-  tar.gz: '~0.1.1'
-
 optionalDependencies:
   # those are native modules that could fail to compile
   # and unavailable on windows
@@ -68,7 +65,6 @@ bundledDependencies:
   - onscroll
   - transition-complete
   - helpers.less
-  - tar.gz
   - sinopia-htpasswd
   - http-errors
 

test/functional/config-1.yaml

@@ -7,6 +7,9 @@ users:
 users_file: ./test-storage/htpasswd
 max_users: 1
 
+web:
+  enable: true
+
 uplinks:
   express:
     url: http://localhost:55550/

test/functional/config-2.yaml

@@ -9,6 +9,9 @@ uplinks:
     url: http://localhost:55551/
     maxage: 0
 
+web:
+  enable: true
+
 logs:
   - {type: stdout, format: pretty, level: trace}
 

test/functional/fixtures/newnpmreg.json

@@ -1 +1 @@
-{"_id":"testpkg-newnpmreg","name":"testpkg-newnpmreg","description":"","dist-tags":{"foo":"0.0.0","latest":"0.0.0"},"versions":{"0.0.0":{"name":"testpkg-newnpmreg","version":"0.0.0","description":"","main":"index.js","scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":""},"author":"","license":"ISC","_id":"[email protected]","dist":{"shasum":"8ee7331cbc641581b1a8cecd9d38d744a8feb863","tarball":"http://localhost:1234/testpkg-newnpmreg/-/testpkg-newnpmreg-0.0.0.tgz"},"_from":".","_npmVersion":"1.3.1","_npmUser":{"name":"alex","email":"[email protected]"},"maintainers":[{"name":"alex","email":"[email protected]"}]}},"readme":"","maintainers":[{"name":"alex","email":"[email protected]"}],"_attachments":{"testpkg-newnpmreg-0.0.0.tgz":{"content_type":"application/octet-stream","data":"H4sIAAAAAAAAA+2TsW7CMBCGM/spTh6YKHUSIJLXqkPnrixWcIMLsS3btCDEu/fs0Ba1SFVVVISUP8Odzqf/zlY+K+qlaOSt7eLo2RudnVmMsel4DBjzasKOY1JZlJDlRVkU5aSspnnG8pIVOZ6fe5FTWvsgHK7yV5/uLvARr0Q7qkUrKadB+mCXzY2Wr9q2TjZ0SF+k88poPGUj/LAyl752yoauioVWqJgpPZcb/Hmw0jV4ynfJEw9lvTAwo/fOGcdBG4h18FbW6knJ+YzCYAByowLkdD+kTlrjVTBumzy2Nq7XqIDea7eKY7FJrMPCuG6Hlaql9rHr4fGO7i/9pFcl+4X/rWhX557xA/9FVZ3gv+j5/w9F+jl8g58c0OeQyCdH3HOglETsObxTTw7McwLJClt+wzz5JD45IPEcEHjMEfg0r8M9pQfaOSDs5NLP16tXr15XqzeJD6m5AAwAAA==","length":352}}}
+{"_id":"testpkg-newnpmreg","name":"testpkg-newnpmreg","description":"","dist-tags":{"foo":"0.0.0","latest":"0.0.0"},"versions":{"0.0.0":{"name":"testpkg-newnpmreg","version":"0.0.0","description":"","main":"index.js","scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":""},"author":"","license":"ISC","_id":"[email protected]","dist":{"shasum":"8ee7331cbc641581b1a8cecd9d38d744a8feb863","tarball":"http://localhost:1234/testpkg-newnpmreg/-/testpkg-newnpmreg-0.0.0.tgz"},"_from":".","_npmVersion":"1.3.1","_npmUser":{"name":"alex","email":"[email protected]"},"maintainers":[{"name":"alex","email":"[email protected]"}]}},"readme":"blah blah blah","maintainers":[{"name":"alex","email":"[email protected]"}],"_attachments":{"testpkg-newnpmreg-0.0.0.tgz":{"content_type":"application/octet-stream","data":"H4sIAAAAAAAAA+2TsW7CMBCGM/spTh6YKHUSIJLXqkPnrixWcIMLsS3btCDEu/fs0Ba1SFVVVISUP8Odzqf/zlY+K+qlaOSt7eLo2RudnVmMsel4DBjzasKOY1JZlJDlRVkU5aSspnnG8pIVOZ6fe5FTWvsgHK7yV5/uLvARr0Q7qkUrKadB+mCXzY2Wr9q2TjZ0SF+k88poPGUj/LAyl752yoauioVWqJgpPZcb/Hmw0jV4ynfJEw9lvTAwo/fOGcdBG4h18FbW6knJ+YzCYAByowLkdD+kTlrjVTBumzy2Nq7XqIDea7eKY7FJrMPCuG6Hlaql9rHr4fGO7i/9pFcl+4X/rWhX557xA/9FVZ3gv+j5/w9F+jl8g58c0OeQyCdH3HOglETsObxTTw7McwLJClt+wzz5JD45IPEcEHjMEfg0r8M9pQfaOSDs5NLP16tXr15XqzeJD6m5AAwAAA==","length":352}}}

test/functional/newnpmreg.js

@@ -68,5 +68,21 @@ module.exports = function() {
 				cb()
 			})
 		})
+
+		it('server1 - readme', function(cb) {
+			server.request({uri:'/-/readme/testpkg-newnpmreg'}, function(err, res, body) {
+				assert.equal(res.statusCode, 200)
+				assert.equal(body, '<p>blah blah blah</p>\n')
+				cb()
+			})
+		})
+
+		it('server2 - readme', function(cb) {
+			server2.request({uri:'/-/readme/testpkg-newnpmreg'}, function(err, res, body) {
+				assert.equal(res.statusCode, 200)
+				assert.equal(body, '<p>blah blah blah</p>\n')
+				cb()
+			})
+		})
 	})
 }