Merge pull request #92 from frankcash/frankcash/91

Adds swagger

app.js

@@ -1,16 +1,20 @@
 /*
  * * Module dependencies
  * */
-var express = require('express');
+let express = require('express');
 require('dotenv').config();
-var lobstersRoute = require('./routes/lobsters.js');
-var redditRoute = require('./routes/reddit.js');
-var hackerRoute = require('./routes/hackernews.js');
+let lobstersRoute = require('./routes/lobsters.js');
+let redditRoute = require('./routes/reddit.js');
+let hackerRoute = require('./routes/hackernews.js');
+let swaggerUi = require('swagger-ui-express'),
+    swaggerDocument = require('./swagger/swagger.json');
 
 var app = express(); // sets up the server
 
 var PORT = process.env.PORT || 3000;
 
+app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument));
+
 app.get('/ycomb', hackerRoute.htop);
 
 

db/schema/crawls.sql

@@ -5,5 +5,4 @@ CREATE TABLE crawls(
     title VARCHAR(255),
     comments VARCHAR(255),
     crawled_at TIMESTAMP
-
-);
+);

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   postgres:
     container_name: hackerqueue-pg
-    image: postgres:9.6.3
+    image: postgres:12-alpine
     environment:
       POSTGRES_PASSWORD: hackerx
       POSTGRES_USER: hacker
@@ -13,7 +13,8 @@ services:
       - ./.data/pg:/var/lib/postgresql/data
   web:
     build: .
-    image: ledgehog:latest
+    image: hackerqueue:latest
+    container_name: hackerqueue-web
     environment:
       DB_HOST: postgres
       DATABASE_URL: "postgres://hacker:hackerx@postgres:5432/hackerqueue"
@@ -22,4 +23,4 @@ services:
     depends_on:
       - postgres
     links:
-      - postgres:postgres
+      - postgres:postgres

helpers/index.js

@@ -3,4 +3,8 @@ exports.url_refer = (url)=>{
         return url+="?utm_source=hackerqueue"
     }
     return url
-};
+};
+
+exports.wrap = (arr)=>{
+	return {Crawls: arr}
+}

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "HackerQueue",
-	"version": "1.3.4",
+	"version": "1.3.5",
 	"private": "false",
 	"license": "MIT",
 	"contributors": [
@@ -33,7 +33,9 @@
 		"jade": "*",
 		"nib": "*",
 		"pg": "^7.4.1",
-		"request": "^2.88.0"
+		"request": "^2.88.0",
+		"swagger-node-express": "^2.1.3",
+		"swagger-ui-express": "^4.1.2"
 	},
 	"devDependencies": {
 		"chai": "*",

public/app.js

@@ -20,50 +20,50 @@ app.controller("AppTest", function($scope, $http, $location, $anchorScroll){
   For top posts
   **/
   $http.get('/ycomb').success(function(data) {
-      $scope.hackerTop = data;
+      $scope.hackerTop = data.Crawls;
   });
   $http.get('/lobster').success(function(data) {
-      $scope.lobTop = data;
+      $scope.lobTop = data.Crawls;
   });
   $http.get('/rp').success(function(data) {
-      $scope.rTop = data;
+      $scope.rTop = data.Crawls;
   });
 
   /**
   For new posts
   **/
   $http.get('/ynew').success(function(data) {
-      $scope.hackerNew = data;
+      $scope.hackerNew = data.Crawls;
   });
   $http.get('/lnew').success(function(data) {
-      $scope.lNew = data;
+      $scope.lNew = data.Crawls;
   });
   $http.get('/rnew').success(function(data) {
-      $scope.rNew = data;
+      $scope.rNew = data.Crawls;
   });
 
   $scope.refresh = function(){
     $http.get('/ycomb').success(function(data) {
-        $scope.hackerTop = data;
+        $scope.hackerTop = data.Crawls;
     });
     $http.get('/lobster').success(function(data) {
-        $scope.lobTop = data;
+        $scope.lobTop = data.Crawls;
     });
     $http.get('/rp').success(function(data) {
-        $scope.rTop = data;
+        $scope.rTop = data.Crawls;
     });
 
     /**
     For new posts
     **/
     $http.get('/ynew').success(function(data) {
-        $scope.hackerNew = data;
+        $scope.hackerNew = data.Crawls;
     });
     $http.get('/lnew').success(function(data) {
-        $scope.lNew = data;
+        $scope.lNew = data.Crawls;
     });
     $http.get('/rnew').success(function(data) {
-        $scope.rNew = data;
+        $scope.rNew = data.Crawls;
     });
   };
 

routes/hackernews.js

@@ -8,12 +8,9 @@ function parse(html){
   let $ = cheerio.load(html);
   $('.athing').each(function(){
     let $storylink = $(this).find('.storylink');
-    const rank = $(this).find('.rank').text();
     const title = $storylink.text();
     const url = helpers.url_refer($storylink.attr('href'));
     let $subtext = $(this).next();
-    const points = $subtext.find('.score').text();
-    const username = $subtext.find('.hnuser').text();
     let $comments = $subtext.find('a').last();
     const comments = $comments.text();
     const YCOMB_COMMENT_URL = "https://news.ycombinator.com/";
@@ -28,12 +25,9 @@ function parse(html){
     });
 
     let metadata = { // creates a new object
-      rank: parseInt(rank),
       site: "HN",
       title: title,
       url: url,
-      points: parseInt(points),
-      username: username,
       comments: parseInt(comments),
       comments_link: comments_link
     };
@@ -45,7 +39,7 @@ function parse(html){
 exports.htop = function(req,res){
   request('https://news.ycombinator.com', function(error, response, html){
       if(!error && response.statusCode === 200){
-        res.send(parse(html, "news.ycombinator.com"));
+        res.send(helpers.wrap(parse(html, "news.ycombinator.com")));
       }
   });
 
@@ -54,7 +48,7 @@ exports.htop = function(req,res){
 exports.hnew = function(req,res){
   request('https://news.ycombinator.com/newest', function(error, response, html){
       if(!error && response.statusCode === 200){
-        res.send(parse(html, "news.ycombinator.com/newest"));
+        res.send(helpers.wrap(parse(html, "news.ycombinator.com/newest")));
       }
   });
 

routes/lobsters.js

@@ -31,7 +31,7 @@ const parseLobsterElement = function(a) {
   const url = helpers.url_refer(fixSelfPost(a.children().attr('href')));
 
   // parses link title
-  const title = a.text();
+  const title = a.text().replace(/(\r\n|\n|\r)/gm, "");
 
   let commentsLabel = a.parent().children('.byline').children('span.comments_label');
   let commentsMatch = commentsLabel.text().match("[0-9]+");
@@ -59,7 +59,7 @@ const parseLobsterElement = function(a) {
 
   const metadata = {
     site: source,
-    title:title,
+    title: title,
     url:url,
     comments:comments,
     comments_link:comments_link
@@ -90,8 +90,8 @@ const parseLobsterResponse = function(html) {
 exports.ltop = function(req, res){
   request('https://lobste.rs', function(error, response, html){
       if(!error && response.statusCode === 200){
-        let metadataArray = parseLobsterResponse(html);
-        res.send(metadataArray);
+        const metadataArray = parseLobsterResponse(html);
+      	res.send(helpers.wrap(metadataArray));
       }
   });
 };
@@ -100,7 +100,7 @@ exports.lnew = function(req, res){
   request('https://lobste.rs/recent', function(error, response, html){
     if(!error && response.statusCode === 200){
       const metadataArray = parseLobsterResponse(html);
-      res.send(metadataArray);
+      res.send(helpers.wrap(metadataArray));
     }
   });
 };

routes/reddit.js

@@ -24,7 +24,6 @@ function parse(html, source){
     const comments_link = "https://www.reddit.com" + comments_tag.attr("href");
     const comments      = parseInt(comments_tag.text());
 
-    const points = post.find('button[aria-label=upvote]').parent().children('div').text();
 
     const title = title_tag.text();
     const url   = helpers.url_refer(fixSelfPost(link_tag.attr('href')));
@@ -44,7 +43,6 @@ function parse(html, source){
       url: url,
       comments: comments,
       comments_link: comments_link,
-      points: points
     };
 
     metadataArray.push(metadata);
@@ -56,7 +54,7 @@ function parse(html, source){
 exports.rtop = function(req,res){
   request('http://www.reddit.com/r/programming', function(error, response, html){
     if(!error && response.statusCode === 200){
-      res.send(parse(html, "reddit.com/r/programming"));
+      res.send(helpers.wrap(parse(html, "reddit.com/r/programming")));
     }
   });
 };
@@ -65,7 +63,7 @@ exports.rtop = function(req,res){
 exports.rnew = function(req,res){
   request('http://www.reddit.com/r/programming/new/', function(error, response, html){
     if(!error && response.statusCode === 200){
-      res.send(parse(html, "reddit.com/r/programming/new"));
+      res.send(helpers.wrap(parse(html, "reddit.com/r/programming/new")));
     }
   });
 };