diff --git a/.gitignore b/.gitignore
index 0b1f484..162840a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -171,4 +171,7 @@ cython_debug/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
-.ruff_cache
\ No newline at end of file
+.ruff_cache
+
+env_variables.yaml
+env.ps1
\ No newline at end of file
diff --git a/app.py b/app.py
index fba1d84..3e3c0d4 100644
--- a/app.py
+++ b/app.py
@@ -1,10 +1,39 @@
-from flask import Flask
+from flask import Flask, request, render_template, jsonify
+from flask_caching import Cache
+
+from env import GOODREADS_URL
+
+from src.book_scraper import BookScraper
+
+"""
+Mounts a prefix for all routes
+"""
+SCRIPT_NAME = "/api"
+
+"""
+Scraper for book API
+"""
+SCRAPER = BookScraper(GOODREADS_URL)
+
+"""
+App configuration
+"""
+config = {
+ "DEBUG": True, # some Flask specific configs
+ "CACHE_TYPE": "SimpleCache", # Flask-Caching related configs
+ "CACHE_DEFAULT_TIMEOUT": 300
+}
"""
The main application
"""
app = Flask(__name__)
+app.config.from_mapping(config)
+
+cache = Cache(app)
-@app.route("/")
-def root():
- return "Hello World!"
+@app.route("/books")
+@cache.cached(timeout=86400)
+def get_books():
+ results = SCRAPER.scrape()
+ return render_template("books.html", results=results), 200
\ No newline at end of file
diff --git a/app.yaml b/app.yaml
index 83a0c28..979c22f 100644
--- a/app.yaml
+++ b/app.yaml
@@ -14,12 +14,12 @@
runtime: python310
+includes:
+ - env_variables.yaml
+
# Commented out because this is going to be the default service
# service: samoyedapi
-# env_variables:
-# PORT: "8000"
-
instance_class: F1
automatic_scaling:
diff --git a/env.py b/env.py
new file mode 100644
index 0000000..d8f909c
--- /dev/null
+++ b/env.py
@@ -0,0 +1,3 @@
+import os
+
+GOODREADS_URL = os.environ.get('GOODREADS_URL')
\ No newline at end of file
diff --git a/main.py b/main.py
deleted file mode 100644
index 1b7a161..0000000
--- a/main.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from flask import Flask #, render_template
-
-app = Flask(__name__)
-
-@app.route("/")
-def root():
- return "Hello World!"
-
-if __name__ == "__main__":
- # This is used when running locally only. When deploying to Google App
- # Engine, a webserver process such as Gunicorn will serve the app. This
- # can be configured by adding an `entrypoint` to app.yaml.
- # Flask's development server will automatically serve static files in
- # the "static" directory. See:
- # http://flask.pocoo.org/docs/1.0/quickstart/#static-files. Once deployed,
- # App Engine itself will serve those files as configured in app.yaml.
- app.run(host="127.0.0.1", port=8080, debug=True)
\ No newline at end of file
diff --git a/src/book_scraper.py b/src/book_scraper.py
index 7059bcd..c56e33d 100644
--- a/src/book_scraper.py
+++ b/src/book_scraper.py
@@ -1,13 +1,16 @@
from typing import List, Dict
import requests
from bs4 import BeautifulSoup
-from cachetools import cached, TTLCache
class BookScraper:
+ """
+ Scrapes the available book data at the given url.
+ """
+
def __init__(self, url: str):
+ assert url is not None
self.url = url
- @cached(cache=TTLCache(maxsize=1, ttl=86400))
def scrape(self) -> List[Dict[str, str]]:
html = self.request()
responses = self.parse(html)
@@ -16,7 +19,15 @@ def scrape(self) -> List[Dict[str, str]]:
def request(self) -> str:
try:
- response = requests.get(self.url)
+ response = requests.get(self.url, headers={
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+ 'Accept-Encoding': 'gzip, deflate, br, zstd',
+ 'Accept-Language': 'en-US',
+ 'Connection': 'keep-alive',
+ 'Host': 'www.goodreads.com',
+ 'Upgrade-Insecure-Requests': '1',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0',
+ })
response.raise_for_status()
except Exception as err:
raise err
@@ -31,17 +42,16 @@ def parse(self, html: str) -> List[Dict[str, str]]:
# Iterate through children
out = []
- for child in root.children:
- if isinstance(child, str) or child["class"] == "clear":
- continue
+ for child in root.select("div.Updates"):
+ assert child.name == "div"
# Get first column tags
- c1 = soup.select_one("div.firstcol")
+ c1 = child.select_one("div.firstcol")
image_tag = c1.img
# Get second column tags
- c2 = soup.select_one("div.secondcol")
+ c2 = child.select_one("div.secondcol")
info_tag = c2.select_one("div.whos-review")
@@ -60,9 +70,16 @@ def parse(self, html: str) -> List[Dict[str, str]]:
image_url = image_tag["src"]
- progress_percent = progress_tag.text.strip("()%")
- assert progress_percent.isdigit()
-
+ # Extract progress percentage
+ progress_text = progress_tag.text.strip("()%")
+ if progress_text.startswith("page"):
+ split = progress_text.split()
+ current = int(split[1])
+ total = int(split[-1])
+ progress_percent = (100*current)//total
+ else:
+ progress_percent = int(progress_text)
+
# Output parsed data
out.append({
"title": title,
diff --git a/src/books.py b/src/books.py
deleted file mode 100644
index 9e5f5f8..0000000
--- a/src/books.py
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-
-
-
-
-
diff --git a/src/courses.py b/src/courses.py
deleted file mode 100644
index e69de29..0000000
diff --git a/templates/books.html b/templates/books.html
new file mode 100644
index 0000000..9c58587
--- /dev/null
+++ b/templates/books.html
@@ -0,0 +1,10 @@
+
+ {% for result in results %}
+
+ {% endfor %}
+
@@ -172,43 +172,6 @@ def test_book_scraper_parser():
—
-Mar 26, 2024 11:47PM
-
-
-
-
-
-
-
-
-
-
"""
@@ -222,5 +185,5 @@ def test_book_scraper_parser():
"author": "Sylvia Plath",
"author_url": "http://goodreads.com/author/show/4379.Sylvia_Plath",
"image_url": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1554582218l/6514._SX98_.jpg",
- "progress": "4"
+ "progress": 4
} == responses[0]
\ No newline at end of file