From 206fabee7d8fc0ad0f80d9f17565575de55b8de9 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Tue, 7 Nov 2017 13:39:05 -0500
Subject: [PATCH 01/17] Add rabbit dockerfile and celery dep

---
 app/requirements/prod.txt | 3 +++
 docker-compose.yml        | 6 ++++++
 queue/Dockerfile          | 1 +
 3 files changed, 10 insertions(+)
 create mode 100644 queue/Dockerfile

diff --git a/app/requirements/prod.txt b/app/requirements/prod.txt
index 50eb636..1e5267e 100644
--- a/app/requirements/prod.txt
+++ b/app/requirements/prod.txt
@@ -40,3 +40,6 @@ Flask-DebugToolbar==0.10.1
 
 # Requests
 requests==2.18.4
+
+# Queue
+celery==4.1.0
diff --git a/docker-compose.yml b/docker-compose.yml
index ae29ea7..3821361 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -14,6 +14,7 @@ services:
       - SERVER_NAME
     depends_on:
       - db
+      - queue
 
   webserver:
     build: ./webserver
@@ -37,6 +38,11 @@ services:
     volumes:
       - db_data:/var/lib/league/db
 
+  queue:
+    build: ./queue
+    ports:
+      - '5672:5672'
+
 volumes:
     uwsgi:
     db_data:
diff --git a/queue/Dockerfile b/queue/Dockerfile
new file mode 100644
index 0000000..5f12a3a
--- /dev/null
+++ b/queue/Dockerfile
@@ -0,0 +1 @@
+FROM rabbitmq:3.6.12

From bb7457c03a8bbd4d212ade7729080561b1a91ce9 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Tue, 7 Nov 2017 16:31:59 -0500
Subject: [PATCH 02/17] Add Flower monitor

---
 .env                      |  1 +
 app/autoapp.py            |  2 ++
 app/league/app.py         |  2 +-
 app/league/celery.py      | 23 +++++++++++++++++++++++
 app/league/settings.py    |  4 ++++
 app/league/tasks.py       |  0
 app/requirements/prod.txt |  3 +++
 docker-compose.yml        | 23 +++++++++++++++++++++--
 queue/Dockerfile          |  1 -
 9 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 app/league/celery.py
 create mode 100644 app/league/tasks.py
 delete mode 100644 queue/Dockerfile

diff --git a/.env b/.env
index 98b1a80..597478a 100644
--- a/.env
+++ b/.env
@@ -1,3 +1,4 @@
 POSTGRES_DB=league
 POSTGRES_USER=league
 POSTGRES_PASSWORD=league
+CELERY_BROKER_URL=redis://cache:6379/0
diff --git a/app/autoapp.py b/app/autoapp.py
index 048b485..6b18c14 100755
--- a/app/autoapp.py
+++ b/app/autoapp.py
@@ -3,8 +3,10 @@
 from flask.helpers import get_debug_flag
 
 from league.app import create_app
+from league.celery import create_celery
 from league.settings import DevConfig, ProdConfig
 
 CONFIG = DevConfig if get_debug_flag() else ProdConfig
 
 app = create_app(CONFIG)
+celery = create_celery(app)
diff --git a/app/league/app.py b/app/league/app.py
index bfba108..de67596 100644
--- a/app/league/app.py
+++ b/app/league/app.py
@@ -15,7 +15,7 @@
 
 def create_app(config_object=ProdConfig):
     """
-    An application factory.
+    Create application using app factory.
 
     See: http://flask.pocoo.org/docs/patterns/appfactories/.
 
diff --git a/app/league/celery.py b/app/league/celery.py
new file mode 100644
index 0000000..d2df339
--- /dev/null
+++ b/app/league/celery.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+"""Celery setup."""
+from celery import Celery
+
+
+def create_celery(app):
+    """Build celery object."""
+    celery = Celery(app.import_name,
+                    backend=app.config['CELERY_RESULT_BACKEND'],
+                    broker=app.config['CELERY_BROKER_URL'])
+    celery.conf.update(app.config)
+    TaskBase = celery.Task
+
+    class ContextTask(TaskBase):
+        abstract = True
+
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return TaskBase.__call__(self, *args, **kwargs)
+
+    celery.Task = ContextTask
+
+    return celery
diff --git a/app/league/settings.py b/app/league/settings.py
index 359d2c1..6432caa 100644
--- a/app/league/settings.py
+++ b/app/league/settings.py
@@ -26,6 +26,10 @@ class Config(object):
             '<a href="https://github.com/massgo/league">GitHub</a>.'
         )
     }
+    CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL',
+                                       'redis://cache:6379/0')
+    CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND',
+                                           'rpc://queue')
 
 
 class ProdConfig(Config):
diff --git a/app/league/tasks.py b/app/league/tasks.py
new file mode 100644
index 0000000..e69de29
diff --git a/app/requirements/prod.txt b/app/requirements/prod.txt
index 1e5267e..4e2e46b 100644
--- a/app/requirements/prod.txt
+++ b/app/requirements/prod.txt
@@ -43,3 +43,6 @@ requests==2.18.4
 
 # Queue
 celery==4.1.0
+
+# Cache
+redis==2.10.6
diff --git a/docker-compose.yml b/docker-compose.yml
index 3821361..0ef63f4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,9 +39,28 @@ services:
       - db_data:/var/lib/league/db
 
   queue:
-    build: ./queue
+    image: rabbitmq:3.6.12
+
+  cache:
+    image: redis:3.2.11-alpine
+
+  worker:
+    build: ./app
+    command: celery -A autoapp.celery worker
+    depends_on:
+      - db
+      - queue
+      - cache
+
+  queue_monitor:
+    environment:
+      - CELERY_BROKER_URL
+    image: mher/flower
+    depends_on:
+      - queue
+      - cache
     ports:
-      - '5672:5672'
+      - '5555:5555'
 
 volumes:
     uwsgi:
diff --git a/queue/Dockerfile b/queue/Dockerfile
deleted file mode 100644
index 5f12a3a..0000000
--- a/queue/Dockerfile
+++ /dev/null
@@ -1 +0,0 @@
-FROM rabbitmq:3.6.12

From 3b9b04d561aef8382b3135f699b39a9a5392e302 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Tue, 7 Nov 2017 16:44:28 -0500
Subject: [PATCH 03/17] Add second worker

---
 docker-compose.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 0ef63f4..e22cd81 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -44,7 +44,15 @@ services:
   cache:
     image: redis:3.2.11-alpine
 
-  worker:
+  worker_1:
+    build: ./app
+    command: celery -A autoapp.celery worker
+    depends_on:
+      - db
+      - queue
+      - cache
+
+  worker_2:
     build: ./app
     command: celery -A autoapp.celery worker
     depends_on:

From 7ea0a5d30c9e182074c4fc81ae3969d4862e1823 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Thu, 9 Nov 2017 13:48:19 -0500
Subject: [PATCH 04/17] Refactor to use celeryext

---
 app/{ => league}/autoapp.py |  8 ++++----
 app/league/celery.py        | 23 -----------------------
 app/migrate_and_run.sh      |  2 +-
 app/requirements/prod.txt   |  1 +
 docker-compose.yml          |  4 ++--
 5 files changed, 8 insertions(+), 30 deletions(-)
 rename app/{ => league}/autoapp.py (55%)
 delete mode 100644 app/league/celery.py

diff --git a/app/autoapp.py b/app/league/autoapp.py
similarity index 55%
rename from app/autoapp.py
rename to app/league/autoapp.py
index 6b18c14..7429a21 100755
--- a/app/autoapp.py
+++ b/app/league/autoapp.py
@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 """Create an application instance."""
 from flask.helpers import get_debug_flag
+from flask_celeryext import create_celery_app
 
-from league.app import create_app
-from league.celery import create_celery
-from league.settings import DevConfig, ProdConfig
+from .app import create_app
+from .settings import DevConfig, ProdConfig
 
 CONFIG = DevConfig if get_debug_flag() else ProdConfig
 
 app = create_app(CONFIG)
-celery = create_celery(app)
+celery = create_celery_app(app)
diff --git a/app/league/celery.py b/app/league/celery.py
deleted file mode 100644
index d2df339..0000000
--- a/app/league/celery.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Celery setup."""
-from celery import Celery
-
-
-def create_celery(app):
-    """Build celery object."""
-    celery = Celery(app.import_name,
-                    backend=app.config['CELERY_RESULT_BACKEND'],
-                    broker=app.config['CELERY_BROKER_URL'])
-    celery.conf.update(app.config)
-    TaskBase = celery.Task
-
-    class ContextTask(TaskBase):
-        abstract = True
-
-        def __call__(self, *args, **kwargs):
-            with app.app_context():
-                return TaskBase.__call__(self, *args, **kwargs)
-
-    celery.Task = ContextTask
-
-    return celery
diff --git a/app/migrate_and_run.sh b/app/migrate_and_run.sh
index 072dad9..68dca1d 100755
--- a/app/migrate_and_run.sh
+++ b/app/migrate_and_run.sh
@@ -8,4 +8,4 @@ do
     sleep 1
 done
 
-uwsgi --chmod-socket=666 -s /tmp/uwsgi/uwsgi.sock --plugin python3 --manage-script-name --mount /=autoapp:app
+uwsgi --chmod-socket=666 -s /tmp/uwsgi/uwsgi.sock --plugin python3 --manage-script-name --mount /=league.autoapp:app
diff --git a/app/requirements/prod.txt b/app/requirements/prod.txt
index 4e2e46b..17a64b3 100644
--- a/app/requirements/prod.txt
+++ b/app/requirements/prod.txt
@@ -43,6 +43,7 @@ requests==2.18.4
 
 # Queue
 celery==4.1.0
+flask-celeryext==0.3.0
 
 # Cache
 redis==2.10.6
diff --git a/docker-compose.yml b/docker-compose.yml
index e22cd81..189197d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -46,7 +46,7 @@ services:
 
   worker_1:
     build: ./app
-    command: celery -A autoapp.celery worker
+    command: celery -A league.autoapp.celery worker
     depends_on:
       - db
       - queue
@@ -54,7 +54,7 @@ services:
 
   worker_2:
     build: ./app
-    command: celery -A autoapp.celery worker
+    command: celery -A league.autoapp.celery worker
     depends_on:
       - db
       - queue

From c827e1f74703cfb77d675081d2f3e0ab5e058479 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Thu, 9 Nov 2017 13:50:26 -0500
Subject: [PATCH 05/17] Fix script app path

---
 app/migrate_and_run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/migrate_and_run.sh b/app/migrate_and_run.sh
index 68dca1d..6a3d49a 100755
--- a/app/migrate_and_run.sh
+++ b/app/migrate_and_run.sh
@@ -1,6 +1,6 @@
 #! /bin/sh
 
-export FLASK_APP=autoapp.py
+export FLASK_APP=league/autoapp.py
 
 while true
 do

From 841e70c1d28aaae991428c156b68e464104ad78c Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Thu, 9 Nov 2017 15:57:23 -0500
Subject: [PATCH 06/17] Get hello world task working

---
 app/league/api/views.py  | 9 +++++++++
 app/league/app.py        | 6 ++++--
 app/league/autoapp.py    | 8 +++++---
 app/league/extensions.py | 2 ++
 app/league/tasks.py      | 9 +++++++++
 app/migrate_and_run.sh   | 2 +-
 docker-compose.yml       | 4 ++--
 7 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/app/league/api/views.py b/app/league/api/views.py
index 82836c5..9b79556 100644
--- a/app/league/api/views.py
+++ b/app/league/api/views.py
@@ -8,6 +8,7 @@
 from league.api.forms import GameCreateForm, GameUpdateForm
 from league.extensions import csrf_protect, messenger
 from league.models import Color, Game, Player
+from league import tasks
 
 blueprint = Blueprint('api', __name__, url_prefix='/api/v1.0',
                       static_folder='../static')
@@ -137,3 +138,11 @@ def delete_game(game_id):
         return '', 204
     else:
         return '', 404
+
+
+@blueprint.route('/helloworld')
+def hello_world():
+    """Hello world task."""
+    result = tasks.hello_world.delay()
+
+    return result.wait()
diff --git a/app/league/app.py b/app/league/app.py
index de67596..7e5f6a6 100644
--- a/app/league/app.py
+++ b/app/league/app.py
@@ -7,8 +7,9 @@
 
 from league import admin, api, commands, dashboard, public
 from league.assets import assets
-from league.extensions import (bcrypt, cache, csrf_protect, db, debug_toolbar,
-                               login_manager, messenger, migrate)
+from league.extensions import (bcrypt, cache, celery, csrf_protect, db,
+                               debug_toolbar, login_manager, messenger,
+                               migrate)
 from league.public.forms import LoginForm
 from league.settings import ProdConfig
 
@@ -38,6 +39,7 @@ def register_extensions(app):
     bcrypt.init_app(app)
     cache.init_app(app)
     db.init_app(app)
+    celery.init_app(app)
     csrf_protect.init_app(app)
     login_manager.init_app(app)
     debug_toolbar.init_app(app)
diff --git a/app/league/autoapp.py b/app/league/autoapp.py
index 7429a21..7a7ba61 100755
--- a/app/league/autoapp.py
+++ b/app/league/autoapp.py
@@ -1,12 +1,14 @@
 # -*- coding: utf-8 -*-
 """Create an application instance."""
+from flask import has_app_context
 from flask.helpers import get_debug_flag
-from flask_celeryext import create_celery_app
 
 from .app import create_app
 from .settings import DevConfig, ProdConfig
 
 CONFIG = DevConfig if get_debug_flag() else ProdConfig
 
-app = create_app(CONFIG)
-celery = create_celery_app(app)
+flaskapp = create_app(CONFIG)
+if not has_app_context():
+    flaskapp.app_context().push()
+celery = flaskapp.extensions['flask-celeryext'].celery
diff --git a/app/league/extensions.py b/app/league/extensions.py
index dc02ddb..4e4c7ee 100644
--- a/app/league/extensions.py
+++ b/app/league/extensions.py
@@ -7,6 +7,7 @@
 from flask_bcrypt import Bcrypt
 from flask_caching import Cache
 from flask_debugtoolbar import DebugToolbarExtension
+from flask_celeryext import FlaskCeleryExt
 from flask_login import LoginManager
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
@@ -15,6 +16,7 @@
 from league.slack_messenger import SlackMessenger
 
 bcrypt = Bcrypt()
+celery = FlaskCeleryExt()
 csrf_protect = CsrfProtect()
 login_manager = LoginManager()
 db = SQLAlchemy()
diff --git a/app/league/tasks.py b/app/league/tasks.py
index e69de29..804079e 100644
--- a/app/league/tasks.py
+++ b/app/league/tasks.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+"""Celery tasks."""
+from celery import current_app
+
+
+@current_app.task
+def hello_world():
+    """Hello world task."""
+    return 'Hello world!'
diff --git a/app/migrate_and_run.sh b/app/migrate_and_run.sh
index 6a3d49a..f7b0ceb 100755
--- a/app/migrate_and_run.sh
+++ b/app/migrate_and_run.sh
@@ -8,4 +8,4 @@ do
     sleep 1
 done
 
-uwsgi --chmod-socket=666 -s /tmp/uwsgi/uwsgi.sock --plugin python3 --manage-script-name --mount /=league.autoapp:app
+uwsgi --chmod-socket=666 -s /tmp/uwsgi/uwsgi.sock --plugin python3 --manage-script-name --mount /=league.autoapp:flaskapp
diff --git a/docker-compose.yml b/docker-compose.yml
index 189197d..1979a89 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -46,7 +46,7 @@ services:
 
   worker_1:
     build: ./app
-    command: celery -A league.autoapp.celery worker
+    command: celery -A league.autoapp worker
     depends_on:
       - db
       - queue
@@ -54,7 +54,7 @@ services:
 
   worker_2:
     build: ./app
-    command: celery -A league.autoapp.celery worker
+    command: celery -A league.autoapp worker
     depends_on:
       - db
       - queue

From 8eb64aecca68d9e4fb9a5d6a31feb4f80af8d34c Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Thu, 9 Nov 2017 16:07:58 -0500
Subject: [PATCH 07/17] Move autoapp back

---
 app/{league => }/autoapp.py | 10 +++++-----
 app/migrate_and_run.sh      |  4 ++--
 docker-compose.yml          |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)
 rename app/{league => }/autoapp.py (59%)

diff --git a/app/league/autoapp.py b/app/autoapp.py
similarity index 59%
rename from app/league/autoapp.py
rename to app/autoapp.py
index 7a7ba61..2e3ad17 100755
--- a/app/league/autoapp.py
+++ b/app/autoapp.py
@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 """Create an application instance."""
-from flask import has_app_context
 from flask.helpers import get_debug_flag
 
-from .app import create_app
-from .settings import DevConfig, ProdConfig
+from league.app import create_app
+from league.settings import DevConfig, ProdConfig
 
 CONFIG = DevConfig if get_debug_flag() else ProdConfig
 
 flaskapp = create_app(CONFIG)
-if not has_app_context():
-    flaskapp.app_context().push()
+
+# push app context so we can reference extensions
+flaskapp.app_context().push()
 celery = flaskapp.extensions['flask-celeryext'].celery
diff --git a/app/migrate_and_run.sh b/app/migrate_and_run.sh
index f7b0ceb..9767477 100755
--- a/app/migrate_and_run.sh
+++ b/app/migrate_and_run.sh
@@ -1,6 +1,6 @@
 #! /bin/sh
 
-export FLASK_APP=league/autoapp.py
+export FLASK_APP=autoapp.py
 
 while true
 do
@@ -8,4 +8,4 @@ do
     sleep 1
 done
 
-uwsgi --chmod-socket=666 -s /tmp/uwsgi/uwsgi.sock --plugin python3 --manage-script-name --mount /=league.autoapp:flaskapp
+uwsgi --chmod-socket=666 -s /tmp/uwsgi/uwsgi.sock --plugin python3 --manage-script-name --mount /=autoapp:flaskapp
diff --git a/docker-compose.yml b/docker-compose.yml
index 1979a89..0e10445 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -46,7 +46,7 @@ services:
 
   worker_1:
     build: ./app
-    command: celery -A league.autoapp worker
+    command: celery -A autoapp worker
     depends_on:
       - db
       - queue
@@ -54,7 +54,7 @@ services:
 
   worker_2:
     build: ./app
-    command: celery -A league.autoapp worker
+    command: celery -A autoapp worker
     depends_on:
       - db
       - queue

From 9618ee10352c456e1e3a2a142e037c2baedfffe6 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Thu, 9 Nov 2017 16:14:03 -0500
Subject: [PATCH 08/17] Line length -> 79

---
 app/league/api/views.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/league/api/views.py b/app/league/api/views.py
index 9b79556..61c3032 100644
--- a/app/league/api/views.py
+++ b/app/league/api/views.py
@@ -21,7 +21,8 @@ def _set_game_create_choices(game_create_form):
     Should allow up to one more than current maxima.
     """
     max_season, max_episode = Game.get_max_season_ep()
-    game_create_form.season.choices = [(s, s) for s in range(1, max_season + 2)]
+    game_create_form.season.choices = [(s, s) for s in
+                                       range(1, max_season + 2)]
     game_create_form.episode.choices = [(e, e) for e in
                                         range(1, max_episode + 2)]
 
@@ -87,10 +88,12 @@ def _slack_game_msg(game):
 
     return result.format(w_name=game.white.full_name,
                          w_url=url_for('dashboard.get_player',
-                                       player_id=game.white.id, _external=True),
+                                       player_id=game.white.id,
+                                       _external=True),
                          b_name=game.black.full_name,
                          b_url=url_for('dashboard.get_player',
-                                       player_id=game.black.id, _external=True),
+                                       player_id=game.black.id,
+                                       _external=True),
                          handicap=game.handicap,
                          komi=game.komi,
                          date_string=game.played_at,

From e219e29a385418e80d0ef0b53b21c7d30ae73909 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Thu, 9 Nov 2017 20:13:34 -0500
Subject: [PATCH 09/17] Add AGA sync task

---
 app/league/api/views.py   | 17 +++++++++++++----
 app/league/tasks.py       | 26 ++++++++++++++++++++++++++
 app/requirements/prod.txt |  3 +++
 docker-compose.yml        | 16 ++++++++++++++++
 4 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/app/league/api/views.py b/app/league/api/views.py
index 61c3032..038a82b 100644
--- a/app/league/api/views.py
+++ b/app/league/api/views.py
@@ -5,10 +5,10 @@
 from flask import Blueprint, jsonify, request, url_for
 from flask_login import login_required
 
+from league import tasks
 from league.api.forms import GameCreateForm, GameUpdateForm
 from league.extensions import csrf_protect, messenger
 from league.models import Color, Game, Player
-from league import tasks
 
 blueprint = Blueprint('api', __name__, url_prefix='/api/v1.0',
                       static_folder='../static')
@@ -143,9 +143,18 @@ def delete_game(game_id):
         return '', 404
 
 
-@blueprint.route('/helloworld')
+@blueprint.route('/hello-world')
 def hello_world():
-    """Hello world task."""
+    """Run and wait for Hello world task."""
     result = tasks.hello_world.delay()
 
-    return result.wait()
+    return result.wait(), 200
+
+
+@blueprint.route('/queue-aga-sync')
+@login_required
+def queue_aga_sync():
+    """Queue AGA sync job."""
+    tasks.sync_aga_data.delay()
+
+    return '', 200
diff --git a/app/league/tasks.py b/app/league/tasks.py
index 804079e..743f55c 100644
--- a/app/league/tasks.py
+++ b/app/league/tasks.py
@@ -1,9 +1,35 @@
 # -*- coding: utf-8 -*-
 """Celery tasks."""
+import math
+
+from bs4 import BeautifulSoup
 from celery import current_app
+from requests import get
+
+from league.models import Player
 
 
 @current_app.task
 def hello_world():
     """Hello world task."""
     return 'Hello world!'
+
+
+@current_app.task
+def sync_aga_data():
+    """Update all local AGA data by scraping the AGA servers."""
+    for player in Player.get_players():
+        url = 'http://www.usgo.org/ratings-lookup-id?PlayerID={}'.format(
+            player.aga_id)
+        data = get(url)
+        soup = BeautifulSoup(data.text, 'html.parser')
+        columns = [col.text for col in soup.findAll('tr')[1].findAll('td')]
+        if int(columns[0]) == player.aga_id:
+            rating = float(columns[2])
+            if rating > 0:
+                rank = math.floor(rating)
+            else:
+                rank = math.ceil(rating)
+            player.update(aga_rank=rank)
+
+    return
diff --git a/app/requirements/prod.txt b/app/requirements/prod.txt
index 17a64b3..5300a20 100644
--- a/app/requirements/prod.txt
+++ b/app/requirements/prod.txt
@@ -47,3 +47,6 @@ flask-celeryext==0.3.0
 
 # Cache
 redis==2.10.6
+
+# BeautifulSoup
+beautifulsoup4==4.6.0
diff --git a/docker-compose.yml b/docker-compose.yml
index 0e10445..dde0ada 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -51,6 +51,14 @@ services:
       - db
       - queue
       - cache
+    environment:
+      - POSTGRES_DB
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - LEAGUE_ROOT_PASS
+      - SLACK_CHANNEL
+      - SLACK_WEBHOOK
+      - SERVER_NAME
 
   worker_2:
     build: ./app
@@ -59,6 +67,14 @@ services:
       - db
       - queue
       - cache
+    environment:
+      - POSTGRES_DB
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - LEAGUE_ROOT_PASS
+      - SLACK_CHANNEL
+      - SLACK_WEBHOOK
+      - SERVER_NAME
 
   queue_monitor:
     environment:

From 9577d527edc2d192878962a6749481bc4984529d Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 13:21:37 -0500
Subject: [PATCH 10/17] Add logging

---
 app/league/api/views.py | 2 +-
 app/league/tasks.py     | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/league/api/views.py b/app/league/api/views.py
index 038a82b..9cd0ac6 100644
--- a/app/league/api/views.py
+++ b/app/league/api/views.py
@@ -157,4 +157,4 @@ def queue_aga_sync():
     """Queue AGA sync job."""
     tasks.sync_aga_data.delay()
 
-    return '', 200
+    return 'Task queued!', 200
diff --git a/app/league/tasks.py b/app/league/tasks.py
index 743f55c..745b8e7 100644
--- a/app/league/tasks.py
+++ b/app/league/tasks.py
@@ -4,10 +4,13 @@
 
 from bs4 import BeautifulSoup
 from celery import current_app
+from celery.utils.log import get_task_logger
 from requests import get
 
 from league.models import Player
 
+logger = get_task_logger(__name__)
+
 
 @current_app.task
 def hello_world():

From 515446c3ba7408bd4d145549cdcfb6f5d0893bd3 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 13:21:49 -0500
Subject: [PATCH 11/17] Add redis dev config for Celery

---
 app/league/settings.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/league/settings.py b/app/league/settings.py
index 6432caa..852f4bd 100644
--- a/app/league/settings.py
+++ b/app/league/settings.py
@@ -63,6 +63,10 @@ class DevConfig(Config):
     ASSETS_DEBUG = True  # Don't bundle/minify static assets
     CACHE_TYPE = 'simple'  # Can be "memcached", "redis", etc.
     LEAGUE_ROOT_PASS = 'root'
+    CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL',
+                                       'redis://localhost:6379/0')
+    CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND',
+                                           'redis://localhost:6379/0')
 
 
 class TestConfig(Config):

From 459890d6103b89250fe32b70806cfe2717c73068 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 13:22:08 -0500
Subject: [PATCH 12/17] Skip players with missing rating from AGA

---
 app/league/tasks.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/app/league/tasks.py b/app/league/tasks.py
index 745b8e7..424cde3 100644
--- a/app/league/tasks.py
+++ b/app/league/tasks.py
@@ -27,12 +27,19 @@ def sync_aga_data():
         data = get(url)
         soup = BeautifulSoup(data.text, 'html.parser')
         columns = [col.text for col in soup.findAll('tr')[1].findAll('td')]
-        if int(columns[0]) == player.aga_id:
-            rating = float(columns[2])
-            if rating > 0:
-                rank = math.floor(rating)
-            else:
-                rank = math.ceil(rating)
-            player.update(aga_rank=rank)
+        logger.debug('AGA Data: {}'.format(columns))
+
+        if not int(columns[0]) == player.aga_id:
+            continue
+        if columns[2] == '':
+            continue
+
+        rating = float(columns[2])
+        if rating > 0:
+            rank = math.floor(rating)
+        else:
+            rank = math.ceil(rating)
+
+        player.update(aga_rank=rank)
 
     return

From 2680981700b18772e05f5859c46f6c8652355d8b Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 14:39:19 -0500
Subject: [PATCH 13/17] Add AGA syncing page and update JS/HTML messaging

---
 app/league/admin/views.py                   |  7 ++++
 app/league/templates/admin/aga_syncing.html | 38 +++++++++++++++++++++
 app/league/templates/admin/settings.html    |  1 +
 app/league/templates/dashboard/games.html   | 34 ++++++++----------
 app/league/templates/layout.html            |  7 ++++
 5 files changed, 67 insertions(+), 20 deletions(-)
 create mode 100644 app/league/templates/admin/aga_syncing.html

diff --git a/app/league/admin/views.py b/app/league/admin/views.py
index 7adbd96..1d06d91 100644
--- a/app/league/admin/views.py
+++ b/app/league/admin/views.py
@@ -84,6 +84,13 @@ def settings():
     return render_template('admin/settings.html')
 
 
+@blueprint.route('/aga-syncing')
+@admin_required
+def aga_syncing():
+    """AGA syncing."""
+    return render_template('admin/aga_syncing.html')
+
+
 @blueprint.route('/site_settings/', methods=['GET', 'POST'])
 @admin_required
 def manage_site_settings():
diff --git a/app/league/templates/admin/aga_syncing.html b/app/league/templates/admin/aga_syncing.html
new file mode 100644
index 0000000..45c658b
--- /dev/null
+++ b/app/league/templates/admin/aga_syncing.html
@@ -0,0 +1,38 @@
+
+{% extends "layout.html" %}
+{% block content %}
+<div class="container-narrow">
+  <h1>AGA Syncing</h1>
+  <br/>
+  <button id="submit-button" type="submit" class="btn btn-primary">Sync AGA Data</button>
+</div>
+{% endblock %}
+
+{% block js %}
+<script>
+
+var csrfToken = '{{ csrf_token() }}';
+
+$msg_container = $('#error-container');
+$msg_template = $('#msg-template');
+
+function display_msgs(msgs, alert_type='warning') {
+  $msg_container.hide();
+  for (var attr in msgs) {
+    $msg_container.append($msg_template.html());
+    $alerts = $msg_container.children('.alert');
+    $new_alert = $alerts.last();
+    $new_alert.addClass('alert-' + alert_type);
+    $new_alert.append(msgs[attr]);
+  }
+  $msg_container.show();
+}
+
+$('#submit-button').click(function() {
+    display_msgs(['wee'], 'danger');
+    display_msgs(['woo'], 'success');
+    display_msgs(['waa']);
+});
+
+</script>
+{% endblock %}
diff --git a/app/league/templates/admin/settings.html b/app/league/templates/admin/settings.html
index 48f33ad..c49599d 100644
--- a/app/league/templates/admin/settings.html
+++ b/app/league/templates/admin/settings.html
@@ -8,5 +8,6 @@ <h1>Admin Settings</h1>
   <a href="{{ url_for('admin.create_user') }}"><h3>Create user</h3></a>
   <a href="{{ url_for('admin.manage_slack_integration') }}"><h3>Manage Slack integration</h3></a>
   <a href="{{ url_for('admin.manage_site_settings') }}"><h3>Manage site settings</h3></a>
+  <a href="{{ url_for('admin.aga_syncing') }}"><h3>AGA Syncing</h3></a>
 </div>
 {% endblock %}
diff --git a/app/league/templates/dashboard/games.html b/app/league/templates/dashboard/games.html
index d7cc4c2..8fb49c1 100644
--- a/app/league/templates/dashboard/games.html
+++ b/app/league/templates/dashboard/games.html
@@ -2,13 +2,6 @@
 {% extends "layout.html" %}
 
 {% block content %}
-<!-- See http://stackoverflow.com/questions/18673860/defining-a-html-template-to-append-using-jquery -->
-<script id="error-template" type="text/x-error-template">
-  <div class="alert alert-warning">
-    <a class="close" title="Close" href="#" data-dismiss="alert">&times;</a>
-  </div><!-- end .alert -->
-</script>
-
 <div class="modal fade" id="editor-modal" tabindex="-1" role="dialog" aria-labelledby="editor-title">
   <div class="modal-dialog" role="document">
     <form class="modal-content form-horizontal" id="editor">
@@ -130,18 +123,19 @@ <h1 class="page-header">Games</h1>
   return moment.utc(remotedate, server_dt_format).utcOffset(local_tz).format(local_dt_format);
 }
 
-$error_container = $('#error-container');
-$error_template = $('#error-template');
+$msg_container = $('#error-container');
+$msg_template = $('#msg-template');
 
-function display_errors(errors) {
-  $error_container.hide();
-  for (var attr in errors) {
-    $error_container.append($error_template.html());
-    alerts = $error_container.children('.alert');
-    $new_alert = alerts[alerts.length -1];
-    $new_alert.append(attr + ' - ' + errors[attr]);
+function display_msgs(msgs, alert_type='warning') {
+  $msg_container.hide();
+  for (var attr in msgs) {
+    $msg_container.append($msg_template.html());
+    $alerts = $msg_container.children('.alert');
+    $new_alert = $alerts.last();
+    $new_alert.addClass('alert-' + alert_type);
+    $new_alert.append(msgs[attr]);
   }
-  $error_container.show();
+  $msg_container.show();
 }
 
 function format_player(value) {
@@ -182,7 +176,7 @@ <h1 class="page-header">Games</h1>
       });
     },
     error: function(jqXHR, textStatus, errorThrown) {
-      display_errors(jqXHR.responseJSON);
+      display_msgs(jqXHR.responseJSON);
     }
   });
 }
@@ -219,7 +213,7 @@ <h1 class="page-header">Games</h1>
       });
     },
     error: function(jqXHR, textStatus, errorThrown) {
-      display_errors(jqXHR.responseJSON);
+      display_msgs(jqXHR.responseJSON);
     }
   });
 }
@@ -233,7 +227,7 @@ <h1 class="page-header">Games</h1>
     },
     error: function(jqXHR, textStatus, errorThrown) {
       if (jqXHR.status == 404) {
-        display_errors({'game_id': 'Game ' + id + ' could not be found and was not deleted.'});
+        display_msgs({'game_id': 'Game ' + id + ' could not be found and was not deleted.'});
       }
     }
   });
diff --git a/app/league/templates/layout.html b/app/league/templates/layout.html
index 848ebfc..518262c 100644
--- a/app/league/templates/layout.html
+++ b/app/league/templates/layout.html
@@ -34,6 +34,13 @@
 {% endwith %}
 
 <header>{% block header %}{% endblock %}</header>
+<!-- See http://stackoverflow.com/questions/18673860/defining-a-html-template-to-append-using-jquery -->
+<script id="msg-template" type="text/x-error-template">
+  <div class="alert">
+    <a class="close" title="Close" href="#" data-dismiss="alert">&times;</a>
+  </div><!-- end .alert -->
+</script>
+
 <div class="{% block content_class %}container{% endblock content_class %}">
 
   <div role="main">

From a8820cd3ce8a847871989f641fc85e4923c023fe Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 14:55:54 -0500
Subject: [PATCH 14/17] Syncing works, though msgs are broken

---
 app/league/api/views.py                     |  2 +-
 app/league/templates/admin/aga_syncing.html | 17 ++++++++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/app/league/api/views.py b/app/league/api/views.py
index 9cd0ac6..70f57e1 100644
--- a/app/league/api/views.py
+++ b/app/league/api/views.py
@@ -151,7 +151,7 @@ def hello_world():
     return result.wait(), 200
 
 
-@blueprint.route('/queue-aga-sync')
+@blueprint.route('/queue-aga-sync', methods=['POST'])
 @login_required
 def queue_aga_sync():
     """Queue AGA sync job."""
diff --git a/app/league/templates/admin/aga_syncing.html b/app/league/templates/admin/aga_syncing.html
index 45c658b..0aee1d9 100644
--- a/app/league/templates/admin/aga_syncing.html
+++ b/app/league/templates/admin/aga_syncing.html
@@ -29,9 +29,20 @@ <h1>AGA Syncing</h1>
 }
 
 $('#submit-button').click(function() {
-    display_msgs(['wee'], 'danger');
-    display_msgs(['woo'], 'success');
-    display_msgs(['waa']);
+    var jqxhr = $.post({
+        url: '{{ url_for('api.queue_aga_sync') }}',
+        data: { csrf_token: csrfToken },
+        success: function(data, textStatus, jqXHR) {
+            display_msgs(jqXHR.responseJSON, 'success');
+        }
+    })
+    .done(function(data, textStatus, jqXHR) {
+        display_msgs(jqXHR.responseJSON);
+    })
+    .fail(function(data, textStatus, jqXHR) {
+        display_msgs(jqXHR.responseJSON, 'danger');
+    })
+    console.log(jqxhr);
 });
 
 </script>

From d2919082e2b4ca8aa71042a05e2a1e0b01f5fa57 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 15:04:06 -0500
Subject: [PATCH 15/17] Fix some crap

---
 app/league/templates/admin/aga_syncing.html | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/app/league/templates/admin/aga_syncing.html b/app/league/templates/admin/aga_syncing.html
index 0aee1d9..d7e4f5b 100644
--- a/app/league/templates/admin/aga_syncing.html
+++ b/app/league/templates/admin/aga_syncing.html
@@ -31,16 +31,13 @@ <h1>AGA Syncing</h1>
 $('#submit-button').click(function() {
     var jqxhr = $.post({
         url: '{{ url_for('api.queue_aga_sync') }}',
-        data: { csrf_token: csrfToken },
-        success: function(data, textStatus, jqXHR) {
-            display_msgs(jqXHR.responseJSON, 'success');
-        }
+        data: { csrf_token: csrfToken }
     })
-    .done(function(data, textStatus, jqXHR) {
-        display_msgs(jqXHR.responseJSON);
+    .done(function() {
+        display_msgs(['Job submitted!'], 'success');
     })
-    .fail(function(data, textStatus, jqXHR) {
-        display_msgs(jqXHR.responseJSON, 'danger');
+    .fail(function() {
+        display_msgs(['Job failed to submit!'], 'danger');
     })
     console.log(jqxhr);
 });

From 030c6ce0468138dc3113484b00ce1a52d0251161 Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 15:12:42 -0500
Subject: [PATCH 16/17] Add debug logging to celery workers

---
 docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index dde0ada..dd58767 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -46,7 +46,7 @@ services:
 
   worker_1:
     build: ./app
-    command: celery -A autoapp worker
+    command: celery -A autoapp worker -l DEBUG
     depends_on:
       - db
       - queue
@@ -62,7 +62,7 @@ services:
 
   worker_2:
     build: ./app
-    command: celery -A autoapp worker
+    command: celery -A autoapp worker -l DEBUG
     depends_on:
       - db
       - queue

From e47ed66113856fe6ae37a1620eeb6b8696c7e5bc Mon Sep 17 00:00:00 2001
From: Andrew Hall <hndrewaall@gmail.com>
Date: Fri, 1 Dec 2017 15:15:42 -0500
Subject: [PATCH 17/17] Fix linting errors

---
 app/league/app.py        | 3 +--
 app/league/extensions.py | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/league/app.py b/app/league/app.py
index 7e5f6a6..dbb9e4a 100644
--- a/app/league/app.py
+++ b/app/league/app.py
@@ -8,8 +8,7 @@
 from league import admin, api, commands, dashboard, public
 from league.assets import assets
 from league.extensions import (bcrypt, cache, celery, csrf_protect, db,
-                               debug_toolbar, login_manager, messenger,
-                               migrate)
+                               debug_toolbar, login_manager, messenger, migrate)
 from league.public.forms import LoginForm
 from league.settings import ProdConfig
 
diff --git a/app/league/extensions.py b/app/league/extensions.py
index 4e4c7ee..6bedbf8 100644
--- a/app/league/extensions.py
+++ b/app/league/extensions.py
@@ -6,8 +6,8 @@
 """
 from flask_bcrypt import Bcrypt
 from flask_caching import Cache
-from flask_debugtoolbar import DebugToolbarExtension
 from flask_celeryext import FlaskCeleryExt
+from flask_debugtoolbar import DebugToolbarExtension
 from flask_login import LoginManager
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy