improved ui, added container with celery

Rakefile

@@ -38,8 +38,9 @@ task :build_ui => [NG, :gen_client] do
   end
 end
 
-task :docker_up => [:build_ui] do
-  sh "docker-compose build ui"
+#task :docker_up => [:build_ui] do
+task :docker_up do
+  sh "docker-compose build"
   sh "docker-compose up"
 end
 

commands.txt

@@ -0,0 +1,4 @@
+../../venv/bin/celery -A bg.clry worker -l info
+rm -rf /tmp/kk-jobs/ && ./agent.py -d /tmp/kk-jobs -s http://localhost:5000/backend
+venv ./scheduler.py
+venv ./server.py

docker-compose.yaml

@@ -43,6 +43,19 @@ services:
     depends_on:
       - server
 
+  celery:
+    restart: always
+    build:
+      context: kraken
+      dockerfile: docker-celery.txt
+    networks:
+      - db_net
+    environment:
+      REDIS_HOST: "redis"
+    depends_on:
+      - postgres
+      - redis
+
   ui:
     build:
       context: ui
@@ -59,6 +72,8 @@ services:
 
   postgres:
     image: postgres:11
+    volumes:
+      - db-data:/var/lib/postgresql/data
     networks:
       - db_net
     environment:

kraken/agent/kraken_rndtest.py

@@ -0,0 +1,70 @@
+import logging
+import random
+
+import tool
+
+log = logging.getLogger(__name__)
+
+
+CITIES = ['Shanghai', 'Lagos', 'Istanbul', 'Karachi', 'Mumbai', 'Moscow', 'Sao Paulo', 'Beijing', 'Guangzhou', 'Delhi',
+          'Lahore', 'Shenzhen', 'Seoul', 'Jakarta', 'Tianjin', 'Chennai', 'Tokyo', 'Cairo', 'Dhaka', 'Mexico', 'Kinshasa',
+          'Bangalore', 'New York', 'London', 'Bangkok', 'Tehran', 'Dongguan', 'Ho Chi Minh City', 'Bogota', 'Lima',
+          'Hong Kong', 'Hanoi', 'Hyderabad', 'Wuhan', 'Rio de Janeiro', 'Foshan', 'Ahmedabad', 'Baghdad', 'Singapore',
+          'Shantou', 'Riyadh', 'Jeddah', 'Santiago', 'Saint Petersburg', 'Qalyubia', 'Chengdu', 'Alexandria', 'Ankara',
+          'Chongqing', 'Kolkata', 'Xian', 'Surat', 'Johannesburg', 'Nanjing', 'Dar es Salaam', 'Yangon', 'Abidjan',
+          'Harbin', 'Zhengzhou', 'Suzhou', 'Sydney', 'New Taipei City', 'Los Angeles', 'Melbourne', 'Cape Town',
+          'Shenyang', 'Yokohama', 'Busan', 'Hangzhou', 'Quanzhou', 'Durban', 'Casablanca', 'Algiers', 'Berlin', 'Nairobi',
+          'Hefei', 'Kabul', 'Pyongyang', 'Madrid', 'Ekurhuleni', 'Pune', 'Addis Ababa', 'Changsha', 'Jaipur', 'Xuzhou',
+          'Wenzhou']
+
+ANIMALS = ['Aardvark', 'Albatross', 'Alligator', 'Alpaca', 'Ant', 'Anteater', 'Antelope', 'Ape', 'Armadillo', 'Donkey',
+           'Baboon', 'Badger', 'Barracuda', 'Bat', 'Bear', 'Beaver', 'Bee', 'Bison', 'Boar', 'Buffalo', 'Galago',
+           'Butterfly', 'Camel', 'Caribou', 'Cat', 'Caterpillar', 'Cattle', 'Chamois', 'Cheetah', 'Chicken', 'Chimpanzee',
+           'Chinchilla', 'Chough', 'Clam', 'Cobra', 'Cockroach', 'Cod', 'Cormorant', 'Coyote', 'Crab', 'Crane', 'Crocodile',
+           'Crow', 'Curlew', 'Deer', 'Dinosaur', 'Dog', 'Dogfish', 'Dolphin', 'Dotterel', 'Dove', 'Dragonfly', 'Duck',
+           'Dugong', 'Dunlin', 'Eagle', 'Echidna', 'Eel', 'Eland', 'Elephant', 'Elk', 'Emu', 'Falcon', 'Ferret', 'Finch',
+           'Fish', 'Flamingo', 'Fly', 'Fox', 'Frog', 'Gaur', 'Gazelle', 'Gerbil', 'Giant Panda', 'Giraffe', 'Gnat', 'Gnu',
+           'Goat', 'Goose', 'Goldfinch', 'Goldfish', 'Gorilla', 'Goshawk', 'Grasshopper', 'Grouse', 'Guanaco', 'Gull',
+           'Hamster', 'Hare', 'Hawk', 'Hedgehog', 'Heron', 'Herring', 'Hippopotamus', 'Hornet', 'Horse', 'Human',
+           'Hummingbird', 'Hyena', 'Jackal', 'Jaguar', 'Jay', 'Jellyfish', 'Kangaroo', 'Koala', 'Kouprey', 'Kudu',
+           'Lapwing', 'Lark', 'Lemur', 'Leopard', 'Lion', 'Llama', 'Lobster', 'Locust', 'Loris', 'Louse', 'Lyrebird',
+           'Magpie', 'Mallard', 'Manatee', 'Marten', 'Meerkat', 'Mink', 'Mole', 'Monkey', 'Moose', 'Mouse', 'Mosquito',
+           'Mule', 'Narwhal', 'Newt', 'Nightingale', 'Octopus', 'Okapi', 'Opossum', 'Oryx', 'Ostrich', 'Otter', 'Owl',
+           'Ox', 'Oyster', 'Panther', 'Parrot', 'Partridge', 'Peafowl', 'Pelican', 'Penguin', 'Pheasant', 'Pig', 'Pigeon',
+           'Porcupine', 'Porpoise', 'Quail', 'Quelea', 'Rabbit', 'Raccoon', 'Rail', 'Ram', 'Rat', 'Raven', 'Reindeer',
+           'Rhinoceros', 'Rook', 'Ruff', 'Salamander', 'Salmon', 'Sandpiper', 'Sardine', 'Scorpion', 'Seahorse', 'Seal',
+           'Shark', 'Sheep', 'Shrew', 'Shrimp', 'Skunk', 'Snail', 'Snake', 'Spider', 'Squid', 'Squirrel', 'Starling',
+           'Stingray', 'Stinkbug', 'Stork', 'Swallow', 'Swan', 'Tapir', 'Tarsier', 'Termite', 'Tiger', 'Toad', 'Trout',
+           'Turkey', 'Turtle', 'Vicuna', 'Viper', 'Vulture', 'Wallaby', 'Walrus', 'Wasp', 'Weasel', 'Whale', 'Wolf',
+           'Wolverine', 'Wombat', 'Woodcock', 'Woodpecker', 'Worm', 'Wren', 'Yak', 'Zebra']
+
+
+def _generate_test_case_name():
+    animal = random.choice(ANIMALS).replace(' ', '-')
+    city = random.choice(CITIES).replace(' ', '-')
+    return "test_%s_%s.from.%s" % (animal, random.randint(0, 100000), city)
+
+
+def collect_tests(step):
+    random.seed(1)
+    count = int(step.get('count', 10))
+    tests = []
+    for _ in range(count):
+        tests.append(_generate_test_case_name())
+    return tests
+
+
+def run_tests(step, report_result=None):
+    tests = step['tests']
+
+    for test in tests:
+        cmd = 'random_test %s' % test
+
+        result = dict(cmd=cmd, test=test, status=random.choice([0, 1, 2, 3, 4, 5]))
+        report_result(result)
+
+    return 0, ''
+
+
+if __name__ == '__main__':
+    tool.main()

kraken/docker-celery.txt

@@ -0,0 +1,22 @@
+# FROM python:3.7-slim
+# RUN groupadd user && useradd --create-home --home-dir /home/user -g user user
+# WORKDIR /home/user
+# RUN pip install redis
+# ENV CELERY_VERSION 4.0.2
+# RUN pip install celery=="$CELERY_VERSION"
+# RUN { \
+# 	echo 'import os'; \
+# 	echo "BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'amqp://')"; \
+# } > celeryconfig.py
+# # --link some-rabbit:rabbit "just works"
+# ENV CELERY_BROKER_URL amqp://guest@rabbit
+# USER user
+# CMD ["celery", "worker"]
+
+FROM ubuntu:19.10
+WORKDIR /server
+COPY server /server
+RUN apt update && apt install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel libpq-dev gcc python3-dev
+RUN pip3 install --no-cache-dir -r /server/requirements.txt
+ENV REDIS_HOST localhost
+CMD celery -A bg.clry worker -l info

kraken/server/bg/clry.py

@@ -1,3 +1,6 @@
+import os
 from celery import Celery
 
-app = Celery('kraken', broker='redis://localhost//', include=['bg.tasks'])
+REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')
+
+app = Celery('kraken', broker='redis://%s//' % REDIS_HOST, include=['bg.tasks'])

kraken/server/bg/jobs.py

@@ -14,7 +14,7 @@
 
 
 def create_app():
-    #logging.basicConfig(format=consts.CONSOLE_LOG_FMT, level=logging.INFO)
+    #logging.basicConfig(format=consts.LOG_FMT, level=logging.INFO)
 
     # Create  Flask app instance
     app = Flask('Kraken Background')

kraken/server/requirements.txt

@@ -1,24 +1,34 @@
+billiard==3.6.1.0
+celery==4.3.0
 certifi==2018.11.29
 chardet==3.0.4
 Click==7.0
 clickclick==1.2.2
 connexion==2.2.0
 Flask==1.0.2
+Flask-Cors==3.0.8
 Flask-SQLAlchemy==2.3.2
 idna==2.8
+importlib-metadata==0.23
 inflection==0.3.1
 itsdangerous==1.1.0
 Jinja2==2.10
 jsonschema==2.6.0
+kombu==4.6.5
 MarkupSafe==1.1.0
+more-itertools==7.2.0
 openapi-spec-validator==0.2.4
 pathlib==1.0.1
+psycopg2==2.8.3
+Pygments==2.4.2
+pytz==2019.2
 PyYAML==4.2b4
+redis==3.3.8
 requests==2.21.0
 six==1.12.0
 SQLAlchemy==1.2.17
 swagger-ui-bundle==0.0.3
 urllib3==1.24.1
+vine==1.3.0
 Werkzeug==0.14.1
-psycopg2==2.8.3
-flask-cors==3.0.8
+zipp==0.6.0