added intallation and updating of kraken agent

Rakefile

@@ -6,6 +6,9 @@ NG = File.expand_path('webui/node_modules/.bin/ng')
 SWAGGER_CODEGEN = "#{TOOLS_DIR}/swagger-codegen-cli-2.4.8.jar"
 SWAGGER_FILE = File.expand_path("server/kraken/server/swagger.yml")
 
+kk_ver = ENV['kk_ver'] || '0.0'
+ENV['KRAKEN_VERSION'] = kk_ver
+
 # prepare env
 task :prepare_env do
   sh 'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y default-jre python3-venv npm libpq-dev libpython3.7-dev'
@@ -39,7 +42,6 @@ file NG => NPX do
 end
 
 task :build_ui => [NG, :gen_client] do
-  kk_ver = ENV['kk_ver'] || '0.0'
   Dir.chdir('ui') do
     sh "sed -e 's/0\.0/#{kk_ver}/g' src/environments/environment.prod.ts.in > src/environments/environment.prod.ts"
     sh 'npx ng build --prod'
@@ -108,6 +110,7 @@ end
 
 file './agent/venv/bin/kkagent' => './agent/venv/bin/python3' do
   Dir.chdir('agent') do
+    sh './venv/bin/pip install -r requirements.txt'
     sh './venv/bin/python3 setup.py develop --upgrade'
   end
 end
@@ -116,17 +119,61 @@ task :run_agent => './agent/venv/bin/kkagent' do
   sh 'cp server/kraken/server/consts.py agent/kraken/agent/'
   sh 'cp server/kraken/server/logs.py agent/kraken/agent/'
   sh 'rm -rf /tmp/kk-jobs/'
-  sh 'bash -c "source ./agent/venv/bin/activate && kkagent -d /tmp/kk-jobs -s http://localhost:8080/backend"'
+  sh 'bash -c "source ./agent/venv/bin/activate && kkagent -d /tmp/kk-jobs -s http://localhost:8080 run"'
 end
 
-task :run_agent_container do
+task :run_agent_in_docker do
   Rake::Task["build_agent"].invoke
   Dir.chdir('agent') do
     sh 'docker build -f docker-agent.txt -t kkagent .'
   end
   sh 'docker run --rm -ti  -v /var/run/docker.sock:/var/run/docker.sock -v /var/snap/lxd/common/lxd/unix.socket:/var/snap/lxd/common/lxd/unix.socket -v `pwd`/agent:/agent -e KRAKEN_AGENT_SLOT=7 -e KRAKEN_SERVER_ADDR=192.168.0.89:8080  kkagent'
 end
 
+task :run_agent_in_lxd do
+#  Rake::Task["build_agent"].invoke
+  Dir.chdir('agent') do
+    systems = [
+      ['ubuntu:20.04', 'u20'],
+      ['images:fedora/32/amd64', 'f32'],
+#      ['images:centos/7/amd64', 'c7'],
+      ['images:centos/8/amd64', 'c8'],
+      ['images:debian/buster/amd64', 'd10'],
+      ['images:debian/bullseye/amd64', 'd11'],
+      ['images:opensuse/15.2/amd64', 's15'],
+    ]
+
+    sh 'lxc network delete kk-net || true'
+    sh 'lxc network create kk-net || true'
+    systems.each do |sys, name|
+      cntr_name = "kk-agent-#{name}"
+      sh "lxc stop #{cntr_name} || true"
+      sh "lxc delete #{cntr_name} || true"
+      sh "lxc launch #{sys} #{cntr_name}"
+      sh "lxc network attach kk-net #{cntr_name}"
+      sh "lxc exec #{cntr_name} -- sleep 5"
+      if sys.include?('centos/7') or sys.include?('debian/buster')
+        sh "lxc exec #{cntr_name} -- dhclient"
+      end
+      if sys.include?('centos')
+        sh "lxc exec #{cntr_name} -- yum install -y python3 sudo"
+      end
+      if sys.include?('debian')
+        sh "lxc exec #{cntr_name} -- apt-get update"
+        sh "lxc exec #{cntr_name} -- apt-get install -y curl python3 sudo"
+      end
+      if sys.include?('opensuse/15.2')
+        sh "lxc exec #{cntr_name} -- zypper install -y curl python3 sudo system-group-wheel"
+      end
+      sh "lxc exec #{cntr_name} -- curl -o agent http://192.168.0.89:8080/install/agent"
+      sh "lxc exec #{cntr_name} -- chmod a+x agent"
+      sh "lxc exec #{cntr_name} -- ./agent -s http://192.168.0.89:8080 install"
+      sh "lxc exec #{cntr_name} -- journalctl -u kraken-agent.service"
+      #sh "lxc exec #{cntr_name} -- journalctl -f -u kraken-agent.service'
+    end
+  end
+end
+
 task :run_celery => './server/venv/bin/kkcelery' do
   sh './server/venv/bin/kkcelery'
 end
@@ -239,14 +286,16 @@ task :build_docker_deploy do
 end
 
 task :docker_release do
-  kk_ver = ENV['kk_ver']
+  # for lab.kraken.ci
   sh "docker-compose -f docker-compose-swarm.yaml config > kraken-docker-stack-#{kk_ver}.yaml"
   sh "sed -i -e s/kk_ver/#{kk_ver}/g kraken-docker-stack-#{kk_ver}.yaml"
-  sh "docker-compose -f docker-compose.yaml config > docker-compose-#{kk_ver}.yaml"
-  sh "sed -i -e s/kk_ver/#{kk_ver}/g docker-compose-#{kk_ver}.yaml"
-  sh "sed -i -e 's#127.0.0.1:5000#eu.gcr.io/kraken-261806#g' docker-compose-#{kk_ver}.yaml"
-  sh "docker-compose -f docker-compose-#{kk_ver}.yaml build --build-arg kkver=#{kk_ver}"
-  sh "docker-compose -f docker-compose-#{kk_ver}.yaml push"
+  # for installing under the desk and for pushing images to docker images repository
+  sh "docker-compose -f docker-compose.yaml config > kraken-docker-compose-#{kk_ver}.yaml"
+  sh "sed -i -e s/kk_ver/#{kk_ver}/g kraken-docker-compose-#{kk_ver}.yaml"
+  sh "sed -i -e 's#127.0.0.1:5000#eu.gcr.io/kraken-261806#g' kraken-docker-compose-#{kk_ver}.yaml"
+  sh "cp agent/kkagent agent/kktool server/"
+  sh "docker-compose -f kraken-docker-compose-#{kk_ver}.yaml build --force-rm --no-cache --pull --build-arg kkver=#{kk_ver}"
+  sh "docker-compose -f kraken-docker-compose-#{kk_ver}.yaml push"
 end
 
 task :prepare_swarm do
@@ -266,12 +315,11 @@ task :run_portainer do
 end
 
 task :deploy_lab do
-  kk_ver = ENV['kk_ver']
   sh "./venv/bin/fab -e -H lab.kraken.ci upgrade --kk-ver #{kk_ver}"
 end
 
 task :release_deploy do
   Rake::Task["build_all"].invoke
   Rake::Task["docker_release"].invoke
-  Rake::Task["deploy_lab"].invoke
+#  Rake::Task["deploy_lab"].invoke
 end

agent/docker-agent.txt

@@ -3,6 +3,6 @@ WORKDIR /agent
 RUN apt-get update && apt-get install -y --no-install-recommends python3 python3-pytest git openssh-client ca-certificates sudo python3-docker
 COPY kkagent /agent
 COPY kktool /agent
-CMD ./kkagent -s http://${KRAKEN_SERVER_ADDR}/backend -d /tmp/kk
+CMD ./kkagent -s http://${KRAKEN_SERVER_ADDR}/backend -d /tmp/kk run
 ARG kkver=0.0
-LABEL kkver=${kkver} kkname=server
+LABEL kkver=${kkver} kkname=agent

agent/kraken/agent/agent.py

@@ -1,28 +1,35 @@
 #!/usr/bin/env python3
 
 import os
+import sys
 import time
 import logging
 import argparse
 import platform
 import traceback
 
+import pkg_resources
+
 from . import logs
+from . import consts
 from . import config
 from . import server
 from . import jobber
+from . import update
+from . import install
 
 log = logging.getLogger('agent')
 
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Kraken Agent')
-    parser.add_argument('-s', '--server', help='Server URL', required=True)
-    parser.add_argument('-d', '--data-dir', help='Directory for presistent data', required=True)
+    parser.add_argument('-s', '--server', help='Server URL')
+    parser.add_argument('-d', '--data-dir', help='Directory for presistent data')
     parser.add_argument('-t', '--tools-dirs', help='List of tools directories')
+    parser.add_argument('command', help="A command to execute")
 
     args = parser.parse_args()
-    return args
+    return parser, args
 
 
 def dispatch_job(srv, job):
@@ -51,21 +58,50 @@ def collect_sys_info():
     sys_info = {}
     s = platform.system().lower()
     sys_info['system'] = s
-    if s == 'linux':
-        distr = platform.linux_distribution(full_distribution_name=False)  # pylint: disable=deprecated-method
-        sys_info['distro_name'] = distr[0].lower()
-        sys_info['distro_version'] = distr[1]
+    # if s == 'linux':
+    #     distr = platform.linux_distribution(full_distribution_name=False)  # pylint: disable=deprecated-method
+    #     sys_info['distro_name'] = distr[0].lower()
+    #     sys_info['distro_version'] = distr[1]
 
     return sys_info
 
 
+def check_integrity():
+    return True
+
+
 def main():
     logs.setup_logging('agent')
+    kraken_version = pkg_resources.get_distribution('kraken-agent').version
+    log.info('Starting Kraken Agent, version %s', kraken_version)
 
-    args = parse_args()
+    parser, args = parse_args()
     cfg = vars(args)
     config.set_config(cfg)
 
+    # check integrity (used during update)
+    if args.command == 'check-integrity':
+        if check_integrity():
+            sys.exit(0)
+        else:
+            sys.exit(1)
+    # install agent
+    elif args.command == 'install':
+        install.install()
+        sys.exit(0)
+
+    # no specific command so start agent main loop
+
+    if args.server is None:
+        parser.print_usage()
+        print('missing required --server option')
+        sys.exit(1)
+
+    if args.data_dir is None:
+        parser.print_usage()
+        print('missing required --data-dir option')
+        sys.exit(1)
+
     # allow running kktool from current dir in container
     os.environ["PATH"] += os.pathsep + os.getcwd()
     os.environ["PATH"] += os.pathsep + os.path.abspath(__file__)
@@ -84,11 +120,15 @@ def main():
 
     while True:
         try:
-            job, cfg_changes = srv.get_job()
+            job, cfg_changes, version = srv.get_job()
 
             if cfg_changes:
                 apply_cfg_changes(cfg_changes)
 
+            if version and version != kraken_version:
+                log.info('new version: %s, was: %s, updating agent' % (version, kraken_version))
+                update.update_agent(version)
+
             if job:
                 log.set_ctx(job=job['id'], run=job['run_id'])
             log.info('received job: %s', str(job)[:200])

agent/kraken/agent/install.py

@@ -0,0 +1,83 @@
+import tempfile
+import platform
+import subprocess
+from pathlib import Path
+
+import distro
+import pkg_resources
+
+from . import config
+from . import update
+from . import consts
+
+SYSTEMD_SERVICE = '''[Unit]
+Description=Kraken Agent
+Wants=network-online.target
+After=network-online.target
+After=time-sync.target
+
+[Service]
+User=kraken
+ExecStart=/opt/kraken/kkagent -s %s -d %s run
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target
+'''
+
+def run(cmd):
+    subprocess.run(cmd, shell=True, check=True)
+
+
+def install_linux():
+    dstr = distro.id()
+
+    if dstr == 'ubuntu':
+        # mute problems with sudo
+        run('sudo bash -c \'echo "Set disable_coredump false" >> /etc/sudo.conf\'')
+
+    # create kraken user
+    if dstr in ['ubuntu', 'debian']:
+        run('sudo useradd kraken -d %s -m -s /bin/bash -G sudo' % update.AGENT_DIR)
+    elif dstr in ['fedora', 'centos']:
+        run('sudo useradd kraken -d %s -m -s /bin/bash -G wheel --system' % update.AGENT_DIR)
+    elif 'suse' in dstr:
+        run('sudo useradd kraken -d %s -m -s /bin/bash -G wheel --system -U' % update.AGENT_DIR)
+    else:
+        raise Exception('distro %s is not supported yet' % dstr)
+
+    # install bin files
+    kraken_version = pkg_resources.get_distribution('kraken-agent').version
+    dest_dir = update.get_dest_dir(kraken_version)
+    if dest_dir.exists():
+        run('sudo rm -rf %s' % dest_dir)
+    run('sudo mkdir -p %s' % dest_dir)
+    data_dir = Path(update.AGENT_DIR) / 'data'
+    run('sudo mkdir -p %s' % data_dir)
+    run('sudo chown kraken:kraken %s' % dest_dir)
+    tmp_dir = Path(tempfile.gettempdir())
+    agent_path, tool_path = update.get_blobs(tmp_dir)
+    run('sudo mv %s %s %s' % (agent_path, tool_path, dest_dir))
+    run('sudo ln -s %s/kkagent /opt/kraken/kkagent' % dest_dir)
+    run('sudo ln -s %s/kktool /opt/kraken/kktool' % dest_dir)
+    run('sudo chown kraken:kraken %s/* /opt/kraken/*' % dest_dir)
+
+    # setup kraken agent service in systemd
+    svc = SYSTEMD_SERVICE % (config.get('server'), data_dir)
+    sysd_dest = '/lib/systemd/system'
+    if 'suse' in dstr:
+        sysd_dest = '/usr/lib/systemd/system'
+    run('sudo bash -c \'echo "%s" > %s/kraken-agent.service\'' % (svc, sysd_dest))
+    run('sudo systemctl daemon-reload')
+    run('sudo systemctl enable kraken-agent.service')
+    run('sudo systemctl start kraken-agent.service')
+    run('sudo systemctl status kraken-agent.service')
+
+
+def install():
+    s = platform.system()
+    if s == 'Linux':
+        install_linux()
+    else:
+        raise Exception('system %s is not supported yet' % s)

agent/kraken/agent/jobber.py

@@ -36,7 +36,7 @@ def _load_tools_list():
     for entry_point in pkg_resources.iter_entry_points('kraken.tools'):
         entry_point.load()
         # log.info("TOOL %s: %s", entry_point.name, entry_point.module_name)
-        tools[entry_point.name] = 'kktool -m %s' % entry_point.module_name
+        tools[entry_point.name] = '/opt/kraken/kktool -m %s' % entry_point.module_name
 
     return tools
 
@@ -145,9 +145,9 @@ async def _async_exec_tool(exec_ctx, proc_coord, tool_path, command, cwd, timeou
     return_addr = "%s:%s" % addr
     log.info('return_addr %s', return_addr)
 
-    subprocess_task = asyncio.create_task(exec_ctx.async_run(
+    subprocess_task = asyncio.ensure_future(exec_ctx.async_run(
         proc_coord, tool_path, return_addr, step_file_path, command, cwd, timeout))
-    tcp_server_task = asyncio.create_task(_async_tcp_server(server))
+    tcp_server_task = asyncio.ensure_future(_async_tcp_server(server))
     done, _ = await asyncio.wait([subprocess_task, tcp_server_task],
                                  return_when=asyncio.FIRST_COMPLETED)
     if tcp_server_task not in done:
@@ -163,7 +163,13 @@ def _exec_tool(kk_srv, exec_ctx, tool_path, command, cwd, timeout, step_file_pat
         tool_path = "%s %s" % (sys.executable, tool_path)
 
     proc_coord = ProcCoord(kk_srv, command, job_id, idx)
-    asyncio.run(_async_exec_tool(exec_ctx, proc_coord, tool_path, command, cwd, timeout, step_file_path))
+    f = _async_exec_tool(exec_ctx, proc_coord, tool_path, command, cwd, timeout, step_file_path)
+    if hasattr(asyncio, 'run'):
+        asyncio.run(f)  # this is available since Python 3.7
+    else:
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(f)
+        loop.close()
     return proc_coord.result
 
 

agent/kraken/agent/local_run.py

@@ -1,8 +1,8 @@
 import asyncio
 import logging
 import datetime
-import netifaces
 
+from . import sysutils
 
 log = logging.getLogger(__name__)
 
@@ -22,16 +22,11 @@ def stop(self):
         pass
 
     def get_return_ip_addr(self):  # pylint: disable=no-self-use
-        for iface in netifaces.interfaces():
+        for iface, ip_addr in sysutils.get_ifaces():
+            log.info('adr: %s', ip_addr)
             if iface == 'lo':
                 continue
-            addrs = netifaces.ifaddresses(iface)
-            if netifaces.AF_INET not in addrs:
-                continue
-            addrs = addrs[netifaces.AF_INET]
-            if len(addrs) == 0:
-                continue
-            return addrs[0]['addr']
+            return ip_addr
         return '0.0.0.0'
 
     async def async_run(self, proc_coord, tool_path, return_addr, step_file_path, command, cwd, timeout):