Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: update build process and ssh in bench.py #1204

Closed
wants to merge 2 commits into from
Closed

*: update build process and ssh in bench.py #1204

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
55e6987
update build process an ssh in bench.py
martingrambow Oct 25, 2019
3270cc7
fixed indentation and updated default ami values
martingrambow Oct 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 32 additions & 28 deletions bench/bench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def ssh_connect_with_retries(host, retries=3, timeout=30):
try:
ssh_client = paramiko.client.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.client.WarningPolicy())
ssh_client.connect(host, username='ubuntu', timeout=timeout)
ssh_client.connect(host, username='ubuntu', timeout=timeout, key_filename=(tornado.options.options.ssh_key_name))
return ssh_client
except (socket.error, paramiko.ssh_exception.SSHException):
if i == retries - 1:
Expand Down Expand Up @@ -55,6 +55,9 @@ def ssh_cmd(ssh_client, cmd, timeout=2):
time.sleep(0.1)

if exit_status != 0:
logging.warning('Got an error code (%i), cmd was %s', exit_status, cmd)
logging.warning('stdout: %s', stdout)
logging.warning('stderr: %s', stderr)
raise Exception('%r' % stderr)

return stdout, stderr
Expand All @@ -76,20 +79,17 @@ def _bootstrap(addr):
'sudo -S tar -C /usr/local -xzf go%s.linux-amd64.tar.gz' % golang_version,
'sudo -S apt-get update',
'sudo -S apt-get -y install git mercurial',
'sudo -S apt-get -y install golang',
'curl https://raw.githubusercontent.com/golang/dep/master/install.sh > installdep.sh',
'chmod +x installdep.sh',
'sudo GOPATH=/usr/local/go ./installdep.sh',
'mkdir -p go/src/github.com/nsqio',
'cd go/src/github.com/nsqio && git clone https://github.com/nsqio/nsq',
'cd go/src/github.com/nsqio/nsq && git checkout %s' % commit,
'sudo -S curl -s -o /usr/local/bin/gpm \
https://raw.githubusercontent.com/pote/gpm/v1.2.3/bin/gpm',
'sudo -S chmod +x /usr/local/bin/gpm',
'cd go/src/github.com/nsqio/nsq && \
GOPATH=/home/ubuntu/go PATH=$PATH:/usr/local/go/bin gpm install',
'cd go/src/github.com/nsqio/nsq/apps/nsqd && \
GOPATH=/home/ubuntu/go /usr/local/go/bin/go build',
'cd go/src/github.com/nsqio/nsq/bench/bench_writer && \
GOPATH=/home/ubuntu/go /usr/local/go/bin/go build',
'cd go/src/github.com/nsqio/nsq/bench/bench_reader && \
GOPATH=/home/ubuntu/go /usr/local/go/bin/go build',
'cd go/src/github.com/nsqio/nsq && GOPATH=/home/ubuntu/go /usr/local/go/bin/dep ensure',
'cd go/src/github.com/nsqio/nsq && make',
'cd go/src/github.com/nsqio/nsq/bench/bench_writer && GOPATH=/home/ubuntu/go /usr/local/go/bin/go build',
'cd go/src/github.com/nsqio/nsq/bench/bench_reader && GOPATH=/home/ubuntu/go /usr/local/go/bin/go build',
martingrambow marked this conversation as resolved.
Show resolved Hide resolved
'sudo -S mkdir -p /mnt/nsq',
'sudo -S chmod 777 /mnt/nsq']:
ssh_cmd(ssh_client, cmd, timeout=10)
Expand Down Expand Up @@ -147,11 +147,12 @@ def run():
for cmd in [
'sudo -S pkill -f nsqd',
'sudo -S rm -f /mnt/nsq/*.dat',
'GOMAXPROCS=32 ./go/src/github.com/nsqio/nsq/apps/nsqd/nsqd \
--data-path=/mnt/nsq --mem-queue-size=10000000 --max-rdy-count=%s' % (
tornado.options.options.rdy
)]:
nsqd_chans.append((ssh_client, ssh_cmd_async(ssh_client, cmd)))
'echo starting nsq &>> nsq.log',
'GOMAXPROCS=32 screen -mdS "nsq" ./go/src/github.com/nsqio/nsq/build/nsqd \
--data-path=/mnt/nsq \
--mem-queue-size=10000000 \
--max-rdy-count=%s &>> nsq.log' % (tornado.options.options.rdy)]:
nsqd_chans.append((ssh_client, ssh_cmd(ssh_client, cmd)))
except Exception:
logging.exception('failed')

Expand Down Expand Up @@ -250,7 +251,8 @@ def run():
max_duration / total_ops * 1000 * 1000)

for ssh_client, chan in nsqd_chans:
chan.close()
if hasattr(chan, 'close'):
chan.close()


def _find_hosts():
Expand Down Expand Up @@ -287,11 +289,11 @@ def decomm():
help='AWS account access key')
tornado.options.define('secret_key', type=str, default='',
help='AWS account secret key')
tornado.options.define('ami', type=str, default='ami-48fd2120',
tornado.options.define('ami', type=str, default='ami-02df9ea15c1778c9c',
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This ami does not seem to exist in the default region, us-east-1 (and I also checked a couple of other common regions and could not find it). Can you find an ami from the default region?

$ aws --region us-east-1 ec2 describe-images --image-ids ami-48fd2120
{
    "Images": [
        {
            "Architecture": "x86_64",
            "CreationDate": "2014-08-16T21:40:48.000Z",
            "ImageId": "ami-48fd2120",
            "ImageLocation": "ubuntu-us-east-1/images/hvm-instance/ubuntu-trusty-14.04-amd64-server-20140816.manifest.xml",
            "ImageType": "machine",
            "Public": true,
            "OwnerId": "099720109477",
            "State": "available",
            "BlockDeviceMappings": [],
            "Hypervisor": "xen",
            "Name": "ubuntu/images/hvm-instance/ubuntu-trusty-14.04-amd64-server-20140816",
            "RootDeviceType": "instance-store",
            "SriovNetSupport": "simple",
            "VirtualizationType": "hvm"
        }
    ]
}
$ aws --region us-east-1 ec2 describe-images --image-ids ami-02df9ea15c1778c9c

An error occurred (InvalidAMIID.NotFound) when calling the DescribeImages operation: The image id '[ami-02df9ea15c1778c9c]' does not exist

(I guess we should update to ubuntu-16.04 or ubuntu-18.04, but you probably did that, I just can't tell because I can't find the ami ...)

help='AMI ID')
tornado.options.define('ssh_key_name', type=str, default='default',
help='SSH key name')
tornado.options.define('instance_type', type=str, default='c3.2xlarge',
tornado.options.define('instance_type', type=str, default='t2.medium',
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think c4.xlarge or c5.xlarge would be more in the spirit of the previous default. A bit expensive, but they'll run for less than an hour, and t2 or t3 instances are probably not good for benchmarking because they are more oversubscribed and throttled.

help='EC2 instance type')
tornado.options.define('msg_size', type=int, default=200,
help='size of message')
Expand All @@ -301,19 +303,21 @@ def decomm():
help='the benchmark mode (pub, pubsub)')
tornado.options.define('commit', type=str, default='master',
help='the git commit')
tornado.options.define('golang_version', type=str, default='1.5.1',
tornado.options.define('golang_version', type=str, default='1.10.8',
help='the go version')
tornado.options.parse_command_line()

logging.getLogger('paramiko').setLevel(logging.WARNING)
warnings.simplefilter('ignore')

cmd_name = sys.argv[-1]
cmd_map = {
'bootstrap': bootstrap,
'run': run,
'decomm': decomm
}
cmd = cmd_map.get(cmd_name, bootstrap)

logging.info('CMD: %s', cmd_name)
cmd = bootstrap
if (cmd_name == 'bootstrap'):
cmd = bootstrap
if (cmd_name == 'run'):
cmd = run
if (cmd_name == 'decomm'):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This part did not have to change. The restructure is OK with me, except that using parentheses like this is odd and unusual for python (they're not needed).

cmd = decomm
logging.info('Running %s', cmd)
sys.exit(cmd())